FEATURE: implement capping of new/unread

We cap new and unread at 2/5th of SiteSetting.max_tracked_new_unread

This dynamic capping is applied under 2 conditions:

1. New capping is applied once every 15 minutes in the periodical job, this effectively ensures that usually even super active sites are capped at 200 new items

2. Unread capping is applied if a user hits max_tracked_new_unread,
  meaning if new + unread == 500, we defer a job that runs within 15 minutes that will cap user at 200 unread

This logic ensures that at worst case a user gets "bad" numbers for 15 minutes and then the system goes ahead and fixes itself up

app/models/topic_tracking_state.rb

@@ -106,11 +106,12 @@ class TopicTrackingState
                   WHEN COALESCE(u.new_topic_duration_minutes, :default_duration) = :always THEN u.created_at
                   WHEN COALESCE(u.new_topic_duration_minutes, :default_duration) = :last_visit THEN COALESCE(u.previous_visit_at,u.created_at)
                   ELSE (:now::timestamp - INTERVAL '1 MINUTE' * COALESCE(u.new_topic_duration_minutes, :default_duration))
-               END, us.new_since)",
+               END, us.new_since, :min_date)",
                 now: DateTime.now,
                 last_visit: User::NewTopicDuration::LAST_VISIT,
                 always: User::NewTopicDuration::ALWAYS,
-                default_duration: SiteSetting.default_other_new_topic_duration_minutes
+                default_duration: SiteSetting.default_other_new_topic_duration_minutes,
+                min_date: Time.at(SiteSetting.min_new_topics_time).to_datetime
               ).where_values[0]
   end
 
@@ -125,19 +126,49 @@ class TopicTrackingState
     # This code needs to be VERY efficient as it is triggered via the message bus and may steal
     #  cycles from usual requests
     #
-
-    unread = TopicQuery.unread_filter(Topic).where_values.join(" AND ")
-    new = TopicQuery.new_filter(Topic, "xxx").where_values.join(" AND ").gsub!("'xxx'", treat_as_new_topic_clause)
+    #
+    sql = report_raw_sql(topic_id: topic_id)
 
     sql = <<SQL
     WITH x AS (
-    SELECT u.id AS user_id,
+      #{sql}
+    ) SELECT * FROM x LIMIT #{SiteSetting.max_tracked_new_unread.to_i}
+SQL
+
+    SqlBuilder.new(sql)
+      .map_exec(TopicTrackingState, user_id: user_id, topic_id: topic_id)
+
+  end
+
+
+  def self.report_raw_sql(opts=nil)
+
+    unread =
+      if opts && opts[:skip_unread]
+        "1=0"
+      else
+        TopicQuery.unread_filter(Topic).where_values.join(" AND ")
+      end
+
+    new =
+      if opts && opts[:skip_new]
+        "1=0"
+      else
+        TopicQuery.new_filter(Topic, "xxx").where_values.join(" AND ").gsub!("'xxx'", treat_as_new_topic_clause)
+      end
+
+    select = (opts && opts[:select]) || "
+           u.id AS user_id,
            topics.id AS topic_id,
            topics.created_at,
            highest_post_number,
            last_read_post_number,
            c.id AS category_id,
-           tu.notification_level
+           tu.notification_level"
+
+
+    sql = <<SQL
+    SELECT #{select}
     FROM topics
     JOIN users u on u.id = :user_id
     JOIN user_stats AS us ON us.user_id = u.id
@@ -162,15 +193,11 @@ class TopicTrackingState
 
 SQL
 
-    if topic_id
+    if opts && opts[:topic_id]
       sql << " AND topics.id = :topic_id"
     end
 
-    sql << " ORDER BY topics.bumped_at DESC ) SELECT * FROM x LIMIT #{SiteSetting.max_tracked_new_unread.to_i}"
-
-    SqlBuilder.new(sql)
-      .map_exec(TopicTrackingState, user_id: user_id, topic_id: topic_id)
-
+    sql << " ORDER BY topics.bumped_at DESC"
   end
 
 end