mirror of
https://github.com/discourse/discourse.git
synced 2026-08-08 17:53:55 +08:00
`UserVisit.count_by_active_users` computed the 30-day rolling MAU with a
correlated subquery that re-scanned user_visits once per day in the
range (O(days * window)). So a one-year range this spends several
seconds of SQL time on large sites. 💥
The PR avoids repeatedly recalculating overlapping 30-day MAU windows.
On meta, the 30-day dashboard range improved from `0.173s` to `0.044s`,
and the 365-day report range improved from `1.312s` to `0.296s`, while
preserving identical output. (details in table)
benchmark run manually, md table generated by AI.
| Range | Previous query | PR query | Improvement |
| --- | ---: | ---: | ---: |
| 7 days | 0.113s | 0.025s | 78% faster |
| 14 days | 0.125s | 0.030s | 76% faster |
| 30 days | 0.173s | 0.044s | 75% faster |
| 45 days | 0.209s | 0.057s | 73% faster |
| 60 days | 0.260s | 0.068s | 74% faster |
| 75 days | 0.311s | 0.084s | 73% faster |
| 90 days | 0.360s | 0.097s | 73% faster |
| 180 days | 0.658s | 0.174s | 74% faster |
| 365 days | 1.312s | 0.296s | 77% faster |
141 lines
4.4 KiB
Ruby
Vendored
141 lines
4.4 KiB
Ruby
Vendored
# frozen_string_literal: true
|
|
|
|
class UserVisit < ActiveRecord::Base
|
|
belongs_to :user
|
|
|
|
def self.counts_by_day_query(start_date, end_date, group_id = nil)
|
|
result = where("visited_at >= ? and visited_at <= ?", start_date.to_date, end_date.to_date)
|
|
|
|
if group_id
|
|
result = result.joins("INNER JOIN users ON users.id = user_visits.user_id")
|
|
result = result.joins("INNER JOIN group_users ON group_users.user_id = users.id")
|
|
result = result.where("group_users.group_id = ?", group_id)
|
|
end
|
|
result.group(:visited_at).order(:visited_at)
|
|
end
|
|
|
|
def self.count_by_active_users(start_date, end_date)
|
|
sql = <<~SQL
|
|
WITH visits_in_mau_range AS (
|
|
SELECT user_id, visited_at AS d
|
|
FROM user_visits
|
|
WHERE visited_at >= :start_date::DATE - 29
|
|
AND visited_at <= :end_date::DATE
|
|
),
|
|
visit_island_starts AS (
|
|
SELECT user_id, d,
|
|
CASE
|
|
WHEN lag(d) OVER (PARTITION BY user_id ORDER BY d) >= d - 30 THEN 0
|
|
ELSE 1
|
|
END AS new_island
|
|
FROM visits_in_mau_range
|
|
),
|
|
visit_islands AS (
|
|
SELECT user_id, d, sum(new_island) OVER (PARTITION BY user_id ORDER BY d) AS grp
|
|
FROM visit_island_starts
|
|
),
|
|
active_ranges AS (
|
|
SELECT min(d) AS start_d, max(d) + 29 AS end_d
|
|
FROM visit_islands
|
|
GROUP BY user_id, grp
|
|
),
|
|
active_range_events AS (
|
|
SELECT start_d AS day, 1 AS delta FROM active_ranges
|
|
UNION ALL
|
|
SELECT end_d + 1 AS day, -1 AS delta FROM active_ranges
|
|
),
|
|
daily_active_deltas AS (
|
|
SELECT day, sum(delta) AS delta
|
|
FROM active_range_events
|
|
GROUP BY day
|
|
),
|
|
days AS (
|
|
SELECT generate_series(
|
|
(SELECT min(day) FROM daily_active_deltas),
|
|
:end_date::DATE,
|
|
INTERVAL '1 day'
|
|
)::DATE AS day
|
|
),
|
|
rolling_mau AS (
|
|
SELECT days.day,
|
|
sum(coalesce(daily_active_deltas.delta, 0)) OVER (ORDER BY days.day) AS mau
|
|
FROM days
|
|
LEFT JOIN daily_active_deltas ON daily_active_deltas.day = days.day
|
|
),
|
|
dau AS (
|
|
SELECT visited_at AS date, count(*) AS dau
|
|
FROM user_visits
|
|
WHERE visited_at >= :start_date::DATE
|
|
AND visited_at <= :end_date::DATE
|
|
GROUP BY visited_at
|
|
)
|
|
SELECT dau.date, dau.dau, rolling_mau.mau
|
|
FROM dau
|
|
JOIN rolling_mau ON rolling_mau.day = dau.date
|
|
ORDER BY dau.date
|
|
SQL
|
|
|
|
DB.query_hash(sql, start_date: start_date, end_date: end_date)
|
|
end
|
|
|
|
# A count of visits in a date range by day
|
|
def self.by_day(start_date, end_date, group_id = nil)
|
|
counts_by_day_query(start_date, end_date, group_id).count
|
|
end
|
|
|
|
def self.mobile_by_day(start_date, end_date, group_id = nil)
|
|
counts_by_day_query(start_date, end_date, group_id).where(mobile: true).count
|
|
end
|
|
|
|
def self.counts_by_day_and_mobile(start_date, end_date, group_id: nil)
|
|
sql = <<~SQL
|
|
SELECT
|
|
visited_at,
|
|
mobile,
|
|
COUNT(*) AS visit_count,
|
|
SUM(COUNT(*)) OVER () AS total
|
|
FROM user_visits
|
|
#{"INNER JOIN group_users ON group_users.user_id = user_visits.user_id" if group_id}
|
|
WHERE visited_at >= :start_date AND visited_at <= :end_date
|
|
#{"AND group_users.group_id = :group_id" if group_id}
|
|
GROUP BY visited_at, mobile
|
|
ORDER BY visited_at
|
|
SQL
|
|
|
|
params = { start_date: start_date, end_date: end_date, prev_start: start_date - 30.days }
|
|
params[:group_id] = group_id.to_i if group_id
|
|
|
|
DB.query(sql, **params)
|
|
end
|
|
|
|
def self.ensure_consistency!
|
|
DB.exec <<~SQL
|
|
UPDATE user_stats u set days_visited =
|
|
(
|
|
SELECT COUNT(*) FROM user_visits v WHERE v.user_id = u.user_id
|
|
)
|
|
WHERE days_visited <>
|
|
(
|
|
SELECT COUNT(*) FROM user_visits v WHERE v.user_id = u.user_id
|
|
)
|
|
SQL
|
|
end
|
|
end
|
|
|
|
# == Schema Information
|
|
#
|
|
# Table name: user_visits
|
|
#
|
|
# id :integer not null, primary key
|
|
# mobile :boolean default(FALSE)
|
|
# posts_read :integer default(0)
|
|
# time_read :integer default(0), not null
|
|
# visited_at :date not null
|
|
# user_id :integer not null
|
|
#
|
|
# Indexes
|
|
#
|
|
# index_user_visits_on_user_id_and_visited_at (user_id,visited_at) UNIQUE
|
|
# index_user_visits_on_user_id_and_visited_at_and_time_read (user_id,visited_at,time_read)
|
|
# index_user_visits_on_visited_at_and_mobile (visited_at,mobile)
|
|
#
|