mirror of
https://github.com/discourse/discourse.git
synced 2026-08-08 17:53:55 +08:00
539eff3f93This PR adds two KPI tiles to the Traffic section — bounce rate and average session duration — measured per browsing session as real engaged time: the seconds a tab is both visible and focused. It builds on the per-session engagement core already captures (`browser_pageview_session_engagements`), so the only new table is its daily summary `browser_pageview_session_engagement_daily_rollups` — additive, no existing schema changes. Key technical changes: 1. Reuse core's capture. Engaged time is already recorded per session by core's `human-activity-tracker` → `/srv/se` beacon → `browser_pageview_session_engagements`; this PR adds no second tracker, endpoint, table, or site setting. 2. Key engagement capture on `persist_browser_pageview_events` alone. `Middleware::RequestTracker#is_engagement_tracking_request?` and the `discourse-engagement-tracking-enabled` meta tag no longer also require the `dashboard_improvements` upcoming change, so engagement is captured wherever pageview events are persisted rather than only where the redesigned dashboard is enabled. 3. Pre-compute in a scheduled job. The existing `Jobs::MaintainBrowserPageviewRollups` job, which already maintains the country and referrer rollups on the same cadence and setting gate, now also rolls sessions into `browser_pageview_session_engagement_daily_rollups`, one row per `(date, logged_in)`, so the dashboard reads this small summary instead of scanning raw events. A session is bounced when it has a single pageview and under ten engaged seconds (no engagement counts as zero). Sessions that started within the last ten minutes are held out of aggregation until a later run, so a live session is not counted as a zero-engagement bounce before the client's first engagement flush (three minutes in) has arrived. Engagement rolls up only from the first engagement row's date forward, because pageview rows predate this feature and have no engagement data — counting them would make all of history look like instant bounces. 4. Show the tiles only when there's data. They render when `persist_browser_pageview_events` is on, alongside the existing Direct traffic KPI, with a neutral placeholder until the first visits are recorded; duration is formatted as `Xm Ys`.
420 lines
15 KiB
Ruby
Vendored
420 lines
15 KiB
Ruby
Vendored
# frozen_string_literal: true
|
|
|
|
RSpec.describe Jobs::MaintainBrowserPageviewRollups do
|
|
subject(:job) { described_class.new }
|
|
|
|
before { SiteSetting.persist_browser_pageview_events = true }
|
|
|
|
describe "#execute" do
|
|
it "does nothing when persist_browser_pageview_events is disabled" do
|
|
SiteSetting.persist_browser_pageview_events = false
|
|
event =
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://reddit.com/",
|
|
country_code: "US",
|
|
)
|
|
|
|
job.execute({})
|
|
|
|
expect(BrowserPageviewCountryDailyRollup.count).to eq(0)
|
|
expect(BrowserPageviewReferrerDailyRollup.count).to eq(0)
|
|
expect(event.reload.normalized_referrer_version).to be_nil
|
|
end
|
|
|
|
context "when aggregating rollups" do
|
|
it "aggregates both country and referrer rollups from recent pageview events" do
|
|
Fabricate(:browser_pageview_event, country_code: "US", normalized_referrer: "google.com")
|
|
|
|
job.execute({})
|
|
|
|
expect(BrowserPageviewCountryDailyRollup.where(country_code: "US").sum(:count)).to eq(1)
|
|
expect(
|
|
BrowserPageviewReferrerDailyRollup.where(normalized_referrer: "google.com").sum(:count),
|
|
).to eq(1)
|
|
end
|
|
|
|
it "uses beacon source for rollups when dashboard_improvements is enabled" do
|
|
SiteSetting.dashboard_improvements = false
|
|
Fabricate(
|
|
:browser_pageview_event,
|
|
country_code: "US",
|
|
normalized_referrer: "google.com",
|
|
source: BrowserPageviewEvent::SOURCE_PIGGYBACK,
|
|
)
|
|
Fabricate(
|
|
:browser_pageview_event,
|
|
country_code: "GB",
|
|
normalized_referrer: "reddit.com",
|
|
source: BrowserPageviewEvent::SOURCE_BEACON,
|
|
)
|
|
|
|
job.execute({})
|
|
|
|
expect(BrowserPageviewCountryDailyRollup.pluck(:country_code, :count)).to eq([["US", 1]])
|
|
expect(BrowserPageviewReferrerDailyRollup.pluck(:normalized_referrer, :count)).to eq(
|
|
[["google.com", 1]],
|
|
)
|
|
|
|
SiteSetting.dashboard_improvements = true
|
|
job.execute({})
|
|
|
|
expect(BrowserPageviewCountryDailyRollup.pluck(:country_code, :count)).to contain_exactly(
|
|
["US", 1],
|
|
["GB", 1],
|
|
)
|
|
expect(
|
|
BrowserPageviewReferrerDailyRollup.pluck(:normalized_referrer, :count),
|
|
).to contain_exactly(["google.com", 1], ["reddit.com", 1])
|
|
end
|
|
|
|
it "backfills from the earliest event date on the first run when rollups are empty" do
|
|
Fabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)
|
|
Fabricate(:browser_pageview_event, country_code: "GB", created_at: 5.days.ago)
|
|
|
|
job.execute({})
|
|
|
|
expect(BrowserPageviewCountryDailyRollup.pluck(:country_code)).to contain_exactly(
|
|
"US",
|
|
"GB",
|
|
)
|
|
end
|
|
|
|
it "only aggregates yesterday and today once historical rollups are populated" do
|
|
Fabricate(:browser_pageview_event, country_code: "US", created_at: 60.days.ago)
|
|
job.execute({}) # first run backfills everything
|
|
|
|
Fabricate(:browser_pageview_event, country_code: "GB", created_at: 60.days.ago) # late old event
|
|
Fabricate(:browser_pageview_event, country_code: "FR") # today
|
|
job.execute({}) # second run only does yesterday + today
|
|
|
|
expect(BrowserPageviewCountryDailyRollup.pluck(:country_code)).to contain_exactly(
|
|
"US",
|
|
"FR",
|
|
)
|
|
end
|
|
end
|
|
|
|
context "when aggregating engagement rollups" do
|
|
before { freeze_time(Time.utc(2026, 6, 20, 12, 0, 0)) }
|
|
|
|
it "does nothing when persist_browser_pageview_events is disabled" do
|
|
SiteSetting.persist_browser_pageview_events = false
|
|
Fabricate(:browser_pageview_session_engagement, created_at: Time.utc(2026, 6, 10))
|
|
Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 10, 9))
|
|
|
|
job.execute({})
|
|
|
|
expect(BrowserPageviewSessionEngagementDailyRollup.count).to eq(0)
|
|
end
|
|
|
|
it "aggregates nothing until the first engagement row exists" do
|
|
Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 10, 9))
|
|
|
|
job.execute({})
|
|
|
|
expect(BrowserPageviewSessionEngagementDailyRollup.count).to eq(0)
|
|
end
|
|
|
|
it "floors aggregation at the earliest engagement row's date" do
|
|
Fabricate(:browser_pageview_session_engagement, created_at: Time.utc(2026, 6, 10, 8))
|
|
Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 9, 9))
|
|
Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 10, 9))
|
|
Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 11, 9))
|
|
|
|
job.execute({})
|
|
|
|
expect(BrowserPageviewSessionEngagementDailyRollup.order(:date).pluck(:date)).to eq(
|
|
[Date.new(2026, 6, 10), Date.new(2026, 6, 11)],
|
|
)
|
|
end
|
|
|
|
it "backfills from the floor forward on the first run" do
|
|
Fabricate(:browser_pageview_session_engagement, created_at: Time.utc(2026, 6, 12, 8))
|
|
Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 12, 9))
|
|
Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 18, 9))
|
|
|
|
job.execute({})
|
|
|
|
expect(BrowserPageviewSessionEngagementDailyRollup.pluck(:date)).to contain_exactly(
|
|
Date.new(2026, 6, 12),
|
|
Date.new(2026, 6, 18),
|
|
)
|
|
end
|
|
|
|
it "re-aggregates days a multi-day failure skipped, not only the previous day" do
|
|
Fabricate(:browser_pageview_session_engagement, created_at: Time.utc(2026, 6, 10, 8))
|
|
Fabricate(:browser_pageview_session_engagement_daily_rollup, date: Date.new(2026, 6, 10))
|
|
Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 15, 9))
|
|
|
|
job.execute({})
|
|
|
|
expect(
|
|
BrowserPageviewSessionEngagementDailyRollup.where(date: Date.new(2026, 6, 15)).sum(
|
|
:sessions,
|
|
),
|
|
).to eq(1)
|
|
end
|
|
|
|
it "does not reach back for a late event on a day behind the last rolled-up day" do
|
|
Fabricate(:browser_pageview_session_engagement, created_at: Time.utc(2026, 6, 18, 8))
|
|
Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 19, 9))
|
|
job.execute({})
|
|
|
|
Fabricate(:browser_pageview_event, created_at: Time.utc(2026, 6, 18, 9))
|
|
job.execute({})
|
|
|
|
expect(BrowserPageviewSessionEngagementDailyRollup.pluck(:date)).to eq(
|
|
[Date.new(2026, 6, 19)],
|
|
)
|
|
end
|
|
end
|
|
|
|
context "when backfilling referrers" do
|
|
it "normalizes historical rows using the inspector and stamps the current version" do
|
|
raw = "https://www.reddit.com/r/discourse/"
|
|
event = Fabricate(:browser_pageview_event_with_unnormalized_referrer, referrer: raw)
|
|
|
|
job.execute({})
|
|
|
|
expect(event.reload.normalized_referrer).to eq(
|
|
BrowserPageviewReferrerInspector.normalize(raw),
|
|
)
|
|
expect(event.normalized_referrer_version).to eq(BrowserPageviewReferrerInspector::VERSION)
|
|
end
|
|
|
|
it "only backfills referrers from the active source" do
|
|
SiteSetting.dashboard_improvements = true
|
|
piggyback_event =
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://www.google.com/",
|
|
source: BrowserPageviewEvent::SOURCE_PIGGYBACK,
|
|
)
|
|
beacon_event =
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://www.reddit.com/",
|
|
source: BrowserPageviewEvent::SOURCE_BEACON,
|
|
)
|
|
|
|
job.execute({})
|
|
|
|
expect(piggyback_event.reload.normalized_referrer_version).to be_nil
|
|
expect(beacon_event.reload.normalized_referrer).to eq("reddit.com")
|
|
expect(beacon_event.normalized_referrer_version).to eq(
|
|
BrowserPageviewReferrerInspector::VERSION,
|
|
)
|
|
end
|
|
|
|
it "leaves rows without a referrer untouched and does not let them block completion" do
|
|
direct_visit = Fabricate(:browser_pageview_event_with_unnormalized_referrer, referrer: nil)
|
|
referred =
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://reddit.com/",
|
|
)
|
|
|
|
job.execute({})
|
|
|
|
expect(direct_visit.reload.normalized_referrer_version).to be_nil
|
|
expect(referred.reload.normalized_referrer_version).to eq(
|
|
BrowserPageviewReferrerInspector::VERSION,
|
|
)
|
|
expect { job.execute({}) }.not_to change { referred.reload.normalized_referrer_version }
|
|
end
|
|
|
|
it "processes an empty-string referrer once, normalizing it to NULL and stamping it" do
|
|
event = Fabricate(:browser_pageview_event_with_unnormalized_referrer, referrer: "")
|
|
|
|
job.execute({})
|
|
|
|
event.reload
|
|
expect(event.normalized_referrer).to be_nil
|
|
expect(event.normalized_referrer_version).to eq(BrowserPageviewReferrerInspector::VERSION)
|
|
end
|
|
|
|
it "is a no-op once every referrer row is current" do
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://reddit.com/",
|
|
)
|
|
job.execute({})
|
|
|
|
expect { job.execute({}) }.not_to change {
|
|
BrowserPageviewReferrerDailyRollup.pluck(:id, :count)
|
|
}
|
|
end
|
|
|
|
it "only processes up to the configured batch size per run" do
|
|
SiteSetting.browser_pageview_referrer_backfill_batch_size = 1
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://reddit.com/",
|
|
)
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://news.ycombinator.com/",
|
|
)
|
|
|
|
job.execute({})
|
|
|
|
expect(BrowserPageviewEvent.where(normalized_referrer_version: nil).count).to eq(1)
|
|
end
|
|
|
|
it "waits to repair a date until every stale referrer row for that date is processed" do
|
|
SiteSetting.browser_pageview_referrer_backfill_batch_size = 1
|
|
date = 3.days.ago.to_date
|
|
2.times do
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://www.google.com/",
|
|
created_at: date,
|
|
)
|
|
end
|
|
|
|
job.execute({})
|
|
|
|
rollups =
|
|
BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count)
|
|
expect(rollups).to eq([[nil, 2]])
|
|
|
|
job.execute({})
|
|
|
|
rollups =
|
|
BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count)
|
|
expect(rollups).to eq([["google.com", 2]])
|
|
end
|
|
|
|
it "skips rows within a day of the retention cutoff when cleanup is enabled" do
|
|
SiteSetting.clean_up_browser_pageview_events = true
|
|
prunable =
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://www.reddit.com/",
|
|
created_at: (BrowserPageviewEvent::RETENTION_PERIOD + 1.day).ago,
|
|
)
|
|
near_cutoff =
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://www.reddit.com/",
|
|
created_at: BrowserPageviewEvent::RETENTION_PERIOD.ago,
|
|
)
|
|
recent =
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://www.reddit.com/",
|
|
)
|
|
|
|
job.execute({})
|
|
|
|
prunable.reload
|
|
expect(prunable.normalized_referrer).to be_nil
|
|
expect(prunable.normalized_referrer_version).to be_nil
|
|
expect(near_cutoff.reload.normalized_referrer_version).to be_nil
|
|
expect(recent.reload.normalized_referrer_version).to eq(
|
|
BrowserPageviewReferrerInspector::VERSION,
|
|
)
|
|
end
|
|
|
|
it "backfills rows older than the retention period when cleanup is disabled" do
|
|
SiteSetting.clean_up_browser_pageview_events = false
|
|
event =
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://www.reddit.com/",
|
|
created_at: (BrowserPageviewEvent::RETENTION_PERIOD + 1.day).ago,
|
|
)
|
|
|
|
job.execute({})
|
|
|
|
expect(event.reload.normalized_referrer_version).to eq(
|
|
BrowserPageviewReferrerInspector::VERSION,
|
|
)
|
|
end
|
|
|
|
it "repairs the affected rollups to match a fresh aggregation, including the NULL bucket" do
|
|
date = 3.days.ago.to_date
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://www.google.com/",
|
|
created_at: date,
|
|
)
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://www.google.com/",
|
|
created_at: date,
|
|
)
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: nil,
|
|
created_at: date,
|
|
)
|
|
|
|
job.execute({})
|
|
|
|
rollups =
|
|
BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count)
|
|
expect(rollups).to contain_exactly(["google.com", 2], [nil, 1])
|
|
end
|
|
|
|
it "re-selects the row and repairs the rollup when a crash happens before the version is stamped" do
|
|
date = 3.days.ago.to_date
|
|
event =
|
|
Fabricate(
|
|
:browser_pageview_event_with_unnormalized_referrer,
|
|
referrer: "https://www.google.com/",
|
|
created_at: date,
|
|
)
|
|
|
|
BrowserPageviewReferrerDailyRollup.stubs(:recompute).raises("rollup boom")
|
|
expect { job.execute({}) }.to raise_error("rollup boom")
|
|
|
|
expect(event.reload.normalized_referrer).to eq("google.com")
|
|
expect(event.normalized_referrer_version).to be_nil
|
|
expect(
|
|
BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count),
|
|
).to eq([[nil, 1]])
|
|
|
|
BrowserPageviewReferrerDailyRollup.unstub(:recompute)
|
|
job.execute({})
|
|
|
|
expect(event.reload.normalized_referrer_version).to eq(
|
|
BrowserPageviewReferrerInspector::VERSION,
|
|
)
|
|
expect(
|
|
BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer, :count),
|
|
).to eq([["google.com", 1]])
|
|
end
|
|
|
|
it "re-normalizes rows stamped with an older version and repairs their rollups" do
|
|
date = 3.days.ago.to_date
|
|
raw = "https://google.com/?utm_source=newsletter"
|
|
event =
|
|
Fabricate(
|
|
:browser_pageview_event,
|
|
referrer: raw,
|
|
normalized_referrer: "google.com/?utm_source=newsletter",
|
|
normalized_referrer_version: BrowserPageviewReferrerInspector::VERSION,
|
|
created_at: date,
|
|
)
|
|
|
|
stub_const(
|
|
BrowserPageviewReferrerInspector,
|
|
"VERSION",
|
|
BrowserPageviewReferrerInspector::VERSION + 1,
|
|
) do
|
|
job.execute({})
|
|
|
|
event.reload
|
|
expect(event.normalized_referrer).to eq(BrowserPageviewReferrerInspector.normalize(raw))
|
|
expect(event.normalized_referrer_version).to eq(BrowserPageviewReferrerInspector::VERSION)
|
|
end
|
|
|
|
expect(BrowserPageviewReferrerDailyRollup.where(date:).pluck(:normalized_referrer)).to eq(
|
|
["google.com"],
|
|
)
|
|
end
|
|
end
|
|
end
|
|
end
|
