discourse/plugins/discourse-ai/spec/jobs/scheduled/summaries_backfill_spec.rb

# frozen_string_literal: true

RSpec.describe Jobs::SummariesBackfill do
  subject(:job) { described_class.new }

  fab!(:topic) do
    Fabricate(:topic, word_count: 200, highest_post_number: 2, last_posted_at: 2.hours.ago)
  end
  let(:limit) { 24 } # guarantee two summaries per batch
  let(:intervals) { 12 } # budget is split into intervals. Job runs every five minutes.

  before do
    enable_current_plugin
    assign_fake_provider_to(:ai_default_llm_model)
    SiteSetting.ai_summarization_enabled = true
    SiteSetting.ai_summary_backfill_maximum_topics_per_hour = limit
    SiteSetting.ai_summary_gists_enabled = true
  end

  def summary_tool_call(summary, id:)
    DiscourseAi::Completions::ToolCall.new(id:, name: "set_topic_summary", parameters: { summary: })
  end

  describe "#current_budget" do
    let(:type) { AiSummary.summary_types[:complete] }

    context "when no summary has been backfilled yet" do
      it "returns the full budget" do
        expect(job.current_budget(type)).to eq(limit / intervals)
      end

      it "ignores summaries generated by users" do
        Fabricate(:ai_summary, target: topic, origin: AiSummary.origins[:human])

        expect(job.current_budget(type)).to eq(limit / intervals)
      end

      it "only accounts for summaries of the given type" do
        Fabricate(:topic_ai_gist, target: topic, origin: AiSummary.origins[:human])

        expect(job.current_budget(type)).to eq(limit / intervals)
      end
    end
  end

  describe "#backfill_candidates" do
    let(:type) { AiSummary.summary_types[:complete] }

    it "only selects posts with enough words" do
      topic.update!(word_count: 100)

      expect(job.backfill_candidates(type, locale: :source)).to be_empty
    end

    it "ignores up to date summaries" do
      Fabricate(
        :ai_summary,
        target: topic,
        locale: SiteSetting.default_locale,
        highest_target_number: 2,
        updated_at: 10.minutes.ago,
      )

      expect(job.backfill_candidates(type, locale: :source)).to be_empty
    end

    it "ignores outdated summaries updated less than five minutes ago" do
      Fabricate(
        :ai_summary,
        target: topic,
        locale: SiteSetting.default_locale,
        highest_target_number: 1,
        updated_at: 4.minutes.ago,
      )

      expect(job.backfill_candidates(type, locale: :source)).to be_empty
    end

    it "orders candidates by topic#last_posted_at" do
      topic.update!(last_posted_at: 1.minute.ago)
      topic_2 = Fabricate(:topic, word_count: 200, last_posted_at: 2.minutes.ago)

      expect(job.backfill_candidates(type, locale: :source).map(&:id)).to contain_exactly(
        topic.id,
        topic_2.id,
      )
    end

    it "prioritizes topics without summaries" do
      topic_2 =
        Fabricate(:topic, word_count: 200, last_posted_at: 2.minutes.ago, highest_post_number: 1)
      topic.update!(last_posted_at: 1.minute.ago)
      Fabricate(
        :ai_summary,
        target: topic,
        locale: SiteSetting.default_locale,
        updated_at: 1.hour.ago,
        highest_target_number: 1,
      )

      expect(job.backfill_candidates(type, locale: :source).map(&:id)).to contain_exactly(
        topic_2.id,
        topic.id,
      )
    end

    it "respects max age setting" do
      SiteSetting.ai_summary_backfill_topic_max_age_days = 1
      topic.update!(last_posted_at: 2.days.ago)

      expect(job.backfill_candidates(type, locale: :source)).to be_empty
    end

    it "selects missing gist locales independently" do
      gist_type = AiSummary.summary_types[:gist]
      Fabricate(:topic_ai_gist, target: topic, locale: "en", highest_target_number: 2)

      expect(job.backfill_candidates(gist_type, locale: "en")).to be_empty
      expect(job.backfill_candidates(gist_type, locale: "ja").map(&:id)).to contain_exactly(
        topic.id,
      )
    end

    it "selects a missing complete source locale independently" do
      topic.update!(locale: "en")
      Fabricate(:ai_summary, target: topic, locale: "he", highest_target_number: 2)

      expect(job.backfill_candidates(type, locale: :source).map(&:id)).to contain_exactly(topic.id)

      Fabricate(:ai_summary, target: topic, locale: "en", highest_target_number: 2)
      expect(job.backfill_candidates(type, locale: :source)).to be_empty
    end

    it "treats configured regional locales as the same base language" do
      gist_type = AiSummary.summary_types[:gist]
      Fabricate(:topic_ai_gist, target: topic, locale: "pt", highest_target_number: 2)

      expect(job.backfill_candidates(gist_type, locale: "pt_BR")).to be_empty
    end

    it "matches source locales by base language and case" do
      gist_type = AiSummary.summary_types[:gist]
      topic.update!(locale: "EN-gb")
      Fabricate(:topic_ai_gist, target: topic, locale: "en", highest_target_number: 2)

      expect(job.backfill_candidates(gist_type, locale: :source)).to be_empty
    end
  end

  describe "#execute" do
    it "backfills a batch" do
      topic_2 =
        Fabricate(:topic, word_count: 200, last_posted_at: 2.minutes.ago, highest_post_number: 1)
      topic.update!(last_posted_at: 1.minute.ago)
      Fabricate(
        :ai_summary,
        target: topic,
        locale: SiteSetting.default_locale,
        updated_at: 3.hours.ago,
        highest_target_number: 1,
      )
      Fabricate(
        :topic_ai_gist,
        target: topic,
        locale: "en",
        updated_at: 3.hours.ago,
        highest_target_number: 1,
      )

      summary_1 = "Summary of topic_2"
      gist_1 = "Gist of topic_2"
      summary_2 = "Updated summary of topic"
      gist_2 = "Updated gist of topic"

      DiscourseAi::Completions::Llm.with_prepared_responses(
        [
          summary_tool_call(gist_1, id: "gist_1"),
          summary_tool_call(gist_2, id: "gist_2"),
          summary_1,
          summary_2,
        ],
      ) { job.execute({}) }

      expect(
        AiSummary
          .complete
          .find_by(target: topic_2, locale: SiteSetting.default_locale)
          .summarized_text,
      ).to eq(summary_1)
      expect(AiSummary.gist.find_by(target: topic_2, locale: "en").summarized_text).to eq(gist_1)
      expect(
        AiSummary
          .complete
          .find_by(target: topic, locale: SiteSetting.default_locale)
          .summarized_text,
      ).to eq(summary_2)
      expect(AiSummary.gist.find_by(target: topic, locale: "en").summarized_text).to eq(gist_2)

      # Queue has to be empty if we just generated all summaries
      expect(
        job.backfill_candidates(AiSummary.summary_types[:complete], locale: :source),
      ).to be_empty
      expect(job.backfill_candidates(AiSummary.summary_types[:gist], locale: "en")).to be_empty

      # Queue still empty when they are up to date and time passes.
      AiSummary.update_all(updated_at: 20.minutes.ago)
      expect(
        job.backfill_candidates(AiSummary.summary_types[:complete], locale: :source),
      ).to be_empty
      expect(job.backfill_candidates(AiSummary.summary_types[:gist], locale: "en")).to be_empty
    end

    it "continues after one gist fails and still backfills complete summaries" do
      successful_topic =
        Fabricate(:topic, word_count: 200, last_posted_at: 2.minutes.ago, highest_post_number: 1)
      topic.update!(last_posted_at: 1.minute.ago)

      successful_gist = "Successful gist"
      DiscourseAi::Completions::Llm.with_prepared_responses(
        [
          "The model skipped the required tool",
          summary_tool_call(successful_gist, id: "successful_gist"),
          "Complete summary after failure",
          "Second complete summary",
        ],
      ) { expect { job.execute({}) }.not_to raise_error }

      expect(AiSummary.gist.find_by(target: topic, locale: "en")).to be_nil
      expect(AiSummary.gist.find_by(target: successful_topic, locale: "en")).to have_attributes(
        summarized_text: successful_gist,
      )
      expect(AiSummary.complete.where(target: [topic, successful_topic]).count).to eq(2)
    end

    it "stops a batch after repeated consecutive failures" do
      3.times do |index|
        Fabricate(
          :topic,
          word_count: 200,
          last_posted_at: (index + 2).minutes.ago,
          highest_post_number: 1,
        )
      end
      SiteSetting.ai_summary_backfill_maximum_topics_per_hour = 48
      48.times do
        Fabricate(:ai_summary, origin: AiSummary.origins[:system], created_at: 1.minute.ago)
      end

      unused_tool_call = summary_tool_call("Should not be generated", id: "unused")
      DiscourseAi::Completions::Llm.with_prepared_responses(
        ["Missing tool 1", "Missing tool 2", "Missing tool 3", unused_tool_call],
      ) do |spy|
        job.execute({})
        expect(spy.completions).to eq(described_class::MAX_CONSECUTIVE_FAILURES)
      end

      expect(AiSummary.gist).to be_empty
    end

    it "updates the highest_target_number if the summary turned to be up to date" do
      SiteSetting.ai_summary_gists_enabled = false
      og_highest_post_number = topic.highest_post_number
      existing_summary =
        Fabricate(
          :ai_summary,
          target: topic,
          locale: SiteSetting.default_locale,
          updated_at: 3.hours.ago,
          highest_target_number: og_highest_post_number,
        )
      topic.update!(highest_post_number: og_highest_post_number + 1)

      # No prepared responses here. We don't perform a completion call.
      job.execute({})

      expect(existing_summary.reload.highest_target_number).to eq(og_highest_post_number + 1)
    end

    it "rotates locale priority between runs" do
      topic.update!(locale: "fr")
      SiteSetting.content_localization_enabled = true
      SiteSetting.content_localization_supported_locales = "en|ja"
      SiteSetting.ai_summary_backfill_maximum_topics_per_hour = 12
      interval = 5.minutes.to_i
      current_slot = Time.zone.now.to_i / interval
      rotation_one = Time.zone.at((current_slot - (current_slot % 3) + 1) * interval)

      freeze_time(rotation_one) do
        DiscourseAi::Completions::Llm.with_prepared_responses(
          [summary_tool_call("Japanese gist", id: "ja_gist"), "Complete summary"],
        ) { job.execute({}) }
      end

      expect(AiSummary.gist.find_by(target: topic)).to have_attributes(locale: "ja")
    end

    it "distributes the gist budget across locales before repeating a locale" do
      second_topic =
        Fabricate(:topic, word_count: 200, last_posted_at: 2.minutes.ago, highest_post_number: 1)
      topic.update!(locale: "fr", last_posted_at: 1.minute.ago)
      second_topic.update!(locale: "fr")
      SiteSetting.content_localization_enabled = true
      SiteSetting.content_localization_supported_locales = "en|ja"
      SiteSetting.ai_summary_backfill_maximum_topics_per_hour = 48
      interval = 5.minutes.to_i
      current_slot = Time.zone.now.to_i / interval
      rotation_zero = Time.zone.at((current_slot - (current_slot % 3)) * interval)

      responses =
        4.times.map { |index| summary_tool_call("Gist #{index}", id: "gist_#{index}") } +
          ["Complete summary 1", "Complete summary 2"]

      freeze_time(rotation_zero) do
        DiscourseAi::Completions::Llm.with_prepared_responses(responses) { job.execute({}) }
      end

      expect(AiSummary.gist.where(target: topic).pluck(:locale)).to contain_exactly(
        "en",
        "ja",
        "fr",
      )
      expect(AiSummary.gist.where(target: second_topic).pluck(:locale)).to eq(["en"])
    end

    it "caches the LlmModel and reuses it for all summaries in a batch" do
      topic_2 =
        Fabricate(:topic, word_count: 200, last_posted_at: 2.minutes.ago, highest_post_number: 1)
      topic.update!(last_posted_at: 1.minute.ago)

      # Track LlmModel.find_by calls
      find_by_call_count = 0
      LlmModel
        .stubs(:find_by)
        .with do
          find_by_call_count += 1
          true
        end
        .returns(LlmModel.last)

      DiscourseAi::Completions::Llm.with_prepared_responses(
        [
          summary_tool_call("gist_1", id: "gist_1"),
          summary_tool_call("gist_2", id: "gist_2"),
          "summary_1",
          "summary_2",
        ],
      ) { job.execute({}) }

      # Should only call LlmModel.find_by once for the entire batch, not once per topic
      expect(find_by_call_count).to eq(1)
    end
  end
end