mirror of
https://github.com/discourse/discourse.git
synced 2026-08-09 21:45:25 +08:00
3a647c8e50Implemented an eval “comparison matrix” that lets you run the same evals across multiple personas or multiple LLMs and have a judge model declare a winner with per-candidate scores. The CLI adds --compare personas|llms, keeps persona selection (auto-prepending default for persona mode), and always ensures a judge is configured. A dedicated ComparisonRunner reuses Workbench results to build candidate outputs and sends them to Judge#compare, which crafts a rubric-aware comparison prompt and parses structured winner/ratings JSON. Outputs are streamed to the console and individual run logs still get written. README documents how to use the new flag and what each mode does.
57 lines
1.6 KiB
Ruby
Vendored
57 lines
1.6 KiB
Ruby
Vendored
# frozen_string_literal: true
|
|
|
|
require_relative "base"
|
|
|
|
module DiscourseAi
|
|
module Evals
|
|
module Runners
|
|
class Spam < Base
|
|
def self.can_handle?(full_feature_name)
|
|
full_feature_name&.start_with?("spam:")
|
|
end
|
|
|
|
def run(eval_case, llm)
|
|
args = eval_case.args
|
|
persona = resolve_persona(persona_class: DiscourseAi::Personas::SpamDetector)
|
|
user = Discourse.system_user
|
|
|
|
content = "- Topic title: #{args[:title]}\nPost content: #{args[:input]}"
|
|
|
|
context =
|
|
DiscourseAi::Personas::BotContext
|
|
.new(
|
|
user: user,
|
|
skip_show_thinking: true,
|
|
feature_name: "evals/spam",
|
|
messages: [{ type: :user, content: content }],
|
|
)
|
|
.tap do |ctx|
|
|
ctx.custom_instructions = args[:custom_instructions] if args[:custom_instructions]
|
|
end
|
|
|
|
verdict = capture_verdict(persona, user, llm, context)
|
|
|
|
wrap_result(verdict.to_s, { feature: feature_name })
|
|
end
|
|
|
|
private
|
|
|
|
def capture_verdict(persona, user, llm, context)
|
|
bot = DiscourseAi::Personas::Bot.as(user, persona: persona, model: llm)
|
|
schema = persona.response_format&.first
|
|
|
|
if schema.present?
|
|
capture_structured_response(
|
|
bot,
|
|
context,
|
|
schema_key: schema["key"],
|
|
schema_type: schema["type"],
|
|
)
|
|
else
|
|
capture_plain_response(bot, context)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
