mirror of
https://gh.wpcy.net/https://github.com/discourse/discourse.git
synced 2026-05-27 18:59:19 +08:00
f5adb47d78
This refactor collapses all eval execution paths into a single Playground orchestrator that understands feature-specific behavior, keeps StructuredRecorder usage consistent, and lets Eval act purely as a data loader. By tightening the flow and upgrading the specs and documentation, we gain clearer logs, simpler CLI wiring, and a cleaner seam for upcoming persona-backed work while preserving the existing eval surface area.
74 lines
1.8 KiB
Ruby
Vendored
74 lines
1.8 KiB
Ruby
Vendored
# frozen_string_literal: true
|
|
require "optparse"
|
|
require_relative "features"
|
|
|
|
class DiscourseAi::Evals::Cli
|
|
class Options
|
|
attr_accessor :eval_name, :model, :list, :list_models, :list_features, :feature_key
|
|
|
|
def initialize(
|
|
eval_name: nil,
|
|
model: nil,
|
|
list: false,
|
|
list_models: false,
|
|
list_features: false,
|
|
feature_key: nil
|
|
)
|
|
@eval_name = eval_name
|
|
@model = model
|
|
@list = list
|
|
@list_models = list_models
|
|
@list_features = list_features
|
|
@feature_key = feature_key
|
|
end
|
|
end
|
|
|
|
def self.parse_options!(features_registry)
|
|
options = Options.new
|
|
|
|
parser =
|
|
OptionParser.new do |opts|
|
|
opts.banner = "Usage: evals/run [options]"
|
|
|
|
opts.on("-e", "--eval NAME", "Name of the evaluation to run") do |eval_name|
|
|
options.eval_name = eval_name
|
|
end
|
|
|
|
opts.on("--list-models", "List models") { |model| options.list_models = true }
|
|
opts.on("--list-features", "List features available for evals") do
|
|
options.list_features = true
|
|
end
|
|
|
|
opts.on(
|
|
"-m",
|
|
"--model NAME",
|
|
"Model to evaluate (will eval all models if not specified)",
|
|
) { |model| options.model = model }
|
|
|
|
opts.on("-l", "--list", "List evals") { |model| options.list = true }
|
|
|
|
opts.on(
|
|
"-f",
|
|
"--feature KEY",
|
|
"Feature key to evaluate (module_name:feature_name)",
|
|
) { |key| options.feature_key = key }
|
|
end
|
|
|
|
show_help = ARGV.empty?
|
|
parser.parse!
|
|
|
|
if show_help
|
|
puts parser
|
|
exit 0
|
|
end
|
|
|
|
if options.feature_key && !features_registry.valid_feature_key?(options.feature_key)
|
|
STDERR.puts(
|
|
"Unknown feature '#{options.feature_key}'. Run with --list-features to view valid keys.",
|
|
)
|
|
exit 1
|
|
end
|
|
|
|
options
|
|
end
|
|
end
|
