mirror of
https://gh.wpcy.net/https://github.com/discourse/discourse.git
synced 2026-06-19 05:35:40 +08:00
fa54f62348
Document attachments (doc, docx, xls, xlsx, rtf, csv, md, txt) are now converted to text before being included in LLM prompts, instead of being forwarded as raw base64 payloads. PDFs remain the only format sent as a raw upload, capped at 10MB. New converters under lib/completions: - DocToText shells out to antiword - DocxToText parses OOXML directly with size and depth limits - XlsToText shells out to xls2csv - XlsxToText parses OOXML and shared strings into CSV-style text - RtfToText is a custom RTF tokenizer with destination/group handling Plain text formats (csv, md, txt) are read with a 1MB byte cap and UTF-8 normalization. Extracted text is truncated to 100k characters, with a preamble noting the original filename and size. Dialect trimming now uses token-aware truncation against a per-message budget so large extracted documents collapse cleanly under the prompt limit, rather than the previous step-based slicing of raw content. Other changes: - LlmModel.normalize_attachment_types is shared with UploadEncoder and collapses "markdown" to "md" so the canonical extension is consistent across model config, UI defaults, and encoder output - ai-llm-attachment-types adds csv, xls, xlsx to the default choices - Locale strings clarify that vision controls images and allowed_attachment_types controls documents --------- Co-authored-by: Rafael Silva <xfalcox@gmail.com>
68 lines
2 KiB
Ruby
Vendored
68 lines
2 KiB
Ruby
Vendored
# frozen_string_literal: true
|
|
|
|
RSpec.describe LlmModel do
|
|
before { enable_current_plugin }
|
|
|
|
describe "api_key" do
|
|
fab!(:llm_model, :seeded_model)
|
|
|
|
before { ENV["DISCOURSE_AI_SEEDED_LLM_API_KEY_2"] = "blabla" }
|
|
|
|
it "should use environment variable over database value if seeded LLM" do
|
|
expect(llm_model.api_key).to eq("blabla")
|
|
end
|
|
end
|
|
|
|
describe "#credit_system_enabled?" do
|
|
fab!(:seeded_model)
|
|
fab!(:regular_model, :llm_model)
|
|
|
|
it "returns false for non-seeded models" do
|
|
expect(regular_model.credit_system_enabled?).to be false
|
|
end
|
|
|
|
it "returns false for seeded models without credit allocation" do
|
|
expect(seeded_model.credit_system_enabled?).to be false
|
|
end
|
|
|
|
it "returns true for seeded models with credit allocation" do
|
|
Fabricate(:llm_credit_allocation, llm_model: seeded_model)
|
|
expect(seeded_model.credit_system_enabled?).to be true
|
|
end
|
|
end
|
|
|
|
describe "AWS Bedrock provider validation" do
|
|
fab!(:bedrock_model, :bedrock_model)
|
|
|
|
it "requires either access_key_id or role_arn" do
|
|
# Should fail with neither
|
|
bedrock_model.provider_params = { region: "us-east-1" }
|
|
expect(bedrock_model.valid?).to be false
|
|
expect(bedrock_model.errors[:base]).to include(
|
|
I18n.t("discourse_ai.llm_models.bedrock_missing_auth"),
|
|
)
|
|
end
|
|
|
|
it "is valid with access_key_id only" do
|
|
bedrock_model.provider_params = { region: "us-east-1", access_key_id: "test_key" }
|
|
expect(bedrock_model.valid?).to be true
|
|
end
|
|
|
|
it "is valid with role_arn only" do
|
|
bedrock_model.provider_params = {
|
|
region: "us-east-1",
|
|
role_arn: "arn:aws:iam::123:role/test",
|
|
}
|
|
expect(bedrock_model.valid?).to be true
|
|
end
|
|
end
|
|
|
|
describe "allowed_attachment_types" do
|
|
it "normalizes markdown attachments to md" do
|
|
model = Fabricate.build(:llm_model)
|
|
model.allowed_attachment_types = %w[pdf markdown md htm text]
|
|
|
|
expect(model.allowed_attachment_types).to eq(%w[pdf md html txt])
|
|
end
|
|
end
|
|
end
|
