mirror of
https://gh.wpcy.net/https://github.com/discourse/discourse.git
synced 2026-05-15 15:29:48 +08:00
fa54f62348
Document attachments (doc, docx, xls, xlsx, rtf, csv, md, txt) are now converted to text before being included in LLM prompts, instead of being forwarded as raw base64 payloads. PDFs remain the only format sent as a raw upload, capped at 10MB. New converters under lib/completions: - DocToText shells out to antiword - DocxToText parses OOXML directly with size and depth limits - XlsToText shells out to xls2csv - XlsxToText parses OOXML and shared strings into CSV-style text - RtfToText is a custom RTF tokenizer with destination/group handling Plain text formats (csv, md, txt) are read with a 1MB byte cap and UTF-8 normalization. Extracted text is truncated to 100k characters, with a preamble noting the original filename and size. Dialect trimming now uses token-aware truncation against a per-message budget so large extracted documents collapse cleanly under the prompt limit, rather than the previous step-based slicing of raw content. Other changes: - LlmModel.normalize_attachment_types is shared with UploadEncoder and collapses "markdown" to "md" so the canonical extension is consistent across model config, UI defaults, and encoder output - ai-llm-attachment-types adds csv, xls, xlsx to the default choices - Locale strings clarify that vision controls images and allowed_attachment_types controls documents --------- Co-authored-by: Rafael Silva <xfalcox@gmail.com>
92 lines
2.6 KiB
Ruby
92 lines
2.6 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require "compression/safe_zip_reader"
|
|
|
|
RSpec.describe Compression::SafeZipReader do
|
|
let(:temp_folder) do
|
|
path = "#{Pathname.new(Dir.tmpdir).realpath}/#{SecureRandom.hex}"
|
|
FileUtils.mkdir(path)
|
|
path
|
|
end
|
|
|
|
let(:zip_path) { File.join(temp_folder, "archive.zip") }
|
|
|
|
after { FileUtils.rm_rf(temp_folder) }
|
|
|
|
def create_zip(entries)
|
|
Zip::File.open(zip_path, create: true) do |zip_file|
|
|
entries.each do |name, content|
|
|
zip_file.get_output_stream(name) { |stream| stream.write(content) }
|
|
end
|
|
end
|
|
end
|
|
|
|
it "reads an entry within the configured limits" do
|
|
create_zip("document.xml" => "hello")
|
|
|
|
described_class.open(zip_path, max_entries: 10, max_total_bytes: 100) do |zip|
|
|
expect(zip.read_entry("document.xml", max_bytes: 10)).to eq("hello")
|
|
expect(zip.remaining_total_bytes).to eq(95)
|
|
end
|
|
end
|
|
|
|
it "returns nil for missing entries" do
|
|
create_zip("document.xml" => "hello")
|
|
|
|
described_class.open(zip_path) do |zip|
|
|
expect(zip.read_entry("missing.xml", max_bytes: 10)).to be_nil
|
|
end
|
|
end
|
|
|
|
it "raises for missing required entries" do
|
|
create_zip("document.xml" => "hello")
|
|
|
|
described_class.open(zip_path) do |zip|
|
|
expect { zip.read_entry("missing.xml", max_bytes: 10, required: true) }.to raise_error(
|
|
described_class::MissingEntryError,
|
|
)
|
|
end
|
|
end
|
|
|
|
it "raises when an entry exceeds its per-entry limit" do
|
|
create_zip("document.xml" => "hello")
|
|
|
|
described_class.open(zip_path) do |zip|
|
|
expect { zip.read_entry("document.xml", max_bytes: 4) }.to raise_error(
|
|
described_class::EntryTooLargeError,
|
|
)
|
|
end
|
|
end
|
|
|
|
it "raises when reads exceed the total inflated byte budget" do
|
|
create_zip("a.xml" => "hello", "b.xml" => "world")
|
|
|
|
described_class.open(zip_path, max_total_bytes: 6) do |zip|
|
|
expect(zip.read_entry("a.xml", max_bytes: 10)).to eq("hello")
|
|
expect { zip.read_entry("b.xml", max_bytes: 10) }.to raise_error(
|
|
described_class::EntryTooLargeError,
|
|
)
|
|
end
|
|
end
|
|
|
|
it "raises when the archive has too many entries" do
|
|
create_zip("a.xml" => "a", "b.xml" => "b")
|
|
|
|
expect { described_class.open(zip_path, max_entries: 1) { nil } }.to raise_error(
|
|
described_class::TooManyEntriesError,
|
|
)
|
|
end
|
|
|
|
it "streams entries to files within the configured limits" do
|
|
create_zip("document.xml" => "hello")
|
|
|
|
Tempfile.create("safe-zip-reader") do |tempfile|
|
|
described_class.open(zip_path) do |zip|
|
|
zip.stream_entry_to_file("document.xml", tempfile, max_bytes: 10)
|
|
end
|
|
|
|
tempfile.rewind
|
|
expect(tempfile.read).to eq("hello")
|
|
end
|
|
end
|
|
end
|
