0
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2026-08-08 17:53:55 +08:00
discourse/spec/lib/tasks/s3_rake_spec.rb
Jake Goldsborough b2e982cadc DEV: Address parallel S3 upload review feedback
- Preload existing assets before worker threads initialize S3 client state.
- Use a fixed eight-worker pool for network-bound uploads.
- Update rake task specs for the new pool size and prewarming order.
2026-07-23 17:11:41 +00:00

138 lines
4.2 KiB
Ruby
Vendored

# frozen_string_literal: true
require "rails_helper"
require "rake"
# rubocop:disable RSpec/DescribeClass
RSpec.describe "s3:upload_assets rake task" do
before do
Rake::Task.clear
Discourse::Application.load_tasks
end
let(:task) { Rake::Task["s3:upload_assets"] }
let(:logger) { instance_double(Logger) }
before do
allow(Logger).to receive(:new).and_return(logger)
allow(logger).to receive(:<<)
allow(logger).to receive(:error)
# Stub S3 configuration check
allow(GlobalSetting).to receive(:use_s3?).and_return(true)
# Stub S3 site settings
SiteSetting.s3_upload_bucket = "test-bucket"
SiteSetting.s3_region = "us-east-1"
allow_any_instance_of(Object).to receive(:assets).and_return([])
allow_any_instance_of(Object).to receive(:existing_assets).and_return(Set.new)
# Skip CORS rules task completely to avoid S3 calls
allow(S3CorsRulesets).to receive(:sync).and_return(nil)
end
describe "error handling" do
let(:test_assets) do
[
%w[/tmp/asset1.js assets/asset1-abc.js application/javascript],
%w[/tmp/asset2.css assets/asset2-def.css text/css],
%w[/tmp/asset3.js assets/asset3-ghi.js application/javascript],
]
end
before { allow_any_instance_of(Object).to receive(:assets).and_return(test_assets) }
it "fails the task when any upload fails" do
call_count = 0
allow_any_instance_of(Object).to receive(:upload) do |*args|
call_count += 1
raise "S3 connection timeout" if call_count == 2 # Second upload fails
end
expect { task.invoke }.to raise_error(/Asset upload failed/)
end
it "reports all failures when multiple uploads fail" do
allow_any_instance_of(Object).to receive(:upload) do
raise "S3 connection timeout"
end
expect { task.invoke }.to raise_error(/Asset upload failed/)
expect(logger).to have_received(:error).with(/3 asset upload\(s\) failed/)
end
it "succeeds when all uploads succeed" do
allow_any_instance_of(Object).to receive(:upload)
expect { task.invoke }.not_to raise_error
end
end
describe "parallelization" do
let(:test_assets) do
Array.new(10) do |i|
["/tmp/asset#{i}.js", "assets/asset#{i}-abc.js", "application/javascript"]
end
end
before do
allow_any_instance_of(Object).to receive(:assets).and_return(test_assets)
allow_any_instance_of(Object).to receive(:upload)
end
it "uses a thread pool for concurrent uploads" do
pool = instance_double(Concurrent::FixedThreadPool)
allow(Concurrent::FixedThreadPool).to receive(:new).and_return(pool)
allow(pool).to receive(:shutdown)
allow(pool).to receive(:wait_for_termination)
# Mock promises
promise = instance_double(Concurrent::Promise, rejected?: false)
allow(Concurrent::Promise).to receive(:execute).and_return(promise)
task.invoke
expect(Concurrent::FixedThreadPool).to have_received(:new)
expect(pool).to have_received(:shutdown)
expect(pool).to have_received(:wait_for_termination)
end
it "uses eight upload threads" do
allow(Concurrent::FixedThreadPool).to receive(:new).with(8).and_call_original
task.invoke
expect(Concurrent::FixedThreadPool).to have_received(:new).with(8)
end
end
describe "S3 pre-warming" do
let(:test_assets) { [%w[/tmp/asset1.js assets/asset1-abc.js application/javascript]] }
before do
allow_any_instance_of(Object).to receive(:assets).and_return(test_assets)
allow_any_instance_of(Object).to receive(:upload)
end
it "loads existing assets before spawning threads" do
existing_assets_loaded = false
threads_spawned = false
allow_any_instance_of(Object).to receive(:existing_assets) do
existing_assets_loaded = true
expect(threads_spawned).to be_falsey
Set.new
end
original_new = Concurrent::FixedThreadPool.method(:new)
allow(Concurrent::FixedThreadPool).to receive(:new) do |*args|
threads_spawned = true
expect(existing_assets_loaded).to be_truthy
original_new.call(*args)
end
task.invoke
end
end
end
# rubocop:enable RSpec/DescribeClass