Discourse/discourse
Discourse/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-10-03 17:21:20 +08:00
Code Packages Activity Actions 7

FEATURE: Add support for aws MediaConvert (#33092)

Browse source 
When enabled this will convert uploaded videos to a standard format that should
be playable on all devices and browsers.

The goal of this feature is to prevent codec playback issues that
sometimes can occur with video uploads.

It uses an adapter pattern, so that other services for video conversion
could be easily added in the future.
This commit is contained in:
Blake Erickson 2025-07-23 11:58:33 -06:00 • committed by GitHub
parent a27e20c300
commit af3abb54e3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 1295 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Gemfile
View file

@ -62,6 +62,7 @@ gem "fastimage"

gem "aws-sdk-s3", require: false
gem "aws-sdk-sns", require: false
gem "aws-sdk-mediaconvert", require: false
gem "excon", require: false
gem "unf", require: false


9
Gemfile.lock
View file

@ -59,7 +59,7 @@ GEM
activesupport (>= 6.0.0)
ast (2.4.3)
aws-eventstream (1.4.0)
aws-partitions (1.1117.0)
aws-partitions (1.1120.0)
aws-sdk-core (3.226.0)
aws-eventstream (~> 1, >= 1.3.0)
aws-partitions (~> 1, >= 1.992.0)
@ -70,6 +70,9 @@ GEM
aws-sdk-kms (1.99.0)
aws-sdk-core (~> 3, >= 3.216.0)
aws-sigv4 (~> 1.5)
aws-sdk-mediaconvert (1.160.0)
aws-sdk-core (~> 3, >= 3.225.0)
aws-sigv4 (~> 1.5)
aws-sdk-s3 (1.182.0)
aws-sdk-core (~> 3, >= 3.216.0)
aws-sdk-kms (~> 1)
@ -723,6 +726,7 @@ DEPENDENCIES
addressable
afm
annotaterb
aws-sdk-mediaconvert
aws-sdk-s3
aws-sdk-sns
better_errors
@ -888,9 +892,10 @@ CHECKSUMS
annotaterb (4.17.0) sha256=f0338f8aaadd5c47fa3deaccb560a54abcdde29aca6f69f4b94726ea9256b4bd
ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383
aws-eventstream (1.4.0) sha256=116bf85c436200d1060811e6f5d2d40c88f65448f2125bc77ffce5121e6e183b
aws-partitions (1.1117.0) sha256=fe6469ff7426b449c0c6d0aa43e45f387a4b26992bf48f6897c0b730b205a8a3
aws-partitions (1.1120.0) sha256=9dd2c4de7780ad0b51f6314a9cefb405e7e5cb678c67c5fecdc8b60367226d6b
aws-sdk-core (3.226.0) sha256=6b2dca6576d965b8c7ddf393fe49dac6aea2b6f4a07ead0c35306bba2b0c90f5
aws-sdk-kms (1.99.0) sha256=ba292fc3ffd672532aae2601fe55ff424eee78da8e23c23ba6ce4037138275a8
aws-sdk-mediaconvert (1.160.0) sha256=48bded7432312a4a3bcb82c03d562a847db9e743edf751e998eee2eca38f402c
aws-sdk-s3 (1.182.0) sha256=d0fc3579395cb6cb69bf6e975240ce031fc673190e74c8dddbdd6c18572b450d
aws-sdk-sns (1.96.0) sha256=f92b3c7203c53181b1cafb3dbbea85f330002ad696175bda829cfef359fa6dd4
aws-sigv4 (1.12.1) sha256=6973ff95cb0fd0dc58ba26e90e9510a2219525d07620c8babeb70ef831826c00

40
app/jobs/regular/check_video_conversion_status.rb Normal file
View file

@ -0,0 +1,40 @@
# frozen_string_literal: true
module Jobs
class CheckVideoConversionStatus < ::Jobs::Base
sidekiq_options queue: "low", concurrency: 5

def execute(args)
return if args[:upload_id].blank? || args[:job_id].blank? || args[:adapter_type].blank?

upload = Upload.find_by(id: args[:upload_id])
return if upload.blank?

adapter =
VideoConversion::AdapterFactory.get_adapter(upload, adapter_type: args[:adapter_type])

status = adapter.check_status(args[:job_id])

case status
when :complete
if adapter.handle_completion(args[:job_id], args[:output_path], args[:new_sha1])
# Successfully completed
Rails.logger.info(
"Completed video conversion for upload ID #{upload.id} and job ID #{args[:job_id]}",
)
else
# Handle completion failed
Rails.logger.error(
"Failed to handle video conversion completion for upload ID #{upload.id} and job ID #{args[:job_id]}",
)
end
when :error
Rails.logger.error(
"Video conversion job failed for upload ID #{upload.id} and job ID #{args[:job_id]}",
)
when :pending
# Re-enqueue the job to check again
Jobs.enqueue_in(30.seconds, :check_video_conversion_status, args)
end
end
end
end

34
app/jobs/regular/convert_video.rb Normal file
View file

@ -0,0 +1,34 @@
# frozen_string_literal: true
module Jobs
class ConvertVideo < ::Jobs::Base
sidekiq_options queue: "low", concurrency: 5
MAX_RETRIES = 5
RETRY_DELAY = 30.seconds

def execute(args)
return if args[:upload_id].blank?

upload = Upload.find_by(id: args[:upload_id])
return if upload.blank?

return if OptimizedVideo.exists?(upload_id: upload.id)

if upload.url.blank?
retry_count = args[:retry_count].to_i
if retry_count < MAX_RETRIES
Jobs.enqueue_in(RETRY_DELAY, :convert_video, args.merge(retry_count: retry_count + 1))
return
else
Rails.logger.error(
"Upload #{upload.id} URL remained blank after #{MAX_RETRIES} retries when optimizing video",
)
return
end
end

adapter = VideoConversion::AdapterFactory.get_adapter(upload)

Rails.logger.error("Video conversion failed for upload #{upload.id}") if !adapter.convert
end
end
end

80
app/models/optimized_video.rb Normal file
View file

@ -0,0 +1,80 @@
# frozen_string_literal: true

class OptimizedVideo < ActiveRecord::Base
belongs_to :upload
belongs_to :optimized_upload, class_name: "Upload"

validates :upload_id, presence: true
validates :optimized_upload_id, presence: true
validates :adapter, presence: true
validates :upload_id, uniqueness: { scope: :adapter }

def self.create_for(upload, filename, user_id, options = {})
return if upload.blank?

optimized_upload =
Upload.create!(
user_id: user_id,
original_filename: filename,
filesize: options[:filesize],
sha1: options[:sha1],
extension: options[:extension] || "mp4",
url: options[:url],
etag: options[:etag],
)

optimized_video =
OptimizedVideo.new(
upload_id: upload.id,
optimized_upload_id: optimized_upload.id,
adapter: options[:adapter],
)

if optimized_video.save
UploadReference.ensure_exist!(upload_ids: [optimized_upload.id], target: upload)
optimized_video
else
optimized_upload.destroy
Rails.logger.error(
"Failed to create optimized video for upload ID #{upload.id}: #{optimized_video.errors.full_messages.join(", ")}",
)
nil
end
end

def destroy
OptimizedVideo.transaction do
Discourse.store.remove_upload(optimized_upload) if optimized_upload
if optimized_upload_id
UploadReference.where(upload_id: optimized_upload_id, target: upload).destroy_all
end
super
optimized_upload&.destroy
end
end

delegate :url, :filesize, :sha1, :extension, to: :optimized_upload
end

# == Schema Information
#
# Table name: optimized_videos
#
# id :bigint not null, primary key
# upload_id :integer not null
# optimized_upload_id :integer not null
# adapter :string
# created_at :datetime not null
# updated_at :datetime not null
#
# Indexes
#
# index_optimized_videos_on_optimized_upload_id (optimized_upload_id)
# index_optimized_videos_on_upload_id (upload_id)
# index_optimized_videos_on_upload_id_and_adapter (upload_id,adapter) UNIQUE
#
# Foreign Keys
#
# fk_rails_... (optimized_upload_id => uploads.id)
# fk_rails_... (upload_id => uploads.id)
#

1
app/models/post.rb
View file

@ -1162,6 +1162,7 @@ class Post < ActiveRecord::Base
"track/@src",
"video/@poster",
"div/@data-video-src",
"div/@data-original-video-src",
)

links =

2
app/models/upload.rb
View file

@ -23,6 +23,8 @@ class Upload < ActiveRecord::Base

has_many :post_hotlinked_media, dependent: :destroy, class_name: "PostHotlinkedMedia"
has_many :optimized_images, dependent: :destroy
has_many :optimized_videos, dependent: :destroy
has_many :optimized_video_uploads, through: :optimized_videos, source: :optimized_upload
has_many :user_uploads, dependent: :destroy
has_many :upload_references, dependent: :destroy
has_many :posts, through: :upload_references, source: :target, source_type: "Post"

16
app/services/video_conversion/adapter_factory.rb Normal file
View file

@ -0,0 +1,16 @@
# frozen_string_literal: true

module VideoConversion
class AdapterFactory
def self.get_adapter(upload, options = {})
adapter_type = options[:adapter_type] || SiteSetting.video_conversion_service

case adapter_type
when "aws_mediaconvert"
AwsMediaConvertAdapter.new(upload, options)
else
raise ArgumentError, "Unknown video conversion service: #{adapter_type}"
end
end
end
end

276
app/services/video_conversion/aws_media_convert_adapter.rb Normal file
View file

@ -0,0 +1,276 @@
# frozen_string_literal: true
require "aws-sdk-mediaconvert"

module VideoConversion
class AwsMediaConvertAdapter < BaseAdapter
ADAPTER_NAME = "aws_mediaconvert"

def convert
return false if !valid_settings?

begin
new_sha1 = SecureRandom.hex(20)

# Use FileStore::BaseStore logic to generate the path
# Create a temporary upload object to leverage the path generation logic
temp_upload = build_temp_upload_for_path_generation(new_sha1)
output_path = Discourse.store.get_path_for_upload(temp_upload).sub(/\.mp4$/, "")

# Extract the path from the URL
# The URL format is: //bucket.s3.dualstack.region.amazonaws.com/path/to/file
# or: //bucket.s3.region.amazonaws.com/path/to/file
url = @upload.url.sub(%r{^//}, "") # Remove leading //

# Split on the first / to separate the domain from the path
domain, path = url.split("/", 2)

# Verify the domain contains our bucket
if !domain&.include?(SiteSetting.s3_upload_bucket)
raise Discourse::InvalidParameters.new(
"Upload URL domain for upload ID #{@upload.id} does not contain expected bucket name: #{SiteSetting.s3_upload_bucket}",
)
end

input_path = "s3://#{SiteSetting.s3_upload_bucket}/#{path}"
settings = build_conversion_settings(input_path, output_path)

begin
response =
mediaconvert_client.create_job(
role: SiteSetting.mediaconvert_role_arn,
settings: settings,
status_update_interval: "SECONDS_10",
user_metadata: {
"upload_id" => @upload.id.to_s,
"new_sha1" => new_sha1,
"output_path" => output_path,
},
)

# Enqueue status check job
Jobs.enqueue_in(
30.seconds,
:check_video_conversion_status,
upload_id: @upload.id,
job_id: response.job.id,
new_sha1: new_sha1,
output_path: output_path,
original_filename: @upload.original_filename,
user_id: @upload.user_id,
adapter_type: "aws_mediaconvert",
)

true # Return true on success
rescue Aws::MediaConvert::Errors::ServiceError => e
Discourse.warn_exception(
e,
message: "MediaConvert job creation failed",
env: {
upload_id: @upload.id,
},
)
false
rescue => e
Discourse.warn_exception(
e,
message: "Unexpected error in MediaConvert job creation",
env: {
upload_id: @upload.id,
},
)
false
end
rescue Discourse::InvalidParameters => e
Rails.logger.error("Invalid parameters for upload #{@upload.id}: #{e.message}")
false
rescue => e
Discourse.warn_exception(
e,
message: "Unexpected error in video conversion",
env: {
upload_id: @upload.id,
},
)
false
end
end

def check_status(job_id)
response = mediaconvert_client.get_job(id: job_id)

case response.job.status
when "COMPLETE"
STATUS_COMPLETE
when "ERROR"
Rails.logger.error("MediaConvert job #{job_id} failed")
STATUS_ERROR
when "SUBMITTED", "PROGRESSING"
STATUS_PENDING
else
Rails.logger.warn(
"Unexpected MediaConvert job status for job #{job_id}: #{response.job.status}",
)
STATUS_ERROR
end
end

def handle_completion(job_id, output_path, new_sha1)
s3_store = FileStore::S3Store.new
path = "#{output_path}.mp4"
object = s3_store.object_from_path(path)

return false if !object&.exists?

begin
url = "//#{s3_store.s3_bucket}.s3.dualstack.#{SiteSetting.s3_region}.amazonaws.com/#{path}"

optimized_video = create_optimized_video_record(output_path, new_sha1, object.size, url)

if optimized_video
update_posts_with_optimized_video
true
else
Rails.logger.error("Failed to create OptimizedVideo record for upload #{@upload.id}")
false
end
rescue => e
Discourse.warn_exception(
e,
message: "Error in video processing completion",
env: {
upload_id: @upload.id,
job_id: job_id,
},
)
false
end
end

private

def valid_settings?
SiteSetting.video_conversion_enabled && SiteSetting.mediaconvert_role_arn.present?
end

def build_temp_upload_for_path_generation(new_sha1)
# Create a temporary upload object to leverage FileStore::BaseStore path generation
# This object is only used for path generation and won't be saved to the database
Upload.new(
id: @upload.id, # Use the same ID to get the same depth calculation
sha1: new_sha1,
extension: "mp4",
)
end

def mediaconvert_client
@mediaconvert_client ||= build_client
end

def build_client
# For some reason the endpoint is not visible in the aws console UI so we need to get it from the API
if SiteSetting.mediaconvert_endpoint.blank?
client = create_basic_client
resp = client.describe_endpoints
SiteSetting.mediaconvert_endpoint = resp.endpoints[0].url
end

# Validate that we have an endpoint before proceeding
if SiteSetting.mediaconvert_endpoint.blank?
error_msg = "MediaConvert endpoint is required but could not be discovered"
Discourse.warn_exception(
StandardError.new(error_msg),
message: error_msg,
env: {
upload_id: @upload.id,
},
)
raise StandardError, error_msg
end

create_basic_client(endpoint: SiteSetting.mediaconvert_endpoint)
end

def create_basic_client(endpoint: nil)
Aws::MediaConvert::Client.new(
region: SiteSetting.s3_region,
credentials:
Aws::Credentials.new(SiteSetting.s3_access_key_id, SiteSetting.s3_secret_access_key),
endpoint: endpoint,
)
end

def update_posts_with_optimized_video
post_ids = UploadReference.where(upload_id: @upload.id, target_type: "Post").pluck(:target_id)

Post
.where(id: post_ids)
.find_each do |post|
Rails.logger.info("Rebaking post #{post.id} to use optimized video")
post.rebake!
end
end

def build_conversion_settings(input_path, output_path)
self.class.build_conversion_settings(input_path, output_path)
end

def self.build_conversion_settings(input_path, output_path)
{
timecode_config: {
source: "ZEROBASED",
},
output_groups: [
{
name: "File Group",
output_group_settings: {
type: "FILE_GROUP_SETTINGS",
file_group_settings: {
destination: "s3://#{SiteSetting.s3_upload_bucket}/#{output_path}",
},
},
outputs: [
{
container_settings: {
container: "MP4",
},
video_description: {
codec_settings: {
codec: "H_264",
h264_settings: {
bitrate: 2_000_000,
rate_control_mode: "CBR",
},
},
},
audio_descriptions: [
{
codec_settings: {
codec: "AAC",
aac_settings: {
bitrate: 96_000,
sample_rate: 48_000,
coding_mode: "CODING_MODE_2_0",
},
},
},
],
},
],
},
],
inputs: [
{
file_input: input_path,
audio_selectors: {
"Audio Selector 1": {
default_selection: "DEFAULT",
},
},
video_selector: {
},
},
],
}
end
end
end

54
app/services/video_conversion/base_adapter.rb Normal file
View file

@ -0,0 +1,54 @@
# frozen_string_literal: true
module VideoConversion
class BaseAdapter
STATUS_COMPLETE = :complete
STATUS_ERROR = :error
STATUS_PENDING = :pending

def initialize(upload, options = {})
@upload = upload
@options = options
end

# Starts the conversion process and returns a job identifier
def convert
raise NotImplementedError, "#{self.class} must implement #convert"
end

# Checks the status of a conversion job
# Returns a symbol: STATUS_COMPLETE, STATUS_ERROR, or STATUS_PENDING
def check_status(job_id)
raise NotImplementedError, "#{self.class} must implement #check_status"
end

# Handles the completion of a successful conversion
# This is called by the job system when status is :complete
def handle_completion(job_id, output_path, new_sha1)
raise NotImplementedError, "#{self.class} must implement #handle_completion"
end

protected

def create_optimized_video_record(output_path, new_sha1, filesize, url)
OptimizedVideo.create_for(
@upload,
@upload.original_filename.sub(/\.[^.]+$/, "_converted.mp4"),
@upload.user_id,
filesize: filesize,
sha1: new_sha1,
url: url,
extension: "mp4",
adapter: adapter_name,
)
end

private

def adapter_name
self.class::ADAPTER_NAME
rescue NameError
# Fallback for adapters that don't define ADAPTER_NAME
self.class.name.demodulize.underscore
end
end
end

6
config/locales/server.en.yml
View file

@ -2121,6 +2121,10 @@ en:
composer_media_optimization_image_resize_width_target: "Images with widths larger than `composer_media_optimization_image_dimensions_resize_threshold` will be resized to this width. Must be >= than `composer_media_optimization_image_dimensions_resize_threshold`."
composer_media_optimization_image_encode_quality: "JPG encode quality used in the re-encode process."

video_conversion_enabled: "Enable video conversion for uploaded video files. This allows videos to be converted to web-compatible formats."
video_conversion_service: "The video conversion service to use for processing uploaded videos."
mediaconvert_role_arn: "AWS IAM role ARN for MediaConvert service. Required when using AWS MediaConvert for video conversion."

min_ratio_to_crop: "Ratio used to crop tall images. Enter the result of width / height."

simultaneous_uploads: "Maximum number of files that can be dragged & dropped in the composer"
@ -2815,6 +2819,8 @@ en:
invalid_search_ranking_weights: "Value is invalid for search_ranking_weights site setting. Example: '{0.1,0.2,0.3,1.0}'. Note that maximum value for each weight is 1.0."
content_localization_locale_limit: "The number of supported locales cannot exceed %{max}."
content_localization_anon_language_switcher_requirements: "The language switcher requires the `set locale from cookie` site setting to be enabled, and the `content localization supported locales` to have at least one language."
mediaconvert_role_arn_required: "AWS IAM role ARN is required when using AWS MediaConvert for video conversion."
s3_credentials_required_for_video_conversion: "S3 credentials are required for video conversion. Please configure S3 access keys or enable IAM profile."

keywords:
clean_up_inactive_users_after_days: "deactivated|inactive|unactivated"

13
config/site_settings.yml
View file

@ -2269,6 +2269,19 @@ files:
default: true
client: true
hidden: true
video_conversion_enabled:
default: false
validator: "VideoConversionEnabledValidator"
video_conversion_service:
type: enum
default: "aws_mediaconvert"
choices:
- "aws_mediaconvert"
mediaconvert_role_arn:
default: ""
mediaconvert_endpoint:
default: ""
hidden: true

trust:
default_trust_level:

19
db/migrate/20250606170129_create_optimized_videos.rb Normal file
View file

@ -0,0 +1,19 @@
# frozen_string_literal: true

class CreateOptimizedVideos < ActiveRecord::Migration[7.2]
def change
create_table :optimized_videos do |t|
t.integer :upload_id, null: false
t.integer :optimized_upload_id, null: false
t.string :adapter
t.timestamps
end

add_index :optimized_videos, :upload_id
add_index :optimized_videos, :optimized_upload_id
add_index :optimized_videos, %i[upload_id adapter], unique: true

add_foreign_key :optimized_videos, :uploads, column: :upload_id
add_foreign_key :optimized_videos, :uploads, column: :optimized_upload_id
end
end

45
lib/cooked_post_processor.rb
View file

@ -43,6 +43,7 @@ class CookedPostProcessor
remove_full_quote_on_direct_reply if new_post
post_process_oneboxes
post_process_images
post_process_videos
add_blocked_hotlinked_media_placeholders
post_process_quotes
optimize_urls
@ -348,7 +349,7 @@ class CookedPostProcessor

%w[src].each do |selector|
@doc
.css("img[#{selector}]")
.css("img[#{selector}], video[#{selector}]")
.each do |img|
custom_emoji = img["class"]&.include?("emoji-custom") && Emoji.custom?(img["title"])
img[selector] = UrlHelper.cook_url(
@ -393,6 +394,48 @@ class CookedPostProcessor
end
end

def post_process_videos
changes_made = false

begin
@doc
.css(".video-placeholder-container")
.each do |container|
src = container["data-video-src"]
next if src.blank?

# Look for optimized video
upload = Upload.get_from_url(src)
if upload && optimized_video = OptimizedVideo.find_by(upload_id: upload.id)
optimized_url = optimized_video.optimized_upload.url
# Only update if the URL is different
if container["data-video-src"] != optimized_url
container["data-original-video-src"] = container["data-video-src"] unless container[
"data-original-video-src"
]
container["data-video-src"] = optimized_url
changes_made = true
end
# Ensure we maintain reference to original upload
@post.link_post_uploads(fragments: @doc)
end
end

# Update the post's cooked content if changes were made
if changes_made
new_cooked = @doc.to_html
@post.cooked = new_cooked
if !@post.save
Rails.logger.error("Failed to save post: #{@post.errors.full_messages.join(", ")}")
end
end
rescue => e
Rails.logger.error("Error in post_process_videos: #{e.message}")
Rails.logger.error(e.backtrace.join("\n"))
raise
end
end

def process_hotlinked_image(img)
onebox = img.ancestors(".onebox, .onebox-body").first


6
lib/file_store/s3_store.rb
View file

@ -40,6 +40,7 @@ module FileStore
content_type: content_type,
cache_locally: true,
private: upload.secure?,
upload_id: upload.id,
)
url
end
@ -115,6 +116,11 @@ module FileStore
path, etag = s3_helper.upload(file, path, options)
end

if opts[:upload_id] && FileHelper.is_supported_video?(opts[:filename]) &&
SiteSetting.video_conversion_enabled
Jobs.enqueue(:convert_video, upload_id: opts[:upload_id])
end

# return the upload url and etag
[File.join(absolute_base_url, path), etag]
end

43
lib/validators/video_conversion_enabled_validator.rb Normal file
View file

@ -0,0 +1,43 @@
# frozen_string_literal: true

class VideoConversionEnabledValidator
def initialize(opts = {})
@opts = opts
end

def valid_value?(val)
return true if val == "f" # Allow disabling video conversion

# Check MediaConvert-specific requirements only when using aws_mediaconvert
if SiteSetting.video_conversion_service == "aws_mediaconvert"
# Check if MediaConvert role ARN is provided
return false if SiteSetting.mediaconvert_role_arn.blank?

# Check if S3 credentials are provided (either access keys or IAM profile)
return false if s3_credentials_missing?
end

true
end

def error_message
# Only check MediaConvert-specific requirements when using aws_mediaconvert
if SiteSetting.video_conversion_service == "aws_mediaconvert"
if SiteSetting.mediaconvert_role_arn.blank?
I18n.t("site_settings.errors.mediaconvert_role_arn_required")
elsif s3_credentials_missing?
I18n.t("site_settings.errors.s3_credentials_required_for_video_conversion")
end
end
end

private

def s3_credentials_missing?
# Check if using IAM profile
return false if SiteSetting.s3_use_iam_profile

# Check if access key and secret are provided
SiteSetting.s3_access_key_id.blank? || SiteSetting.s3_secret_access_key.blank?
end
end

7
spec/fabricators/optimized_video_fabricator.rb Normal file
View file

@ -0,0 +1,7 @@
# frozen_string_literal: true

Fabricator(:optimized_video) do
upload
optimized_upload { Fabricate(:optimized_video_upload) }
adapter "aws_mediaconvert"
end

12
spec/fabricators/upload_fabricator.rb
View file

@ -66,6 +66,9 @@ Fabricator(:video_upload, from: :upload) do
thumbnail_width nil
thumbnail_height nil
extension "mp4"
url do |attrs|
sequence(:url) { |n| Discourse.store.get_path_for("original", n + 1, attrs[:sha1], ".mp4") }
end
end

Fabricator(:secure_upload, from: :upload) do
@ -108,3 +111,12 @@ Fabricator(:upload_reference) do
target
upload
end

Fabricator(:optimized_video_upload, from: :upload) do
original_filename "video_converted.mp4"
filesize 1024
extension "mp4"
url do |attrs|
sequence(:url) { |n| "//bucket.s3.region.amazonaws.com/original/1X/#{attrs[:sha1]}.mp4" }
end
end

78
spec/jobs/regular/convert_video_spec.rb Normal file
View file

@ -0,0 +1,78 @@
# frozen_string_literal: true

RSpec.describe Jobs::ConvertVideo do
subject(:job) { described_class.new }

before(:each) do
extensions = SiteSetting.authorized_extensions.split("|")
SiteSetting.authorized_extensions = (extensions | ["mp4"]).join("|")
end

let!(:upload) { Fabricate(:video_upload) }
let(:args) { { upload_id: upload.id } }

describe "#execute" do
it "does nothing if upload_id is blank" do
expect { job.execute({}) }.not_to change { OptimizedVideo.count }
end

it "does nothing if upload does not exist" do
expect { job.execute(upload_id: -1) }.not_to change { OptimizedVideo.count }
end

it "does nothing if optimized video already exists" do
Fabricate(:optimized_video, upload: upload)
expect { job.execute(args) }.not_to change { OptimizedVideo.count }
end

context "when upload url is blank" do
before do
upload.stubs(:url).returns("")
Upload.stubs(:find_by).with(id: upload.id).returns(upload)
Jobs::ConvertVideo.jobs.clear
end

it "retries the job if under max retries" do
expect { job.execute(args.merge(retry_count: 0)) }.to change {
Jobs::ConvertVideo.jobs.size
}.by(1)

enqueued_job = Jobs::ConvertVideo.jobs.last
expect(enqueued_job["args"].first["retry_count"]).to eq(1)
end

it "logs error and stops retrying after max retries" do
expect {
job.execute(args.merge(retry_count: Jobs::ConvertVideo::MAX_RETRIES))
}.not_to change { Jobs::ConvertVideo.jobs.size }
Rails
.logger
.expects(:error)
.with(
"Upload #{upload.id} URL remained blank after #{Jobs::ConvertVideo::MAX_RETRIES} retries when optimizing video",
)
job.execute(args.merge(retry_count: Jobs::ConvertVideo::MAX_RETRIES))
end
end

context "when upload has a url" do
let(:adapter) { mock }

before do
VideoConversion::AdapterFactory.stubs(:get_adapter).with(upload).returns(adapter)
Upload.stubs(:find_by).with(id: upload.id).returns(upload)
end

it "converts the video using the adapter" do
adapter.expects(:convert).returns(true)
job.execute(args)
end

it "logs error if conversion fails" do
adapter.expects(:convert).returns(false)
Rails.logger.expects(:error).with("Video conversion failed for upload #{upload.id}")
job.execute(args)
end
end
end
end

61
spec/lib/file_store/s3_store_spec.rb
View file

@ -204,6 +204,67 @@ RSpec.describe FileStore::S3Store do
)
end
end

context "when video conversion is enabled" do
let(:video_file) { file_from_fixtures("small.mp4", "media") }
let(:video_upload) do
Fabricate.build(:upload, original_filename: "small.mp4", extension: "mp4", id: 42)
end

before do
# Set up required MediaConvert settings
SiteSetting.video_conversion_service = "aws_mediaconvert"
SiteSetting.mediaconvert_role_arn = "arn:aws:iam::123456789012:role/MediaConvertRole"
SiteSetting.video_conversion_enabled = true
# Default stub that returns false for any argument
allow(FileHelper).to receive(:is_supported_video?).and_return(false)
# Override for the specific video file case
allow(FileHelper).to receive(:is_supported_video?).with("small.mp4").and_return(true)
allow(store.s3_helper).to receive(:upload).and_return(["some/path.mp4", "\"etag\""])
# Setup Jobs as a spy
allow(Jobs).to receive(:enqueue)
end

it "enqueues a convert_video job for supported video files" do
store.store_upload(video_file, video_upload)

expect(Jobs).to have_received(:enqueue).with(:convert_video, upload_id: video_upload.id)
end

it "does not enqueue a convert_video job for unsupported video files" do
allow(FileHelper).to receive(:is_supported_video?).with("small.mp4").and_return(false)

store.store_upload(video_file, video_upload)

expect(Jobs).not_to have_received(:enqueue).with(
:convert_video,
upload_id: video_upload.id,
)
end

it "does not enqueue a convert_video job when video conversion is disabled" do
SiteSetting.video_conversion_enabled = false

store.store_upload(video_file, video_upload)

expect(Jobs).not_to have_received(:enqueue).with(
:convert_video,
upload_id: video_upload.id,
)
end

it "does not enqueue a convert_video job for non-video files" do
non_video_upload =
Fabricate.build(:upload, original_filename: "image.png", extension: "png", id: 43)

store.store_upload(uploaded_file, non_video_upload)

expect(Jobs).not_to have_received(:enqueue).with(
:convert_video,
upload_id: non_video_upload.id,
)
end
end
end

describe "#store_optimized_image" do

61
spec/models/optimized_video_spec.rb Normal file
View file

@ -0,0 +1,61 @@
# frozen_string_literal: true

RSpec.describe OptimizedVideo do
before(:each) do
# Add video extensions to authorized extensions
extensions = SiteSetting.authorized_extensions.split("|")
SiteSetting.authorized_extensions = (extensions | %w[mp4 mov avi mkv]).join("|")
end

describe ".create_for" do
let(:options) do
{
filesize: 1024,
sha1: "test-sha1-hash",
url: "//bucket.s3.region.amazonaws.com/original/1X/test.mp4",
extension: "mp4",
adapter: "aws_mediaconvert",
}
end

it "returns nil if upload is blank" do
expect(OptimizedVideo.create_for(nil, "test.mp4", 1, options)).to be_nil
end

it "creates an optimized video record with associated upload" do
# Create upload manually without fabricators
user = User.create!(username: "testuser", email: "test@example.com")
upload =
Upload.create!(
user: user,
original_filename: "original.mp4",
filesize: 2048,
sha1: "original-sha1",
extension: "mp4",
url: "//bucket.s3.region.amazonaws.com/original.mp4",
)

expect { OptimizedVideo.create_for(upload, "test.mp4", upload.user_id, options) }.to change {
Upload.count
}.by(1).and change { OptimizedVideo.count }.by(1)

optimized_video = OptimizedVideo.last
optimized_upload = optimized_video.optimized_upload

expect(optimized_video.upload).to eq(upload)
expect(optimized_video.adapter).to eq("aws_mediaconvert")
expect(optimized_upload.filesize).to eq(1024)
expect(optimized_upload.sha1).to eq("test-sha1-hash")
expect(optimized_upload.url).to eq("//bucket.s3.region.amazonaws.com/original/1X/test.mp4")
expect(optimized_upload.extension).to eq("mp4")
end
end

describe "#destroy" do
it "should destroy the optimized video and its associated upload" do
optimized_video = Fabricate(:optimized_video)
expect { optimized_video.destroy }.to change(OptimizedVideo, :count).by(-1)
expect(Upload.exists?(optimized_video.optimized_upload_id)).to be(false)
end
end
end

334
spec/services/video_conversion/aws_media_convert_adapter_spec.rb Normal file
View file

@ -0,0 +1,334 @@
# frozen_string_literal: true

require "rails_helper"
require "aws-sdk-mediaconvert"
require "aws-sdk-s3" # so Aws::S3::Object::Acl is loaded

# Dummy context struct for verifying double
FakeContext = Struct.new(:request_id)

RSpec.describe VideoConversion::AwsMediaConvertAdapter do
fab!(:user)

before(:each) do
extensions = SiteSetting.authorized_extensions.split("|")
SiteSetting.authorized_extensions = (extensions | ["mp4"]).join("|")
end

let!(:upload) { Fabricate(:video_upload, user: user) }
fab!(:post) { Fabricate(:post, user: user) }
let(:options) { { quality: "high" } }
let(:adapter) { described_class.new(upload, options) }
let(:mediaconvert_client) { instance_double(Aws::MediaConvert::Client) }
let(:s3_store) { instance_double(FileStore::S3Store) }
let(:s3_object) { instance_double(Aws::S3::Object) }
let(:s3_bucket) { "test-bucket" }
let(:s3_region) { "us-east-1" }
let(:new_sha1) { "a" * 40 } # A valid SHA1 is 40 characters
let(:mediaconvert_job) { instance_double(Aws::MediaConvert::Types::Job) }
let(:mediaconvert_job_response) do
instance_double(Aws::MediaConvert::Types::CreateJobResponse, job: mediaconvert_job)
end
let(:mediaconvert_context) { instance_double(FakeContext, request_id: "test-request-id") }
let(:post_relation) { instance_double(ActiveRecord::Relation) }
# The ACL resource class is Aws::S3::ObjectAcl in aws-sdk-s3 v3
let(:acl_object) { instance_double(Aws::S3::ObjectAcl) }

before do
upload.update!(sha1: new_sha1)

allow(SecureRandom).to receive(:hex).with(20).and_return(new_sha1)

allow(SiteSetting).to receive(:video_conversion_enabled).and_return(true)
allow(SiteSetting).to receive(:mediaconvert_role_arn).and_return(
"arn:aws:iam::123456789012:role/MediaConvertRole",
)
allow(SiteSetting).to receive(:mediaconvert_endpoint).and_return(
"https://mediaconvert.endpoint",
)
allow(SiteSetting).to receive(:s3_upload_bucket).and_return(s3_bucket)
allow(SiteSetting).to receive(:s3_region).and_return(s3_region)
allow(SiteSetting).to receive(:s3_access_key_id).and_return("test-key")
allow(SiteSetting).to receive(:s3_secret_access_key).and_return("test-secret")
allow(SiteSetting).to receive(:s3_use_acls).and_return(true)

allow(Aws::MediaConvert::Client).to receive(:new).and_return(mediaconvert_client)
allow(FileStore::S3Store).to receive(:new).and_return(s3_store)
allow(s3_store).to receive(:s3_bucket).and_return(s3_bucket)
allow(s3_store).to receive(:object_from_path).and_return(s3_object)
allow(s3_object).to receive(:exists?).and_return(true)
allow(s3_object).to receive(:size).and_return(1024)
allow(s3_object).to receive(:acl).and_return(acl_object)
allow(acl_object).to receive(:put).with(acl: "public-read").and_return(true)

allow(UploadReference).to receive(:where).with(
upload_id: upload.id,
target_type: "Post",
).and_return(instance_double(ActiveRecord::Relation, pluck: [post.id]))

allow(Post).to receive(:where).with(id: [post.id]).and_return(post_relation)
allow(post_relation).to receive(:find_each).and_yield(post)
allow(post).to receive(:rebake!)
allow(Rails.logger).to receive(:error)
allow(Rails.logger).to receive(:warn)
allow(Rails.logger).to receive(:info)
allow(Discourse).to receive(:warn_exception)
allow(Jobs).to receive(:enqueue_in)
allow(OptimizedVideo).to receive(:create_for)
end

describe "#convert" do
let(:output_path) { "/uploads/default/test_0/original/1X/#{new_sha1}" }
let(:job_id) { "job-123" }

before { allow(Jobs).to receive(:enqueue_in) }

context "when settings are valid" do
before do
upload.update!(
url: "//#{s3_bucket}.s3.#{s3_region}.amazonaws.com/uploads/default/original/test.mp4",
original_filename: "test.mp4",
)

allow(mediaconvert_job).to receive(:id).and_return(job_id)
allow(mediaconvert_client).to receive(:create_job).and_return(mediaconvert_job_response)
end

it "creates a MediaConvert job and enqueues status check" do
input_path = "s3://#{s3_bucket}/uploads/default/original/test.mp4"
expected_settings = described_class.build_conversion_settings(input_path, output_path)

expected_job_params = {
role: SiteSetting.mediaconvert_role_arn,
settings: expected_settings,
status_update_interval: "SECONDS_10",
user_metadata: {
"upload_id" => upload.id.to_s,
"new_sha1" => new_sha1,
"output_path" => output_path,
},
}

adapter.convert

expect(mediaconvert_client).to have_received(:create_job).with(expected_job_params)

expected_args = {
adapter_type: "aws_mediaconvert",
job_id: job_id,
new_sha1: new_sha1,
output_path: output_path,
original_filename: upload.original_filename,
upload_id: upload.id,
user_id: upload.user_id,
}

expect(Jobs).to have_received(:enqueue_in).with(
30.seconds,
:check_video_conversion_status,
expected_args,
)

expect(adapter.convert).to be true
end

it "handles MediaConvert service errors" do
error = Aws::MediaConvert::Errors::ServiceError.new(mediaconvert_context, "Test error")
allow(error).to receive(:code).and_return("InvalidParameter")
allow(mediaconvert_client).to receive(:create_job).and_raise(error)

adapter.convert

expect(Discourse).to have_received(:warn_exception).with(
error,
message: "MediaConvert job creation failed",
env: {
upload_id: upload.id,
},
)
expect(adapter.convert).to be false
end

it "handles unexpected errors" do
error = StandardError.new("Unexpected error")
allow(mediaconvert_client).to receive(:create_job).and_raise(error)

adapter.convert

expect(Discourse).to have_received(:warn_exception).with(
error,
message: "Unexpected error in MediaConvert job creation",
env: {
upload_id: upload.id,
},
)
expect(adapter.convert).to be false
end
end

context "when settings are invalid" do
before { allow(SiteSetting).to receive(:video_conversion_enabled).and_return(false) }

it "returns false" do
expect(adapter.convert).to be false
end
end

context "with invalid upload URL" do
before { upload.update!(url: "//wrong-bucket.s3.region.amazonaws.com/path/to/file") }

it "returns false and logs error" do
adapter.convert
expect(Rails.logger).to have_received(:error).with(
"Invalid parameters for upload #{upload.id}: Upload URL domain for upload ID #{upload.id} does not contain expected bucket name: #{s3_bucket}",
)
expect(adapter.convert).to be false
end
end
end

describe "#check_status" do
let(:job_id) { "job-123" }

context "when job is complete" do
before do
allow(mediaconvert_job).to receive(:status).and_return("COMPLETE")
allow(mediaconvert_client).to receive(:get_job).and_return(mediaconvert_job_response)
end

it "returns :complete" do
expect(adapter.check_status(job_id)).to eq(:complete)
end
end

context "when job has error" do
before do
allow(mediaconvert_job).to receive(:status).and_return("ERROR")
allow(mediaconvert_client).to receive(:get_job).and_return(mediaconvert_job_response)
end

it "returns :error and logs the error" do
adapter.check_status(job_id)
expect(Rails.logger).to have_received(:error).with(/MediaConvert job #{job_id} failed/)
expect(adapter.check_status(job_id)).to eq(:error)
end
end

context "when job is in progress" do
before do
allow(mediaconvert_job).to receive(:status).and_return("PROGRESSING")
allow(mediaconvert_client).to receive(:get_job).and_return(mediaconvert_job_response)
end

it "returns :pending" do
expect(adapter.check_status(job_id)).to eq(:pending)
end
end

context "when job has unexpected status" do
before do
allow(mediaconvert_job).to receive(:status).and_return("UNKNOWN")
allow(mediaconvert_client).to receive(:get_job).and_return(mediaconvert_job_response)
end

it "returns :error and logs warning" do
adapter.check_status(job_id)
expect(Rails.logger).to have_received(:warn).with(/Unexpected MediaConvert job status/)
expect(adapter.check_status(job_id)).to eq(:error)
end
end
end

describe "#handle_completion" do
let(:job_id) { "job-123" }
let(:output_path) { "optimized/videos/test-sha1" }
let(:expected_url) do
"//#{s3_bucket}.s3.dualstack.#{s3_region}.amazonaws.com/#{output_path}.mp4"
end

it "creates optimized video record and rebakes posts" do
allow(s3_object).to receive(:exists?).and_return(true)
allow(s3_object).to receive(:size).and_return(1024)
allow(OptimizedVideo).to receive(:create_for).and_return(true)

adapter.handle_completion(job_id, output_path, new_sha1)

expect(s3_object).to have_received(:exists?)
expect(s3_object).to have_received(:size)
expect(OptimizedVideo).to have_received(:create_for).with(
upload,
"video_converted.mp4",
upload.user_id,
{
extension: "mp4",
filesize: 1024,
sha1: new_sha1,
url: expected_url,
adapter: "aws_mediaconvert",
},
)
expect(post).to have_received(:rebake!)
expect(Rails.logger).to have_received(:info).with(/Rebaking post #{post.id}/)

expect(adapter.handle_completion(job_id, output_path, new_sha1)).to be true
end

context "when S3 object doesn't exist" do
before { allow(s3_object).to receive(:exists?).and_return(false) }

it "returns false" do
expect(adapter.handle_completion(job_id, output_path, new_sha1)).to be false
end
end

context "when optimized video creation fails" do
before do
allow(s3_object).to receive(:exists?).and_return(true)
allow(OptimizedVideo).to receive(:create_for).and_return(false)
end

it "returns false and logs error" do
adapter.handle_completion(job_id, output_path, new_sha1)
expect(Rails.logger).to have_received(:error).with(/Failed to create OptimizedVideo record/)
expect(adapter.handle_completion(job_id, output_path, new_sha1)).to be false
end
end

context "when an error occurs" do
let(:error) { StandardError.new("Test error") }

before do
allow(s3_object).to receive(:exists?).and_return(true)
allow(s3_object).to receive(:size).and_return(1024)
allow(OptimizedVideo).to receive(:create_for).and_raise(error)
end

it "returns false and logs error" do
adapter.handle_completion(job_id, output_path, new_sha1)
expect(Discourse).to have_received(:warn_exception).with(
error,
message: "Error in video processing completion",
env: {
upload_id: upload.id,
job_id: job_id,
},
)
expect(adapter.handle_completion(job_id, output_path, new_sha1)).to be false
end
end

context "when ACL update is disabled" do
before do
allow(SiteSetting).to receive(:s3_use_acls).and_return(false)
allow(s3_object).to receive(:exists?).and_return(true)
allow(OptimizedVideo).to receive(:create_for).and_return(true)
end

it "skips ACL update and completes successfully" do
adapter.handle_completion(job_id, output_path, new_sha1)
expect(s3_object).not_to have_received(:acl)
expect(adapter.handle_completion(job_id, output_path, new_sha1)).to be true
end
end
end
end

100
spec/services/video_conversion/base_adapter_spec.rb Normal file
View file

@ -0,0 +1,100 @@
# frozen_string_literal: true

RSpec.describe VideoConversion::BaseAdapter do
before(:each) do
extensions = SiteSetting.authorized_extensions.split("|")
SiteSetting.authorized_extensions = (extensions | ["mp4"]).join("|")
end

let(:upload) { Fabricate(:video_upload) }
let(:options) { { quality: "high" } }
let(:adapter) { described_class.new(upload, options) }

describe "#initialize" do
it "sets the upload and options" do
expect(adapter.instance_variable_get(:@upload)).to eq(upload)
expect(adapter.instance_variable_get(:@options)).to eq(options)
end
end

describe "#convert" do
it "raises NotImplementedError" do
expect { adapter.convert }.to raise_error(
NotImplementedError,
"#{described_class} must implement #convert",
)
end
end

describe "#check_status" do
it "raises NotImplementedError" do
expect { adapter.check_status("job-123") }.to raise_error(
NotImplementedError,
"#{described_class} must implement #check_status",
)
end
end

describe "#handle_completion" do
it "raises NotImplementedError" do
expect {
adapter.handle_completion("job-123", "/path/to/output.mp4", "new-sha1")
}.to raise_error(NotImplementedError, "#{described_class} must implement #handle_completion")
end
end

describe "#create_optimized_video_record" do
let(:output_path) { "/path/to/output.mp4" }
let(:new_sha1) { "new-sha1-hash" }
let(:filesize) { 1024 }
let(:url) { "//bucket.s3.region.amazonaws.com/optimized/videos/new-sha1-hash.mp4" }

before { allow(OptimizedVideo).to receive(:create_for) }

context "with adapter that defines ADAPTER_NAME" do
let(:adapter_name) { "test_adapter_#{SecureRandom.hex(4)}" }
let(:test_adapter_class) do
adapter_name_const = adapter_name
Class.new(VideoConversion::BaseAdapter) do
const_set(:ADAPTER_NAME, adapter_name_const)
def self.name
"TestAdapter"
end
end
end
let(:adapter) { test_adapter_class.new(upload) }

it "creates an optimized video record with correct attributes" do
adapter.send(:create_optimized_video_record, output_path, new_sha1, filesize, url)

expect(OptimizedVideo).to have_received(:create_for).with(
upload,
"video_converted.mp4",
upload.user_id,
filesize: filesize,
sha1: new_sha1,
url: url,
extension: "mp4",
adapter: adapter_name,
)
end

it "handles filenames with multiple extensions" do
upload.update!(original_filename: "video.original.mp4")

adapter.send(:create_optimized_video_record, output_path, new_sha1, filesize, url)

expect(OptimizedVideo).to have_received(:create_for).with(
upload,
"video.original_converted.mp4",
upload.user_id,
filesize: filesize,
sha1: new_sha1,
url: url,
extension: "mp4",
adapter: adapter_name,
)
end
end
end
end