diff --git a/migrations/config/importer.yml b/migrations/config/importer.yml index c085cda8acf..8339b8f999a 100644 --- a/migrations/config/importer.yml +++ b/migrations/config/importer.yml @@ -1,2 +1,3 @@ intermediate_db: /shared/import/intermediate.db mappings_db: /shared/import/mappings.db +uploads_db: /shared/import/uploads.db diff --git a/migrations/lib/cli/import_command.rb b/migrations/lib/cli/import_command.rb index 44465aa784b..941416e8ec9 100644 --- a/migrations/lib/cli/import_command.rb +++ b/migrations/lib/cli/import_command.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require "extralite" - module Migrations::CLI class ImportCommand def initialize(options) diff --git a/migrations/lib/cli/upload_command.rb b/migrations/lib/cli/upload_command.rb index 843cb1f70d5..49fb6a7ea76 100644 --- a/migrations/lib/cli/upload_command.rb +++ b/migrations/lib/cli/upload_command.rb @@ -9,6 +9,10 @@ module Migrations::CLI def execute puts "Starting uploads..." + ::Migrations.load_rails_environment(quiet: true) + + adjust_db_pool_size + validate_settings_file! settings = load_settings @@ -21,7 +25,7 @@ module Migrations::CLI def load_settings settings = ::Migrations::SettingsParser.parse!(@options.settings) - merge_settings_from_cli_args!(@options, settings) + merge_settings_from_cli_args!(settings) settings end @@ -36,5 +40,20 @@ module Migrations::CLI raise ::Migrations::NoSettingsFound, "Settings file not found: #{path}" if !File.exist?(path) end + + def adjust_db_pool_size + max_db_connections = DB.query_single("SHOW max_connections").first.to_i + current_size = ActiveRecord::Base.connection_pool.size + + if current_size < max_db_connections + db_config = ActiveRecord::Base.connection_db_config.configuration_hash.dup + db_config[:pool] = max_db_connections + ActiveRecord::Base.establish_connection(db_config) + + puts "Adjusted DB pool size from #{current_size} to #{ActiveRecord::Base.connection_pool.size}" + else + puts "DB pool size: #{current_size} (max connections: #{max_db_connections})" + end + end end end diff --git a/migrations/lib/importer/executor.rb b/migrations/lib/importer/executor.rb index 1f1860d07a0..178a54e5230 100644 --- a/migrations/lib/importer/executor.rb +++ b/migrations/lib/importer/executor.rb @@ -8,6 +8,7 @@ module Migrations::Importer @shared_data = SharedData.new(@discourse_db) attach_mappings_db(config[:mappings_db]) + attach_uploads_db(config[:uploads_db]) end def start @@ -24,12 +25,16 @@ module Migrations::Importer private def attach_mappings_db(db_path) - # ::Migrations::Database.reset!(db_path) - ::Migrations::Database.migrate( - db_path, - migrations_path: ::Migrations::Database::MAPPINGS_DB_SCHEMA_PATH, - ) - @intermediate_db.execute("ATTACH DATABASE ? AS mapped", db_path) + migrate_and_attach(db_path, ::Migrations::Database::MAPPINGS_DB_SCHEMA_PATH, "mapped") + end + + def attach_uploads_db(db_path) + migrate_and_attach(db_path, ::Migrations::Database::UPLOADS_DB_SCHEMA_PATH, "files") + end + + def migrate_and_attach(db_path, schema_path, alias_name) + ::Migrations::Database.migrate(db_path, migrations_path: schema_path) + @intermediate_db.execute("ATTACH DATABASE ? AS #{alias_name}", db_path) end def step_classes diff --git a/migrations/lib/importer/mapping_type.rb b/migrations/lib/importer/mapping_type.rb index 5e3aef50615..bd1eff2a201 100644 --- a/migrations/lib/importer/mapping_type.rb +++ b/migrations/lib/importer/mapping_type.rb @@ -3,5 +3,6 @@ module Migrations::Importer module MappingType USERS = 1 + UPLOADS = 10 end end diff --git a/migrations/lib/importer/steps/uploads.rb b/migrations/lib/importer/steps/uploads.rb new file mode 100644 index 00000000000..04bf75e12ec --- /dev/null +++ b/migrations/lib/importer/steps/uploads.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Migrations::Importer::Steps + class Uploads < ::Migrations::Importer::CopyStep + depends_on :users + store_mapped_ids true + + requires_set :existing_sha1s, "SELECT sha1 FROM uploads" + + column_names %i[ + user_id + original_filename + filesize + width + height + url + created_at + updated_at + sha1 + origin + retain_hours + extension + thumbnail_width + thumbnail_height + etag + secure + access_control_post_id + original_sha1 + animated + verification_status + security_last_changed_at + security_last_changed_reason + dominant_color + ] + + total_rows_query <<~SQL, MappingType::UPLOADS + SELECT COUNT(*) + FROM files.uploads up + LEFT JOIN mapped.ids mup ON up.id = mup.original_id AND mup.type = ? + WHERE up.upload IS NOT NULL + AND mup.original_id IS NULL + SQL + + rows_query <<~SQL, MappingType::UPLOADS + SELECT up.id, up.upload + FROM files.uploads up + LEFT JOIN mapped.ids mup ON up.id = mup.original_id AND mup.type = ? + WHERE up.upload IS NOT NULL + AND mup.original_id IS NULL + ORDER BY up.ROWID + SQL + + private + + def transform_row(row) + upload = JSON.parse(row[:upload], symbolize_names: true) + return nil unless @existing_sha1s.add?(upload[:sha1]) + + upload[:original_id] = row[:id] + upload.delete(:id) + + super(upload) + end + end +end diff --git a/migrations/lib/uploader/uploads.rb b/migrations/lib/uploader/uploads.rb index 931de237695..192ae8d7809 100644 --- a/migrations/lib/uploader/uploads.rb +++ b/migrations/lib/uploader/uploads.rb @@ -62,7 +62,7 @@ module Migrations::Uploader end def configure_logging - @original_exifr_logger = EXIFR.logger + @original_exifr_logger = ::EXIFR.logger # disable logging for EXIFR which is used by ImageOptim EXIFR.logger = Logger.new(nil) @@ -74,7 +74,7 @@ module Migrations::Uploader def cleanup_resources databases.values.each(&:close) - EXIFR.logger = @original_exifr_logger + ::EXIFR.logger = @original_exifr_logger end end end