DEV: Import uploads from UploadsDB (#33259)

migrations/config/importer.yml

@@ -1,2 +1,3 @@
 intermediate_db: /shared/import/intermediate.db
 mappings_db: /shared/import/mappings.db
+uploads_db: /shared/import/uploads.db

migrations/lib/cli/import_command.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "extralite"
-
 module Migrations::CLI
   class ImportCommand
     def initialize(options)

migrations/lib/cli/upload_command.rb

@@ -9,6 +9,10 @@ module Migrations::CLI
     def execute
       puts "Starting uploads..."
 
+      ::Migrations.load_rails_environment(quiet: true)
+
+      adjust_db_pool_size
+
       validate_settings_file!
       settings = load_settings
 
@@ -21,7 +25,7 @@ module Migrations::CLI
 
     def load_settings
       settings = ::Migrations::SettingsParser.parse!(@options.settings)
-      merge_settings_from_cli_args!(@options, settings)
+      merge_settings_from_cli_args!(settings)
 
       settings
     end
@@ -36,5 +40,20 @@ module Migrations::CLI
 
       raise ::Migrations::NoSettingsFound, "Settings file not found: #{path}" if !File.exist?(path)
     end
+
+    def adjust_db_pool_size
+      max_db_connections = DB.query_single("SHOW max_connections").first.to_i
+      current_size = ActiveRecord::Base.connection_pool.size
+
+      if current_size < max_db_connections
+        db_config = ActiveRecord::Base.connection_db_config.configuration_hash.dup
+        db_config[:pool] = max_db_connections
+        ActiveRecord::Base.establish_connection(db_config)
+
+        puts "Adjusted DB pool size from #{current_size} to #{ActiveRecord::Base.connection_pool.size}"
+      else
+        puts "DB pool size: #{current_size} (max connections: #{max_db_connections})"
+      end
+    end
   end
 end

migrations/lib/importer/executor.rb

@@ -8,6 +8,7 @@ module Migrations::Importer
       @shared_data = SharedData.new(@discourse_db)
 
       attach_mappings_db(config[:mappings_db])
+      attach_uploads_db(config[:uploads_db])
     end
 
     def start
@@ -24,12 +25,16 @@ module Migrations::Importer
     private
 
     def attach_mappings_db(db_path)
-      # ::Migrations::Database.reset!(db_path)
-      ::Migrations::Database.migrate(
-        db_path,
-        migrations_path: ::Migrations::Database::MAPPINGS_DB_SCHEMA_PATH,
-      )
-      @intermediate_db.execute("ATTACH DATABASE ? AS mapped", db_path)
+      migrate_and_attach(db_path, ::Migrations::Database::MAPPINGS_DB_SCHEMA_PATH, "mapped")
+    end
+
+    def attach_uploads_db(db_path)
+      migrate_and_attach(db_path, ::Migrations::Database::UPLOADS_DB_SCHEMA_PATH, "files")
+    end
+
+    def migrate_and_attach(db_path, schema_path, alias_name)
+      ::Migrations::Database.migrate(db_path, migrations_path: schema_path)
+      @intermediate_db.execute("ATTACH DATABASE ? AS #{alias_name}", db_path)
     end
 
     def step_classes

migrations/lib/importer/mapping_type.rb

@@ -3,5 +3,6 @@
 module Migrations::Importer
   module MappingType
     USERS = 1
+    UPLOADS = 10
   end
 end

migrations/lib/importer/steps/uploads.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Migrations::Importer::Steps
+  class Uploads < ::Migrations::Importer::CopyStep
+    depends_on :users
+    store_mapped_ids true
+
+    requires_set :existing_sha1s, "SELECT sha1 FROM uploads"
+
+    column_names %i[
+                   user_id
+                   original_filename
+                   filesize
+                   width
+                   height
+                   url
+                   created_at
+                   updated_at
+                   sha1
+                   origin
+                   retain_hours
+                   extension
+                   thumbnail_width
+                   thumbnail_height
+                   etag
+                   secure
+                   access_control_post_id
+                   original_sha1
+                   animated
+                   verification_status
+                   security_last_changed_at
+                   security_last_changed_reason
+                   dominant_color
+                 ]
+
+    total_rows_query <<~SQL, MappingType::UPLOADS
+      SELECT COUNT(*)
+      FROM files.uploads up
+           LEFT JOIN mapped.ids mup ON up.id = mup.original_id AND mup.type = ?
+      WHERE up.upload IS NOT NULL
+        AND mup.original_id IS NULL
+    SQL
+
+    rows_query <<~SQL, MappingType::UPLOADS
+      SELECT up.id, up.upload
+      FROM files.uploads up
+           LEFT JOIN mapped.ids mup ON up.id = mup.original_id AND mup.type = ?
+      WHERE up.upload IS NOT NULL
+        AND mup.original_id IS NULL
+      ORDER BY up.ROWID
+    SQL
+
+    private
+
+    def transform_row(row)
+      upload = JSON.parse(row[:upload], symbolize_names: true)
+      return nil unless @existing_sha1s.add?(upload[:sha1])
+
+      upload[:original_id] = row[:id]
+      upload.delete(:id)
+
+      super(upload)
+    end
+  end
+end

migrations/lib/uploader/uploads.rb

@@ -62,7 +62,7 @@ module Migrations::Uploader
     end
 
     def configure_logging
-      @original_exifr_logger = EXIFR.logger
+      @original_exifr_logger = ::EXIFR.logger
 
       # disable logging for EXIFR which is used by ImageOptim
       EXIFR.logger = Logger.new(nil)
@@ -74,7 +74,7 @@ module Migrations::Uploader
 
     def cleanup_resources
       databases.values.each(&:close)
-      EXIFR.logger = @original_exifr_logger
+      ::EXIFR.logger = @original_exifr_logger
     end
   end
 end