From 01bc465db819d5ba0e433a8226c80a9bd38b593d Mon Sep 17 00:00:00 2001 From: Roman Rizzi Date: Fri, 11 Oct 2019 14:38:10 -0300 Subject: [PATCH] DEV: Split max decompressed setting for themes and backups (#8179) --- config/site_settings.yml | 5 ++++- ...1_migrate_decompressed_file_max_size_mb.rb | 19 +++++++++++++++++++ lib/backup_restore/restorer.rb | 11 +++++++++-- lib/compression/pipeline.rb | 4 ++-- lib/compression/strategy.rb | 8 ++++---- lib/theme_store/zip_importer.rb | 3 ++- .../theme_store/zip_exporter_spec.rb | 3 ++- spec/lib/compression/engine_spec.rb | 8 +++++--- 8 files changed, 47 insertions(+), 14 deletions(-) create mode 100644 db/migrate/20191011131041_migrate_decompressed_file_max_size_mb.rb diff --git a/config/site_settings.yml b/config/site_settings.yml index 2d5daec3975..2f3962e4c01 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -1201,9 +1201,12 @@ files: default: 5 min: 0 max: 20 - decompressed_file_max_size_mb: + decompressed_theme_max_file_size_mb: default: 1000 hidden: true + decompressed_backup_max_file_size_mb: + default: 100000 + hidden: true trust: default_trust_level: diff --git a/db/migrate/20191011131041_migrate_decompressed_file_max_size_mb.rb b/db/migrate/20191011131041_migrate_decompressed_file_max_size_mb.rb new file mode 100644 index 00000000000..f3db85da390 --- /dev/null +++ b/db/migrate/20191011131041_migrate_decompressed_file_max_size_mb.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class MigrateDecompressedFileMaxSizeMb < ActiveRecord::Migration[6.0] + def up + current_value = DB.query_single("SELECT value FROM site_settings WHERE name ='decompressed_file_max_size_mb' ").first + + if current_value && current_value != '1000' + DB.exec <<~SQL + INSERT INTO site_settings (name, data_type, value, created_at, updated_at) + VALUES + ('decompressed_theme_max_file_size_mb', 3, #{current_value}, current_timestamp, current_timestamp), + ('decompressed_backup_max_file_size_mb', 3, #{current_value}, current_timestamp, current_timestamp) + SQL + end + end + + def down + end +end diff --git a/lib/backup_restore/restorer.rb b/lib/backup_restore/restorer.rb index 2480941ceb6..42fd2db39f9 100644 --- a/lib/backup_restore/restorer.rb +++ b/lib/backup_restore/restorer.rb @@ -92,6 +92,9 @@ module BackupRestore extract_uploads after_restore_hook + rescue Compression::Strategy::ExtractFailed + log "The uncompressed file is too big. Consider increasing the decompressed_theme_max_file_size_mb hidden setting." + rollback rescue SystemExit log "Restore process was cancelled!" rollback @@ -138,7 +141,7 @@ module BackupRestore pipeline = Compression::Pipeline.new([Compression::Tar.new, Compression::Gzip.new]) - unzipped_path = pipeline.decompress(@tmp_directory, @archive_filename) + unzipped_path = pipeline.decompress(@tmp_directory, @archive_filename, available_size) pipeline.strip_directory(unzipped_path, @tmp_directory) end @@ -170,11 +173,15 @@ module BackupRestore log "Extracting dump file..." - Compression::Gzip.new.decompress(@tmp_directory, @dump_filename) + Compression::Gzip.new.decompress(@tmp_directory, @dump_filename, available_size) end protected + def available_size + SiteSetting.decompressed_backup_max_file_size_mb + end + def ensure_restore_is_enabled raise BackupRestore::RestoreDisabledError unless Rails.env.development? || SiteSetting.allow_restore? end diff --git a/lib/compression/pipeline.rb b/lib/compression/pipeline.rb index f955268cd5a..98a93cce414 100644 --- a/lib/compression/pipeline.rb +++ b/lib/compression/pipeline.rb @@ -20,10 +20,10 @@ module Compression end end - def decompress(dest_path, compressed_file_path, allow_non_root_folder: false) + def decompress(dest_path, compressed_file_path, max_size, allow_non_root_folder: false) @strategies.reverse.reduce(compressed_file_path) do |to_decompress, strategy| last_extension = strategy.extension - strategy.decompress(dest_path, to_decompress, allow_non_root_folder: allow_non_root_folder) + strategy.decompress(dest_path, to_decompress, max_size, allow_non_root_folder: allow_non_root_folder) to_decompress.gsub(last_extension, '') end end diff --git a/lib/compression/strategy.rb b/lib/compression/strategy.rb index 921ebd80135..47151a3bf07 100644 --- a/lib/compression/strategy.rb +++ b/lib/compression/strategy.rb @@ -9,11 +9,11 @@ module Compression file_name.include?(extension) end - def decompress(dest_path, compressed_file_path, allow_non_root_folder: false) + def decompress(dest_path, compressed_file_path, max_size, allow_non_root_folder: false) sanitized_compressed_file_path = sanitize_path(compressed_file_path) get_compressed_file_stream(sanitized_compressed_file_path) do |compressed_file| - available_size = calculate_available_size + available_size = calculate_available_size(max_size) entries_of(compressed_file).each do |entry| entry_path = build_entry_path( @@ -59,8 +59,8 @@ module Compression end end - def calculate_available_size - 1024**2 * (SiteSetting.decompressed_file_max_size_mb / 1.049) # Mb to Mib + def calculate_available_size(max_size) + 1024**2 * (max_size / 1.049) # Mb to Mib end def entries_of(compressed_file) diff --git a/lib/theme_store/zip_importer.rb b/lib/theme_store/zip_importer.rb index c1a4aea0989..d252037ace6 100644 --- a/lib/theme_store/zip_importer.rb +++ b/lib/theme_store/zip_importer.rb @@ -18,8 +18,9 @@ class ThemeStore::ZipImporter FileUtils.mkdir(@temp_folder) Dir.chdir(@temp_folder) do + available_size = SiteSetting.decompressed_theme_max_file_size_mb Compression::Engine.engine_for(@original_filename).tap do |engine| - engine.decompress(@temp_folder, @filename) + engine.decompress(@temp_folder, @filename, available_size) engine.strip_directory(@temp_folder, @temp_folder, relative: true) end end diff --git a/spec/components/theme_store/zip_exporter_spec.rb b/spec/components/theme_store/zip_exporter_spec.rb index e7df513aa84..922ae3081d7 100644 --- a/spec/components/theme_store/zip_exporter_spec.rb +++ b/spec/components/theme_store/zip_exporter_spec.rb @@ -63,7 +63,8 @@ describe ThemeStore::ZipExporter do file = 'discourse-header-icons.zip' dest = 'discourse-header-icons' Dir.chdir(dir) do - Compression::Zip.new.decompress(dir, file, allow_non_root_folder: true) + available_size = SiteSetting.decompressed_theme_max_file_size_mb + Compression::Zip.new.decompress(dir, file, available_size, allow_non_root_folder: true) `rm #{file}` folders = Dir.glob("**/*").reject { |f| File.file?(f) } diff --git a/spec/lib/compression/engine_spec.rb b/spec/lib/compression/engine_spec.rb index 04b6f7d42e8..27a161934ae 100644 --- a/spec/lib/compression/engine_spec.rb +++ b/spec/lib/compression/engine_spec.rb @@ -3,6 +3,8 @@ require 'rails_helper' describe Compression::Engine do + let(:available_size) { SiteSetting.decompressed_theme_max_file_size_mb } + before do @temp_folder = "#{Pathname.new(Dir.tmpdir).realpath}/#{SecureRandom.hex}" @folder_name = 'test' @@ -36,7 +38,7 @@ describe Compression::Engine do it 'decompress the folder and inspect files correctly' do engine = described_class.engine_for(@compressed_path) - engine.decompress(@temp_folder, "#{@temp_folder}/#{@folder_name}.zip") + engine.decompress(@temp_folder, "#{@temp_folder}/#{@folder_name}.zip", available_size) expect(read_file("test/hello.txt")).to eq("hello world") expect(read_file("test/a/inner")).to eq("hello world inner") @@ -49,7 +51,7 @@ describe Compression::Engine do it 'decompress the folder and inspect files correctly' do engine = described_class.engine_for(@compressed_path) - engine.decompress(@temp_folder, "#{@temp_folder}/#{@folder_name}.tar.gz") + engine.decompress(@temp_folder, "#{@temp_folder}/#{@folder_name}.tar.gz", available_size) expect(read_file("test/hello.txt")).to eq("hello world") expect(read_file("test/a/inner")).to eq("hello world inner") @@ -62,7 +64,7 @@ describe Compression::Engine do it 'decompress the folder and inspect files correctly' do engine = described_class.engine_for(@compressed_path) - engine.decompress(@temp_folder, "#{@temp_folder}/#{@folder_name}.tar") + engine.decompress(@temp_folder, "#{@temp_folder}/#{@folder_name}.tar", available_size) expect(read_file("test/hello.txt")).to eq("hello world") expect(read_file("test/a/inner")).to eq("hello world inner")