diff --git a/script/import_scripts/base.rb b/script/import_scripts/base.rb index 2f076adc2fb..98fc685b1fc 100644 --- a/script/import_scripts/base.rb +++ b/script/import_scripts/base.rb @@ -366,6 +366,7 @@ class ImportScripts::Base # try based on email if e.try(:record).try(:errors).try(:messages).try(:[], :primary_email).present? if existing = User.find_by_email(opts[:email].downcase) + existing.created_at = opts[:created_at] if opts[:created_at] existing.custom_fields["import_id"] = import_id existing.save! u = existing @@ -630,6 +631,35 @@ class ImportScripts::Base [created, skipped] end + def create_likes(results, opts = {}) + created = 0 + skipped = 0 + total = opts[:total] || results.count + + results.each do |result| + params = yield(result) + + if params.nil? + skipped += 1 + else + created_by = User.find_by(id: user_id_from_imported_user_id(params[:user_id])) + post = Post.find_by(id: post_id_from_imported_post_id(params[:post_id])) + + if created_by && post + PostActionCreator.create(created_by, post, :like, created_at: params[:created_at]) + created += 1 + else + skipped += 1 + puts "Skipping like for user id #{params[:user_id]} and post id #{params[:post_id]}" + end + end + + print_status(created + skipped + (opts[:offset] || 0), total, get_start_time("likes")) + end + + [created, skipped] + end + def close_inactive_topics(opts = {}) num_days = opts[:days] || 30 puts '', "Closing topics that have been inactive for more than #{num_days} days." diff --git a/script/import_scripts/phpbb3/database/database_3_0.rb b/script/import_scripts/phpbb3/database/database_3_0.rb index a69a9f9fe23..89d5439b05e 100644 --- a/script/import_scripts/phpbb3/database/database_3_0.rb +++ b/script/import_scripts/phpbb3/database/database_3_0.rb @@ -13,7 +13,7 @@ module ImportScripts::PhpBB3 SQL end - def fetch_users(last_user_id) + def fetch_users(last_user_id, _profile_fields) query(<<-SQL, :user_id) SELECT u.user_id, u.user_email, u.username, u.user_password, u.user_regdate, u.user_lastvisit, u.user_ip, u.user_type, u.user_inactive_reason, g.group_name, b.ban_start, b.ban_end, b.ban_reason, @@ -227,12 +227,31 @@ module ImportScripts::PhpBB3 SELECT b.user_id, t.topic_first_post_id FROM #{@table_prefix}bookmarks b JOIN #{@table_prefix}topics t ON (b.topic_id = t.topic_id) - WHERE b.user_id > #{last_user_id} + WHERE (b.user_id, b.topic_id) > (#{last_user_id}, #{last_topic_id}) ORDER BY b.user_id, b.topic_id LIMIT #{@batch_size} SQL end + def count_likes + count(<<-SQL) + SELECT COUNT(*) AS count + FROM #{@table_prefix}thanks + WHERE user_id <> poster_id + SQL + end + + def fetch_likes(last_post_id, last_user_id) + query(<<-SQL, :post_id, :user_id) + SELECT post_id, user_id, thanks_time + FROM #{@table_prefix}thanks + WHERE user_id <> poster_id + AND (post_id, user_id) > (#{last_post_id}, #{last_user_id}) + ORDER BY post_id, user_id + LIMIT #{@batch_size} + SQL + end + def get_smiley(smiley_code) query(<<-SQL).first SELECT emotion, smiley_url diff --git a/script/import_scripts/phpbb3/database/database_3_1.rb b/script/import_scripts/phpbb3/database/database_3_1.rb index 95481429026..ee666bbbc01 100644 --- a/script/import_scripts/phpbb3/database/database_3_1.rb +++ b/script/import_scripts/phpbb3/database/database_3_1.rb @@ -5,7 +5,7 @@ require_relative '../support/constants' module ImportScripts::PhpBB3 class Database_3_1 < Database_3_0 - def fetch_users(last_user_id) + def fetch_users(last_user_id, profile_fields) query(<<-SQL, :user_id) SELECT u.user_id, u.user_email, u.username, CASE WHEN u.user_password LIKE '$2y$%' @@ -15,6 +15,7 @@ module ImportScripts::PhpBB3 u.user_type, u.user_inactive_reason, g.group_name, b.ban_start, b.ban_end, b.ban_reason, u.user_posts, f.pf_phpbb_website AS user_website, f.pf_phpbb_location AS user_from, u.user_birthday, u.user_avatar_type, u.user_avatar + #{profile_fields_query(profile_fields)} FROM #{@table_prefix}users u LEFT OUTER JOIN #{@table_prefix}profile_fields_data f ON (u.user_id = f.user_id) JOIN #{@table_prefix}groups g ON (g.group_id = u.group_id) @@ -27,5 +28,18 @@ module ImportScripts::PhpBB3 LIMIT #{@batch_size} SQL end + + private + + def profile_fields_query(profile_fields) + @profile_fields_query ||= begin + if profile_fields.present? + columns = profile_fields.map { |field| "pf_#{field[:phpbb_field_name]}" } + ", #{columns.join(', ')}" + else + "" + end + end + end end end diff --git a/script/import_scripts/phpbb3/importer.rb b/script/import_scripts/phpbb3/importer.rb index f9ee6e04b98..b8b84e29e69 100644 --- a/script/import_scripts/phpbb3/importer.rb +++ b/script/import_scripts/phpbb3/importer.rb @@ -38,6 +38,7 @@ module ImportScripts::PhpBB3 import_posts import_private_messages if @settings.import_private_messages import_bookmarks if @settings.import_bookmarks + import_likes if @settings.import_likes end def change_site_settings @@ -71,7 +72,7 @@ module ImportScripts::PhpBB3 last_user_id = 0 batches do |offset| - rows, last_user_id = @database.fetch_users(last_user_id) + rows, last_user_id = @database.fetch_users(last_user_id, @settings.custom_fields) rows = rows.to_a.uniq { |row| row[:user_id] } break if rows.size < 1 @@ -173,7 +174,7 @@ module ImportScripts::PhpBB3 importer = @importers.category_importer create_categories(rows) do |row| - next if @settings.category_mappings[row[:forum_id].to_s] == 'SKIP' + next if @settings.category_mappings.dig(row[:forum_id].to_s, :skip) importer.map_category(row) end @@ -241,6 +242,25 @@ module ImportScripts::PhpBB3 end end + def import_likes + puts '', 'importing likes' + total_count = @database.count_likes + last_post_id = last_user_id = 0 + + batches do |offset| + rows, last_post_id, last_user_id = @database.fetch_likes(last_post_id, last_user_id) + break if rows.size < 1 + + create_likes(rows, total: total_count, offset: offset) do |row| + { + post_id: @settings.prefix(row[:post_id]), + user_id: @settings.prefix(row[:user_id]), + created_at: Time.zone.at(row[:thanks_time]) + } + end + end + end + def update_last_seen_at # no need for this since the importer sets last_seen_at for each user during the import end diff --git a/script/import_scripts/phpbb3/importers/category_importer.rb b/script/import_scripts/phpbb3/importers/category_importer.rb index d7ad577ff3b..6e5725b97e5 100644 --- a/script/import_scripts/phpbb3/importers/category_importer.rb +++ b/script/import_scripts/phpbb3/importers/category_importer.rb @@ -17,7 +17,7 @@ module ImportScripts::PhpBB3 return if @settings.category_mappings[row[:forum_id].to_s] if row[:parent_id] && @settings.category_mappings[row[:parent_id].to_s] - puts "parent category (#{row[:parent_id]}) was mapped, but children was not (#{row[:forum_id]})" + puts "parent category (#{row[:parent_id]}) was mapped, but child was not (#{row[:forum_id]})" end { diff --git a/script/import_scripts/phpbb3/importers/post_importer.rb b/script/import_scripts/phpbb3/importers/post_importer.rb index 8355835568f..8f41e9ed669 100644 --- a/script/import_scripts/phpbb3/importers/post_importer.rb +++ b/script/import_scripts/phpbb3/importers/post_importer.rb @@ -22,7 +22,7 @@ module ImportScripts::PhpBB3 end def map_post(row) - return if @settings.category_mappings[row[:forum_id].to_s] == 'SKIP' + return if @settings.category_mappings.dig(row[:forum_id].to_s, :skip) imported_user_id = @settings.prefix(row[:post_username].blank? ? row[:poster_id] : row[:post_username]) user_id = @lookup.user_id_from_imported_user_id(imported_user_id) || -1 @@ -56,8 +56,13 @@ module ImportScripts::PhpBB3 def map_first_post(row, mapped) poll_data = add_poll(row, mapped) if @settings.import_polls - mapped[:category] = @lookup.category_id_from_imported_category_id(@settings.prefix(@settings.category_mappings[row[:forum_id].to_s])) || - @lookup.category_id_from_imported_category_id(@settings.prefix(row[:forum_id])) + mapped[:category] = if category_mapping = @settings.category_mappings[row[:forum_id].to_s] + category_mapping[:discourse_category_id] || + @lookup.category_id_from_imported_category_id(@settings.prefix(category_mapping[:target_category_id])) + else + @lookup.category_id_from_imported_category_id(@settings.prefix(row[:forum_id])) + end + mapped[:title] = CGI.unescapeHTML(row[:topic_title]).strip[0...255] mapped[:pinned_at] = mapped[:created_at] unless row[:topic_type] == Constants::POST_NORMAL mapped[:pinned_globally] = row[:topic_type] == Constants::POST_GLOBAL diff --git a/script/import_scripts/phpbb3/importers/user_importer.rb b/script/import_scripts/phpbb3/importers/user_importer.rb index bf12bbc5d62..3fa61d6e17e 100644 --- a/script/import_scripts/phpbb3/importers/user_importer.rb +++ b/script/import_scripts/phpbb3/importers/user_importer.rb @@ -42,6 +42,7 @@ module ImportScripts::PhpBB3 website: row[:user_website], location: row[:user_from], date_of_birth: parse_birthdate(row), + custom_fields: custom_fields(row), post_create_action: proc do |user| suspend_user(user, row) @avatar_importer.import_avatar(user, row) if row[:user_avatar_type].present? @@ -83,6 +84,45 @@ module ImportScripts::PhpBB3 birthdate && birthdate.year > 0 ? birthdate : nil end + def user_fields + @user_fields ||= begin + Hash[UserField.all.map { |field| [field.name, field] }] + end + end + + def field_mappings + @field_mappings ||= begin + @settings.custom_fields.map do |field| + { + phpbb_field_name: "pf_#{field[:phpbb_field_name]}".to_sym, + discourse_user_field: user_fields[field[:discourse_field_name]] + } + end + end + end + + def custom_fields(row) + return nil if @settings.custom_fields.blank? + + custom_fields = {} + + field_mappings.each do |field| + value = row[field[:phpbb_field_name]] + user_field = field[:discourse_user_field] + + case user_field.field_type + when "confirm" + value = value == 1 ? true : nil + when "dropdown" + value = user_field.user_field_options.find { |option| option.value == value } ? value : nil + end + + custom_fields["user_field_#{user_field.id}"] = value if value.present? + end + + custom_fields + end + # Suspends the user if it is currently banned. def suspend_user(user, row, disable_email = false) if row[:user_inactive_reason] == Constants::INACTIVE_MANUAL diff --git a/script/import_scripts/phpbb3/settings.yml b/script/import_scripts/phpbb3/settings.yml index e7c438a45b1..8727e67f594 100644 --- a/script/import_scripts/phpbb3/settings.yml +++ b/script/import_scripts/phpbb3/settings.yml @@ -36,20 +36,25 @@ import: # Category mappings # - # For example, topics from phpBB category 1 and 2 will be imported - # in the new "Foo Category" category, topics from phpBB category 3 - # will be imported in subcategory "Bar category", topics from phpBB - # category 4 will be merged into category 5 and category 6 will be - # skipped. + # * "source_category_id" is the forum ID in phpBB3 + # * "target_category_id" is either a forum ID from phpBB3 or a "forum_id" + # from the "new_categories" setting (see above) + # * "discourse_category_id" is a category ID from Discourse + # * "skip" allows you to ignore a category during import # - # category_mappings: - # 1: foo - # 2: foo - # 3: bar - # 4: 5 - # 6: SKIP + # Use "target_category_id" if you want to merge categories and use + # "discourse_category_id" if you want to import a forum into an existing + # category in Discourse. # - category_mappings: {} + # category_mappings: + # - source_category_id: 1 + # target_category_id: foo + # - source_category_id: 2 + # discourse_category_id: 42 + # - source_category_id: 6 + # skip: true + # + category_mappings: [] # Tag mappings # @@ -122,6 +127,9 @@ import: private_messages: true polls: true + # Import likes from the phpBB's "Thanks for posts" extension + likes: false + # When true: each imported user will have the original username from phpBB as its name # When false: the name of each imported user will be blank unless the username was changed during import username_as_name: false @@ -134,3 +142,12 @@ import: # here are two example mappings... smiley: [':D', ':-D', ':grin:'] heart: ':love:' + + # Map custom profile fields from phpBB to custom user fields in Discourse (works for phpBB 3.1+) + # + # custom_fields: + # - phpbb_field_name: "company_name" + # discourse_field_name: "Company" + # - phpbb_field_name: "facebook" + # discourse_field_name: "Facebook" + custom_fields: [] diff --git a/script/import_scripts/phpbb3/support/settings.rb b/script/import_scripts/phpbb3/support/settings.rb index ee01a66e3ac..b259821ab3c 100644 --- a/script/import_scripts/phpbb3/support/settings.rb +++ b/script/import_scripts/phpbb3/support/settings.rb @@ -24,6 +24,7 @@ module ImportScripts::PhpBB3 attr_reader :import_polls attr_reader :import_bookmarks attr_reader :import_passwords + attr_reader :import_likes attr_reader :import_uploaded_avatars attr_reader :import_remote_avatars @@ -38,6 +39,7 @@ module ImportScripts::PhpBB3 attr_reader :username_as_name attr_reader :emojis + attr_reader :custom_fields attr_reader :database @@ -47,7 +49,7 @@ module ImportScripts::PhpBB3 @site_name = import_settings['site_name'] @new_categories = import_settings['new_categories'] - @category_mappings = import_settings['category_mappings'] + @category_mappings = import_settings.fetch('category_mappings', []).to_h { |m| [m[:source_category_id].to_s, m] } @tag_mappings = import_settings['tag_mappings'] @rank_mapping = import_settings['rank_mapping'] @@ -57,6 +59,7 @@ module ImportScripts::PhpBB3 @import_polls = import_settings['polls'] @import_bookmarks = import_settings['bookmarks'] @import_passwords = import_settings['passwords'] + @import_likes = import_settings['likes'] avatar_settings = import_settings['avatars'] @import_uploaded_avatars = avatar_settings['uploaded'] @@ -72,6 +75,7 @@ module ImportScripts::PhpBB3 @username_as_name = import_settings['username_as_name'] @emojis = import_settings.fetch('emojis', []) + @custom_fields = import_settings.fetch('custom_fields', []) @database = DatabaseSettings.new(yaml['database']) end