DEV: Add support for converting and importing linked user_custom_fields (#34339)

First pass at converting and importing user customer fields linked to
user fields


Depends on https://github.com/discourse/discourse/pull/34739

migrations/config/intermediate_db.yml

@@ -123,6 +123,11 @@ schema:
         modify:
           - name: "last_used"
             nullable: true
+    user_custom_fields:
+      columns:
+        exclude:
+          - "id"
+      primary_key_column_names: [ "user_id" , "name", "value" ]
     user_emails:
       columns:
         include:
@@ -136,6 +141,27 @@ schema:
       columns:
         exclude:
           - "id"
+    user_field_values:
+      copy_of: user_custom_fields
+      columns:
+        exclude:
+          - "id"
+          - "name"
+        add:
+          - name: "field_id"
+            datatype: numeric
+            nullable: false
+          - name: "is_multiselect_field"
+            datatype: boolean
+      indexes:
+        - name: "user_field_values_multiselect_index"
+          columns: [ "user_id", "field_id", "value" ]
+          unique: true
+          condition: "WHERE is_multiselect_field = TRUE"
+        - name: "user_field_values_not_multiselect_index"
+          columns: [ "user_id", "field_id" ]
+          unique: true
+          condition: "WHERE is_multiselect_field = FALSE"
     user_fields:
       columns:
         exclude:
@@ -428,7 +454,6 @@ schema:
         - "user_badges"
         - "user_chat_channel_memberships"
         - "user_chat_thread_memberships"
-        - "user_custom_fields"
         - "user_exports"
         - "user_histories"
         - "user_ip_address_histories"

migrations/db/intermediate_db_schema/100-base-schema.sql

@@ -202,6 +202,15 @@ CREATE TABLE user_associated_accounts
     PRIMARY KEY (user_id, provider_name)
 );
 
+CREATE TABLE user_custom_fields
+(
+    name       TEXT     NOT NULL,
+    user_id    NUMERIC  NOT NULL,
+    value      TEXT,
+    created_at DATETIME,
+    PRIMARY KEY (user_id, name, value)
+);
+
 CREATE TABLE user_emails
 (
     email      TEXT     NOT NULL,
@@ -219,6 +228,18 @@ CREATE TABLE user_field_options
     PRIMARY KEY (user_field_id, value)
 );
 
+CREATE TABLE user_field_values
+(
+    created_at           DATETIME,
+    field_id             NUMERIC  NOT NULL,
+    is_multiselect_field BOOLEAN,
+    user_id              NUMERIC  NOT NULL,
+    value                TEXT
+);
+
+CREATE UNIQUE INDEX user_field_values_multiselect_index ON user_field_values (user_id, field_id, value) WHERE is_multiselect_field = TRUE;
+CREATE UNIQUE INDEX user_field_values_not_multiselect_index ON user_field_values (user_id, field_id) WHERE is_multiselect_field = FALSE;
+
 CREATE TABLE user_fields
 (
     original_id       NUMERIC  NOT NULL PRIMARY KEY,

migrations/lib/converters/discourse/steps/user_field_values.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Migrations::Converters::Discourse
+  class UserFieldValues < ::Migrations::Converters::Base::ProgressStep
+    USER_FIELD_PREFIX = "user_field_"
+
+    attr_accessor :source_db
+
+    def max_progress
+      @source_db.count <<~SQL
+        SELECT COUNT(*)
+        FROM user_custom_fields
+        WHERE user_id >= 0
+          AND name LIKE '#{USER_FIELD_PREFIX}%'
+      SQL
+    end
+
+    def items
+      @source_db.query <<~SQL
+        SELECT user_custom_fields.*,
+               CAST(REPLACE(name, '#{USER_FIELD_PREFIX}', '') AS INTEGER) AS field_id,
+               (COUNT(*) OVER (PARTITION BY user_id, name) > 1)           AS is_multiselect_field
+        FROM user_custom_fields
+        WHERE user_id >= 0
+          AND name LIKE '#{USER_FIELD_PREFIX}%'
+        ORDER BY user_id, name
+      SQL
+    end
+
+    def process_item(item)
+      IntermediateDB::UserFieldValue.create(
+        created_at: item[:created_at],
+        field_id: item[:field_id],
+        user_id: item[:user_id],
+        value: item[:value],
+        is_multiselect_field: item[:is_multiselect_field],
+      )
+    end
+  end
+end

migrations/lib/database/intermediate_db/user_custom_field.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the "config/intermediate_db.yml" configuration file and then run
+# `bin/cli schema generate` to regenerate this file.
+
+module Migrations::Database::IntermediateDB
+  module UserCustomField
+    SQL = <<~SQL
+      INSERT INTO user_custom_fields (
+        name,
+        user_id,
+        value,
+        created_at
+      )
+      VALUES (
+        ?, ?, ?, ?
+      )
+    SQL
+    private_constant :SQL
+
+    def self.create(name:, user_id:, value:, created_at: nil)
+      ::Migrations::Database::IntermediateDB.insert(
+        SQL,
+        name,
+        user_id,
+        value,
+        ::Migrations::Database.format_datetime(created_at),
+      )
+    end
+  end
+end

migrations/lib/database/intermediate_db/user_field_value.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the "config/intermediate_db.yml" configuration file and then run
+# `bin/cli schema generate` to regenerate this file.
+
+module Migrations::Database::IntermediateDB
+  module UserFieldValue
+    SQL = <<~SQL
+      INSERT INTO user_field_values (
+        created_at,
+        field_id,
+        is_multiselect_field,
+        user_id,
+        value
+      )
+      VALUES (
+        ?, ?, ?, ?, ?
+      )
+    SQL
+    private_constant :SQL
+
+    def self.create(created_at: nil, field_id:, is_multiselect_field: nil, user_id:, value: nil)
+      ::Migrations::Database::IntermediateDB.insert(
+        SQL,
+        ::Migrations::Database.format_datetime(created_at),
+        field_id,
+        ::Migrations::Database.format_boolean(is_multiselect_field),
+        user_id,
+        value,
+      )
+    end
+  end
+end

migrations/lib/importer/steps/user_custom_fields.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Migrations::Importer::Steps
+  class UserCustomFields < ::Migrations::Importer::CopyStep
+    depends_on :users
+
+    requires_set :existing_user_custom_fields, <<~SQL
+      SELECT user_id, name, value
+      FROM user_custom_fields
+      WHERE user_id > 0 AND name NOT LIKE '#{User::USER_FIELD_PREFIX}%'
+    SQL
+
+    column_names %i[created_at updated_at user_id name value]
+
+    total_rows_query <<~SQL, MappingType::USERS
+      SELECT COUNT(*)
+      FROM user_custom_fields
+           JOIN mapped.ids mapped_user
+             ON user_custom_fields.user_id = mapped_user.original_id
+               AND mapped_user.type = ?
+    SQL
+
+    rows_query <<~SQL, MappingType::USERS
+      SELECT user_custom_fields.*,
+              mapped_user.discourse_id AS discourse_user_id
+      FROM user_custom_fields
+           JOIN mapped.ids mapped_user
+              ON user_custom_fields.user_id = mapped_user.original_id
+               AND mapped_user.type = ?
+      ORDER BY user_custom_fields.user_id
+    SQL
+
+    private
+
+    def transform_row(row)
+      name = row[:name]
+      user_id = row[:discourse_user_id]
+
+      if name.start_with?(User::USER_FIELD_PREFIX)
+        puts "    '#{name}': Name cannot start with #{User::USER_FIELD_PREFIX}"
+        return nil
+      end
+
+      return nil unless @existing_user_custom_fields.add?(user_id, name, row[:value])
+
+      row[:user_id] = user_id
+
+      super
+    end
+  end
+end

migrations/lib/importer/steps/user_field_values.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Migrations::Importer::Steps
+  class UserFieldValues < ::Migrations::Importer::CopyStep
+    depends_on :users, :user_fields
+
+    requires_set :existing_user_field_values, <<~SQL
+      SELECT user_id, name, value
+      FROM user_custom_fields
+      WHERE user_id > 0 AND name LIKE '#{User::USER_FIELD_PREFIX}%'
+    SQL
+
+    table_name :user_custom_fields
+    column_names %i[created_at updated_at user_id name value]
+
+    total_rows_query <<~SQL, MappingType::USER_FIELDS, MappingType::USERS
+      SELECT COUNT(*)
+      FROM user_field_values
+           JOIN mapped.ids mapped_user_field
+             ON user_field_values.field_id = mapped_user_field.original_id
+               AND mapped_user_field.type =  ?1
+           JOIN mapped.ids mapped_user
+             ON user_field_values.user_id = mapped_user.original_id
+               AND mapped_user.type = ?2
+    SQL
+
+    rows_query <<~SQL, MappingType::USER_FIELDS, MappingType::USERS
+      SELECT user_field_values.*,
+              mapped_user_field.discourse_id AS discourse_user_field_id,
+              mapped_user.discourse_id       AS discourse_user_id
+      FROM user_field_values
+           JOIN mapped.ids mapped_user_field
+             ON user_field_values.field_id = mapped_user_field.original_id
+               AND mapped_user_field.type = ?1
+            JOIN mapped.ids mapped_user
+              ON user_field_values.user_id = mapped_user.original_id
+               AND mapped_user.type = ?2
+      ORDER BY user_field_values.user_id
+    SQL
+
+    private
+
+    def transform_row(row)
+      name = "#{User::USER_FIELD_PREFIX}#{row[:discourse_user_field_id]}"
+      user_id = row[:discourse_user_id]
+
+      return nil unless @existing_user_field_values.add?(user_id, name, row[:value])
+
+      row[:name] = name
+      row[:user_id] = user_id
+
+      super
+    end
+  end
+end