Discourse/discourse
Discourse/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-09-04 08:47:37 +08:00
Code Packages Activity Actions 12

Improvements to mbox import script

Browse source 
* Ignore errors during indexing and show information about the message causing the problem
* Always activate imported users if they aren't staged
This commit is contained in:
Gerhard Schlager 2018-03-06 11:32:12 +01:00
parent bdb25338e5
commit dc32ee5cbf
2 changed files with 38 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

2
script/import_scripts/mbox/importer.rb
View file

@ -70,7 +70,7 @@ module ImportScripts::Mbox
name: row['name'], name: row['name'],
trust_level: @settings.trust_level, trust_level: @settings.trust_level,
staged: @settings.staged, staged: @settings.staged,
active: false, active: !@settings.staged,
created_at: to_time(row['date_of_first_message']) created_at: to_time(row['date_of_first_message'])
} }
end end

63
script/import_scripts/mbox/support/indexer.rb
View file

@ -62,34 +62,45 @@ module ImportScripts::Mbox
def index_emails(directory, category_name) def index_emails(directory, category_name)
all_messages(directory, category_name) do |receiver, filename, opts| all_messages(directory, category_name) do |receiver, filename, opts|
msg_id = receiver.message_id begin
parsed_email = receiver.mail msg_id = receiver.message_id
from_email, from_display_name = receiver.parse_from_field(parsed_email) parsed_email = receiver.mail
body, elided, format = receiver.select_body from_email, from_display_name = receiver.parse_from_field(parsed_email)
reply_message_ids = extract_reply_message_ids(parsed_email) body, elided, format = receiver.select_body
reply_message_ids = extract_reply_message_ids(parsed_email)
email = { email = {
msg_id: msg_id, msg_id: msg_id,
from_email: from_email, from_email: from_email,
from_name: from_display_name, from_name: from_display_name,
subject: extract_subject(receiver, category_name), subject: extract_subject(receiver, category_name),
email_date: parsed_email.date&.to_s, email_date: parsed_email.date&.to_s,
raw_message: receiver.raw_email, raw_message: receiver.raw_email,
body: body, body: body,
elided: elided, elided: elided,
format: format, format: format,
attachment_count: receiver.attachments.count, attachment_count: receiver.attachments.count,
charset: parsed_email.charset&.downcase, charset: parsed_email.charset&.downcase,
category: category_name, category: category_name,
filename: File.basename(filename), filename: File.basename(filename),
first_line_number: opts[:first_line_number], first_line_number: opts[:first_line_number],
last_line_number: opts[:last_line_number], last_line_number: opts[:last_line_number],
index_duration: (monotonic_time - opts[:start_time]).round(4) index_duration: (monotonic_time - opts[:start_time]).round(4)
} }
@database.transaction do |db| @database.transaction do |db|
db.insert_email(email) db.insert_email(email)
db.insert_replies(msg_id, reply_message_ids) unless reply_message_ids.empty? db.insert_replies(msg_id, reply_message_ids) unless reply_message_ids.empty?
end
rescue StandardError => e
if opts[:first_line_number] && opts[:last_line_number]
STDERR.puts "Failed to index message in #{filename} at lines #{opts[:first_line_number]}-#{opts[:last_line_number]}"
else
STDERR.puts "Failed to index message in #{filename}"
end
STDERR.puts e.message
STDERR.puts e.backtrace.inspect
end end
end end
end end