Feature: CommonMark support

This adds the markdown.it engine to Discourse.
https://github.com/markdown-it/markdown-it

As the migration is going to take a while the new engine is default
disabled. To enable it you must change the hidden site setting:
enable_experimental_markdown_it.

This commit is a squash of many other commits, it also includes some
improvements to autospec (ability to run plugins), and a dev dependency
on the og gem for html normalization.

lib/pretty_text.rb

@@ -50,19 +50,10 @@ module PrettyText
     end
   end
 
-  def self.create_es6_context
-    ctx = MiniRacer::Context.new(timeout: 15000)
-
-    ctx.eval("window = {}; window.devicePixelRatio = 2;") # hack to make code think stuff is retina
-
-    if Rails.env.development? || Rails.env.test?
-      ctx.attach("console.log", proc { |l| p l })
-    end
-
-    ctx_load(ctx, "#{Rails.root}/app/assets/javascripts/discourse-loader.js")
-    ctx_load(ctx, "vendor/assets/javascripts/lodash.js")
-    manifest = File.read("#{Rails.root}/app/assets/javascripts/pretty-text-bundle.js")
+  def self.ctx_load_manifest(ctx, name)
+    manifest = File.read("#{Rails.root}/app/assets/javascripts/#{name}")
     root_path = "#{Rails.root}/app/assets/javascripts/"
+
     manifest.each_line do |l|
       l = l.chomp
       if l =~ /\/\/= require (\.\/)?(.*)$/
@@ -74,6 +65,22 @@ module PrettyText
         end
       end
     end
+  end
+
+  def self.create_es6_context
+    ctx = MiniRacer::Context.new(timeout: 15000)
+
+    ctx.eval("window = {}; window.devicePixelRatio = 2;") # hack to make code think stuff is retina
+
+    if Rails.env.development? || Rails.env.test?
+      ctx.attach("console.log", proc { |l| p l })
+    end
+
+    ctx_load(ctx, "#{Rails.root}/app/assets/javascripts/discourse-loader.js")
+    ctx_load(ctx, "vendor/assets/javascripts/lodash.js")
+    ctx_load_manifest(ctx, "pretty-text-bundle.js")
+
+    root_path = "#{Rails.root}/app/assets/javascripts/"
 
     apply_es6_file(ctx, root_path, "discourse/lib/utilities")
 
@@ -140,52 +147,62 @@ module PrettyText
         paths[:S3BaseUrl] = Discourse.store.absolute_base_url
       end
 
-      context.eval("__optInput = {};")
-      context.eval("__optInput.siteSettings = #{SiteSetting.client_settings_json};")
-      context.eval("__paths = #{paths.to_json};")
-
-      if opts[:topicId]
-        context.eval("__optInput.topicId = #{opts[:topicId].to_i};")
+      if SiteSetting.enable_experimental_markdown_it
+        unless context.eval("window.markdownit")
+          ctx_load_manifest(context, "markdown-it-bundle.js")
+        end
       end
 
-      context.eval("__optInput.userId = #{opts[:user_id].to_i};") if opts[:user_id]
-
-      context.eval("__optInput.getURL = __getURL;")
-      context.eval("__optInput.getCurrentUser = __getCurrentUser;")
-      context.eval("__optInput.lookupAvatar = __lookupAvatar;")
-      context.eval("__optInput.getTopicInfo = __getTopicInfo;")
-      context.eval("__optInput.categoryHashtagLookup = __categoryLookup;")
-      context.eval("__optInput.mentionLookup = __mentionLookup;")
-
       custom_emoji = {}
       Emoji.custom.map { |e| custom_emoji[e.name] = e.url }
-      context.eval("__optInput.customEmoji = #{custom_emoji.to_json};")
 
-      context.eval('__textOptions = __buildOptions(__optInput);')
+      buffer = <<~JS
+        __optInput = {};
+        __optInput.siteSettings = #{SiteSetting.client_settings_json};
+        __paths = #{paths.to_json};
+        __optInput.getURL = __getURL;
+        __optInput.getCurrentUser = __getCurrentUser;
+        __optInput.lookupAvatar = __lookupAvatar;
+        __optInput.getTopicInfo = __getTopicInfo;
+        __optInput.categoryHashtagLookup = __categoryLookup;
+        __optInput.mentionLookup = __mentionLookup;
+        __optInput.customEmoji = #{custom_emoji.to_json};
+      JS
+
+      if opts[:topicId]
+        buffer << "__optInput.topicId = #{opts[:topicId].to_i};\n"
+      end
+
+      if opts[:user_id]
+        buffer << "__optInput.userId = #{opts[:user_id].to_i};\n"
+      end
+
+      buffer << "__textOptions = __buildOptions(__optInput);\n"
 
       # Be careful disabling sanitization. We allow for custom emails
       if opts[:sanitize] == false
-        context.eval('__textOptions.sanitize = false;')
+        buffer << ('__textOptions.sanitize = false;')
       end
 
-      opts = context.eval("__pt = new __PrettyText(__textOptions);")
+      buffer << ("__pt = new __PrettyText(__textOptions);")
+      opts = context.eval(buffer)
 
       DiscourseEvent.trigger(:markdown_context, context)
       baked = context.eval("__pt.cook(#{text.inspect})")
     end
 
-    if baked.blank? && !(opts || {})[:skip_blank_test]
-      # we may have a js engine issue
-      test = markdown("a", skip_blank_test: true)
-      if test.blank?
-        Rails.logger.warn("Markdown engine appears to have crashed, resetting context")
-        reset_context
-        opts ||= {}
-        opts = opts.dup
-        opts[:skip_blank_test] = true
-        baked = markdown(text, opts)
-      end
-    end
+    # if baked.blank? && !(opts || {})[:skip_blank_test]
+    #   # we may have a js engine issue
+    #   test = markdown("a", skip_blank_test: true)
+    #   if test.blank?
+    #     Rails.logger.warn("Markdown engine appears to have crashed, resetting context")
+    #     reset_context
+    #     opts ||= {}
+    #     opts = opts.dup
+    #     opts[:skip_blank_test] = true
+    #     baked = markdown(text, opts)
+    #   end
+    # end
 
     baked
   end