From 884bdf7240cf12c232d30ddd169045e8d525ee21 Mon Sep 17 00:00:00 2001 From: Robin Ward Date: Mon, 22 Aug 2016 12:43:02 -0400 Subject: [PATCH] FEATURE: Ability to scrub titles when importing embeddable content --- .../admin/controllers/admin-embedding.js.es6 | 4 +-- .../javascripts/admin/templates/embedding.hbs | 3 ++ app/models/embedding.rb | 1 + app/models/topic_embed.rb | 9 +++++- config/locales/client.en.yml | 1 + config/locales/server.en.yml | 1 + config/site_settings.yml | 3 ++ spec/models/topic_embed_spec.rb | 29 +++++++++++++++++-- 8 files changed, 45 insertions(+), 6 deletions(-) diff --git a/app/assets/javascripts/admin/controllers/admin-embedding.js.es6 b/app/assets/javascripts/admin/controllers/admin-embedding.js.es6 index 780c61f3f91..b73a8c03544 100644 --- a/app/assets/javascripts/admin/controllers/admin-embedding.js.es6 +++ b/app/assets/javascripts/admin/controllers/admin-embedding.js.es6 @@ -38,9 +38,7 @@ export default Ember.Controller.extend({ const updates = embedding.getProperties(embedding.get('fields')); this.set('saved', false); - this.get('embedding').update(updates).then(() => { - this.set('saved', true); - }).catch(popupAjaxError); + this.get('embedding').update(updates).then(() => this.set('saved', true)).catch(popupAjaxError); }, addHost() { diff --git a/app/assets/javascripts/admin/templates/embedding.hbs b/app/assets/javascripts/admin/templates/embedding.hbs index 6fbe31c124c..196ada30f8c 100644 --- a/app/assets/javascripts/admin/templates/embedding.hbs +++ b/app/assets/javascripts/admin/templates/embedding.hbs @@ -30,6 +30,9 @@ {{embedding-setting field="embed_by_username" value=embedding.embed_by_username}} {{embedding-setting field="embed_post_limit" value=embedding.embed_post_limit}} + {{embedding-setting field="embed_title_scrubber" + value=embedding.embed_title_scrubber + placeholder="- site.com$"}} {{embedding-setting field="embed_truncate" value=embedding.embed_truncate type="checkbox"}} diff --git a/app/models/embedding.rb b/app/models/embedding.rb index 796f80671b8..7e28a7694f9 100644 --- a/app/models/embedding.rb +++ b/app/models/embedding.rb @@ -6,6 +6,7 @@ class Embedding < OpenStruct def self.settings %i(embed_by_username embed_post_limit + embed_title_scrubber embed_truncate embed_whitelist_selector embed_blacklist_selector diff --git a/app/models/topic_embed.rb b/app/models/topic_embed.rb index dc9165a1666..f98bd984c8c 100644 --- a/app/models/topic_embed.rb +++ b/app/models/topic_embed.rb @@ -79,7 +79,14 @@ class TopicEmbed < ActiveRecord::Base doc = Readability::Document.new(open(url).read, opts) tags = {'img' => 'src', 'script' => 'src', 'a' => 'href'} - title = doc.title + title = doc.title || '' + title.strip! + + if SiteSetting.embed_title_scrubber.present? + title.sub!(Regexp.new(SiteSetting.embed_title_scrubber), '') + title.strip! + end + doc = Nokogiri::HTML(doc.content) doc.search(tags.keys.join(',')).each do |node| url_param = tags[node.name] diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index dfbf3900599..9406cf25ad1 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -3120,6 +3120,7 @@ en: embed_by_username: "Username for topic creation" embed_post_limit: "Maximum number of posts to embed" embed_username_key_from_feed: "Key to pull discourse username from feed" + embed_title_scrubber: "Regular expression used to scrub the title of posts" embed_truncate: "Truncate the embedded posts" embed_whitelist_selector: "CSS selector for elements that are allowed in embeds" embed_blacklist_selector: "CSS selector for elements that are removed from embeds" diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 67fbc70f1ff..13c243ff3f1 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -1317,6 +1317,7 @@ en: feed_polling_url: "EMBEDDING ONLY: URL of RSS/ATOM feed to embed." embed_by_username: "Discourse username of the user who creates the embedded topics." embed_username_key_from_feed: "Key to pull discourse username from feed." + embed_title_scrubber: "Regular expression for scrubbing embeddable titles." embed_truncate: "Truncate the embedded posts." embed_post_limit: "Maximum number of posts to embed." embed_username_required: "The username for topic creation is required." diff --git a/config/site_settings.yml b/config/site_settings.yml index 9ddb39fb003..9f2dc2bedba 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -973,6 +973,9 @@ embedding: embed_post_limit: default: 100 hidden: true + embed_title_scrubber: + default: '' + hidden: true embed_truncate: default: true hidden: true diff --git a/spec/models/topic_embed_spec.rb b/spec/models/topic_embed_spec.rb index 564259c4919..35a1d9ae596 100644 --- a/spec/models/topic_embed_spec.rb +++ b/spec/models/topic_embed_spec.rb @@ -62,6 +62,31 @@ describe TopicEmbed do describe '.find_remote' do + context ".title_scrub" do + + let(:url) { 'http://eviltrout.com/123' } + let(:contents) { "Through the Looking Glass - Classic Bookssome content here" } + let!(:embeddable_host) { Fabricate(:embeddable_host) } + let!(:file) { StringIO.new } + + before do + file.stubs(:read).returns contents + TopicEmbed.stubs(:open).returns file + end + + it "doesn't scrub the title by default" do + title, _ = TopicEmbed.find_remote(url) + expect(title).to eq("Through the Looking Glass - Classic Books") + end + + it "scrubs the title when the option is enabled" do + SiteSetting.embed_title_scrubber = " - Classic Books$" + title, _ = TopicEmbed.find_remote(url) + expect(title).to eq("Through the Looking Glass") + end + + end + context 'post with allowed classes "foo" and "emoji"' do let(:user) { Fabricate(:user) } @@ -76,7 +101,7 @@ describe TopicEmbed do SiteSetting.stubs(:embed_classname_whitelist).returns 'emoji , foo' file.stubs(:read).returns contents TopicEmbed.stubs(:open).returns file - title, content = TopicEmbed.find_remote(url) + _, content = TopicEmbed.find_remote(url) end it 'img node has emoji class' do @@ -111,7 +136,7 @@ describe TopicEmbed do SiteSetting.stubs(:embed_classname_whitelist).returns ' ' file.stubs(:read).returns contents TopicEmbed.stubs(:open).returns file - title, content = TopicEmbed.find_remote(url) + _, content = TopicEmbed.find_remote(url) end it 'img node doesn\'t have emoji class' do