FEATURE: Replace SimpleRSS with Ruby RSS module (#5311)

* SPEC: PollFeedJob parsing atom feed

* add FeedItemAccessor

It is to provide a consistent interface to access a feed item's tag
content.

* add FeedElementInstaller

to install non-standard and non-namespaced feed elements

* FEATURE: replace SimpleRSS with Ruby RSS module

* get FinalDestination and download with Excon

* support namespaced element with FeedElementInstaller

spec/components/feed_element_installer_spec.rb

@@ -0,0 +1,40 @@
+require 'feed_element_installer'
+require 'rails_helper'
+
+describe FeedElementInstaller do
+  describe '#install_rss_element' do
+    let(:raw_feed) { file_from_fixtures('feed.rss', 'feed').read }
+
+    it 'creates parsing for a non-standard, namespaced element' do
+      FeedElementInstaller.install('discourse:username', raw_feed)
+      feed = RSS::Parser.parse(raw_feed)
+
+      expect(feed.items.first.discourse_username).to eq('xrav3nz')
+    end
+
+    it 'does not create parsing for a non-standard, non-namespaced element' do
+      FeedElementInstaller.install('username', raw_feed)
+      feed = RSS::Parser.parse(raw_feed)
+
+      expect { feed.items.first.username }.to raise_error(NoMethodError)
+    end
+  end
+
+  describe '#install_atom_element' do
+    let(:raw_feed) { file_from_fixtures('feed.atom', 'feed').read }
+
+    it 'creates parsing for a non-standard, namespaced element' do
+      FeedElementInstaller.install('discourse:username', raw_feed)
+      feed = RSS::Parser.parse(raw_feed)
+
+      expect(feed.items.first.discourse_username).to eq('xrav3nz')
+    end
+
+    it 'does not create parsing for a non-standard, non-namespaced element' do
+      FeedElementInstaller.install('username', raw_feed)
+      feed = RSS::Parser.parse(raw_feed)
+
+      expect { feed.items.first.username }.to raise_error(NoMethodError)
+    end
+  end
+end

spec/components/feed_item_accessor_spec.rb

@@ -0,0 +1,33 @@
+require 'rss'
+require 'feed_item_accessor'
+require 'rails_helper'
+
+describe FeedItemAccessor do
+  context 'for ATOM feed' do
+    let(:atom_feed) { RSS::Parser.parse(file_from_fixtures('feed.atom', 'feed'), false) }
+    let(:atom_feed_item) { atom_feed.items.first }
+    let(:item_accessor) { FeedItemAccessor.new(atom_feed_item) }
+
+    describe '#element_content' do
+      it { expect(item_accessor.element_content('title')).to eq(atom_feed_item.title.content) }
+    end
+
+    describe '#link' do
+      it { expect(item_accessor.link).to eq(atom_feed_item.link.href) }
+    end
+  end
+
+  context 'for RSS feed' do
+    let(:rss_feed) { RSS::Parser.parse(file_from_fixtures('feed.rss', 'feed'), false) }
+    let(:rss_feed_item) { rss_feed.items.first }
+    let(:item_accessor) { FeedItemAccessor.new(rss_feed_item) }
+
+    describe '#element_content' do
+      it { expect(item_accessor.element_content('title')).to eq(rss_feed_item.title) }
+    end
+
+    describe '#link' do
+      it { expect(item_accessor.link).to eq(rss_feed_item.link) }
+    end
+  end
+end

spec/fixtures/feed/feed.atom

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<feed
+  xmlns="http://www.w3.org/2005/Atom"
+  xmlns:thr="http://purl.org/syndication/thread/1.0"
+  xmlns:discourse="http://discourse.org/rss/modules/discourse/"
+  xml:lang="en-US"
+  xml:base="https://blog.discourse.org/wp-atom.php"
+   >
+  <title type="text">Discourse</title>
+  <subtitle type="text">Official blog for the open source Discourse project</subtitle>
+  <updated>2017-10-23T23:45:37Z</updated>
+  <link rel="alternate" type="text/html" href="https://blog.discourse.org" />
+  <id>https://blog.discourse.org/feed/atom/</id>
+  <link rel="self" type="application/atom+xml" href="https://blog.discourse.org/feed/atom/" />
+  <generator uri="https://wordpress.org/" version="4.8.2">WordPress</generator>
+  <entry>
+    <discourse:username><![CDATA[xrav3nz]]></discourse:username>
+    <author>
+      <name>xrav3nz</name>
+    </author>
+    <title type="html"><![CDATA[Poll Feed Spec Fixture]]></title>
+    <link rel="alternate" type="text/html" href="https://blog.discourse.org/2017/09/poll-feed-spec-fixture/" />
+    <id>https://blog.discourse.org/?p=pollfeedspec</id>
+    <updated>2017-09-14T15:22:33Z</updated>
+    <published>2017-09-14T15:22:33Z</published>
+    <category scheme="https://blog.discourse.org" term="design" />
+    <summary type="html"><![CDATA[Here are some random descriptions... [&#8230;]]]></summary>
+    <content type="html" xml:base="https://blog.discourse.org/2017/09/poll-feed-spec-fixture/"><![CDATA[<p>This is the body &amp; content. </p>]]></content>
+  </entry>
+</feed>

spec/fixtures/feed/feed.rss

@@ -5,6 +5,7 @@
   xmlns:atom="http://www.w3.org/2005/Atom"
   xmlns:sy="http://purl.org/rss/1.0/modules/syndication/"
   xmlns:slash="http://purl.org/rss/1.0/modules/slash/"
+  xmlns:discourse="http://discourse.org/rss/modules/discourse/"
   >
 <channel>
   <title>Discourse</title>
@@ -21,6 +22,7 @@
     <link>https://blog.discourse.org/2017/09/poll-feed-spec-fixture/</link>
     <pubDate>Thu, 14 Sep 2017 15:22:33 +0000</pubDate>
     <dc:creator><![CDATA[xrav3nz]]></dc:creator>
+    <discourse:username><![CDATA[xrav3nz]]></discourse:username>
     <category><![CDATA[spec]]></category>
     <guid isPermaLink="false">https://blog.discourse.org/?p=pollfeedspec</guid>
     <description><![CDATA[Here are some random descriptions... [&#8230;]]]></description>

spec/jobs/poll_feed_spec.rb

@@ -40,76 +40,91 @@ describe Jobs::PollFeed do
       poller.execute({})
       poller.execute({})
     end
-
   end
 
   describe '#poll_feed' do
     let(:embed_by_username) { 'eviltrout' }
-    let(:embed_username_key_from_feed) { 'dc_creator' }
+    let(:embed_username_key_from_feed) { 'discourse:username' }
     let!(:default_user) { Fabricate(:evil_trout) }
     let!(:feed_author) { Fabricate(:user, username: 'xrav3nz', email: 'hi@bye.com') }
 
-    before do
-      SiteSetting.feed_polling_enabled = true
-      SiteSetting.feed_polling_url = 'https://blog.discourse.org/feed/'
-      SiteSetting.embed_by_username = embed_by_username
+    shared_examples 'topic creation based on the the feed' do
+      describe 'author username parsing' do
+        context 'when neither embed_by_username nor embed_username_key_from_feed is set' do
+          before do
+            SiteSetting.embed_by_username = ""
+            SiteSetting.embed_username_key_from_feed = ""
+          end
 
-      stub_request(:get, SiteSetting.feed_polling_url).to_return(
-        status: 200,
-        body: file_from_fixtures('feed.rss', 'feed').read,
-        headers: { "Content-Type" => "application/rss+xml" }
-      )
-    end
-
-    describe 'author username parsing' do
-      context 'when neither embed_by_username nor embed_username_key_from_feed is set' do
-        before do
-          SiteSetting.embed_by_username = ""
-          SiteSetting.embed_username_key_from_feed = ""
+          it 'does not import topics' do
+            expect { poller.poll_feed }.not_to change { Topic.count }
+          end
         end
 
-        it 'does not import topics' do
-          expect { poller.poll_feed }.not_to change { Topic.count }
+        context 'when embed_by_username is set' do
+          before do
+            SiteSetting.embed_by_username = embed_by_username
+            SiteSetting.embed_username_key_from_feed = ""
+          end
+
+          it 'creates the new topics under embed_by_username' do
+            expect { poller.poll_feed }.to change { Topic.count }.by(1)
+            expect(Topic.last.user).to eq(default_user)
+          end
+        end
+
+        context 'when embed_username_key_from_feed is set' do
+          before do
+            SiteSetting.embed_username_key_from_feed = embed_username_key_from_feed
+          end
+
+          it 'creates the new topics under the username found' do
+            expect { poller.poll_feed }.to change { Topic.count }.by(1)
+            expect(Topic.last.user).to eq(feed_author)
+          end
         end
       end
 
-      context 'when embed_by_username is set' do
-        before do
-          SiteSetting.embed_by_username = embed_by_username
-          SiteSetting.embed_username_key_from_feed = ""
-        end
-
-        it 'creates the new topics under embed_by_username' do
-          expect { poller.poll_feed }.to change { Topic.count }.by(1)
-          expect(Topic.last.user).to eq(default_user)
-        end
-      end
-
-      context 'when embed_username_key_from_feed is set' do
-        before do
-          SiteSetting.embed_username_key_from_feed = embed_username_key_from_feed
-        end
-
-        it 'creates the new topics under the username found' do
-          expect { poller.poll_feed }.to change { Topic.count }.by(1)
-          expect(Topic.last.user).to eq(feed_author)
-        end
+      it 'parses creates a new post correctly' do
+        expect { poller.poll_feed }.to change { Topic.count }.by(1)
+        expect(Topic.last.title).to eq('Poll Feed Spec Fixture')
+        expect(Topic.last.first_post.raw).to include('<p>This is the body &amp; content. </p>')
+        expect(Topic.last.topic_embed.embed_url).to eq('https://blog.discourse.org/2017/09/poll-feed-spec-fixture')
       end
     end
 
-    it 'parses the title correctly' do
-      expect { poller.poll_feed }.to change { Topic.count }.by(1)
-      expect(Topic.last.title).to eq('Poll Feed Spec Fixture')
+    context 'when parsing RSS feed' do
+      before do
+        SiteSetting.feed_polling_enabled = true
+        SiteSetting.feed_polling_url = 'https://blog.discourse.org/feed/'
+        SiteSetting.embed_by_username = embed_by_username
+
+        stub_request(:head, SiteSetting.feed_polling_url).to_return(status: 200)
+        stub_request(:get, SiteSetting.feed_polling_url).to_return(
+          status: 200,
+          body: file_from_fixtures('feed.rss', 'feed').read,
+          headers: { "Content-Type" => "application/rss+xml" }
+        )
+      end
+
+      include_examples 'topic creation based on the the feed'
     end
 
-    it 'parses the content correctly' do
-      expect { poller.poll_feed }.to change { Topic.count }.by(1)
-      expect(Topic.last.first_post.raw).to include('<p>This is the body &amp; content. </p>')
-    end
+    context 'when parsing ATOM feed' do
+      before do
+        SiteSetting.feed_polling_enabled = true
+        SiteSetting.feed_polling_url = 'https://blog.discourse.org/feed/atom/'
+        SiteSetting.embed_by_username = embed_by_username
 
-    it 'parses the link correctly' do
-      expect { poller.poll_feed }.to change { Topic.count }.by(1)
-      expect(Topic.last.topic_embed.embed_url).to eq('https://blog.discourse.org/2017/09/poll-feed-spec-fixture')
+        stub_request(:head, SiteSetting.feed_polling_url).to_return(status: 200)
+        stub_request(:get, SiteSetting.feed_polling_url).to_return(
+          status: 200,
+          body: file_from_fixtures('feed.atom', 'feed').read,
+          headers: { "Content-Type" => "application/atom+xml" }
+        )
+      end
+
+      include_examples 'topic creation based on the the feed'
     end
   end
 end