Discourse/discourse
Discourse/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-09-07 12:02:53 +08:00
Code Packages Activity Actions 5

Improved crawler detection: add Twitterbot, Facebook, curl, Bing, Baidu.

Browse source
This commit is contained in:
Vikhyat Korrapati 2014-03-15 20:01:46 +05:30
parent 5e93d60e9b
commit e3702ecb30
2 changed files with 5 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
spec/components/crawler_detection_spec.rb
View file

@ -15,6 +15,10 @@ describe CrawlerDetection do
described_class.crawler?("(compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)").should == true
described_class.crawler?("Mediapartners-Google").should == true
described_class.crawler?("AdsBot-Google (+http://www.google.com/adsbot.html)").should == true
described_class.crawler?("Twitterbot").should == true
described_class.crawler?("facebookexternalhit/1.1 (+http(s)://www.facebook.com/externalhit_uatext.php)").should == true
described_class.crawler?("Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)").should == true
described_class.crawler?("Baiduspider+(+http://www.baidu.com/search/spider.htm)").should == true
end
it "returns false for non-crawler user agents" do