mirror of
https://github.com/discourse/discourse.git
synced 2026-08-05 12:50:27 +08:00
9527868295This replaces the old ember-cli build with a modern Rolldown build. In local testing, this provides an 80% improvement in build times, while remaining 100% backwards compatible for themes and plugins. As part of this move, we have decided to stop using a proxy in front of Discourse for development. Development should now be done directly against the Rails server. `bin/ember-cli -u` has been replaced with `bin/dev`. This will launch Rails on `:3000`, and will run the rolldown build in the background. Log output from both processes will be shown with an appropriate prefix. You should visit `:3000` in your browser. `:4200` will no longer serve anything. To help with migration, `bin/ember-cli` is now a backwards-compatible shim. It will print help information, and will launch a lightweight server on `:4200` with instructions to move to `:3000`. If you prefer to launch Rails and the JS build as separate commands, you can still do that. Rails boot commands are unchanged, and the rolldown development builder can be run using `bin/dev --only ember`. https://meta.discourse.org/t/403908 --------- Co-authored-by: Jarek Radosz <jarek@cvx.dev> Co-authored-by: Chris Manson <chris@manson.ie>
89 lines
2.9 KiB
Ruby
Vendored
89 lines
2.9 KiB
Ruby
Vendored
# frozen_string_literal: true
|
|
|
|
class RobotsTxtController < ApplicationController
|
|
layout false
|
|
skip_before_action :preload_json,
|
|
:check_xhr,
|
|
:redirect_to_login_if_required,
|
|
:redirect_to_profile_if_required
|
|
|
|
OVERRIDDEN_HEADER = "# This robots.txt file has been customized at /admin/customize/robots\n"
|
|
|
|
# NOTE: order is important!
|
|
DISALLOWED_PATHS = %w[
|
|
/admin/
|
|
/auth/
|
|
/assets/js/browser-update*.js
|
|
/email/
|
|
/session
|
|
/user-api-key
|
|
/*?api_key*
|
|
/*?*api_key*
|
|
]
|
|
|
|
DISALLOWED_WITH_HEADER_PATHS = %w[/badges /my /search /tag/*/l /g /t/*/*.rss /c/*.rss]
|
|
|
|
def index
|
|
if (overridden = SiteSetting.overridden_robots_txt.dup).present?
|
|
overridden.prepend(OVERRIDDEN_HEADER) if guardian.is_admin? && !is_api?
|
|
render plain: overridden
|
|
return
|
|
end
|
|
if SiteSetting.allow_index_in_robots_txt?
|
|
@robots_info = self.class.fetch_default_robots_info
|
|
render :index, content_type: "text/plain"
|
|
else
|
|
render :no_index, content_type: "text/plain"
|
|
end
|
|
end
|
|
|
|
# If you are hosting Discourse in a subfolder, you will need to create your robots.txt
|
|
# in the root of your web server with the appropriate paths. This method will return
|
|
# JSON that can be used by a script to create a robots.txt that works well with your
|
|
# existing site.
|
|
def builder
|
|
result = self.class.fetch_default_robots_info
|
|
overridden = SiteSetting.overridden_robots_txt
|
|
result[:overridden] = overridden if overridden.present?
|
|
render json: result
|
|
end
|
|
|
|
def self.fetch_default_robots_info
|
|
deny_paths_googlebot = DISALLOWED_PATHS.map { |p| Discourse.base_path + p }
|
|
deny_paths =
|
|
deny_paths_googlebot + DISALLOWED_WITH_HEADER_PATHS.map { |p| Discourse.base_path + p }
|
|
deny_all = ["#{Discourse.base_path}/"]
|
|
|
|
result = { header: <<~ROBOTS, agents: [] }
|
|
# See https://datatracker.ietf.org/doc/rfc9309 for documentation on how to use the robots.txt file
|
|
# Google uses the same format as the standard above. More info at https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt
|
|
ROBOTS
|
|
|
|
if SiteSetting.allowed_crawler_user_agents.present?
|
|
SiteSetting
|
|
.allowed_crawler_user_agents
|
|
.split("|")
|
|
.each do |agent|
|
|
paths = agent == "Googlebot" ? deny_paths_googlebot : deny_paths
|
|
result[:agents] << { name: agent, disallow: paths }
|
|
end
|
|
|
|
result[:agents] << { name: "*", disallow: deny_all }
|
|
else
|
|
if SiteSetting.blocked_crawler_user_agents.present?
|
|
SiteSetting
|
|
.blocked_crawler_user_agents
|
|
.split("|")
|
|
.each { |agent| result[:agents] << { name: agent, disallow: deny_all } }
|
|
end
|
|
|
|
result[:agents] << { name: "*", disallow: deny_paths }
|
|
|
|
result[:agents] << { name: "Googlebot", disallow: deny_paths_googlebot }
|
|
end
|
|
|
|
DiscourseEvent.trigger(:robots_info, result)
|
|
|
|
result
|
|
end
|
|
end
|
