diff --git a/app/views/robots_txt/index.erb b/app/views/robots_txt/index.erb index f81e8128ce8..2d35051cf3c 100644 --- a/app/views/robots_txt/index.erb +++ b/app/views/robots_txt/index.erb @@ -1,4 +1,7 @@ -# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file +# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file +<% if Discourse.base_uri.present? %> +# This robots.txt file is not used. Please append the content below in the robots.txt file located at the root +<% end %> # <% @allowed_user_agents.each do |user_agent| %> User-agent: <%= user_agent %>