robots

2024-07-15 19:41:58 -04:00
parent d91b8c7128
commit eddbda5443
2 changed files with 263 additions and 0 deletions
--- a/frontend/static/robots.txt
+++ b/frontend/static/robots.txt
@@ -0,0 +1,10 @@
+User-agent: *
+Allow: /
+
+# Google adsbot ignores robots.txt unless specifically named!
+User-agent: AdsBot-Google
+Allow: /
+
+
+User-agent: GPTBot
+Disallow: /