robots added

modified: mysite/settings.py idk a dockploy modified: mysite/settings.py
2025-08-07 14:00:20 -04:00
parent b984991cce
commit 815d3c9194
4 changed files with 98 additions and 39 deletions
@@ -2,9 +2,14 @@
 {% load static %}
 <html>

+
 <head>
    <link rel="icon" type="image/x-icon" href="{% static 'favicon.ico' %}" />
    <link rel="stylesheet" href="{% static 'styles.css' %}" />
+
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <meta charset="UTF-8">
+
    <title>Who are you?</title>
 </head>

@@ -16,6 +21,7 @@
        WHO R U?
    </h1>

+
    <div class="container">
        <div class="info">
            <h2>IP Address:</h2>
@@ -8,6 +8,42 @@ User-agent: Googlebot
 User-agent: AdsBot-Google
 Disallow: /

+# AI
+User-agent: GPTBot
+Disallow: /
+User-agent: ChatGPT-User
+Disallow: /
+User-agent: Google-Extended
+Disallow: /
+User-agent: PerplexityBot
+Disallow: /
+User-agent: Amazonbot
+Disallow: /
+User-agent: ClaudeBot
+Disallow: /
+User-agent: Omgilibot
+Disallow: /
+User-Agent: FacebookBot
+Disallow: /
+User-Agent: Applebot
+Disallow: /
+User-agent: anthropic-ai
+Disallow: /
+User-agent: Bytespider
+Disallow: /
+User-agent: Claude-Web
+Disallow: /
+User-agent: Diffbot
+Disallow: /
+User-agent: ImagesiftBot
+Disallow: /
+User-agent: Omgilibot
+Disallow: /
+User-agent: Omgili
+Disallow: /
+User-agent: YouBot
+Disallow: /
+
 #Example of how to block all crawlers
 User-agent: *
 Disallow: /