Block more crawlers and update webmanifest (#778)

Fixes: #776

Author: rjt-rockx

lib/banchan_web/components/layouts/root.sface

@@ -11,7 +11,6 @@
       />
     {/if}
     <meta name="banchan-git-rev" content={Application.get_env(:banchan, :git_rev)}>
-    <meta name="robots" content="noai">
     <.live_title prefix={"Banchan Art |" <> " "}>
       {assigns[:page_title] || "The Co-operative Commissions Platform"}
     </.live_title>
@@ -39,6 +38,12 @@
     {/if}
     <meta name="twitter:site" content="@BanchanArt">
 
+    {!-- The Common Crawl dataset. Used by GPT-3 (and GPT-3.5) and available for others. --}
+    <meta name="CCBot" content="nofollow">
+
+    {!-- Used by DeviantArt, ArtStation, etc. based on opt-in or opt-out --}
+    <meta name="robots" content="noai, noimageai">
+
     {!-- Icons and favicons --}
     <link rel="apple-touch-icon" sizes="76x76" href="/apple-touch-icon.png">
     <link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">

priv/static/robots.txt

@@ -7,3 +7,15 @@
 # Fuck AI
 User-agent: GPTBot
 Disallow: /
+
+# The Common Crawl dataset. Original source for GPT and others.
+User-agent: CCBot
+Disallow: /
+
+# The example for img2dataset, although the default is *None*
+User-agent: img2dataset
+Disallow: /
+
+# Google's Bard and Vertex AI generative APIs
+User-agent: Google-Extended
+Disallow: /

priv/static/site.webmanifest

@@ -1,14 +1,14 @@
 {
-    "name": "",
-    "short_name": "",
-    "icons": [
-        {
-            "src": "/android-chrome-96x96.png",
-            "sizes": "96x96",
-            "type": "image/png"
-        }
-    ],
-    "theme_color": "#ffffff",
-    "background_color": "#ffffff",
-    "display": "standalone"
+  "name": "Banchan Art",
+  "short_name": "Banchan Art",
+  "icons": [
+    {
+      "src": "/android-chrome-96x96.png",
+      "sizes": "96x96",
+      "type": "image/png"
+    }
+  ],
+  "theme_color": "#ffffff",
+  "background_color": "#ffffff",
+  "display": "standalone"
 }