MNT Only enable indexing of stable doc and dev developers doc (#22)

lesteve · web-flow · commit 73d7416416f0 · 2025-01-21T11:22:55.000+01:00
diff --git a/robots.txt b/robots.txt
@@ -1,5 +1,4 @@
-# Do not let search engines index the PyData theme preview site
-# during the live testing phase.
-# https://github.com/scikit-learn/scikit-learn/pull/28353
 User-agent: *
-Disallow: /_pst_preview/
+Disallow: /
+Allow: /stable
+Allow: /dev/developers
