From d1606a78f7674befb9c2e658fe38b5854e0ced3a Mon Sep 17 00:00:00 2001
From: Grant Slater <github@firefishy.com>
Date: Thu, 22 May 2025 19:47:41 +0100
Subject: [PATCH] wiki: make robots.txt stricter

---
 .../wiki/templates/default/robots.txt.erb     | 32 +++++++------------
 1 file changed, 12 insertions(+), 20 deletions(-)

diff --git a/cookbooks/wiki/templates/default/robots.txt.erb b/cookbooks/wiki/templates/default/robots.txt.erb
index 8d167108f..45c43d8ff 100644
--- a/cookbooks/wiki/templates/default/robots.txt.erb
+++ b/cookbooks/wiki/templates/default/robots.txt.erb
@@ -8,29 +8,21 @@ Allow: /
 User-agent: 008
 Disallow: /
 
+User-agent: ZvonilBot
+Disallow: /
+
+User-agent: HTTrack
+Disallow: /
+
 User-agent: *
+Allow: /w/api.php?action=mobileview&
+Allow: /w/load.php?
+Disallow: /w/
+Disallow: /api/
+Disallow: /wiki/Special:
+Disallow: /wiki/Special%3A
 Disallow: /index.php/
 Disallow: /wiki/Spam
-Allow: /w/load.php
-Allow: /w/images/
-Allow: /w/skins/
-Disallow: /w/
-Disallow: /wiki/Special:Collection
-Disallow: /wiki/Special:Random
-Disallow: /wiki/Special%3ARandom
-Disallow: /wiki/Special:Search
-
-User-agent: Exabot
-Crawl-delay: 60
-
-User-Agent: ImagesiftBot
-Crawl-delay: 60
-
-User-agent: SemrushBot
-Crawl-delay: 60
-
-User-agent: SeznamBot
-Crawl-delay: 60
 
 Sitemap: https://wiki.openstreetmap.org/sitemap-index-wiki.xml
 
-- 
2.39.5