From 07a2b7f027419088378597d9a770bcf59a226aa3 Mon Sep 17 00:00:00 2001
From: Grant Slater <github@firefishy.com>
Date: Tue, 8 Sep 2026 22:00:20 +0100
Subject: [PATCH] foundation: meta-externalagent crawler is mad

---
 .../templates/default/robots.txt.erb          | 25 ++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/cookbooks/foundation/templates/default/robots.txt.erb b/cookbooks/foundation/templates/default/robots.txt.erb
index bea20e8a7..c5e25335f 100644
--- a/cookbooks/foundation/templates/default/robots.txt.erb
+++ b/cookbooks/foundation/templates/default/robots.txt.erb
@@ -5,7 +5,7 @@ User-agent: 008
 Disallow: /
 
 User-agent: *
-Disallow: /index.php/
+Disallow: /w/index.php
 Disallow: /wiki/Spam
 Disallow: /wiki/Donate/International_Bank_Transfers
 Allow: /w/load.php
@@ -19,3 +19,26 @@ Disallow: /wiki/Special:Search
 
 User-agent: Exabot
 Crawl-delay: 60
+
+User-agent: SemrushBot
+Crawl-delay: 5
+
+User-agent: meta-externalagent
+Disallow: /w/
+Disallow: /w/index.php
+Disallow: /wiki/*?*
+Disallow: /wiki/Special:
+Disallow: /wiki/Special%3A
+Disallow: /wiki/MediaWiki:
+Disallow: /wiki/MediaWiki%3A
+Disallow: /wiki/Talk:
+Disallow: /wiki/Talk%3A
+Disallow: /wiki/User_talk:
+Disallow: /wiki/User_talk%3A
+Disallow: /wiki/User:
+Disallow: /wiki/User%3A
+Disallow: /wiki/Template:
+Disallow: /wiki/Template%3A
+Disallow: /wiki/Module:
+Disallow: /wiki/Module%3A
+Allow: /wiki/
-- 
2.47.3