From 0245b11dd605649c868e75ef27fa01655ecd3281 Mon Sep 17 00:00:00 2001
From: Grant Slater <github@firefishy.com>
Date: Sun, 10 Mar 2024 15:37:16 +0000
Subject: [PATCH] wiki: Delay abusive scrapers

---
 cookbooks/wiki/templates/default/robots.txt.erb | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/cookbooks/wiki/templates/default/robots.txt.erb b/cookbooks/wiki/templates/default/robots.txt.erb
index de552f049..932d07eba 100644
--- a/cookbooks/wiki/templates/default/robots.txt.erb
+++ b/cookbooks/wiki/templates/default/robots.txt.erb
@@ -19,6 +19,15 @@ Disallow: /wiki/Special:Search
 User-agent: Exabot
 Crawl-delay: 60
 
+User-Agent: ImagesiftBot
+Crawl-delay: 60
+
+User-agent: SemrushBot
+Crawl-delay: 60
+
+User-agent: SeznamBot
+Crawl-delay: 60
+
 Sitemap: https://wiki.openstreetmap.org/sitemap-index-wiki.xml
 
 Host: wiki.openstreetmap.org
-- 
2.45.1