From 7d6eddbe9e8217e6596f068c61b80d9183a2e56c Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Fri, 23 May 2025 11:27:16 +0100 Subject: [PATCH] wiki: update bots in robots.txt --- cookbooks/wiki/templates/default/robots.txt.erb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cookbooks/wiki/templates/default/robots.txt.erb b/cookbooks/wiki/templates/default/robots.txt.erb index 45c43d8ff..d8612d6f7 100644 --- a/cookbooks/wiki/templates/default/robots.txt.erb +++ b/cookbooks/wiki/templates/default/robots.txt.erb @@ -8,10 +8,13 @@ Allow: / User-agent: 008 Disallow: / -User-agent: ZvonilBot +User-agent: HTTrack Disallow: / -User-agent: HTTrack +User-agent: DataForSeoBot +Disallow: / + +User-agent: SeznamBot Disallow: / User-agent: * -- 2.39.5