]> git.openstreetmap.org Git - chef.git/blob - cookbooks/wiki/templates/default/robots.txt.erb
wiki: Delay abusive scrapers
[chef.git] / cookbooks / wiki / templates / default / robots.txt.erb
1 User-agent: ia_archiver
2 Allow: /
3
4 User-agent: 008
5 Disallow: /
6
7 User-agent: *
8 Disallow: /index.php/
9 Disallow: /wiki/Spam
10 Allow: /w/load.php
11 Allow: /w/images/
12 Allow: /w/skins/
13 Disallow: /w/
14 Disallow: /wiki/Special:Collection
15 Disallow: /wiki/Special:Random
16 Disallow: /wiki/Special%3ARandom
17 Disallow: /wiki/Special:Search
18
19 User-agent: Exabot
20 Crawl-delay: 60
21
22 User-Agent: ImagesiftBot
23 Crawl-delay: 60
24
25 User-agent: SemrushBot
26 Crawl-delay: 60
27
28 User-agent: SeznamBot
29 Crawl-delay: 60
30
31 Sitemap: https://wiki.openstreetmap.org/sitemap-index-wiki.xml
32
33 Host: wiki.openstreetmap.org