cookbooks/wiki/templates/default/robots.txt.erb

   1 <% if node[:wiki][:test_mode] -%>
   2 User-agent: *
   3 Disallow: /
   4 <% else -%>
   5 User-agent: ia_archiver
   6 Allow: /
   7
   8 User-agent: 008
   9 Disallow: /
  10
  11 User-agent: *
  12 Disallow: /index.php/
  13 Disallow: /wiki/Spam
  14 Allow: /w/load.php
  15 Allow: /w/images/
  16 Allow: /w/skins/
  17 Disallow: /w/
  18 Disallow: /wiki/Special:Collection
  19 Disallow: /wiki/Special:Random
  20 Disallow: /wiki/Special%3ARandom
  21 Disallow: /wiki/Special:Search
  22
  23 User-agent: Exabot
  24 Crawl-delay: 60
  25
  26 User-Agent: ImagesiftBot
  27 Crawl-delay: 60
  28
  29 User-agent: SemrushBot
  30 Crawl-delay: 60
  31
  32 User-agent: SeznamBot
  33 Crawl-delay: 60
  34
  35 Sitemap: https://wiki.openstreetmap.org/sitemap-index-wiki.xml
  36
  37 Host: wiki.openstreetmap.org
  38 <% end -%>