cookbooks/wiki/templates/default/robots.txt.erb

   1 <% if node[:wiki][:test_mode] -%>
   2 User-agent: *
   3 Disallow: /
   4 <% else -%>
   5 User-agent: ia_archiver
   6 Allow: /
   7
   8 User-agent: 008
   9 Disallow: /
  10
  11 User-agent: HTTrack
  12 Disallow: /
  13
  14 User-agent: DataForSeoBot
  15 Disallow: /
  16
  17 User-agent: SeznamBot
  18 Disallow: /
  19
  20 User-agent: *
  21 Allow: /w/api.php?action=mobileview&
  22 Allow: /w/load.php?
  23 Disallow: /w/
  24 Disallow: /api/
  25 Disallow: /wiki/Special:
  26 Disallow: /wiki/Special%3A
  27 Disallow: /index.php/
  28 Disallow: /wiki/Spam
  29
  30 Sitemap: https://wiki.openstreetmap.org/sitemap-index-wiki.xml
  31
  32 Host: wiki.openstreetmap.org
  33 <% end -%>