cookbooks/wiki/templates/default/robots.txt.erb

   1 User-agent: ia_archiver
   2 Allow: /
   3
   4 User-agent: 008
   5 Disallow: /
   6
   7 User-agent: *
   8 Disallow: /index.php/
   9 Disallow: /wiki/Lacking_proper_attribution
  10 Disallow: /wiki/Spam
  11 Allow: /w/load.php
  12 Allow: /w/images/
  13 Allow: /w/skins/
  14 Disallow: /w/
  15 Disallow: /wiki/Special:Collection
  16 Disallow: /wiki/Special:Random
  17 Disallow: /wiki/Special%3ARandom
  18 Disallow: /wiki/Special:Search
  19
  20 User-agent: Exabot
  21 Crawl-delay: 60
  22
  23 Sitemap: http://wiki.openstreetmap.org/sitemap-index-wiki.xml
  24
  25 Host: wiki.openstreetmap.org