From: Tom Hughes <tom@compton.nu>
Date: Tue, 15 Dec 2009 22:39:51 +0000 (+0000)
Subject: Use a script to pull in host blocks and add some robot blocks from
X-Git-Tag: live~6364
X-Git-Url: https://git.openstreetmap.org/rails.git/commitdiff_plain/6b24ce4de98ac4038e2396e0b8163804c38457bb?hp=614fef3c130988eeb932301a7098977e3d3893fe;ds=sidebyside

Use a script to pull in host blocks and add some robot blocks from
the live server.
---

diff --git a/config/lighttpd-host-blocks.sh b/config/lighttpd-host-blocks.sh
new file mode 100644
index 000000000..3022e74db
--- /dev/null
+++ b/config/lighttpd-host-blocks.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+if [ -d /etc/firewall ]
+then
+  for ip in $(ls /etc/firewall)
+  do
+    echo "\$HTTP[\"remoteip\"] == \"$ip\" { url.access-deny = (\"\") }"
+  done
+fi
diff --git a/config/lighttpd.conf b/config/lighttpd.conf
index 4d7a7f4ab..a224f4562 100644
--- a/config/lighttpd.conf
+++ b/config/lighttpd.conf
@@ -39,10 +39,17 @@ $HTTP["remoteip"] == "128.40.168.98" {
 }
 
 #
-# IP blocked at SteveC's request as it was trying to download the
-# history of every object in the database one at a time
+# Pull in host blocks
 #
-$HTTP["remoteip"] == "143.210.16.160" { url.access-deny = ("") }
+include_shell "lighttpd-host-blocks.sh"
+
+#
+# Block some troublesome robots
+#
+$HTTP["useragent"] =~ "msnbot" { url.access-deny = ("") }
+$HTTP["useragent"] =~ "Twiceler" { url.access-deny = ("") }
+$HTTP["useragent"] =~ "Baiduspider" { url.access-deny = ("") }
+$HTTP["useragent"] =~ "Sosospider+" { url.access-deny = ("") }
 
 #
 # Rule to block tilesAtHome when somebody decides to queue vast