bring nominatim under chef control

diff --git a/cookbooks/nominatim/metadata.rb b/cookbooks/nominatim/metadata.rb
index 2806429bc06c58b449770c26a1f22cfb188ab497..2f8e20ee41358797eca46a13758ab44699e4a042 100644 (file)
--- a/cookbooks/nominatim/metadata.rb
+++ b/cookbooks/nominatim/metadata.rb
@@ -6,3 +6,4 @@ long_description  IO.read(File.join(File.dirname(__FILE__), 'README.rdoc'))
 version           "1.0.0"
 depends           "apache"
 depends           "postgresql"
+depends           "git"

diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb
index 0d653c6381a37f99ba1ff20b8d8e1dd823b8f310..a33fe85c24073c1a6682cde02d8fc8efdc2d2f7d 100644 (file)
--- a/cookbooks/nominatim/recipes/default.rb
+++ b/cookbooks/nominatim/recipes/default.rb
@@ -19,16 +19,22 @@
 
 include_recipe "apache"
 include_recipe "postgresql"
+include_recipe "git"
 
 package "php5"
 package "php5-cli"
 package "php5-pgsql"
-
+package "php5-fpm"
+package "php-pear"
 package "php-apc"
 
 apache_module "rewrite"
 apache_module "fastcgi-handler"
 
+home_directory = data_bag_item("accounts", "nominatim")["home"]
+source_directory = "#{home_directory}/nominatim"
+email_errors = data_bag_item("accounts", "lonvia")["email"]
+
 service "php5-fpm" do
   action [ :enable, :start ]
   supports :status => true, :restart => true, :reload => true
@@ -37,24 +43,19 @@ end
 apache_site "nominatim.openstreetmap.org" do
   template "apache.erb"
   directory "/home/lonvia/nominatim"
+  variables :pools => node[:nominatim][:fpm_pools]
 end
 
-template "/etc/php5/fpm/pool.d/www.conf" do
-  source "fpm.conf.erb"
-  owner "root"
-  group "root"
-  mode 0644
-  variables :name => "www", :pm => "dynamic", :max_children => "50"
-  notifies :reload, resources(:service => "php5-fpm")
-end
+node[:nominatim][:fpm_pools].each do |name,data|
 
-template "/etc/php5/fpm/pool.d/bulk.conf" do
-  source "fpm.conf.erb"
-  owner "root"
-  group "root"
-  mode 0644
-  variables :name => "bulk", :pm => "static", :max_children => "7"
-  notifies :reload, resources(:service => "php5-fpm")
+  template "/etc/php5/fpm/pool.d/#{name}.conf" do
+    source "fpm.conf.erb"
+    owner "root"
+    group "root"
+    mode 0644
+    variables data.merge(:name => name)
+    notifies :reload, resources(:service => "php5-fpm")
+  end
 end
 
 postgresql_user "tomh" do
@@ -72,6 +73,11 @@ postgresql_user "twain" do
   superuser true
 end
 
+postgresql_user "nominatim" do
+  cluster "9.1/main"
+  superuser true
+end
+
 postgresql_user "www-data" do
   cluster "9.1/main"
 end
@@ -80,3 +86,140 @@ postgresql_munin "nominatim" do
   cluster "9.1/main"
   database "nominatim"
 end
+
+directory "/var/log/nominatim" do
+  owner "nominatim"
+  group "nominatim"
+  mode 0755
+end
+
+package "osmosis"
+package "gcc"
+package "proj-bin"
+package "libgeos-c1"
+package "postgresql-9.1-postgis"
+package "postgresql-server-dev-9.1"
+package "build-essential"
+package "libxml2-dev"
+package "libgeos-dev"
+package "libgeos++-dev"
+package "libpq-dev"
+package "libbz2-dev"
+package "libtool"
+package "automake"
+package "libproj-dev"
+package "libprotobuf-c0-dev"
+package "protobuf-c-compiler"
+
+execute "php-pear-db" do
+  command "pear install DB"
+  not_if { File.exists?("/usr/share/php/DB") }
+end
+
+execute "compile_nominatim" do
+  action :nothing
+  command "cd #{source_directory} && ./autogen.sh && ./configure && make"
+  user "nominatim"
+end
+
+git source_directory do
+  action :checkout
+  repository node[:nominatim][:repository]
+  enable_submodules true
+  user "nominatim"
+  group "nominatim"
+  notifies :run, "execute[compile_nominatim]"
+end
+
+directory "#{source_directory}/log" do
+  owner "nominatim"
+  group "nominatim"
+  mode 0755
+end
+
+
+template "#{source_directory}/.git/hooks/post-merge" do
+  source "update_source.erb"
+  owner  "nominatim"
+  group  "nominatim"
+  mode   0755
+  variables :source_directory => source_directory
+end
+
+template "#{source_directory}/settings/local.php" do
+  source "nominatim.erb"
+  owner "nominatim"
+  group "nominatim"
+  mode 0664
+end
+
+template "#{source_directory}/settings/ip_blocks.conf" do
+  action :create_if_missing
+  source "ipblocks.erb"
+  owner "nominatim"
+  group "nominatim"
+  mode 0664
+end
+
+file "#{source_directory}/settings/apache_blocks.conf" do
+  action :create_if_missing
+  owner "nominatim"
+  group "nominatim"
+  mode 0664
+end
+
+file "#{source_directory}/settings/ip_blocks.map" do
+  action :create_if_missing
+  owner "nominatim"
+  group "nominatim"
+  mode 0664
+end
+
+cron "nominatim_logrotate" do
+  hour "5"
+  minute "30"
+  weekday "0"
+  command "#{source_directory}/utils/cron_logrotate.sh"
+  user "nominatim"
+  mailto email_errors
+end
+
+cron "nominatim_banip" do
+  command "#{source_directory}/utils/cron_banip.sh"
+  user "nominatim"
+  mailto email_errors
+end
+
+cron "nominatim_vacuum" do
+  hour "2"
+  minute "00"
+  command "#{source_directory}/utils/cron_vacuum.sh"
+  user "nominatim"
+  mailto email_errors
+end
+
+['search', 'reverse'].each do |filename|
+  ['phpj', 'phpx'].each do |ext|
+    link "#{source_directory}/website/#{filename}.#{ext}" do
+      to "#{source_directory}/website/#{filename}.php"
+      user "nominatim"
+      group "nominatim"
+    end
+  end
+end
+
+template "#{source_directory}/utils/nominatim-update" do
+  source "updater.erb"
+  user   "nominatim"
+  group  "nominatim"
+  mode   0755
+end
+
+template "/etc/init.d/nominatim-update" do
+  source "updater.init.erb"
+  user   "nominatim"
+  group  "nominatim"
+  mode   0755
+  variables :source_directory => source_directory
+end
+

diff --git a/cookbooks/nominatim/templates/default/apache.erb b/cookbooks/nominatim/templates/default/apache.erb
index de4fb525eefe0cf5ff51a81f4767f8ab3bfa7d9c..add708d69556ab8a393674c7aa7872c120b6a542 100644 (file)
--- a/cookbooks/nominatim/templates/default/apache.erb
+++ b/cookbooks/nominatim/templates/default/apache.erb
@@ -24,18 +24,15 @@
         AddHandler fcgi:/var/run/php5-fpm-www.sock .phpj
     </Directory>
 
-    Alias /pool-www/ "<%= @directory %>/website/"
-    <Location /pool-www>
+    <% @pools.each do |name,details| -%>
+    Alias /pool-<%= name %>/ "<%= @directory %>/website/"
+    <Location /pool-<%= name %>>
         AddHandler fcgi:/var/run/php5-fpm-www.sock .php
         AddHandler fcgi:/var/run/php5-fpm-www.sock .phpx
         AddHandler fcgi:/var/run/php5-fpm-www.sock .phpj
     </Location>
-    Alias /pool-bulk/ "<%= @directory %>/website/"
-    <Location /pool-bulk>
-        AddHandler fcgi:/var/run/php5-fpm-bulk.sock .php
-        AddHandler fcgi:/var/run/php5-fpm-bulk.sock .phpx
-        AddHandler fcgi:/var/run/php5-fpm-bulk.sock .phpj
-    </Location>
+    <% end -%>
+
     Redirect 509 /pool-block/
     ErrorDocument 509 /509.html
     <Location /pool-block>

diff --git a/cookbooks/nominatim/templates/default/ipblocks.erb b/cookbooks/nominatim/templates/default/ipblocks.erb
new file mode 100644 (file)
index 0000000..5d0f87f
--- /dev/null
+++ b/cookbooks/nominatim/templates/default/ipblocks.erb
@@ -0,0 +1,23 @@
+LOGFILE=/var/log/nominatim/restricted_ips.log
+# space-separated list of IPs that are never banned
+WHITELIST = ''
+# space-separated list of IPs manually blocked
+BLACKLIST = ''
+# user-agents that should be blocked from bulk mode
+# (matched with startswith)
+UA_BLOCKLIST = ()
+
+# time before a automatically blocked IP is allowed back
+BLOCKCOOLOFF_PERIOD='1 hour'
+# quiet time before an IP is released from the bulk pool
+BULKCOOLOFF_PERIOD='15 min'
+
+BULKLONG_LIMIT=8000
+BULKSHORT_LIMIT=2000
+BLOCK_UPPER=19000
+BLOCK_LOWER=4000
+BLOCK_LOADFAC=380
+BULK_LOADFAC=160
+BULK_LOWER=1500
+MAX_BULK_IPS=85
+

diff --git a/cookbooks/nominatim/templates/default/nominatim.erb b/cookbooks/nominatim/templates/default/nominatim.erb
new file mode 100644 (file)
index 0000000..ea486ad
--- /dev/null
+++ b/cookbooks/nominatim/templates/default/nominatim.erb
@@ -0,0 +1,6 @@
+<?php
+# DO NOT EDIT - This file is being maintained by Chef
+
+@define('CONST_Database_DSN', 'pgsql://@/<%= node[:nominatim][:database][:dbname] %>');
+@define('CONST_Website_BaseURL', 'http://nominatim.openstreetmap.org/');
+

diff --git a/cookbooks/nominatim/templates/default/update_source.erb b/cookbooks/nominatim/templates/default/update_source.erb
new file mode 100644 (file)
index 0000000..49e285d
--- /dev/null
+++ b/cookbooks/nominatim/templates/default/update_source.erb
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+cd <%= @source_directory %>
+
+./autogen.sh &&
+./configure &&
+make
+
+psql -U postgres <%= node[:nominatim][:database][:dbname] %> -c "SELECT version();" >/dev/null 2>&1
+if [ ! $? ]; then
+  ./utils/setup.php --create-functions --create-partition-functions --enable-diff-updates
+fi

diff --git a/cookbooks/nominatim/templates/default/updater.erb b/cookbooks/nominatim/templates/default/updater.erb
new file mode 100644 (file)
index 0000000..2c553ad
--- /dev/null
+++ b/cookbooks/nominatim/templates/default/updater.erb
@@ -0,0 +1,30 @@
+#!/bin/bash -e
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+# Send output to the log
+exec > /var/log/nominatim/update.log 2>&1
+
+# Change to Nominatim directory
+DIR="$( cd "$( dirname "$0" )" && pwd )"
+cd $DIR/..
+
+while true
+do
+    starttime=`date +%s`
+
+    ./utils/update.php --no-npi --import-osmosis
+
+    # will trigger recompilation if necessary
+    git pull
+
+    # sleep a bit if updates take less than a minute
+    endtime=`date +%s`
+    elapsed=$((endtime - starttime))
+    if [[ $elapsed < 60 ]]
+    then
+        sleepy=$((60 - $elapsed))
+        echo "Sleeping for ${sleepy}s..."
+        sleep $sleepy
+    fi
+done

diff --git a/cookbooks/nominatim/templates/default/updater.init.erb b/cookbooks/nominatim/templates/default/updater.init.erb
new file mode 100644 (file)
index 0000000..0afbf97
--- /dev/null
+++ b/cookbooks/nominatim/templates/default/updater.init.erb
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+start() {
+  start-stop-daemon --start --chuid suzuki --background --make-pidfile --pidfile /var/run/nominatim-update.pid --exec <%= @source_directory %>/utils/nominatim-update
+}
+
+stop() {
+  start-stop-daemon --stop --retry 3600 --pidfile /var/run/nominatim-update.pid
+}
+
+case "$1" in
+  start)
+    start
+    ;;
+  stop)
+    stop
+    ;;
+  restart)
+    stop || exit $?
+    start
+    ;;
+esac
+

diff --git a/roles/nominatim.rb b/roles/nominatim.rb
index 401f1f75596fcdaff28d50ac698f90ce990958da..ccb9cf0d5120b21b68923e1324dcbaf422192098 100644 (file)
--- a/roles/nominatim.rb
+++ b/roles/nominatim.rb
@@ -5,7 +5,11 @@ default_attributes(
   :accounts => {
     :users => {
       :lonvia => { :status => :administrator },
-      :twain => { :status => :administrator }
+      :twain => { :status => :administrator },
+      :nominatim => {
+        :status => :role,
+        :members => [ :lonvia, :tomh, :twain ]
+      },
     }
   },
   :apache => {
@@ -41,6 +45,23 @@ default_attributes(
         "kernel.shmall" => 16 * 1024 * 1024 * 1024 / 4096
       }
     }
+  },
+  :nominatim => {
+    :repository => "git://git.openstreetmap.org/nominatim.git",
+    :database => {
+        :cluster => "9.1/main",
+        :dbname => "nominatim"
+    },
+    :fpm_pools => {
+        :www => {
+            :pm => "dynamic",
+            :max_children => "50"
+        },
+        :bulk => {
+            :pm => "static",
+            :max_children => "7"
+        }
+    }
   }
 )