nominatim: extend WAL archive delete script to check slave delay

diff --git a/cookbooks/nominatim/recipes/base.rb b/cookbooks/nominatim/recipes/base.rb
index 390de61f93ed3653b6a7b1441156a182e4642aa8..2e6a8c12550cb6d34e6c62e399bba4b2a7c75c1a 100644 (file)
--- a/cookbooks/nominatim/recipes/base.rb
+++ b/cookbooks/nominatim/recipes/base.rb
@@ -70,6 +70,12 @@ directory "/var/log/nominatim" do
   mode 0755
 end
 
   mode 0755
 end
 

+directory "#{home_directory}/status" do
+  owner "nominatim"
+  group "postgres"
+  mode 0775
+end
+

 template "/etc/logrotate.d/nominatim" do
   source "logrotate.nominatim.erb"
   owner "root"
 template "/etc/logrotate.d/nominatim" do
   source "logrotate.nominatim.erb"
   owner "root"


@@ -221,7 +227,7 @@ external_data.each do |fname|
   end
 end
 
   end
 end
 

-additional_scripts = %w(backup-nominatim clean-db-nominatim)
+additional_scripts = %w(backup-nominatim vacuum-db-nominatim)

 
 additional_scripts.each do |fname|
   template "/usr/local/bin/#{fname}" do
 
 additional_scripts.each do |fname|
   template "/usr/local/bin/#{fname}" do


@@ -239,13 +245,6 @@ directory File.dirname(node[:nominatim][:flatnode_file]) do
   recursive true
 end
 
   recursive true
 end
 

-directory "/data/postgresql-archive" do
-  owner "postgres"
-  group "postgres"
-  mode 0700
-  only_if { node[:postgresql][:settings][:defaults][:archive_mode] == "on" }
-end
-

 apache_site "nominatim.openstreetmap.org" do
   template "apache.erb"
   directory source_directory
 apache_site "nominatim.openstreetmap.org" do
   template "apache.erb"
   directory source_directory

diff --git a/cookbooks/nominatim/recipes/master.rb b/cookbooks/nominatim/recipes/master.rb
index 341f3d664dc1befe1ffd11941e8d1ece02da21f5..a3200312bc787aaaabdf47d2cd57ef083b3934ea 100644 (file)
--- a/cookbooks/nominatim/recipes/master.rb
+++ b/cookbooks/nominatim/recipes/master.rb
@@ -23,6 +23,9 @@ passwords = data_bag_item("nominatim", "passwords")
 database_cluster = node[:nominatim][:database][:cluster]
 home_directory = data_bag_item("accounts", "nominatim")["home"]
 
 database_cluster = node[:nominatim][:database][:cluster]
 home_directory = data_bag_item("accounts", "nominatim")["home"]
 

+wal_archives = node[:rsyncd][:modules][:archive][:path]
+slaves = search(:node, 'role:nominatim-slave').map{ |result| result[:fqdn] }.join(' ')
+

 git "#{home_directory}/nominatim" do
   action :checkout
   repository node[:nominatim][:repository]
 git "#{home_directory}/nominatim" do
   action :checkout
   repository node[:nominatim][:repository]


@@ -52,3 +55,23 @@ postgresql_user "replication" do
   password passwords["replication"]
   replication true
 end
   password passwords["replication"]
   replication true
 end

+
+directory wal_archives do
+  owner "postgres"
+  group "postgres"
+  mode 0700
+  only_if { node[:postgresql][:settings][:defaults][:archive_mode] == "on" }
+end
+
+template "/usr/local/bin/clean-db-nominatim" do
+  source "clean-db-nominatim.erb"
+  owner "root"
+  group "root"
+  mode 0755
+  variables :archive_dir => wal_archives,
+            :update_stop_file => "{home_directory}/status/updates_disabled",
+            :streaming_clients => slaves
+  only_if { node[:postgresql][:settings][:defaults][:archive_mode] == "on" }
+end
+
+

diff --git a/cookbooks/nominatim/templates/default/clean-db-nominatim.erb b/cookbooks/nominatim/templates/default/clean-db-nominatim.erb
index 39297e7fe0a9e6a366db4cb71ea57bafd54cb6c1..202240e3f210b85ca5ebe514a8b9dbde230a59ca 100644 (file)
--- a/cookbooks/nominatim/templates/default/clean-db-nominatim.erb
+++ b/cookbooks/nominatim/templates/default/clean-db-nominatim.erb
@@ -12,9 +12,37 @@ for i in `seq 0 246`; do
   psql -q -d nominatim -c "VACUUM ANALYSE search_name_${i}"
 done
 
   psql -q -d nominatim -c "VACUUM ANALYSE search_name_${i}"
 done
 

-<% if node[:postgresql][:settings][:defaults][:archive_mode] == "on" -%>
-

 # Cleanup archive directory
 # Cleanup archive directory

-#find -L /data/postgresql-archive -mmin +1440 -print0 | xargs -0r rm
+oldest=`date +%s`
+
+for fl in <% @streaming_clients %>; do
+  lastupdate=`wget -O - -q http://$fl/last_update.php`
+  if [[ "$?" != "0" ]]; then
+    oldest=0
+    break
+  else
+    epoch=`date +%s -d "$lastupdate"`
+    if [[ "$epoch" -lt "$oldest" ]]; then
+        oldest=$epoch
+    fi
+  fi
+done
+
+
+if [[ "$oldest" != "0" ]]; then
+    now=`date +%s`
+    timeago=$(( ( $now - $oldest ) / 60 + 180 ))
+    if [[ "$timeago" -le "180" ]]; then
+        echo "Oldest is in the future, something is wrong."
+        exit(1)
+    fi
+
+    echo "find -L <%= @archive_dir %> -mmin +$timeago -print0 | xargs -0r rm"
+fi
+
+diskfree=`df --output=pcent <%= @archive_dir %> | tail -n 1 | sed s:[^0-9.]::g

 
 

-<% end -%>
+if [[ "$diskfree" -lt 10 ]]; then
+    echo "WAL archives running out of space. Stopping updates."
+    touch <%= @update_stop_file %>
+fi

diff --git a/cookbooks/nominatim/templates/default/cron.erb b/cookbooks/nominatim/templates/default/cron.erb
index 02894ffc506841d8a371a84ef2bb3ea36646ce6a..956a2d7869329a6b9986c7c188a182a1e938f3b7 100644 (file)
--- a/cookbooks/nominatim/templates/default/cron.erb
+++ b/cookbooks/nominatim/templates/default/cron.erb
@@ -6,4 +6,4 @@ MAILTO=<%= @mailto %>
 <% if node[:nominatim][:enable_backup] -%>
 00 3 1 * * nominatim /usr/local/bin/backup-nominatim
 <% end -%>
 <% if node[:nominatim][:enable_backup] -%>
 00 3 1 * * nominatim /usr/local/bin/backup-nominatim
 <% end -%>

-30 1 * * * postgres /usr/local/bin/clean-db-nominatim
+30 1 * * * postgres /usr/local/bin/vacuum-db-nominatim

diff --git a/cookbooks/nominatim/templates/default/updater.erb b/cookbooks/nominatim/templates/default/updater.erb
index beea8c8ac5e896cd99a25dfc0b9d086e3bb21c48..aba7f72aa9dea61a2281126b13dc2589be18d06f 100644 (file)
--- a/cookbooks/nominatim/templates/default/updater.erb
+++ b/cookbooks/nominatim/templates/default/updater.erb
@@ -6,7 +6,7 @@
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 cd $DIR/..
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 cd $DIR/..
 

-while true
+while [ ! -f "<%= @update_stop_file %>" ]

 do
     # Send output to the log (logrotatable)
     exec >> /var/log/nominatim/update.log 2>&1
 do
     # Send output to the log (logrotatable)
     exec >> /var/log/nominatim/update.log 2>&1

diff --git a/cookbooks/nominatim/templates/default/vacuum-db-nominatim b/cookbooks/nominatim/templates/default/vacuum-db-nominatim
new file mode 100644 (file)
index 0000000..77b1257
--- /dev/null
+++ b/cookbooks/nominatim/templates/default/vacuum-db-nominatim
@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# DO NOT EDIT - This file is being maintained by Chef
+
+
+# Vaccum all tables with indices on integer arrays.
+# Agressive vacuuming seems to help against index bloat.
+psql -q -d nominatim -c 'VACUUM ANALYSE search_name'
+psql -q -d nominatim -c 'VACUUM ANALYSE search_name_country'
+
+for i in `seq 0 246`; do
+  psql -q -d nominatim -c "VACUUM ANALYSE search_name_${i}"
+done
+