From: Sarah Hoffmann <lonvia@denofr.de>
Date: Wed, 18 Nov 2015 13:10:45 +0000 (+0100)
Subject: nominatim: extend WAL archive delete script to check slave delay
X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/8aa57233f9034b901d7a652bd6276d0d7c3ea4f0

nominatim: extend WAL archive delete script to check slave delay
---

diff --git a/cookbooks/nominatim/recipes/base.rb b/cookbooks/nominatim/recipes/base.rb
index 390de61f9..2e6a8c125 100644
--- a/cookbooks/nominatim/recipes/base.rb
+++ b/cookbooks/nominatim/recipes/base.rb
@@ -70,6 +70,12 @@ directory "/var/log/nominatim" do
   mode 0755
 end
 
+directory "#{home_directory}/status" do
+  owner "nominatim"
+  group "postgres"
+  mode 0775
+end
+
 template "/etc/logrotate.d/nominatim" do
   source "logrotate.nominatim.erb"
   owner "root"
@@ -221,7 +227,7 @@ external_data.each do |fname|
   end
 end
 
-additional_scripts = %w(backup-nominatim clean-db-nominatim)
+additional_scripts = %w(backup-nominatim vacuum-db-nominatim)
 
 additional_scripts.each do |fname|
   template "/usr/local/bin/#{fname}" do
@@ -239,13 +245,6 @@ directory File.dirname(node[:nominatim][:flatnode_file]) do
   recursive true
 end
 
-directory "/data/postgresql-archive" do
-  owner "postgres"
-  group "postgres"
-  mode 0700
-  only_if { node[:postgresql][:settings][:defaults][:archive_mode] == "on" }
-end
-
 apache_site "nominatim.openstreetmap.org" do
   template "apache.erb"
   directory source_directory
diff --git a/cookbooks/nominatim/recipes/master.rb b/cookbooks/nominatim/recipes/master.rb
index 341f3d664..a3200312b 100644
--- a/cookbooks/nominatim/recipes/master.rb
+++ b/cookbooks/nominatim/recipes/master.rb
@@ -23,6 +23,9 @@ passwords = data_bag_item("nominatim", "passwords")
 database_cluster = node[:nominatim][:database][:cluster]
 home_directory = data_bag_item("accounts", "nominatim")["home"]
 
+wal_archives = node[:rsyncd][:modules][:archive][:path]
+slaves = search(:node, 'role:nominatim-slave').map{ |result| result[:fqdn] }.join(' ')
+
 git "#{home_directory}/nominatim" do
   action :checkout
   repository node[:nominatim][:repository]
@@ -52,3 +55,23 @@ postgresql_user "replication" do
   password passwords["replication"]
   replication true
 end
+
+directory wal_archives do
+  owner "postgres"
+  group "postgres"
+  mode 0700
+  only_if { node[:postgresql][:settings][:defaults][:archive_mode] == "on" }
+end
+
+template "/usr/local/bin/clean-db-nominatim" do
+  source "clean-db-nominatim.erb"
+  owner "root"
+  group "root"
+  mode 0755
+  variables :archive_dir => wal_archives,
+            :update_stop_file => "{home_directory}/status/updates_disabled",
+            :streaming_clients => slaves
+  only_if { node[:postgresql][:settings][:defaults][:archive_mode] == "on" }
+end
+
+
diff --git a/cookbooks/nominatim/templates/default/clean-db-nominatim.erb b/cookbooks/nominatim/templates/default/clean-db-nominatim.erb
index 39297e7fe..202240e3f 100644
--- a/cookbooks/nominatim/templates/default/clean-db-nominatim.erb
+++ b/cookbooks/nominatim/templates/default/clean-db-nominatim.erb
@@ -12,9 +12,37 @@ for i in `seq 0 246`; do
   psql -q -d nominatim -c "VACUUM ANALYSE search_name_${i}"
 done
 
-<% if node[:postgresql][:settings][:defaults][:archive_mode] == "on" -%>
-
 # Cleanup archive directory
-#find -L /data/postgresql-archive -mmin +1440 -print0 | xargs -0r rm
+oldest=`date +%s`
+
+for fl in <% @streaming_clients %>; do
+  lastupdate=`wget -O - -q http://$fl/last_update.php`
+  if [[ "$?" != "0" ]]; then
+    oldest=0
+    break
+  else
+    epoch=`date +%s -d "$lastupdate"`
+    if [[ "$epoch" -lt "$oldest" ]]; then
+        oldest=$epoch
+    fi
+  fi
+done
+
+
+if [[ "$oldest" != "0" ]]; then
+    now=`date +%s`
+    timeago=$(( ( $now - $oldest ) / 60 + 180 ))
+    if [[ "$timeago" -le "180" ]]; then
+        echo "Oldest is in the future, something is wrong."
+        exit(1)
+    fi
+
+    echo "find -L <%= @archive_dir %> -mmin +$timeago -print0 | xargs -0r rm"
+fi
+
+diskfree=`df --output=pcent <%= @archive_dir %> | tail -n 1 | sed s:[^0-9.]::g
 
-<% end -%>
+if [[ "$diskfree" -lt 10 ]]; then
+    echo "WAL archives running out of space. Stopping updates."
+    touch <%= @update_stop_file %>
+fi
diff --git a/cookbooks/nominatim/templates/default/cron.erb b/cookbooks/nominatim/templates/default/cron.erb
index 02894ffc5..956a2d786 100644
--- a/cookbooks/nominatim/templates/default/cron.erb
+++ b/cookbooks/nominatim/templates/default/cron.erb
@@ -6,4 +6,4 @@ MAILTO=<%= @mailto %>
 <% if node[:nominatim][:enable_backup] -%>
 00 3 1 * * nominatim /usr/local/bin/backup-nominatim
 <% end -%>
-30 1 * * * postgres /usr/local/bin/clean-db-nominatim
+30 1 * * * postgres /usr/local/bin/vacuum-db-nominatim
diff --git a/cookbooks/nominatim/templates/default/updater.erb b/cookbooks/nominatim/templates/default/updater.erb
index beea8c8ac..aba7f72aa 100644
--- a/cookbooks/nominatim/templates/default/updater.erb
+++ b/cookbooks/nominatim/templates/default/updater.erb
@@ -6,7 +6,7 @@
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 cd $DIR/..
 
-while true
+while [ ! -f "<%= @update_stop_file %>" ]
 do
     # Send output to the log (logrotatable)
     exec >> /var/log/nominatim/update.log 2>&1
diff --git a/cookbooks/nominatim/templates/default/vacuum-db-nominatim b/cookbooks/nominatim/templates/default/vacuum-db-nominatim
new file mode 100644
index 000000000..77b1257ef
--- /dev/null
+++ b/cookbooks/nominatim/templates/default/vacuum-db-nominatim
@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# DO NOT EDIT - This file is being maintained by Chef
+
+
+# Vaccum all tables with indices on integer arrays.
+# Agressive vacuuming seems to help against index bloat.
+psql -q -d nominatim -c 'VACUUM ANALYSE search_name'
+psql -q -d nominatim -c 'VACUUM ANALYSE search_name_country'
+
+for i in `seq 0 246`; do
+  psql -q -d nominatim -c "VACUUM ANALYSE search_name_${i}"
+done
+