From: Grant Slater <git@firefishy.com>
Date: Fri, 1 Aug 2014 08:51:06 +0000 (+0100)
Subject: Add planet-notes-dump code
X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/e995ca991a42a599d07fa87ae19a3ab650652de1?hp=bd08729fe0c15a4769332a352cc3723dee13aa24

Add planet-notes-dump code
---

diff --git a/cookbooks/planet/recipes/default.rb b/cookbooks/planet/recipes/default.rb
index 11e90edfb..1edc46fdd 100644
--- a/cookbooks/planet/recipes/default.rb
+++ b/cookbooks/planet/recipes/default.rb
@@ -61,6 +61,24 @@ remote_directory "/store/planet" do
   files_mode 0755
 end
 
+directory "/store/planet/planet" do
+  owner "www-data"
+  group "planet"
+  mode 0775
+end
+
+directory "/store/planet/pbf" do
+  owner "www-data"
+  group "planet"
+  mode 0775
+end
+
+directory "/store/planet/notes" do
+  owner "www-data"
+  group "planet"
+  mode 0775
+end
+
 apache_module "rewrite" do
   conf "rewrite.conf.erb"
 end
diff --git a/cookbooks/planet/recipes/dump-notes.rb b/cookbooks/planet/recipes/dump-notes.rb
new file mode 100644
index 000000000..ac2e07301
--- /dev/null
+++ b/cookbooks/planet/recipes/dump-notes.rb
@@ -0,0 +1,53 @@
+#
+# Cookbook Name:: planet
+# Recipe:: dump
+#
+# Copyright 2013, OpenStreetMap Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "git"
+
+db_passwords = data_bag_item("db", "passwords")
+
+package "python-psycopg2"
+package "python-lxml"
+
+directory "/opt/planet-notes-dump" do
+  owner "root"
+  group "root"
+  mode 0755
+end
+
+git "/opt/planet-notes-dump" do
+  action :sync
+  repository "git://github.com/openstreetmap/planet-notes-dump.git"
+  user "root"
+  group "root"
+end
+
+template "/usr/local/bin/planet-notes-dump" do
+  source "planet-notes-dump.erb"
+  owner "root"
+  group "root"
+  mode 0755
+  variables :password => db_passwords["planetdump"]
+end
+
+template "/etc/cron.d/planet-notes-dump" do
+  source "planet-notes-dump.cron.erb"
+  owner "root"
+  group "root"
+  mode 0644
+end
diff --git a/cookbooks/planet/templates/default/planet-notes-dump.cron.erb b/cookbooks/planet/templates/default/planet-notes-dump.cron.erb
new file mode 100644
index 000000000..0db348698
--- /dev/null
+++ b/cookbooks/planet/templates/default/planet-notes-dump.cron.erb
@@ -0,0 +1,2 @@
+MAILTO=grant-smaug@firefishy.com
+0 3 * * * www-data /usr/local/bin/planet-notes-dump
diff --git a/cookbooks/planet/templates/default/planet-notes-dump.erb b/cookbooks/planet/templates/default/planet-notes-dump.erb
new file mode 100644
index 000000000..6363c0bde
--- /dev/null
+++ b/cookbooks/planet/templates/default/planet-notes-dump.erb
@@ -0,0 +1,59 @@
+#!/bin/bash
+ulimit -m 4194304 -v 4194304
+set -e
+
+if [ -f /tmp/planet-notes-dump.lock ]; then
+	if [ "$(ps -p `cat /tmp/planet-notes-dump.lock` | wc -l)" -gt 1 ]; then
+		# process is still running
+		echo Error: Another planet-notes-dump is running
+		exit 1
+	else
+		# process not running, but lock file not deleted?
+		rm /tmp/planet-notes-dump.lock
+	fi
+fi
+
+#Create Lock
+echo $$ > /tmp/planet-notes-dump.lock
+
+
+cur_date=`date +%y%m%d`
+cur_year=`date +%Y`
+cur_planet_notes=planet-notes-${cur_date}.osn
+planet_dir=/store/planet/
+
+export PATH='/usr/local/bin:/usr/bin:/bin:/usr/bin/X11'
+
+
+if [ \! -d ${planet_dir}/notes/ ]
+	then
+		echo ${planet_dir}notes/ does not exist
+		exit 1
+	fi
+if [ \! -d ${planet_dir}/notes/${cur_year}/ ]; then mkdir ${planet_dir}/notes/${cur_year}/; fi
+cd ${planet_dir}/notes/${cur_year}/
+
+/usr/bin/python /opt/planet-notes-dump/dump.py --database openstreetmap --host db-slave --user planetdump --password '<%= @password %>' .${cur_planet_notes}
+pbzip2 -p6 -9 .${cur_planet_notes}
+
+planet_notes_size=$(du -sb .${cur_planet_notes}.bz2 | awk '{ print $1 }')
+if ((planet_notes_size<12000000)); then
+	echo Planet .${cur_planet_notes}.bz2 too small
+	exit 1
+fi
+
+mv .${cur_planet_notes}.bz2.new ${cur_planet_notes}.bz2
+md5sum ${cur_planet_notes}.bz2 > ${cur_planet_notes}.bz2.md5
+
+#link planet latest to the new file
+cd ${planet_dir}/notes/
+
+ln -fs ${cur_year}/${cur_planet_notes}.bz2 planet-notes-latest.osn.bz2
+
+# mangle md5 files for 'latest' ones
+rm -f planet-notes-latest.osn.bz2.md5
+
+sed -e "s/${cur_planet_notes}.bz2/planet-notes-latest.osn.bz2/" ${cur_year}/${cur_planet_notes}.bz2.md5 > planet-notes-latest.osn.bz2.md5
+
+rm /tmp/planet-notes-dump.lock
+