From: Tom Hughes <tom@compton.nu>
Date: Sun, 16 Feb 2014 12:01:48 +0000 (+0000)
Subject: More work on taginfo
X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/86edbbca8af9903fca18dbb20a6ea2ca62e7d97c

More work on taginfo
---

diff --git a/cookbooks/taginfo/recipes/default.rb b/cookbooks/taginfo/recipes/default.rb
index 51ddbe96a..31592b9ba 100644
--- a/cookbooks/taginfo/recipes/default.rb
+++ b/cookbooks/taginfo/recipes/default.rb
@@ -46,6 +46,19 @@ gem_package "sinatra"
 gem_package "sinatra-r18n"
 gem_package "rack-contrib"
 
+directory "/var/log/taginfo" do
+  owner "taginfo"
+  group "taginfo"
+  mode 0755
+end
+
+template "/etc/logrotate.d/taginfo" do
+  source "logrotate.erb"
+  owner "root"
+  group "root"
+  mode 0644
+end
+
 node[:taginfo][:sites].each do |site|
   name = site[:name]
   directory = site[:directory] || "/srv/#{name}"
@@ -53,6 +66,12 @@ node[:taginfo][:sites].each do |site|
   icon = site[:icon]
   contact = site[:contact]
 
+  directory "/var/log/taginfo/#{site}" do
+    owner "taginfo"
+    group "taginfo"
+    mode 0755
+  end
+
   directory directory do
     owner "taginfo"
     group "taginfo"
@@ -81,8 +100,10 @@ node[:taginfo][:sites].each do |site|
   settings["instance"]["description"] = description
   settings["instance"]["icon"] = "/img/logo/#{icon}.png"
   settings["instance"]["contact"] = contact
+  settings["logging"]["directory"] = "/var/log/taginfo/#{site}"
   settings["opensearch"]["shortname"] = "Taginfo"
   settings["opensearch"]["contact"] = "webmaster@openstreetmap.org"
+  settings["sources"]["db"]["planetfile"] = "#{directory}/planet/planet.pbf"
   settings["tagstats"]["cxxflags"] = "-I../../osmium/include"
 
   file "#{directory}/taginfo-config.json" do
@@ -103,6 +124,30 @@ node[:taginfo][:sites].each do |site|
     notifies :restart, "service[apache2]"
   end
 
+  directory "#{directory}/taginfo/web/tmp" do
+    owner "taginfo"
+    group "taginfo"
+    mode 0755
+  end
+
+  directory "#{directory}/data" do
+    owner "taginfo"
+    group "taginfo"
+    mode 0755
+  end
+
+  directory "#{directory}/download" do
+    owner "taginfo"
+    group "taginfo"
+    mode 0755
+  end
+
+  directory "#{directory}/sources" do
+    owner "taginfo"
+    group "taginfo"
+    mode 0755
+  end
+
   directory "#{directory}/planet" do
     owner "taginfo"
     group "taginfo"
@@ -117,14 +162,26 @@ node[:taginfo][:sites].each do |site|
     mode 0644
   end
 
-  template "#{directory}/planet/configuration.txt" do
+  directory "#{directory}/planet/log" do
+    owner "taginfo"
+    group "taginfo"
+    mode 0755
+  end
+
+  directory "#{directory}/planet/replication" do
+    owner "taginfo"
+    group "taginfo"
+    mode 0755
+  end
+
+  template "#{directory}/planet/replication/configuration.txt" do
     source "configuration.txt.erb"
     owner "taginfo"
     group "taginfo"
     mode 0644
   end
 
-  file "#{directory}/planet/download.lock" do
+  file "#{directory}/planet/replication/download.lock" do
     owner "taginfo"
     group "taginfo"
     mode 0644
@@ -144,10 +201,20 @@ node[:taginfo][:sites].each do |site|
     variables :directory => directory
   end
 
-  directory "#{directory}/data" do
+  template "#{directory}/bin/update-taginfo" do
+    source "update-taginfo.erb"
+    owner "taginfo"
+    group "taginfo"
+    mode 0755
+    variables :directory => directory
+  end
+
+  template "#{directory}/bin/update" do
+    source "update.erb"
     owner "taginfo"
     group "taginfo"
     mode 0755
+    variables :directory => directory
   end
 
   apache_site name do
diff --git a/cookbooks/taginfo/templates/default/configuration.txt.erb b/cookbooks/taginfo/templates/default/configuration.txt.erb
index 6f5bf935f..bb2f62624 100644
--- a/cookbooks/taginfo/templates/default/configuration.txt.erb
+++ b/cookbooks/taginfo/templates/default/configuration.txt.erb
@@ -1,2 +1,4 @@
+# DO NOT EDIT - This file is being maintained by Chef
+
 baseUrl=http://planet.openstreetmap.org/replication/hour
 maxInterval=0
diff --git a/cookbooks/taginfo/templates/default/logrotate.erb b/cookbooks/taginfo/templates/default/logrotate.erb
new file mode 100644
index 000000000..bf6f4a842
--- /dev/null
+++ b/cookbooks/taginfo/templates/default/logrotate.erb
@@ -0,0 +1,14 @@
+# DO NOT EDIT - This file is being maintained by Chef
+<% node[:taginfo][:sites].each do |site| -%>
+
+/var/log/taginfo/<%= site[:name] %>/*.log {
+  daily
+  missingok
+  rotate 0
+  dateext
+  dateformat %Y-%m-%d
+  maxage 28
+  compress
+  delaycompress
+}
+<% end -%>
diff --git a/cookbooks/taginfo/templates/default/update-planet.erb b/cookbooks/taginfo/templates/default/update-planet.erb
index 8fd69ff9e..3a96141a7 100644
--- a/cookbooks/taginfo/templates/default/update-planet.erb
+++ b/cookbooks/taginfo/templates/default/update-planet.erb
@@ -1,26 +1,39 @@
 #!/bin/sh
 
+# DO NOT EDIT - This file is being maintained by Chef
+
+# abort on error
+
 set -e
 
-cd "<%= @directory %>/planet"
+# setup
+
+PLANETDIR="<%= @directory %>/planet"
+LOGDIR="${PLANETDIR}/log"
+PLANETOLD="${PLANETDIR}/planet.pbf"
+PLANETNEW="${PLANETDIR}/planet-new.pbf"
+STATEDIR="${PLANETDIR}/replication"
+OSCFILE="${PLANETDIR}/$(date +%Y%m%d-%H%M).osc"
+
+# start logging
 
-OSCFILE="$(date +%Y%m%d-%H%M).osc"
+exec > "${LOGDIR}/$(date +%Y%m%d-%H%M).log" 2>&1
 
 # if there are old .osc files, they have to be taken into account
 
-OLDFILE=2*-*.osc
+OLDFILE="$(find "${PLANETDIR}" -name "2*-*.osc" -print)"
 if [ -z "$OLDFILE" ]
 then
-   osmosis --rri --simc --write-xml-change "$OSCFILE"
+   osmosis --rri "$STATEDIR" --simc --write-xml-change "$OSCFILE"
 else
-   osmosis --rxc "$OLDFILE" --rri --mc --simc --write-xml-change "$OSCFILE" && rm "$OLDFILE"
+   osmosis --rxc "$OLDFILE" --rri "$STATEDIR" --mc --simc --write-xml-change "$OSCFILE" && rm "$OLDFILE"
 fi
 
 # osc file is ready, create new planet file
 
-osmosis --read-xml-change "$OSCFILE" --read-bin planet.pbf --buffer bufferCapacity=12000 --apply-change --buffer bufferCapacity=12000 --write-pbf planet-new.pbf
+osmosis --read-xml-change "$OSCFILE" --read-bin "$PLANETOLD" --buffer bufferCapacity=12000 --apply-change --buffer bufferCapacity=12000 --write-pbf omitmetadata=true file="$PLANETNEW"
 
 # cleanup
 
-mv planet-new.pbf planet.pbf
+mv "$PLANETNEW" "$PLANETOLD"
 rm "$OSCFILE"
diff --git a/cookbooks/taginfo/templates/default/update-taginfo.erb b/cookbooks/taginfo/templates/default/update-taginfo.erb
new file mode 100644
index 000000000..cf30b4178
--- /dev/null
+++ b/cookbooks/taginfo/templates/default/update-taginfo.erb
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+set -e
+
+ROOT="<%= @directory %>"
+
+cp $ROOT/data/taginfo-history.db $ROOT/sources
+$ROOT/taginfo/sources/update_all.sh $ROOT/sources
+mv $ROOT/data/taginfo-* $ROOT/data/old
+mv $ROOT/sources/taginfo-*.db $ROOT/sources/*/taginfo-*.db $ROOT/data
+mv $ROOT/sources/download/* $ROOT/download
+
+touch $ROOT/taginfo/web/tmp/restart.txt
diff --git a/cookbooks/taginfo/templates/default/update.erb b/cookbooks/taginfo/templates/default/update.erb
new file mode 100644
index 000000000..7050d1617
--- /dev/null
+++ b/cookbooks/taginfo/templates/default/update.erb
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+<%= @directory %>/bin/update-planet
+<%= @directory %>/bin/update-taginfo