From: Tom Hughes Date: Mon, 22 Dec 2014 14:38:59 +0000 (+0000) Subject: Merge branch 'planetdump-ng' X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/e9d337c2a49d120a8dcb508879d74f6bab5b84d9?hp=8b72732cd79ec76f34a560cfd9f1d526d7a7ce7e Merge branch 'planetdump-ng' --- diff --git a/cookbooks/incron/README.md b/cookbooks/incron/README.md new file mode 100644 index 000000000..205739199 --- /dev/null +++ b/cookbooks/incron/README.md @@ -0,0 +1,34 @@ +Cookbook +======== +TODO: Enter the cookbook description here. + +e.g. +This cookbook makes your favorite breakfast sandwich. + +Requirements +------------ +TODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc. + +Attributes +---------- +TODO: List you cookbook attributes here. + +Usage +----- +TODO: Write usage instructions for each cookbook. + +Contributing +------------ +TODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section. + +e.g. +1. Fork the repository on Github +2. Create a named feature branch (like `add_component_x`) +3. Write your change +4. Write tests for your change (if applicable) +5. Run the tests, ensuring they all pass +6. Submit a Pull Request using Github + +License and Authors +------------------- +Authors: TODO: List authors diff --git a/cookbooks/incron/attributes/default.rb b/cookbooks/incron/attributes/default.rb new file mode 100644 index 000000000..4d2ce58b6 --- /dev/null +++ b/cookbooks/incron/attributes/default.rb @@ -0,0 +1 @@ +default[:incron] = {} diff --git a/cookbooks/incron/metadata.rb b/cookbooks/incron/metadata.rb new file mode 100644 index 000000000..3e21a533e --- /dev/null +++ b/cookbooks/incron/metadata.rb @@ -0,0 +1,7 @@ +name "incron" +maintainer "OpenStreetMap Administrators" +maintainer_email "admins@openstreetmap.org" +license "Apache 2.0" +description "Configures incron" +long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) +version "1.0.0" diff --git a/cookbooks/incron/recipes/default.rb b/cookbooks/incron/recipes/default.rb new file mode 100644 index 000000000..d4a176179 --- /dev/null +++ b/cookbooks/incron/recipes/default.rb @@ -0,0 +1,54 @@ +# +# Cookbook Name:: incron +# Recipe:: default +# +# Copyright 2014, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +package "incron" + +service "incron" do + action [ :enable, :start ] + supports :status => true, :reload => true, :restart => true +end + +incrontabs = {} + +node[:incron].each do |name,details| + user = details[:user] + path = details[:path] + mask = details[:events].join(",") + command = details[:command] + + incrontabs[user] ||= [] + + incrontabs[user].push("#{path} #{mask} #{command}") +end + +incrontabs.each do |user,lines| + file "/var/spool/incron/#{user}" do + owner user + group "incron" + mode 0600 + content lines.join("\n") + end +end + +file "/etc/incron.allow" do + owner "root" + group "incron" + mode "0640" + content incrontabs.keys.sort.join("\n") +end diff --git a/cookbooks/planet/attributes/default.rb b/cookbooks/planet/attributes/default.rb new file mode 100644 index 000000000..649d9b399 --- /dev/null +++ b/cookbooks/planet/attributes/default.rb @@ -0,0 +1,4 @@ +default[:planet][:dump][:xml_directory] = "/store/planet/planet" +default[:planet][:dump][:xml_history_directory] = "/store/planet/planet/full-history" +default[:planet][:dump][:pbf_directory] = "/store/planet/pbf" +default[:planet][:dump][:pbf_history_directory] = "/store/planet/pbf/full-history" diff --git a/cookbooks/planet/files/default/bin/planet2pbf b/cookbooks/planet/files/default/bin/planet2pbf deleted file mode 100644 index d7003d5ae..000000000 --- a/cookbooks/planet/files/default/bin/planet2pbf +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -# DO NOT EDIT - This file is being maintained by Chef - -set -e -user=`whoami` -if [ "$user" != "www-data" ]; - then - echo 'Error: run as www-data' - exit 1 - fi - -if [ -f /tmp/planet2pbf.lock ]; then - if [ "$(ps -p `cat /tmp/planet2pbf.lock` | wc -l)" -gt 1 ]; then - echo planet2pbf process is still running - exit 1 - else - # process not running, but lock file not deleted? - rm /tmp/planet2pbf.lock - fi -fi -echo $$ >/tmp/planet2pbf.lock - -YEARFULL=`TZ=GMT+24 date +%Y` - -cd /store/planet/planet/ -if [ -z $1 ] -then - YEAR=`TZ=GMT+24 date +%y` - MONTH=`TZ=GMT+24 date +%m` - DAY=`TZ=GMT+24 date +%d` - if [ -f ${YEARFULL}/planet-${YEAR}${MONTH}${DAY}.osm.bz2 -o -f ${YEARFULL}/.planet-${YEAR}${MONTH}${DAY}.osm.bz2.new ]; - then - planet=planet-${YEAR}${MONTH}${DAY}.osm - else - echo Error /store/planet/planet/${YEARFULL}/planet-${YEAR}${MONTH}${DAY}.osm.bz2 not available - exit 1 - fi -else - planet=$1 -fi -while [ -f ${YEARFULL}/.${planet}.bz2.new ]; -do - echo sleeping - ${planet}.bz2 - sleep 300 -done -if [ -f /store/planet/planet/${YEARFULL}/${planet}.bz2 ] -then - echo $planet - if [ -f /store/planet/pbf/${planet}.pbf ]; - then - echo Error: file /store/planet/pbf/${planet}.pbf exists - exit 1 - fi - JAVACMD_OPTIONS=-Xmx2048M - export JAVACMD_OPTIONS - pbzip2 -dc /store/planet/planet/${YEARFULL}/${planet}.bz2 | osmosis --read-xml - --buffer bufferCapacity=18000 --write-pbf /store/planet/pbf/${planet}.pbf - cd /store/planet/pbf/ - md5sum ${planet}.pbf >${planet}.pbf.md5 - ln -fs ${planet}.pbf planet-latest.osm.pbf - ln -fs ${planet}.pbf.md5 planet-latest.osm.pbf.md5 -else - echo Error no file - ${planet}.bz2 -fi - -rm /tmp/planet2pbf.lock - diff --git a/cookbooks/planet/metadata.rb b/cookbooks/planet/metadata.rb index 7d868d03e..3e2f5f3cb 100644 --- a/cookbooks/planet/metadata.rb +++ b/cookbooks/planet/metadata.rb @@ -8,3 +8,4 @@ version "1.0.0" depends "apache" depends "git" depends "osmosis" +depends "incron" diff --git a/cookbooks/planet/recipes/default.rb b/cookbooks/planet/recipes/default.rb index 706e766e8..51470380d 100644 --- a/cookbooks/planet/recipes/default.rb +++ b/cookbooks/planet/recipes/default.rb @@ -61,16 +61,13 @@ remote_directory "/store/planet" do files_mode 0755 end -directory "/store/planet/planet" do - owner "www-data" - group "planet" - mode 0775 -end - -directory "/store/planet/pbf" do - owner "www-data" - group "planet" - mode 0775 +[:xml_directory, :xml_history_directory, + :pbf_directory, :pbf_history_directory].each do |dir| + directory dir do + owner "www-data" + group "planet" + mode 0775 + end end directory "/store/planet/notes" do diff --git a/cookbooks/planet/recipes/dump.rb b/cookbooks/planet/recipes/dump.rb index 197ed10b4..1425fbec3 100644 --- a/cookbooks/planet/recipes/dump.rb +++ b/cookbooks/planet/recipes/dump.rb @@ -17,48 +17,83 @@ # limitations under the License. # -include_recipe "git" +node.default[:incron][:planetdump] = { + :user => "www-data", + :path => "/store/backup", + :events => [ "IN_CREATE", "IN_MOVED_TO" ], + :command => "/usr/local/bin/planetdump $#" +} -db_passwords = data_bag_item("db", "passwords") +include_recipe "git" +include_recipe "incron" package "gcc" package "make" -package "libpqxx3-dev" +package "autoconf" +package "automake" +package "libxml2-dev" +package "libboost-dev" +package "libboost-program-options-dev" +package "libboost-date-time-dev" +package "libboost-filesystem-dev" +package "libboost-thread-dev" +package "libboost-iostreams-dev" +package "libosmpbf-dev" +package "libprotobuf-dev" +package "osmpbf-bin" -directory "/opt/planetdump" do +directory "/opt/planet-dump-ng" do owner "root" group "root" mode 0755 end -git "/opt/planetdump" do +git "/opt/planet-dump-ng" do action :sync - repository "git://git.openstreetmap.org/planetdump.git" - revision "live" + repository "git://github.com/zerebubuth/planet-dump-ng.git" + revision "master" user "root" group "root" end -execute "/opt/planetdump/Makefile" do +execute "/opt/planet-dump-ng/autogen.sh" do action :nothing - command "make planet06_pg" - cwd "/opt/planetdump" + command "./autogen.sh" + cwd "/opt/planet-dump-ng" user "root" group "root" - subscribes :run, "git[/opt/planetdump]" + subscribes :run, "git[/opt/planet-dump-ng]" end -template "/usr/local/bin/planetdump" do - source "planetdump.erb" - owner "root" +execute "/opt/planet-dump-ng/configure" do + action :nothing + command "./configure" + cwd "/opt/planet-dump-ng" + user "root" group "root" - mode 0755 - variables :password => db_passwords["planetdump"] + subscribes :run, "execute[/opt/planet-dump-ng/autogen.sh]" end -template "/etc/cron.d/planetdump" do - source "planetdump.cron.erb" - owner "root" +execute "/opt/planet-dump-ng/Makefile" do + action :nothing + command "make" + cwd "/opt/planet-dump-ng" + user "root" group "root" - mode 0644 + subscribes :run, "execute[/opt/planet-dump-ng/configure]" +end + +directory "/store/planetdump" do + owner "www-data" + group "www-data" + mode 0755 +end + +["planetdump", "planet-mirror-redirect-update", "apache-latest-planet-filename"].each do |program| + template "/usr/local/bin/#{program}" do + source "#{program}.erb" + owner "root" + group "root" + mode 0755 + end end diff --git a/cookbooks/planet/files/default/bin/apache-latest-planet-filename b/cookbooks/planet/templates/default/apache-latest-planet-filename.erb similarity index 64% rename from cookbooks/planet/files/default/bin/apache-latest-planet-filename rename to cookbooks/planet/templates/default/apache-latest-planet-filename.erb index e5e35819a..d7c8695c7 100644 --- a/cookbooks/planet/files/default/bin/apache-latest-planet-filename +++ b/cookbooks/planet/templates/default/apache-latest-planet-filename.erb @@ -4,7 +4,7 @@ $| = 1; while () { - my $where = readlink("/store/planet/planet/planet-latest.osm.bz2"); + my $where = readlink("<%= node[:planet][:dump][:xml_directory] %>/planet-latest.osm.bz2"); s/planet\/planet\-latest\.osm\.bz2/planet\/$where/g; print $_; } diff --git a/cookbooks/planet/templates/default/cron.erb b/cookbooks/planet/templates/default/cron.erb index 70ec98c04..3feec3f49 100644 --- a/cookbooks/planet/templates/default/cron.erb +++ b/cookbooks/planet/templates/default/cron.erb @@ -1,4 +1,3 @@ # DO NOT EDIT - This file is being maintained by Chef MAILTO=horntail-www-data-cron@firefishy.com 20 */2 * * * www-data /usr/local/bin/planet-mirror-redirect-update -39 10 * * 4 www-data /usr/local/bin/planet2pbf diff --git a/cookbooks/planet/files/default/bin/planet-mirror-redirect-update b/cookbooks/planet/templates/default/planet-mirror-redirect-update.erb similarity index 96% rename from cookbooks/planet/files/default/bin/planet-mirror-redirect-update rename to cookbooks/planet/templates/default/planet-mirror-redirect-update.erb index 6d772ec2b..1f37ecd23 100644 --- a/cookbooks/planet/files/default/bin/planet-mirror-redirect-update +++ b/cookbooks/planet/templates/default/planet-mirror-redirect-update.erb @@ -8,7 +8,7 @@ Modifies .htaccess */ $_YEAR = date('Y'); -$_PLANET_FOLDER = '/store/planet/planet/'.$_YEAR.'/'; +$_PLANET_FOLDER = '<%= node[:planet][:dump][:xml_directory] %>/'.$_YEAR.'/'; $_PLANET_REGEX = "/^(planet|changesets)\-\d{6}(\-nolt)?\.osm\.(bz2|gz)$/"; $_MIRROR = 'http://ftp.heanet.ie/mirrors/openstreetmap.org/planet/'.$_YEAR.'/'; $_PLANET_HTACCESS = $_PLANET_FOLDER.'../../.htaccess'; diff --git a/cookbooks/planet/templates/default/planetdump.cron.erb b/cookbooks/planet/templates/default/planetdump.cron.erb deleted file mode 100644 index b7403d22c..000000000 --- a/cookbooks/planet/templates/default/planetdump.cron.erb +++ /dev/null @@ -1,2 +0,0 @@ -MAILTO=grant-smaug@firefishy.com -10 0 * * 3 www-data /usr/local/bin/planetdump diff --git a/cookbooks/planet/templates/default/planetdump.erb b/cookbooks/planet/templates/default/planetdump.erb index 9df4a6f03..b2b2a6ea4 100644 --- a/cookbooks/planet/templates/default/planetdump.erb +++ b/cookbooks/planet/templates/default/planetdump.erb @@ -1,71 +1,67 @@ #!/bin/bash -ulimit -m 4194304 -v 4194304 -set -e -if [ -f /tmp/planetdump.lock ]; then - if [ "$(ps -p `cat /tmp/planetdump.lock` | wc -l)" -gt 1 ]; then - # process is still running - echo Error: Another planetdump is running - exit 1 - else - # process not running, but lock file not deleted? - rm /tmp/planetdump.lock - fi -fi - -#Create Lock -echo $$ > /tmp/planetdump.lock +# DO NOT EDIT - This file is being maintained by Chef +# Exit on error +set -e -cur_date=`date +%y%m%d` -cur_year=`date +%Y` -cur_planet=planet-${cur_date}.osm -cur_changeset=changesets-${cur_date}.osm -planet_dir=/store/planet/ - -export CONNECTION_PARAMS='dbname=openstreetmap host=db user=planetdump password=<%= @password %>' -export PATH='/usr/local/bin:/usr/bin:/bin:/usr/bin/X11' - +# Get the name of the file and the expected pattern +file="$1" +pattern="^osm-([0-9]{4})-([0-9]{2}-[0-9]{2})\.dmp$" -if [ \! -d ${planet_dir}/planet/ ] - then - echo ${planet_dir}planet/ does not exist - exit 1 - fi -if [ \! -d ${planet_dir}/planet/${cur_year}/ ]; then mkdir ${planet_dir}/planet/${cur_year}/; fi -cd ${planet_dir}/planet/${cur_year}/ +# Give up now if the file isn't a database dump +[[ $file =~ $pattern ]] || exit 0 -/opt/planetdump/planet06_pg | pbzip2 -p6 -6c > .${cur_planet}.bz2.new -/opt/planetdump/planet06_pg --changesets | pbzip2 -p6 -9c > .${cur_changeset}.bz2.new +# Save the year and date from the file name +year="${BASH_REMATCH[1]}" +date="${year}-${BASH_REMATCH[2]}" -planet_size=$(du -sb .${cur_planet}.bz2.new | awk '{ print $1 }') -changeset_size=$(du -sb .${cur_changeset}.bz2.new | awk '{ print $1 }') -if ((planet_size<32000000000)); then - echo Planet .${cur_planet}.bz2.new too small - exit 1 -fi -if ((changeset_size<720000000)); then - echo Changeset .${cur_changeset}.bz2.new too small +# Check the lock +if [ -f /tmp/planetdump.lock ]; then + if [ "$(ps -p `cat /tmp/planetdump.lock` | wc -l)" -gt 1 ]; then + echo "Error: Another planetdump is running" exit 1 + else + rm /tmp/planetdump.lock + fi fi -mv .${cur_planet}.bz2.new ${cur_planet}.bz2 -mv .${cur_changeset}.bz2.new ${cur_changeset}.bz2 -md5sum ${cur_planet}.bz2 > ${cur_planet}.bz2.md5 -md5sum ${cur_changeset}.bz2 > ${cur_changeset}.bz2.md5 - -#link planet latest to the new file -cd ${planet_dir}/planet/ - -ln -fs ${cur_year}/${cur_planet}.bz2 planet-latest.osm.bz2 -ln -fs ${cur_year}/${cur_changeset}.bz2 changesets-latest.osm.bz2 - -# mangle md5 files for 'latest' ones -rm -f planet-latest.osm.bz2.md5 -rm -f changesets-latest.osm.bz2.md5 - -sed -e "s/${cur_planet}.bz2/planet-latest.osm.bz2/" ${cur_year}/${cur_planet}.bz2.md5 > planet-latest.osm.bz2.md5 -sed -e "s/${cur_changeset}.bz2/changesets-latest.osm.bz2/" ${cur_year}/${cur_changeset}.bz2.md5 > changesets-latest.osm.bz2.md5 - -rm /tmp/planetdump.lock +# Create Lock +echo $$ > /tmp/planetdump.lock +# Define cleanup function +function cleanup { + rm /tmp/planetdump.lock +} + +# Remove lock on exit +trap cleanup EXIT + +# Change to working directory +cd /store/planetdump + +# Cleanup +rm -rf users +rm -rf changesets changeset_tags +rm -rf nodes node_tags +rm -rf ways way_tags way_nodes +rm -rf relations relation_tags relation_members + +# Run the dump +time nice -n 19 /store/planet-dump-ng/planet-dump-ng \ + -c "pbzip2 -c" -f "${file}" --dense-nodes=1 \ + -C "changesets-${date}.osm.bz2" \ + -x "planet-${date}.osm.bz2" -X "history-${date}.osm.bz2" \ + -p "planet-${date}.osm.pbf" -P "history-${date}.osm.pbf" + +# Move XML dumps into place +<% { + "changesets-${date}.osm.bz2" => "#{node[:planet][:dump][:xml_directory]}/${year}", + "planet-${date}.osm.bz2" => "#{node[:planet][:dump][:xml_directory]}/${year}", + "history-${date}.osm.bz2" => "#{node[:planet][:dump][:xml_history_directory]}/${year}", + "planet-${date}.osm.pbf" => node[:planet][:dump][:pbf_directory], + "history-${date}.osm.pbf" => node[:planet][:dump][:pbf_history_directory] +}.each do |file,dir| -%> +md5sum "<%= file %>" > "<%= file %>.md5" +mv "<%= file %>" "<%= file %>.md5" "<%= dir %>" +<% end -%>