From 7609b4c7dadb461d16c7c14f18e78782a19ea13d Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Tue, 2 Jul 2013 17:49:26 +0100 Subject: [PATCH 01/16] Send renderd logs to a separate file --- cookbooks/tile/metadata.rb | 1 + cookbooks/tile/recipes/default.rb | 9 +++++++++ cookbooks/tile/templates/default/renderd.rsyslog.erb | 4 ++++ cookbooks/tools/recipes/default.rb | 7 +++++++ 4 files changed, 21 insertions(+) create mode 100644 cookbooks/tile/templates/default/renderd.rsyslog.erb diff --git a/cookbooks/tile/metadata.rb b/cookbooks/tile/metadata.rb index 6d261cf0d..3626f3189 100644 --- a/cookbooks/tile/metadata.rb +++ b/cookbooks/tile/metadata.rb @@ -8,3 +8,4 @@ depends "apache" depends "git" depends "nodejs" depends "postgresql" +depends "tools" diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb index 67aeeaa11..f39c31623 100644 --- a/cookbooks/tile/recipes/default.rb +++ b/cookbooks/tile/recipes/default.rb @@ -21,6 +21,7 @@ include_recipe "apache" include_recipe "git" include_recipe "nodejs" include_recipe "postgresql" +include_recipe "tools" blocks = data_bag_item("tile", "blocks") @@ -415,6 +416,14 @@ template "/etc/cron.d/render-lowzoom" do mode 0755 end +template "/etc/rsyslog.d/20-renderd.conf" do + source "renderd.rsyslog.erb" + owner "root" + group "root" + mode 0644 + notifies :restart, "service[rsyslog]" +end + munin_plugin "mod_tile_fresh" munin_plugin "mod_tile_response" munin_plugin "mod_tile_zoom" diff --git a/cookbooks/tile/templates/default/renderd.rsyslog.erb b/cookbooks/tile/templates/default/renderd.rsyslog.erb new file mode 100644 index 000000000..59c62e6b6 --- /dev/null +++ b/cookbooks/tile/templates/default/renderd.rsyslog.erb @@ -0,0 +1,4 @@ +# DO NOT EDIT - This file is being maintained by Chef + +if $programname == 'renderd' then -/var/log/tile/renderd.log +& ~ diff --git a/cookbooks/tools/recipes/default.rb b/cookbooks/tools/recipes/default.rb index c5e85a6ab..45fc074a4 100644 --- a/cookbooks/tools/recipes/default.rb +++ b/cookbooks/tools/recipes/default.rb @@ -37,3 +37,10 @@ package "iotop" if node[:lsb][:release].to_f <= 11.04 package "lslk" end + +package "rsyslog" + +service "rsyslog" do + action [ :enable, :start ] + supports :status => true, :restart => true, :reload => true +end -- 2.43.2 From 8db11f71f84c2048e0bbabe9e179dd2409a9df05 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Wed, 3 Jul 2013 18:26:35 +0100 Subject: [PATCH 02/16] Reorder supybot plugins to match what it generates --- cookbooks/supybot/templates/default/supybot.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/supybot/templates/default/supybot.conf.erb b/cookbooks/supybot/templates/default/supybot.conf.erb index 212dab1a2..7bb524ed9 100644 --- a/cookbooks/supybot/templates/default/supybot.conf.erb +++ b/cookbooks/supybot/templates/default/supybot.conf.erb @@ -667,7 +667,7 @@ supybot.directories.log: /var/log/supybot # # Default value: ### -supybot.plugins: Git Admin Misc Services User Owner Config Channel +supybot.plugins: Git Admin Misc Owner User Services Config Channel ### # Determines whether this plugin is loaded by default. -- 2.43.2 From d358821303962f093e734708a59193e09098a09a Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Wed, 3 Jul 2013 19:13:13 +0100 Subject: [PATCH 03/16] Add an osmosis cookbook to manage an install of osmosis --- cookbooks/osmosis/README.rdoc | 8 ++++ cookbooks/osmosis/attributes/default.rb | 2 + cookbooks/osmosis/metadata.rb | 7 +++ cookbooks/osmosis/recipes/default.rb | 64 +++++++++++++++++++++++++ 4 files changed, 81 insertions(+) create mode 100644 cookbooks/osmosis/README.rdoc create mode 100644 cookbooks/osmosis/attributes/default.rb create mode 100644 cookbooks/osmosis/metadata.rb create mode 100644 cookbooks/osmosis/recipes/default.rb diff --git a/cookbooks/osmosis/README.rdoc b/cookbooks/osmosis/README.rdoc new file mode 100644 index 000000000..3de2ec7a3 --- /dev/null +++ b/cookbooks/osmosis/README.rdoc @@ -0,0 +1,8 @@ += DESCRIPTION: + += REQUIREMENTS: + += ATTRIBUTES: + += USAGE: + diff --git a/cookbooks/osmosis/attributes/default.rb b/cookbooks/osmosis/attributes/default.rb new file mode 100644 index 000000000..8d4f9a3b6 --- /dev/null +++ b/cookbooks/osmosis/attributes/default.rb @@ -0,0 +1,2 @@ +# Set the default version +default[:osmosis][:version] = "0.43.1" diff --git a/cookbooks/osmosis/metadata.rb b/cookbooks/osmosis/metadata.rb new file mode 100644 index 000000000..6439a5578 --- /dev/null +++ b/cookbooks/osmosis/metadata.rb @@ -0,0 +1,7 @@ +maintainer "OpenStreetMap Administrators" +maintainer_email "admins@openstreetmap.org" +license "Apache 2.0" +description "Installs and configures osmosis" +long_description IO.read(File.join(File.dirname(__FILE__), 'README.rdoc')) +version "1.0.0" +depends "chef" diff --git a/cookbooks/osmosis/recipes/default.rb b/cookbooks/osmosis/recipes/default.rb new file mode 100644 index 000000000..42679ac3f --- /dev/null +++ b/cookbooks/osmosis/recipes/default.rb @@ -0,0 +1,64 @@ +# +# Cookbook Name:: osmosis +# Recipe:: default +# +# Copyright 2013, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "chef" + +osmosis_package = "osmosis-#{node[:osmosis][:version]}.zip" +osmosis_directory = "/opt/osmosis-#{node[:osmosis][:version]}" + +Dir.glob("/var/cache/chef/osmosis-*.zip").each do |zip| + if zip != "/var/cache/chef/#{osmosis_package}" + file zip do + action :delete + backup false + end + end +end + +directory osmosis_directory do + owner "root" + group "root" + mode 0755 +end + +execute "/var/cache/chef/#{osmosis_package}" do + action :nothing + command "unzip -q /var/cache/chef/#{osmosis_package}" + cwd osmosis_directory + user "root" + group "root" +end + +remote_file "/var/cache/chef/#{osmosis_package}" do + action :create_if_missing + source "http://bretth.dev.openstreetmap.org/osmosis-build/#{osmosis_package}" + owner "root" + group "root" + mode 0644 + backup false + notifies :run, "execute[/var/cache/chef/#{osmosis_package}]" +end + +link "/usr/local/bin/osmosis" do + to "#{osmosis_directory}/bin/osmosis" +end + +link "/usr/local/bin/osmosis-extract-apidb-0.6" do + to "#{osmosis_directory}/bin/osmosis-extract-apidb-0.6" +end -- 2.43.2 From d6577fc4c50eae4f698f176a8bc1b53996241627 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Wed, 3 Jul 2013 22:15:15 +0100 Subject: [PATCH 04/16] Bring replication diffs under chef control --- cookbooks/planet/metadata.rb | 2 + cookbooks/planet/recipes/replication.rb | 85 +++++++++++++++++++ .../templates/default/replication.auth.erb | 16 ++++ .../templates/default/replication.config.erb | 11 +++ .../templates/default/replication.cron.erb | 8 ++ roles/planet.rb | 1 + 6 files changed, 123 insertions(+) create mode 100644 cookbooks/planet/recipes/replication.rb create mode 100644 cookbooks/planet/templates/default/replication.auth.erb create mode 100644 cookbooks/planet/templates/default/replication.config.erb create mode 100644 cookbooks/planet/templates/default/replication.cron.erb diff --git a/cookbooks/planet/metadata.rb b/cookbooks/planet/metadata.rb index 23b90473d..ea3eecd2e 100644 --- a/cookbooks/planet/metadata.rb +++ b/cookbooks/planet/metadata.rb @@ -5,3 +5,5 @@ description "Installs and configures a planet server" long_description IO.read(File.join(File.dirname(__FILE__), 'README.rdoc')) version "1.0.0" depends "apache" +depends "git" +depends "osmosis" diff --git a/cookbooks/planet/recipes/replication.rb b/cookbooks/planet/recipes/replication.rb new file mode 100644 index 000000000..f055280f1 --- /dev/null +++ b/cookbooks/planet/recipes/replication.rb @@ -0,0 +1,85 @@ +# +# Cookbook Name:: planet +# Recipe:: dump +# +# Copyright 2013, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "osmosis" + +db_passwords = data_bag_item("db", "passwords") + +directory "/etc/replication" do + owner "root" + group "root" + mode 0755 +end + +template "/etc/replication/auth.conf" do + source "replication.auth.erb" + user "root" + group "planet" + mode 0640 + variables :password => db_passwords["planetdiff"] +end + +directory "/var/lib/replication" do + owner "planet" + group "planet" + mode 0755 +end + +directory "/var/lib/replication/hour" do + owner "planet" + group "planet" + mode 0755 +end + +template "/var/lib/replication/hour/configuration.txt" do + source "replication.config.erb" + owner "planet" + group "planet" + mode 0644 + variables :base => "minute", :interval => 3600 +end + +link "/var/lib/replication/hour/data" do + to "/store/planet/replication/hour" +end + +directory "/var/lib/replication/day" do + owner "planet" + group "planet" + mode 0755 +end + +template "/var/lib/replication/hour/configuration.txt" do + source "replication.config.erb" + owner "planet" + group "planet" + mode 0644 + variables :base => "hour", :interval => 86400 +end + +link "/var/lib/replication/day/data" do + to "/store/planet/replication/day" +end + +template "/etc/cron.d/replication" do + source "replication.cron.erb" + owner "root" + group "root" + mode 0644 +end diff --git a/cookbooks/planet/templates/default/replication.auth.erb b/cookbooks/planet/templates/default/replication.auth.erb new file mode 100644 index 000000000..8486e0ee3 --- /dev/null +++ b/cookbooks/planet/templates/default/replication.auth.erb @@ -0,0 +1,16 @@ +# DO NOT EDIT - This file is being maintained by Chef + +# The database host system +host=db + +# The database instance +database=openstreetmap + +# The database user +user=planetdiff + +# The database password +password=<%= @password =%> + +# The database type +dbType=postgresql diff --git a/cookbooks/planet/templates/default/replication.config.erb b/cookbooks/planet/templates/default/replication.config.erb new file mode 100644 index 000000000..7ac5ac58a --- /dev/null +++ b/cookbooks/planet/templates/default/replication.config.erb @@ -0,0 +1,11 @@ +# DO NOT EDIT - This file is being maintained by Chef + +# The URL of the directory containing change files. +baseUrl=http://planet.openstreetmap.org/replication/<%= @base %> + +# The length of an extraction interval in seconds (3600 = 1 hour). +intervalLength=<%= @interval %> + +# Defines the maximum time interval in seconds to download in a single invocation. +# Setting to 0 disables this feature. +maxInterval=<%= @interval * 10 %> diff --git a/cookbooks/planet/templates/default/replication.cron.erb b/cookbooks/planet/templates/default/replication.cron.erb new file mode 100644 index 000000000..9fb284a00 --- /dev/null +++ b/cookbooks/planet/templates/default/replication.cron.erb @@ -0,0 +1,8 @@ +# DO NOT EDIT - This file is being maintained by Chef + +MAILTO=brett@bretth.com +TZ=UTC + +* * * * * planet /usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf allowIncorrectSchemaVersion=true --write-replication workingDirectory=/store/planet/replication/minute +2 * * * * planet /usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour +5 * * * * planet /usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day diff --git a/roles/planet.rb b/roles/planet.rb index 88cc5c1b5..4c2045388 100644 --- a/roles/planet.rb +++ b/roles/planet.rb @@ -44,6 +44,7 @@ default_attributes( run_list( "recipe[planet]", + "recipe[planet::replication]", "recipe[nfs::server]", "recipe[rsyncd]" ) -- 2.43.2 From cb62c8a4bb4ea656b5e48d2d5a0f8c6894d42c11 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Wed, 3 Jul 2013 22:31:48 +0100 Subject: [PATCH 05/16] Stop the replication task moaning about schema mismatches --- cookbooks/planet/templates/default/replication.cron.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/planet/templates/default/replication.cron.erb b/cookbooks/planet/templates/default/replication.cron.erb index 9fb284a00..6993b1505 100644 --- a/cookbooks/planet/templates/default/replication.cron.erb +++ b/cookbooks/planet/templates/default/replication.cron.erb @@ -3,6 +3,6 @@ MAILTO=brett@bretth.com TZ=UTC -* * * * * planet /usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf allowIncorrectSchemaVersion=true --write-replication workingDirectory=/store/planet/replication/minute +* * * * * planet /usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute 2 * * * * planet /usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour 5 * * * * planet /usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day -- 2.43.2 From dd43ee307485fbac969dac6944373b2158003211 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Wed, 3 Jul 2013 22:54:46 +0100 Subject: [PATCH 06/16] Bring changeset replication task under chef control --- cookbooks/planet/recipes/replication.rb | 20 +++ .../templates/default/changesets.bin.erb | 152 ++++++++++++++++++ .../templates/default/changesets.conf.erb | 3 + .../templates/default/replication.cron.erb | 7 +- 4 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 cookbooks/planet/templates/default/changesets.bin.erb create mode 100644 cookbooks/planet/templates/default/changesets.conf.erb diff --git a/cookbooks/planet/recipes/replication.rb b/cookbooks/planet/recipes/replication.rb index f055280f1..2d9dc4f4b 100644 --- a/cookbooks/planet/recipes/replication.rb +++ b/cookbooks/planet/recipes/replication.rb @@ -21,6 +21,18 @@ include_recipe "osmosis" db_passwords = data_bag_item("db", "passwords") +package "ruby" +package "ruby-libxml" + +gem_package "pg" + +template "/usr/local/bin/replicate-changesets" do + source "changesets.bin.erb" + owner "root" + group "root" + mode 0755 +end + directory "/etc/replication" do owner "root" group "root" @@ -35,6 +47,14 @@ template "/etc/replication/auth.conf" do variables :password => db_passwords["planetdiff"] end +template "/etc/replication/changesets.conf" do + source "changesets.conf.erb" + user "root" + group "planet" + mode 0640 + variables :password => db_passwords["planetdiff"] +end + directory "/var/lib/replication" do owner "planet" group "planet" diff --git a/cookbooks/planet/templates/default/changesets.bin.erb b/cookbooks/planet/templates/default/changesets.bin.erb new file mode 100644 index 000000000..443d8ede3 --- /dev/null +++ b/cookbooks/planet/templates/default/changesets.bin.erb @@ -0,0 +1,152 @@ +#!/usr/bin/ruby + +require 'rubygems' +require 'pg' +require 'yaml' +require 'time' +require 'fileutils' +require 'xml/libxml' +require 'zlib' + +# after this many changes, a changeset will be closed +CHANGES_LIMIT=50000 + +# this is the scale factor for lat/lon values stored as integers in the database +GEO_SCALE=10000000 + +## +# changeset class keeps some information about changesets downloaded from the +# database - enough to let us know which changesets are closed/open & recently +# closed. +class Changeset + attr_reader :id, :created_at, :closed_at, :num_changes + + def initialize(row) + @id = row['id'].to_i + @created_at = Time.parse(row['created_at']) + @closed_at = Time.parse(row['closed_at']) + @num_changes = row['num_changes'].to_i + end + + def closed?(t) + (@closed_at < t) || (@num_changes >= CHANGES_LIMIT) + end + + def open?(t) + not closed?(t) + end + + def activity_between?(t1, t2) + ((@closed_at >= t1) && (@closed_at < t2)) || ((@created_at >= t1) && (@created_at < t2)) + end +end + +## +# state and connections associated with getting changeset data +# replicated to a file. +class Replicator + def initialize(config) + @config = YAML.load(File.read(config)) + @state = YAML.load(File.read(@config['state_file'])) + @conn = PGconn.connect(@config['db']) + @now = Time.now.getutc + end + + def open_changesets + last_run = @state['last_run'] + last_run = (@now - 60) if last_run.nil? + @state['last_run'] = @now + # pretty much all operations on a changeset will modify its closed_at + # time (see rails_port's changeset model). so it is probably enough + # for us to look at anything that was closed recently, and filter from + # there. + @conn. + exec("select id, created_at, closed_at, num_changes from changesets where closed_at > ((now() at time zone 'utc') - '1 hour'::interval)"). + map {|row| Changeset.new(row) }. + select {|cs| cs.activity_between?(last_run, @now) } + end + + # creates an XML file containing the changeset information from the + # list of changesets output by open_changesets. + def changeset_dump(changesets) + doc = XML::Document.new + doc.root = XML::Node.new("osm") + { 'version' => '0.6', + 'generator' => 'replicate_changesets.rb', + 'copyright' => "OpenStreetMap and contributors", + 'attribution' => "http://www.openstreetmap.org/copyright", + 'license' => "http://opendatacommons.org/licenses/odbl/1-0/" }. + each { |k,v| doc.root[k] = v } + + changesets.each do |cs| + xml = XML::Node.new("changeset") + xml['id'] = cs.id.to_s + xml['created_at'] = cs.created_at.getutc.xmlschema + xml['closed_at'] = cs.closed_at.getutc.xmlschema if cs.closed?(@now) + xml['open'] = cs.open?(@now).to_s + + res = @conn.exec("select u.id, u.display_name, c.min_lat, c.max_lat, c.min_lon, c.max_lon from users u join changesets c on u.id=c.user_id where c.id=#{cs.id}") + xml['user'] = res[0]['display_name'] + xml['uid'] = res[0]['id'] + + unless (res[0]['min_lat'].nil? || + res[0]['max_lat'].nil? || + res[0]['min_lon'].nil? || + res[0]['max_lon'].nil?) + xml['min_lat'] = (res[0]['min_lat'].to_f / GEO_SCALE).to_s + xml['max_lat'] = (res[0]['max_lat'].to_f / GEO_SCALE).to_s + xml['min_lon'] = (res[0]['min_lon'].to_f / GEO_SCALE).to_s + xml['max_lon'] = (res[0]['max_lon'].to_f / GEO_SCALE).to_s + end + + res = @conn.exec("select k, v from changeset_tags where changeset_id=#{cs.id}") + res.each do |row| + tag = XML::Node.new("tag") + tag['k'] = row['k'] + tag['v'] = row['v'] + xml << tag + end + + doc.root << xml + end + + doc.to_s + end + + # saves new state (including the changeset dump xml) + def save! + File.open(@config['state_file'], "r") do |fl| + fl.flock(File::LOCK_EX) + + sequence = (@state.has_key?('sequence') ? @state['sequence'] + 1 : 0) + data_file = @config['data_dir'] + sprintf("/%03d/%03d/%03d.osm.gz", sequence / 1000000, (sequence / 1000) % 1000, (sequence % 1000)); + tmp_state = @config['state_file'] + ".tmp" + tmp_data = "/tmp/changeset_data.osm.tmp" + # try and write the files to tmp locations and then + # move them into place later, to avoid in-progress + # clashes, or people seeing incomplete files. + begin + FileUtils.mkdir_p(File.dirname(data_file)) + Zlib::GzipWriter.open(tmp_data) do |fh| + fh.write(changeset_dump(open_changesets)) + end + @state['sequence'] = sequence + File.open(tmp_state, "w") do |fh| + fh.write(YAML.dump(@state)) + end + FileUtils.mv(tmp_data, data_file) + FileUtils.mv(tmp_state, @config['state_file']) + fl.flock(File::LOCK_UN) + + rescue + STDERR.puts("Error! Couldn't update state.") + fl.flock(File::LOCK_UN) + raise + end + end + end +end + +rep = Replicator.new(ARGV[0]) +rep.save! + diff --git a/cookbooks/planet/templates/default/changesets.conf.erb b/cookbooks/planet/templates/default/changesets.conf.erb new file mode 100644 index 000000000..72ac2a067 --- /dev/null +++ b/cookbooks/planet/templates/default/changesets.conf.erb @@ -0,0 +1,3 @@ +state_file: /store/planet/replication/changesets/state.yaml +db: host=db dbname=openstreetmap user=planetdiff password=<%= @password %> +data_dir: /store/planet/replication/changesets diff --git a/cookbooks/planet/templates/default/replication.cron.erb b/cookbooks/planet/templates/default/replication.cron.erb index 6993b1505..dead114ef 100644 --- a/cookbooks/planet/templates/default/replication.cron.erb +++ b/cookbooks/planet/templates/default/replication.cron.erb @@ -1,8 +1,13 @@ # DO NOT EDIT - This file is being maintained by Chef -MAILTO=brett@bretth.com TZ=UTC +MAILTO=brett@bretth.com + * * * * * planet /usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute 2 * * * * planet /usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour 5 * * * * planet /usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day + +MAILTO=zerebubuth@gmail.com + +* * * * * planet /usr/local/bin/replicate-changesets /etc/replication/changesets.conf -- 2.43.2 From 1669af3b24f375c239508dd5e5201650371d450c Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Wed, 3 Jul 2013 23:07:20 +0100 Subject: [PATCH 07/16] Fix hour and day replication configs --- cookbooks/planet/recipes/replication.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/planet/recipes/replication.rb b/cookbooks/planet/recipes/replication.rb index 2d9dc4f4b..027aba96b 100644 --- a/cookbooks/planet/recipes/replication.rb +++ b/cookbooks/planet/recipes/replication.rb @@ -85,7 +85,7 @@ directory "/var/lib/replication/day" do mode 0755 end -template "/var/lib/replication/hour/configuration.txt" do +template "/var/lib/replication/day/configuration.txt" do source "replication.config.erb" owner "planet" group "planet" -- 2.43.2 From 6da11d5333d3a636b7ee9861838e24ae845a3694 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Thu, 4 Jul 2013 00:40:14 +0100 Subject: [PATCH 08/16] Manage the replication directory --- cookbooks/osmosis/recipes/default.rb | 3 ++ .../default/bin/apache-latest-planet-filename | 0 .../default/bin/planet-mirror-redirect-update | 0 cookbooks/planet/files/default/bin/planet2pbf | 0 cookbooks/planet/files/default/cgi/HEADER.cgi | 0 .../replication-bin/replicate-changesets} | 0 .../files/default/replication-cgi/HEADER.cgi | 14 +++++++ cookbooks/planet/recipes/replication.rb | 41 ++++++++++++++++++- 8 files changed, 56 insertions(+), 2 deletions(-) mode change 100755 => 100644 cookbooks/planet/files/default/bin/apache-latest-planet-filename mode change 100755 => 100644 cookbooks/planet/files/default/bin/planet-mirror-redirect-update mode change 100755 => 100644 cookbooks/planet/files/default/bin/planet2pbf mode change 100755 => 100644 cookbooks/planet/files/default/cgi/HEADER.cgi rename cookbooks/planet/{templates/default/changesets.bin.erb => files/default/replication-bin/replicate-changesets} (100%) create mode 100644 cookbooks/planet/files/default/replication-cgi/HEADER.cgi diff --git a/cookbooks/osmosis/recipes/default.rb b/cookbooks/osmosis/recipes/default.rb index 42679ac3f..7532cd7ac 100644 --- a/cookbooks/osmosis/recipes/default.rb +++ b/cookbooks/osmosis/recipes/default.rb @@ -19,6 +19,9 @@ include_recipe "chef" +package "unzip" +package "openjdk-6-jre" + osmosis_package = "osmosis-#{node[:osmosis][:version]}.zip" osmosis_directory = "/opt/osmosis-#{node[:osmosis][:version]}" diff --git a/cookbooks/planet/files/default/bin/apache-latest-planet-filename b/cookbooks/planet/files/default/bin/apache-latest-planet-filename old mode 100755 new mode 100644 diff --git a/cookbooks/planet/files/default/bin/planet-mirror-redirect-update b/cookbooks/planet/files/default/bin/planet-mirror-redirect-update old mode 100755 new mode 100644 diff --git a/cookbooks/planet/files/default/bin/planet2pbf b/cookbooks/planet/files/default/bin/planet2pbf old mode 100755 new mode 100644 diff --git a/cookbooks/planet/files/default/cgi/HEADER.cgi b/cookbooks/planet/files/default/cgi/HEADER.cgi old mode 100755 new mode 100644 diff --git a/cookbooks/planet/templates/default/changesets.bin.erb b/cookbooks/planet/files/default/replication-bin/replicate-changesets similarity index 100% rename from cookbooks/planet/templates/default/changesets.bin.erb rename to cookbooks/planet/files/default/replication-bin/replicate-changesets diff --git a/cookbooks/planet/files/default/replication-cgi/HEADER.cgi b/cookbooks/planet/files/default/replication-cgi/HEADER.cgi new file mode 100644 index 000000000..5432bd9f4 --- /dev/null +++ b/cookbooks/planet/files/default/replication-cgi/HEADER.cgi @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +print """ + + + OpenStreetMap replication diffs + + + + +

planet.openstreetmap.org - replication diffs

+

OpenStreetMap is open data, licensed under the Open Data Commons Open Database License (ODbL)

+

 

+""" diff --git a/cookbooks/planet/recipes/replication.rb b/cookbooks/planet/recipes/replication.rb index 027aba96b..f1d06b5e1 100644 --- a/cookbooks/planet/recipes/replication.rb +++ b/cookbooks/planet/recipes/replication.rb @@ -26,11 +26,48 @@ package "ruby-libxml" gem_package "pg" -template "/usr/local/bin/replicate-changesets" do - source "changesets.bin.erb" +remote_directory "/usr/local/bin" do + source "replication-bin" owner "root" group "root" mode 0755 + files_owner "root" + files_group "root" + files_mode 0755 +end + +remote_directory "/store/planet/replication" do + source "replication-cgi" + owner "root" + group "root" + mode 0755 + files_owner "root" + files_group "root" + files_mode 0755 +end + +directory "/store/planet/replication/changesets" do + owner "planet" + group "planet" + mode 0755 +end + +directory "/store/planet/replication/day" do + owner "planet" + group "planet" + mode 0755 +end + +directory "/store/planet/replication/hour" do + owner "planet" + group "planet" + mode 0755 +end + +directory "/store/planet/replication/minute" do + owner "planet" + group "planet" + mode 0755 end directory "/etc/replication" do -- 2.43.2 From 0e385c19cb1db5b56b79d2860326f52ca2f0ae2a Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Thu, 4 Jul 2013 00:48:20 +0100 Subject: [PATCH 09/16] Use Brett's osmosis build for the hourly/daily replication for now --- cookbooks/planet/templates/default/replication.cron.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cookbooks/planet/templates/default/replication.cron.erb b/cookbooks/planet/templates/default/replication.cron.erb index dead114ef..cb019c0fb 100644 --- a/cookbooks/planet/templates/default/replication.cron.erb +++ b/cookbooks/planet/templates/default/replication.cron.erb @@ -5,8 +5,8 @@ TZ=UTC MAILTO=brett@bretth.com * * * * * planet /usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute -2 * * * * planet /usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour -5 * * * * planet /usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day +2 * * * * planet /home/bretth/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour +5 * * * * planet /home/bretth/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day MAILTO=zerebubuth@gmail.com -- 2.43.2 From a77a17afdd8ca38aa431c0f4fad1736ea9b9d91f Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Thu, 4 Jul 2013 11:21:23 +0100 Subject: [PATCH 10/16] The supybot tmp directory needs to be inside the data directory --- cookbooks/supybot/recipes/default.rb | 6 ------ cookbooks/supybot/templates/default/supybot.conf.erb | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/cookbooks/supybot/recipes/default.rb b/cookbooks/supybot/recipes/default.rb index 7dd59bf36..20385c774 100644 --- a/cookbooks/supybot/recipes/default.rb +++ b/cookbooks/supybot/recipes/default.rb @@ -96,12 +96,6 @@ directory "/var/lib/supybot/git" do mode 0755 end -directory "/var/tmp/supybot" do - owner "supybot" - group "supybot" - mode 0755 -end - directory "/var/log/supybot" do owner "supybot" group "supybot" diff --git a/cookbooks/supybot/templates/default/supybot.conf.erb b/cookbooks/supybot/templates/default/supybot.conf.erb index 7bb524ed9..a6bee5686 100644 --- a/cookbooks/supybot/templates/default/supybot.conf.erb +++ b/cookbooks/supybot/templates/default/supybot.conf.erb @@ -635,7 +635,7 @@ supybot.directories.data: /var/lib/supybot/data # # Default value: tmp ### -supybot.directories.data.tmp: /var/tmp/supybot +supybot.directories.data.tmp: /var/lib/supybot/data/tmp ### # Determines what directory backup data is put into. -- 2.43.2 From ab81dbbb42f18e06653625b09c571e58bbac77ad Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Thu, 4 Jul 2013 11:21:52 +0100 Subject: [PATCH 11/16] Stop supybot flushing it's config so it doesn't fight with chef --- cookbooks/supybot/templates/default/supybot.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/supybot/templates/default/supybot.conf.erb b/cookbooks/supybot/templates/default/supybot.conf.erb index a6bee5686..fe3c63eb6 100644 --- a/cookbooks/supybot/templates/default/supybot.conf.erb +++ b/cookbooks/supybot/templates/default/supybot.conf.erb @@ -451,7 +451,7 @@ supybot.upkeepInterval: 3600 # # Default value: True ### -supybot.flush: True +supybot.flush: False ### # Determines what characters are valid for quoting arguments to commands -- 2.43.2 From 3db44fd1698b5de08358c5d94ddde7271f4b3333 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Fri, 5 Jul 2013 08:36:57 +0100 Subject: [PATCH 12/16] Create the file directory for each style --- cookbooks/tile/recipes/default.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb index f39c31623..bb7459ece 100644 --- a/cookbooks/tile/recipes/default.rb +++ b/cookbooks/tile/recipes/default.rb @@ -220,6 +220,12 @@ node[:tile][:styles].each do |name,details| supports :restart => true end + directory tile_directory do + owner "tile" + group "www-data" + mode 0775 + end + file "#{tile_directory}/planet-import-complete" do action :create_if_missing owner "tile" -- 2.43.2 From 55e94b06f8e0d01c6c91b584b9ffe37d809e066a Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Fri, 5 Jul 2013 09:21:03 +0100 Subject: [PATCH 13/16] Only run awstats for planet on Sunday --- cookbooks/stats/templates/default/awstats.cron.erb | 2 +- roles/stats.rb | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cookbooks/stats/templates/default/awstats.cron.erb b/cookbooks/stats/templates/default/awstats.cron.erb index 819287074..5d081e619 100644 --- a/cookbooks/stats/templates/default/awstats.cron.erb +++ b/cookbooks/stats/templates/default/awstats.cron.erb @@ -1,5 +1,5 @@ # DO NOT EDIT - This file is being maintained by Chef <% @sites.each_with_index do |site,index| -%> -0 <%= "%02d" % [ 8 + index * 2 ] %> * * * www-data /usr/lib/cgi-bin/awstats.pl -config=<%= site[:name] %> -update > /dev/null +0 <%= "%02d" % [ 8 + index * 2 ] %> * * <%= site[:days] %> www-data /usr/lib/cgi-bin/awstats.pl -config=<%= site[:name] %> -update > /dev/null <% end -%> diff --git a/roles/stats.rb b/roles/stats.rb index 214c8b0b8..ae2e02b01 100644 --- a/roles/stats.rb +++ b/roles/stats.rb @@ -6,11 +6,13 @@ default_attributes( :sites => [ { :name => "planet.openstreetmap.org", - :log_pattern => "%YYYY-168-%MM-168-%DD-168.gz" + :log_pattern => "%YYYY-168-%MM-168-%DD-168.gz", + :days => "7" }, { :name => "www.openstreetmap.org", - :log_pattern => "*-%YYYY-48-%MM-48-%DD-48.gz" + :log_pattern => "*-%YYYY-48-%MM-48-%DD-48.gz", + :days => "*" } ] } -- 2.43.2 From 518ae814ad880885f1f7b7a0f12a2da2eff071ce Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Fri, 5 Jul 2013 09:24:52 +0100 Subject: [PATCH 14/16] Manage rotation of planet logs with archiving --- cookbooks/planet/recipes/default.rb | 7 +++++++ .../templates/default/logrotate.apache.erb | 21 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 cookbooks/planet/templates/default/logrotate.apache.erb diff --git a/cookbooks/planet/recipes/default.rb b/cookbooks/planet/recipes/default.rb index c9e998524..11e90edfb 100644 --- a/cookbooks/planet/recipes/default.rb +++ b/cookbooks/planet/recipes/default.rb @@ -70,3 +70,10 @@ apache_module "proxy_http" apache_site "planet.openstreetmap.org" do template "apache.erb" end + +template "/etc/logrotate.d/apache2" do + source "logrotate.apache.erb" + owner "root" + group "root" + mode 0644 +end diff --git a/cookbooks/planet/templates/default/logrotate.apache.erb b/cookbooks/planet/templates/default/logrotate.apache.erb new file mode 100644 index 000000000..e7d9e3c23 --- /dev/null +++ b/cookbooks/planet/templates/default/logrotate.apache.erb @@ -0,0 +1,21 @@ +# DO NOT EDIT - This file is being maintained by Chef + +/var/log/apache2/*.log { + weekly + missingok + rotate 52 + compress + delaycompress + notifempty + create 640 root adm + sharedscripts + postrotate + /etc/init.d/apache2 reload > /dev/null + rsync /var/log/apache2/planet.openstreetmap.org-access.log.2.gz horntail::logs/planet.openstreetmap.org/`date -d "-7 days" +%Y-%m-%d`.gz + endscript + prerotate + if [ -d /etc/logrotate.d/httpd-prerotate ]; then \ + run-parts /etc/logrotate.d/httpd-prerotate; \ + fi; \ + endscript +} -- 2.43.2 From cfbff6dc34bd927a931c16d72ae6ce4a880acd11 Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Fri, 5 Jul 2013 10:46:34 +0100 Subject: [PATCH 15/16] Tile cookbook: Add ModTileMissingRequestTimeout + typo fix --- cookbooks/tile/templates/default/tile.conf.erb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cookbooks/tile/templates/default/tile.conf.erb b/cookbooks/tile/templates/default/tile.conf.erb index 9748774c2..cf0f9fd19 100644 --- a/cookbooks/tile/templates/default/tile.conf.erb +++ b/cookbooks/tile/templates/default/tile.conf.erb @@ -9,11 +9,14 @@ ModTileTileDir /srv/tile.openstreetmap.org/tiles # Time to wait for a re-render before serving a dirty tile ModTileRequestTimeout 3 +# Timeout before giving up for a tile to be rendered that is otherwise missing +ModTileMissingRequestTimeout 10 + # Don't try and re-render dirty tiles if the load is higher than this ModTileMaxLoadOld 36 # Don't try and render missing tiles if the load is higher than this -ModTileMaxLoadOld 72 +ModTileMaxLoadMissing 72 # Maximum expiry to set on a tile ModTileCacheDurationMax 604800 -- 2.43.2 From 559b624a1171efd0e59a991f0f5f4386a1c09712 Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Fri, 5 Jul 2013 11:03:10 +0100 Subject: [PATCH 16/16] orm: Set new disk serial --- roles/orm.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/orm.rb b/roles/orm.rb index 7968002bc..69e002ff5 100644 --- a/roles/orm.rb +++ b/roles/orm.rb @@ -47,7 +47,7 @@ default_attributes( :comment => "Tune scheduler for Areca", :type => "block", :bus => "scsi", - :serial => "2001b4d2031365276", + :serial => "2001b4d2060246956", :attrs => { "queue/scheduler" => "deadline", "queue/nr_requests" => "512" -- 2.43.2