From eac24219806acd267147e066299a84528bf2a4d6 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 3 Oct 2016 10:55:21 +0200 Subject: [PATCH] remove autoconf-based nominatim roles and recipes --- cookbooks/nominatim/recipes/base.rb | 286 ------------------ cookbooks/nominatim/recipes/master.rb | 36 --- cookbooks/nominatim/recipes/slave.rb | 31 -- cookbooks/nominatim/recipes/standalone.rb | 48 --- .../nominatim/templates/default/cron.erb | 12 - .../nominatim/templates/default/nominatim.erb | 21 -- .../templates/default/update_source.erb | 15 - roles/nominatim-master.rb | 38 --- roles/nominatim-slave.rb | 23 -- roles/nominatim-standalone.rb | 16 - roles/nominatim.rb | 107 ------- roles/poldi.rb | 35 +-- 12 files changed, 1 insertion(+), 667 deletions(-) delete mode 100644 cookbooks/nominatim/recipes/base.rb delete mode 100644 cookbooks/nominatim/recipes/master.rb delete mode 100644 cookbooks/nominatim/recipes/slave.rb delete mode 100644 cookbooks/nominatim/recipes/standalone.rb delete mode 100644 cookbooks/nominatim/templates/default/cron.erb delete mode 100644 cookbooks/nominatim/templates/default/nominatim.erb delete mode 100644 cookbooks/nominatim/templates/default/update_source.erb delete mode 100644 roles/nominatim-master.rb delete mode 100644 roles/nominatim-slave.rb delete mode 100644 roles/nominatim-standalone.rb delete mode 100644 roles/nominatim.rb diff --git a/cookbooks/nominatim/recipes/base.rb b/cookbooks/nominatim/recipes/base.rb deleted file mode 100644 index fe6895c78..000000000 --- a/cookbooks/nominatim/recipes/base.rb +++ /dev/null @@ -1,286 +0,0 @@ -# -# Cookbook Name:: nominatim -# Recipe:: base -# -# Copyright 2015, OpenStreetMap Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include_recipe "apache::ssl" -include_recipe "postgresql" - -package "php" -package "php-cli" -package "php-pgsql" -package "php-fpm" -package "php-pear" -package "php-db" - -apache_module "rewrite" -apache_module "proxy" -apache_module "proxy_fcgi" -apache_module "proxy_http" -apache_module "headers" - -home_directory = data_bag_item("accounts", "nominatim")["home"] -source_directory = "#{home_directory}/nominatim" -email_errors = data_bag_item("accounts", "lonvia")["email"] - -database_cluster = node[:nominatim][:database][:cluster] -database_version = database_cluster.sub(%r{/.*}, "") -database_name = node[:nominatim][:database][:dbname] - -postgis_version = node[:nominatim][:database][:postgis] - -service "php7.0-fpm" do - action [:enable, :start] -end - -node[:nominatim][:fpm_pools].each do |name, data| - template "/etc/php/7.0/fpm/pool.d/#{name}.conf" do - source "fpm.conf.erb" - owner "root" - group "root" - mode 0o644 - variables data.merge(:name => name, :port => data[:port]) - notifies :reload, "service[php7.0-fpm]" - end -end - -postgresql_munin "nominatim" do - cluster database_cluster - database database_name -end - -directory node[:nominatim][:logdir] do - owner "nominatim" - group "nominatim" - mode 0o755 - recursive true -end - -file "#{node[:nominatim][:logdir]}/query.log" do - action :create_if_missing - owner "www-data" - group "adm" - mode 0o664 -end - -file "#{node[:nominatim][:logdir]}/update.log" do - action :create_if_missing - owner "nominatim" - group "adm" - mode 0o664 -end - -directory "#{home_directory}/status" do - owner "nominatim" - group "postgres" - mode 0o775 -end - -template "/etc/logrotate.d/nominatim" do - source "logrotate.nominatim.erb" - owner "root" - group "root" - mode 0o644 -end - -template "/etc/logrotate.d/apache2" do - source "logrotate.apache.erb" - owner "root" - group "root" - mode 0o644 -end - -package "osmosis" -package "gcc" -package "proj-bin" -package "libgeos-c1v5" -package "postgresql-#{database_version}-postgis-#{postgis_version}" -package "postgresql-server-dev-#{database_version}" -package "build-essential" -package "libxml2-dev" -package "libexpat-dev" -package "libgeos-dev" -package "libgeos++-dev" -package "libpq-dev" -package "libbz2-dev" -package "libtool" -package "automake" -package "libproj-dev" -package "libprotobuf-c0-dev" -package "protobuf-c-compiler" -package "python-psycopg2" -package "libboost-dev" -package "libboost-system-dev" -package "libboost-filesystem-dev" -package "libboost-thread-dev" - -execute "compile_nominatim" do - action :nothing - command "cd #{source_directory} && ./autogen.sh && ./configure && make" - user "nominatim" -end - -template "#{source_directory}/.git/hooks/post-merge" do - source "update_source.erb" - owner "nominatim" - group "nominatim" - mode 0o755 - variables :source_directory => source_directory -end - -template "#{source_directory}/settings/local.php" do - source "nominatim.erb" - owner "nominatim" - group "nominatim" - mode 0o664 - variables :postgres_version => database_version -end - -template "#{source_directory}/settings/ip_blocks.conf" do - action :create_if_missing - source "ipblocks.erb" - owner "nominatim" - group "nominatim" - mode 0o664 -end - -file "#{source_directory}/settings/apache_blocks.conf" do - action :create_if_missing - owner "nominatim" - group "nominatim" - mode 0o664 -end - -file "#{source_directory}/settings/ip_blocks.map" do - action :create_if_missing - owner "nominatim" - group "nominatim" - mode 0o664 -end - -cron_action = if node[:nominatim][:enabled] - :create - else - :delete - end - -template "/etc/cron.d/nominatim" do - action cron_action - source "cron.erb" - owner "root" - group "root" - mode "0644" - variables :bin_directory => "#{source_directory}/utils", :mailto => email_errors -end - -template "#{source_directory}/utils/nominatim-update" do - source "updater.erb" - user "nominatim" - group "nominatim" - mode 0o755 - variables :bindir => source_directory, - :srcdir => source_directory, - :logfile => "#{node[:nominatim][:logdir]}/update.log", - :branch => node[:nominatim][:revision], - :update_stop_file => "#{home_directory}/status/updates_disabled" -end - -template "/etc/init.d/nominatim-update" do - source "updater.init.erb" - user "nominatim" - group "nominatim" - mode 0o755 - variables :source_directory => source_directory -end - -munin_plugin_conf "nominatim" do - template "munin.erb" - variables :db => node[:nominatim][:database][:dbname], - :querylog => "#{node[:nominatim][:logdir]}/query.log" -end - -munin_plugin "nominatim_importlag" do - target "#{source_directory}/munin/nominatim_importlag" -end - -munin_plugin "nominatim_query_speed" do - target "#{source_directory}/munin/nominatim_query_speed_querylog" -end - -munin_plugin "nominatim_requests" do - target "#{source_directory}/munin/nominatim_requests_querylog" -end - -munin_plugin "nominatim_throttled_ips" do - target "#{source_directory}/munin/nominatim_throttled_ips" -end - -external_data = [ - "wikipedia_article.sql.bin", - "wikipedia_redirect.sql.bin", - "gb_postcode_data.sql.gz" -] - -external_data.each do |fname| - remote_file "#{source_directory}/data/#{fname}" do - action :create_if_missing - source "http://www.nominatim.org/data/#{fname}" - owner "nominatim" - group "nominatim" - mode 0o644 - end -end - -additional_scripts = %w(backup-nominatim vacuum-db-nominatim) - -additional_scripts.each do |fname| - template "/usr/local/bin/#{fname}" do - source "#{fname}.erb" - owner "root" - group "root" - mode 0o755 - variables :db => node[:nominatim][:database][:dbname] - end -end - -directory File.dirname(node[:nominatim][:flatnode_file]) do - owner "nominatim" - group "nominatim" - mode 0o755 - recursive true -end - -apache_site "nominatim.openstreetmap.org" do - template "apache.erb" - directory source_directory - variables :pools => node[:nominatim][:fpm_pools] -end - -apache_site "default" do - action [:disable] -end - -fail2ban_filter "nominatim" do - failregex '^ - - \[[^]]+\] "[^"]+" 429 ' -end - -fail2ban_jail "nominatim" do - filter "nominatim" - logpath "/var/log/apache2/nominatim.openstreetmap.org-access.log" - ports [80, 443] - maxretry 100 -end diff --git a/cookbooks/nominatim/recipes/master.rb b/cookbooks/nominatim/recipes/master.rb deleted file mode 100644 index 6dd15bf96..000000000 --- a/cookbooks/nominatim/recipes/master.rb +++ /dev/null @@ -1,36 +0,0 @@ -# -# Cookbook Name:: nominatim -# Recipe:: master -# -# Copyright 2015, OpenStreetMap Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -slaves = search(:node, "roles:nominatim-slave") # ~FC010 - -node.default[:postgresql][:settings][:defaults][:late_authentication_rules] = [] -node.default[:rsyncd][:modules] = { :archive => { :hosts_allow => [] } } - -slaves.each do |slave| - # set up DB access for each slave - node.default[:postgresql][:settings][:defaults][:late_authentication_rules].push( - :database => "replication", - :user => "replication", - :address => "#{slave[:networking][:internal_ipv4][:address]}/32" - ) - # allow slaves access to the WAL logs - node.default[:rsyncd][:modules][:archive][:hosts_allow].push( - slave[:networking][:internal_ipv4][:address] - ) -end diff --git a/cookbooks/nominatim/recipes/slave.rb b/cookbooks/nominatim/recipes/slave.rb deleted file mode 100644 index 22d1c0bf7..000000000 --- a/cookbooks/nominatim/recipes/slave.rb +++ /dev/null @@ -1,31 +0,0 @@ -# -# Cookbook Name:: nominatim -# Recipe:: slave -# -# Copyright 2015, OpenStreetMap Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -master = search(:node, "roles:nominatim-master")[0] # ~FC010 -host = master[:nominatim][:master_host] - -node.default[:postgresql][:settings][:defaults][:primary_conninfo] = { - :host => host, - :port => "5432", - :user => "replication", - :passwords => { :bag => "nominatim", :item => "passwords" } -} - -node.default[:postgresql][:settings][:defaults][:restore_command] = - "/usr/bin/rsync #{host}::archive/%f %p" diff --git a/cookbooks/nominatim/recipes/standalone.rb b/cookbooks/nominatim/recipes/standalone.rb deleted file mode 100644 index 0df3aafae..000000000 --- a/cookbooks/nominatim/recipes/standalone.rb +++ /dev/null @@ -1,48 +0,0 @@ -# -# Cookbook Name:: nominatim -# Recipe:: standalone -# -# Copyright 2015, OpenStreetMap Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include_recipe "git" - -database_cluster = node[:nominatim][:database][:cluster] -home_directory = data_bag_item("accounts", "nominatim")["home"] - -git "#{home_directory}/nominatim" do - action :checkout - repository node[:nominatim][:repository] - revision node[:nominatim][:revision] - enable_submodules true - user "nominatim" - group "nominatim" - notifies :run, "execute[compile_nominatim]" -end - -include_recipe "nominatim::base" - -superusers = %w(tomh lonvia twain nominatim) - -superusers.each do |user| - postgresql_user user do - cluster database_cluster - superuser true - end -end - -postgresql_user "www-data" do - cluster database_cluster -end diff --git a/cookbooks/nominatim/templates/default/cron.erb b/cookbooks/nominatim/templates/default/cron.erb deleted file mode 100644 index 68961e54e..000000000 --- a/cookbooks/nominatim/templates/default/cron.erb +++ /dev/null @@ -1,12 +0,0 @@ -# Maintained by chef. DO NOT EDIT. - -MAILTO=<%= @mailto %> - -* * * * * lonvia <%= @bin_directory %>/cron_ipanalyse.py /var/log/apache2/nominatim.openstreetmap.org-access.log -<% if node[:nominatim][:enable_backup] -%> -00 3 1 * * nominatim /usr/local/bin/backup-nominatim -<% end -%> -30 1 * * * postgres /usr/local/bin/vacuum-db-nominatim -<% if node[:postgresql][:settings][:defaults][:archive_mode] == "on" -%> -05 */4 * * * postgres /usr/local/bin/clean-db-nominatim -<% end -%> diff --git a/cookbooks/nominatim/templates/default/nominatim.erb b/cookbooks/nominatim/templates/default/nominatim.erb deleted file mode 100644 index 6e911223d..000000000 --- a/cookbooks/nominatim/templates/default/nominatim.erb +++ /dev/null @@ -1,21 +0,0 @@ -'); -if (isset($_SERVER['HTTPS']) && !empty($_SERVER['HTTPS'])) - @define('CONST_Website_BaseURL', 'https://nominatim.openstreetmap.org/'); -else - @define('CONST_Website_BaseURL', 'http://nominatim.openstreetmap.org/'); - -@define('CONST_Postgresql_Version', '<%= @postgres_version %>'); -@define('CONST_Postgis_Version', '<%= node[:nominatim][:database][:postgis] %>'); - -@define('CONST_Osm2pgsql_Flatnode_File', '<%= node[:nominatim][:flatnode_file] %>'); -@define('CONST_Search_NameOnlySearchFrequencyThreshold', 500); - -<% node[:nominatim][:tablespaces].each do |part,name| -%> -@define('CONST_Tablespace_<%= part %>', '<%= name %>'); -<% end -%> - -@define('CONST_Log_File', '<%= node[:nominatim][:logdir] %>/query.log'); -@define('CONST_Log_DB', false); diff --git a/cookbooks/nominatim/templates/default/update_source.erb b/cookbooks/nominatim/templates/default/update_source.erb deleted file mode 100644 index ec8f0176b..000000000 --- a/cookbooks/nominatim/templates/default/update_source.erb +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -# DO NOT EDIT - This file is being maintained by Chef - -cd <%= @source_directory %> - -git submodule update && -./autogen.sh && -./configure && -make - -psql -d <%= node[:nominatim][:database][:dbname] %> -c "SELECT version();" >/dev/null 2>&1 -if [[ "$?" == "0" ]]; then - ./utils/setup.php --create-functions --create-partition-functions --enable-diff-updates -fi diff --git a/roles/nominatim-master.rb b/roles/nominatim-master.rb deleted file mode 100644 index 1961d835c..000000000 --- a/roles/nominatim-master.rb +++ /dev/null @@ -1,38 +0,0 @@ -name "nominatim-master" -description "Role applied to the master nominatim server" - -default_attributes( - :postgresql => { - :settings => { - :defaults => { - :wal_level => "hot_standby", - :archive_mode => "on", - :archive_command => "/bin/cp %p /data/postgresql-archive/%f", - :max_wal_senders => "5" - } - } - }, - :nominatim => { - :state => "master", - :enable_backup => true - }, - :rsyncd => { - :modules => { - :archive => { - :comment => "WAL Archive", - :read_only => true, - :write_only => false, - :list => false, - :uid => "postgres", - :gid => "postgres", - :transfer_logging => false - } - } - } -) - -run_list( - "recipe[rsyncd]", - "recipe[nominatim::master]", - "role[nominatim-base]" -) diff --git a/roles/nominatim-slave.rb b/roles/nominatim-slave.rb deleted file mode 100644 index 0ef9a9f51..000000000 --- a/roles/nominatim-slave.rb +++ /dev/null @@ -1,23 +0,0 @@ -name "nominatim-slave" -description "Role applied to all slave nominatim servers" - -default_attributes( - :postgresql => { - :settings => { - :defaults => { - :hot_standby => "on", - :hot_standby_feedback => "on", - :standby_mode => "on" - } - } - }, - :nominatim => { - :state => "slave", - :enable_backup => false - } -) - -run_list( - "recipe[nominatim::slave]", - "role[nominatim-base]" -) diff --git a/roles/nominatim-standalone.rb b/roles/nominatim-standalone.rb deleted file mode 100644 index 561692c8f..000000000 --- a/roles/nominatim-standalone.rb +++ /dev/null @@ -1,16 +0,0 @@ -name "nominatim-standalone" -description "Role applied to all stand-alone nominatim servers" - -default_attributes( - :apt => { - :sources => ["postgresql"] - }, - :nominatim => { - :enable_backup => false - } -) - -run_list( - "role[nominatim]", - "recipe[nominatim::standalone]" -) diff --git a/roles/nominatim.rb b/roles/nominatim.rb deleted file mode 100644 index 81fc388be..000000000 --- a/roles/nominatim.rb +++ /dev/null @@ -1,107 +0,0 @@ -name "nominatim" -description "Role applied to all nominatim servers" - -default_attributes( - :accounts => { - :users => { - :lonvia => { :status => :administrator }, - :twain => { :status => :administrator }, - :nominatim => { - :status => :role, - :members => [:lonvia, :tomh, :twain] - } - } - }, - :apache => { - :mpm => "event", - :timeout => 60, - :keepalive => false, - :event => { - :server_limit => 32, - :max_clients => 1600, - :threads_per_child => 50 - } - }, - :postgresql => { - :settings => { - :defaults => { - :max_connections => "450", - :synchronous_commit => "off", - :checkpoint_segments => "32", - :checkpoint_timeout => "10min", - :max_wal_size => "1536MB", - :checkpoint_completion_target => "0.9", - :shared_buffers => "2GB", - :autovacuum_max_workers => "1" - } - } - }, - :sysctl => { - :postgres => { - :comment => "Increase shared memory for postgres", - :parameters => { - "kernel.shmmax" => 26 * 1024 * 1024 * 1024, - "kernel.shmall" => 26 * 1024 * 1024 * 1024 / 4096 - } - }, - :kernel_scheduler_tune => { - :comment => "Tune kernel scheduler preempt", - :parameters => { - "kernel.sched_min_granularity_ns" => 10000000, - "kernel.sched_wakeup_granularity_ns" => 15000000 - } - }, - :swappiness => { - :comment => "Reduce swap usage", - :parameters => { - "vm.swappiness" => 10 - } - }, - :network_conntrack_time_wait => { - :comment => "Only track completed connections for 30 seconds", - :parameters => { - "net.netfilter.nf_conntrack_tcp_timeout_time_wait" => "30" - } - }, - :network_conntrack_max => { - :comment => "Increase max number of connections tracked", - :parameters => { - "net.netfilter.nf_conntrack_max" => "131072" - } - } - }, - :nominatim => { - :enabled => false, - :repository => "git://git.openstreetmap.org/nominatim.git", - :tablespaces => { - "Osm2pgsql_Data" => "data", - "Osm2pgsql_Index" => "data", - "Place_Data" => "data", - "Place_Index" => "ssd2", - "Address_Data" => "data", - "Address_Index" => "ssd2", - "Search_Data" => "ssd1", - "Search_Index" => "ssd1", - "Aux_Data" => "aux", - "Aux_Index" => "aux" - }, - :fpm_pools => { - :www => { - :port => "8000", - :pm => "dynamic", - :max_children => "60" - }, - :bulk => { - :port => "8001", - :pm => "static", - :max_children => "10" - }, - :details => { - :port => "8002", - :pm => "static", - :max_children => "2" - } - }, - :redirects => {} - } -) diff --git a/roles/poldi.rb b/roles/poldi.rb index 7fc730a31..379e35f20 100644 --- a/roles/poldi.rb +++ b/roles/poldi.rb @@ -55,43 +55,10 @@ default_attributes( :address => "2001:630:12:500:2e0:81ff:fec5:333e" } } - }, - :postgresql => { - :versions => ["9.3"], - :settings => { - :defaults => { - :work_mem => "160MB", - :maintenance_work_mem => "10GB", - :random_page_cost => "1.5", - :effective_cache_size => "40GB", - :fsync => "on" - } - } - }, - :nominatim => { - :flatnode_file => "/ssd-old/nominatim/nodes.store", - :database => { - :cluster => "9.3/main", - :dbname => "nominatim", - :postgis => "2.1" - }, - :fpm_pools => { - :www => { - :port => "8000", - :pm => "dynamic", - :max_children => "60" - }, - :bulk => { - :port => "8001", - :pm => "static", - :max_children => "10" - } - } } ) run_list( "role[ic]", - "role[tyan-s7010]", - "role[nominatim-standalone]" + "role[tyan-s7010]" ) -- 2.43.2