From: Tom Hughes Date: Wed, 10 May 2017 07:27:11 +0000 (+0100) Subject: Merge remote-tracking branch 'github/pull/122' X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/6fdba5edc17c0ccbb5dc3bd15690dcba6708e1fe?hp=e61e02b852c45c5688194e4693e5b5cee7028b7f Merge remote-tracking branch 'github/pull/122' --- diff --git a/cookbooks/chef/attributes/default.rb b/cookbooks/chef/attributes/default.rb index 4aa59baa7..06db9bf6d 100644 --- a/cookbooks/chef/attributes/default.rb +++ b/cookbooks/chef/attributes/default.rb @@ -5,4 +5,4 @@ default[:apt][:sources] = node[:apt][:sources] | ["opscode"] default[:chef][:server][:version] = "12.13.0-1" # Set the default client version -default[:chef][:client][:version] = "12.19.36" +default[:chef][:client][:version] = "12.20.3" diff --git a/cookbooks/chef/recipes/default.rb b/cookbooks/chef/recipes/default.rb index 2e2e345a1..35ed0d87e 100644 --- a/cookbooks/chef/recipes/default.rb +++ b/cookbooks/chef/recipes/default.rb @@ -112,7 +112,6 @@ if node[:lsb][:release].to_f >= 15.10 description "Chef client" after "network.target" exec_start "/usr/bin/chef-client -i 1800 -s 20" - success_exit_status 3 restart "on-failure" end else @@ -126,6 +125,9 @@ end service "chef-client" do action [:enable, :start] + if node[:lsb][:release].to_f >= 15.10 + restart_command "systemctl kill --signal=TERM chef-client.service" + end supports :status => true, :restart => true, :reload => true subscribes :restart, "dpkg_package[chef]" subscribes :restart, "template[/etc/init/chef-client.conf]" diff --git a/cookbooks/devices/templates/default/udev.rules.erb b/cookbooks/devices/templates/default/udev.rules.erb index d37fcb9eb..114de0d51 100644 --- a/cookbooks/devices/templates/default/udev.rules.erb +++ b/cookbooks/devices/templates/default/udev.rules.erb @@ -27,3 +27,21 @@ ACTION=="add", SUBSYSTEM=="block", ENV{ID_BUS}=="<%= device[:bus] %>", ENV{ID_SE # Disable scatter-gather offload for HP NC362i network controllers SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x10c9", ATTRS{subsystem_vendor}=="0x103c", ATTRS{subsystem_device}=="0x323f", RUN+="/sbin/ethtool -K $name gso off tso off sg off gro off" + +# Workaround unreliable Western Digital WD RE3/RE4 disks (ATA only) +# Set sufficent Linux subsystem timeout +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", ATTR{device/timeout}="90" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", ATTR{device/timeout}="90" +# Disable Disk Write Cache, Set AAM and Power Management correctly +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", RUN+="/sbin/hdparm -q -W0 -q -M254 $env{DEVNAME}" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", RUN+="/sbin/hdparm -q -W0 -q -M254 -q -B254 $env{DEVNAME}" + +# Set Disks TLED / SCT Error Recovery Control +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD5000AAKS-00A7B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD2000FYYZ-01UL1B2", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="TOSHIBA_DT01ACA300", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST31000340NS", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="HGST_HTS725050A7E630", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="HGST_HTE721010A9E630", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" diff --git a/cookbooks/dhcpd/templates/default/dhcpd.conf.erb b/cookbooks/dhcpd/templates/default/dhcpd.conf.erb index 6728745ba..d57fd109d 100644 --- a/cookbooks/dhcpd/templates/default/dhcpd.conf.erb +++ b/cookbooks/dhcpd/templates/default/dhcpd.conf.erb @@ -56,6 +56,12 @@ host draco.oob.openstreetmap.org { fixed-address draco.oob.openstreetmap.org; } +host eddie.oob.openstreetmap.org { + hardware ethernet 0c:c4:7a:d5:8c:c0; + server-name "eddie.oob.openstreetmap.org"; + fixed-address eddie.oob.openstreetmap.org; +} + host errol.oob.openstreetmap.org { hardware ethernet 00:e0:81:c0:8d:01; server-name "errol.oob.openstreetmap.org"; @@ -164,12 +170,66 @@ host spike-03.oob.openstreetmap.org { fixed-address spike-03.oob.openstreetmap.org; } +host tiamat-00.oob.openstreetmap.org { + hardware ethernet 00:25:90:1a:76:01; + server-name "tiamat-00.oob.openstreetmap.org"; + fixed-address tiamat-00.oob.openstreetmap.org; +} + +host tiamat-01.oob.openstreetmap.org { + hardware ethernet 00:25:90:1a:75:78; + server-name "tiamat-01.oob.openstreetmap.org"; + fixed-address tiamat-01.oob.openstreetmap.org; +} + +host tiamat-02.oob.openstreetmap.org { + hardware ethernet 00:25:90:1f:10:e3; + server-name "tiamat-02.oob.openstreetmap.org"; + fixed-address tiamat-02.oob.openstreetmap.org; +} + +host tiamat-03.oob.openstreetmap.org { + hardware ethernet 00:25:90:1a:75:74; + server-name "tiamat-03.oob.openstreetmap.org"; + fixed-address tiamat-03.oob.openstreetmap.org; +} + +host tiamat-11.oob.openstreetmap.org { + hardware ethernet 00:25:90:2c:cd:68; + server-name "tiamat-11.oob.openstreetmap.org"; + fixed-address tiamat-11.oob.openstreetmap.org; +} + +host tiamat-12.oob.openstreetmap.org { + hardware ethernet 00:25:90:1f:0a:9c; + server-name "tiamat-12.oob.openstreetmap.org"; + fixed-address tiamat-12.oob.openstreetmap.org; +} + +host tiamat-13.oob.openstreetmap.org { + hardware ethernet 00:25:90:1f:17:ed; + server-name "tiamat-13.oob.openstreetmap.org"; + fixed-address tiamat-13.oob.openstreetmap.org; +} + +host tiamat-21.oob.openstreetmap.org { + hardware ethernet 00:25:90:29:a8:d7; + server-name "tiamat-21.oob.openstreetmap.org"; + fixed-address tiamat-21.oob.openstreetmap.org; +} + host tiamat-22.oob.openstreetmap.org { hardware ethernet 00:25:90:29:a8:01; server-name "tiamat-22.oob.openstreetmap.org"; fixed-address tiamat-22.oob.openstreetmap.org; } +host tiamat-23.oob.openstreetmap.org { + hardware ethernet 00:25:90:29:a7:ff; + server-name "tiamat-23.oob.openstreetmap.org"; + fixed-address tiamat-23.oob.openstreetmap.org; +} + host thorn-01.oob.openstreetmap.org { hardware ethernet 00:19:bb:35:87:94; server-name "thorn-01.oob.openstreetmap.org"; diff --git a/cookbooks/exim/templates/default/exim4.conf.erb b/cookbooks/exim/templates/default/exim4.conf.erb index 4b6071920..d9d12f83b 100644 --- a/cookbooks/exim/templates/default/exim4.conf.erb +++ b/cookbooks/exim/templates/default/exim4.conf.erb @@ -498,11 +498,10 @@ acl_check_data: <% end -%> # Deny spammy messages with headers of the form: - # X-PHP-Originating-Script: :SendMail.php - # X-PHP-Originating-Script: :SendMail.class.php - # X-PHP-Originating-Script: :ExtendedMail.php - # X-PHP-Originating-Script: :ExtendedMail.class.php - deny condition = ${if match {$h_X-PHP-Originating-Script:}{^[0-9]+:(Send|Extended)[Mm]ail(\\.class)?\\.php\$}} + # X-PHP-Originating-Script: :.php + # X-PHP-Originating-Script: :.class.php + deny condition = ${if match {$h_X-PHP-Originating-Script:}{^[0-9]+:[A-Za-z]+(\\.class)?\\.php\$}} + !hosts = +relay_from_hosts message = This message failed local spam checks. # Accept the message. diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb index 19e48909f..a8009759f 100644 --- a/cookbooks/nominatim/recipes/default.rb +++ b/cookbooks/nominatim/recipes/default.rb @@ -306,6 +306,7 @@ apache_module "proxy" apache_module "proxy_fcgi" apache_module "proxy_http" apache_module "headers" +apache_module "reqtimeout" service "php7.0-fpm" do action [:enable, :start] @@ -357,7 +358,7 @@ web_servers = search(:node, "recipes:web\\:\\:frontend").collect do |n| # ~FC010 end.flatten fail2ban_filter "nominatim" do - failregex '^ - - \[\] "[^"]+" (400|429) ' + failregex '^ - - \[\] "[^"]+" (408|429) ' end fail2ban_jail "nominatim" do diff --git a/cookbooks/nominatim/templates/default/apache.erb b/cookbooks/nominatim/templates/default/apache.erb index b79e76dbe..652fd93a3 100644 --- a/cookbooks/nominatim/templates/default/apache.erb +++ b/cookbooks/nominatim/templates/default/apache.erb @@ -25,6 +25,8 @@ # Remove Proxy request header to mitigate https://httpoxy.org/ RequestHeader unset Proxy early + RequestReadTimeout header=15-30,MinRate=500 body=15-30,MinRate=500 + CustomLog /var/log/apache2/nominatim.openstreetmap.org-access.log combined ErrorLog /var/log/apache2/nominatim.openstreetmap.org-error.log diff --git a/cookbooks/planet/templates/default/replication.cron.erb b/cookbooks/planet/templates/default/replication.cron.erb index 84ee83c7c..8b81f03e0 100644 --- a/cookbooks/planet/templates/default/replication.cron.erb +++ b/cookbooks/planet/templates/default/replication.cron.erb @@ -11,5 +11,5 @@ MAILTO=brett@bretth.com LD_PRELOAD=/opt/flush/flush.so * * * * * planet /usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute -2 * * * * planet /home/bretth/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour -5 * * * * planet /home/bretth/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day +5 * * * * planet /home/bretth/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour +10 * * * * planet /home/bretth/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day diff --git a/roles/angor.rb b/roles/angor.rb index 143228f9a..da5d426ce 100644 --- a/roles/angor.rb +++ b/roles/angor.rb @@ -39,6 +39,5 @@ default_attributes( ) run_list( - "role[inxza]", - "role[tilecache]" + "role[inxza]" ) diff --git a/roles/eddie.rb b/roles/eddie.rb new file mode 100644 index 000000000..eb6753f45 --- /dev/null +++ b/roles/eddie.rb @@ -0,0 +1,59 @@ +name "eddie" +description "Master role applied to eddie" + +default_attributes( + :apt => { + :sources => ["postgresql"] + }, + :db => { + :cluster => "9.5/main" + }, + :networking => { + :interfaces => { + :internal_ipv4 => { + :interface => "enp1s0f0.2801", + :role => :internal, + :family => :inet, + :address => "10.0.0.10" + } + } + }, + :postgresql => { + :settings => { + :defaults => { + :shared_buffers => "64GB", + :work_mem => "64MB", + :maintenance_work_mem => "1GB", + :effective_cache_size => "180GB", + :effective_io_concurrency => "256" + } + } + }, + :sysctl => { + :postgres => { + :comment => "Increase shared memory for postgres", + :parameters => { + "kernel.shmmax" => 66 * 1024 * 1024 * 1024, + "kernel.shmall" => 66 * 1024 * 1024 * 1024 / 4096 + } + } + }, + :sysfs => { + :md_tune => { + :comment => "Enable request merging for NVMe devices", + :parameters => { + "block/nvme0n1/queue/nomerges" => "1", + "block/nvme1n1/queue/nomerges" => "1", + "block/nvme2n1/queue/nomerges" => "1", + "block/nvme3n1/queue/nomerges" => "1", + "block/nvme4n1/queue/nomerges" => "1", + "block/nvme5n1/queue/nomerges" => "1", + "block/nvme6n1/queue/nomerges" => "1" + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/ironbelly.rb b/roles/ironbelly.rb index d5bb0ee33..ab306c6a2 100644 --- a/roles/ironbelly.rb +++ b/roles/ironbelly.rb @@ -23,7 +23,7 @@ default_attributes( } }, :git => { - :allowed_nodes => "*:*", + :allowed_nodes => "fqdn:*", :user => "chefrepo", :group => "chefrepo", :backup => "chef-git" diff --git a/roles/pummelzacken.rb b/roles/pummelzacken.rb index 70c90a0ca..f5a573d47 100644 --- a/roles/pummelzacken.rb +++ b/roles/pummelzacken.rb @@ -27,7 +27,7 @@ default_attributes( :maintenance_work_mem => "10GB", :random_page_cost => "1.5", :effective_cache_size => "60GB", - :fsync => "off" + :fsync => "on" } } }, diff --git a/roles/tiamat-00.rb b/roles/tiamat-00.rb index 5e0f10478..0deb7ca89 100644 --- a/roles/tiamat-00.rb +++ b/roles/tiamat-00.rb @@ -17,6 +17,9 @@ default_attributes( :address => "193.60.236.40" } } + }, + :hardware => { + :watchdog => "w83627hf_wdt" } ) diff --git a/roles/tiamat-01.rb b/roles/tiamat-01.rb index 2d3aff407..b2b0eb19d 100644 --- a/roles/tiamat-01.rb +++ b/roles/tiamat-01.rb @@ -17,6 +17,9 @@ default_attributes( :address => "193.60.236.41" } } + }, + :hardware => { + :watchdog => "w83627hf_wdt" } ) diff --git a/roles/tiamat-02.rb b/roles/tiamat-02.rb index d8a2ed0c9..de959961d 100644 --- a/roles/tiamat-02.rb +++ b/roles/tiamat-02.rb @@ -17,6 +17,9 @@ default_attributes( :address => "193.60.236.42" } } + }, + :hardware => { + :watchdog => "w83627hf_wdt" } ) diff --git a/roles/tiamat-10.rb b/roles/tiamat-10.rb index 2d8b6e557..a0d1c3994 100644 --- a/roles/tiamat-10.rb +++ b/roles/tiamat-10.rb @@ -17,6 +17,9 @@ default_attributes( :address => "193.60.236.44" } } + }, + :hardware => { + :watchdog => "w83627hf_wdt" } ) diff --git a/roles/tiamat-11.rb b/roles/tiamat-11.rb index b698d11c1..dce0c7870 100644 --- a/roles/tiamat-11.rb +++ b/roles/tiamat-11.rb @@ -17,6 +17,9 @@ default_attributes( :address => "193.60.236.45" } } + }, + :hardware => { + :watchdog => "w83627hf_wdt" } ) diff --git a/roles/tiamat-12.rb b/roles/tiamat-12.rb index 9dc371962..66145154c 100644 --- a/roles/tiamat-12.rb +++ b/roles/tiamat-12.rb @@ -17,6 +17,9 @@ default_attributes( :address => "193.60.236.46" } } + }, + :hardware => { + :watchdog => "w83627hf_wdt" } ) diff --git a/roles/tiamat-13.rb b/roles/tiamat-13.rb index 90b83797d..a7ad81f0c 100644 --- a/roles/tiamat-13.rb +++ b/roles/tiamat-13.rb @@ -17,6 +17,9 @@ default_attributes( :address => "193.60.236.47" } } + }, + :hardware => { + :watchdog => "w83627hf_wdt" } ) diff --git a/roles/tiamat-20.rb b/roles/tiamat-20.rb index 48a33794f..576d5c878 100644 --- a/roles/tiamat-20.rb +++ b/roles/tiamat-20.rb @@ -17,6 +17,9 @@ default_attributes( :address => "193.60.236.48" } } + }, + :hardware => { + :watchdog => "w83627hf_wdt" } ) diff --git a/roles/tiamat-21.rb b/roles/tiamat-21.rb index 33badf508..082dd40ef 100644 --- a/roles/tiamat-21.rb +++ b/roles/tiamat-21.rb @@ -17,6 +17,9 @@ default_attributes( :address => "193.60.236.49" } } + }, + :hardware => { + :watchdog => "w83627hf_wdt" } ) diff --git a/roles/tiamat-22.rb b/roles/tiamat-22.rb index f2359438b..09f9238ac 100644 --- a/roles/tiamat-22.rb +++ b/roles/tiamat-22.rb @@ -17,6 +17,9 @@ default_attributes( :address => "193.60.236.50" } } + }, + :hardware => { + :watchdog => "w83627hf_wdt" } ) diff --git a/roles/tiamat-23.rb b/roles/tiamat-23.rb index 886e5f611..dacc1ce2c 100644 --- a/roles/tiamat-23.rb +++ b/roles/tiamat-23.rb @@ -17,6 +17,9 @@ default_attributes( :address => "193.60.236.51" } } + }, + :hardware => { + :watchdog => "w83627hf_wdt" } ) diff --git a/roles/tilecache.rb b/roles/tilecache.rb index 8e6da21ee..313c262f4 100644 --- a/roles/tilecache.rb +++ b/roles/tilecache.rb @@ -12,13 +12,6 @@ default_attributes( :apt => { :sources => ["nginx"] }, - :munin => { - :plugins => { - :cpu => { - :user => { :warning => 200, :critical => 400 } - } - } - }, :sysctl => { :network_conntrack_time_wait => { :comment => "Only track completed connections for 30 seconds",