From: Grant <git@firefishy.com>
Date: Thu, 3 Dec 2020 08:38:19 +0000 (+0000)
Subject: Merge pull request #364 from osm-hr/html-torrents
X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/cc7dc2d10bf3457a9d6a28931bf96539bfac6491?hp=6d2f80bde89988a3b228de16152e599add107aab

Merge pull request #364 from osm-hr/html-torrents

Mention .torrent files too in header (instead of only mirrors)
---

diff --git a/Gemfile.lock b/Gemfile.lock
index dceed86bd..d4dbea51a 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -4,18 +4,18 @@ GEM
     ast (2.4.1)
     bcrypt_pbkdf (1.0.1)
     builder (3.2.4)
-    chef-utils (16.6.14)
-    cookstyle (7.2.1)
-      rubocop (= 1.3.1)
+    chef-utils (16.7.61)
+    cookstyle (7.3.10)
+      rubocop (= 1.5.0)
     diff-lcs (1.4.4)
     docker-api (2.0.0)
       excon (>= 0.47.0)
       multi_json
     ed25519 (1.2.4)
-    erubi (1.9.0)
+    erubi (1.10.0)
     excon (0.78.0)
     ffi (1.13.1)
-    gssapi (1.3.0)
+    gssapi (1.3.1)
       ffi (>= 1.0.1)
     gyoku (1.3.1)
       builder (>= 2.1.2)
@@ -34,11 +34,11 @@ GEM
     logging (2.3.0)
       little-plugger (~> 1.1)
       multi_json (~> 1.14)
-    mixlib-install (3.12.3)
+    mixlib-install (3.12.5)
       mixlib-shellout
       mixlib-versioning
       thor
-    mixlib-shellout (3.1.6)
+    mixlib-shellout (3.2.2)
       chef-utils
     mixlib-versioning (1.2.12)
     multi_json (1.15.0)
@@ -49,13 +49,13 @@ GEM
       net-ssh (>= 4.0.0)
     net-telnet (0.1.1)
     nori (2.6.0)
-    parallel (1.20.0)
+    parallel (1.20.1)
     parser (2.7.2.0)
       ast (~> 2.4.1)
     pastel (0.8.0)
       tty-color (~> 0.5)
     rainbow (3.0.0)
-    regexp_parser (1.8.2)
+    regexp_parser (2.0.0)
     rexml (3.2.4)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -73,16 +73,16 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
-    rubocop (1.3.1)
+    rubocop (1.5.0)
       parallel (~> 1.10)
       parser (>= 2.7.1.5)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8)
+      regexp_parser (>= 2.0)
       rexml
-      rubocop-ast (>= 1.1.1)
+      rubocop-ast (>= 1.2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (1.1.1)
+    rubocop-ast (1.3.0)
       parser (>= 2.7.1.5)
     ruby-progressbar (1.10.1)
     rubyntlm (0.6.2)
@@ -103,7 +103,7 @@ GEM
       unicode-display_width (~> 1.5)
       unicode_utils (~> 1.4)
     strings-ansi (0.2.0)
-    test-kitchen (2.7.2)
+    test-kitchen (2.8.0)
       bcrypt_pbkdf (~> 1.0)
       ed25519 (~> 1.2)
       license-acceptance (>= 1.0.11, < 3.0)
@@ -122,7 +122,7 @@ GEM
       pastel (~> 0.8)
       strings (~> 0.2.0)
       tty-cursor (~> 0.7)
-    tty-color (0.5.2)
+    tty-color (0.6.0)
     tty-cursor (0.7.1)
     tty-prompt (0.22.0)
       pastel (~> 0.8)
@@ -143,7 +143,7 @@ GEM
       logging (>= 1.6.1, < 3.0)
       nori (~> 2.0)
       rubyntlm (~> 0.6.0, >= 0.6.1)
-    winrm-elevated (1.2.2)
+    winrm-elevated (1.2.3)
       erubi (~> 1.8)
       winrm (~> 2.0)
       winrm-fs (~> 1.0)
diff --git a/cookbooks/munin/templates/default/munin.conf.erb b/cookbooks/munin/templates/default/munin.conf.erb
index fbedbb70b..3aa31b432 100644
--- a/cookbooks/munin/templates/default/munin.conf.erb
+++ b/cookbooks/munin/templates/default/munin.conf.erb
@@ -519,7 +519,6 @@ unknown_limit 144
     nginx_requests.graph_args --lower-limit 0
 <% @tilecaches.each do |tc| -%>
     nginx_requests.<%= tc[:name].tr("-", "_") %>.label <%= tc[:name] %>
-    nginx_requests.<%= tc[:name].tr("-", "_") %>.cdef <%= tc[:name].tr("-", "_") %>,8,*
     nginx_requests.<%= tc[:name].tr("-", "_") %>.draw AREASTACK
     nginx_requests.<%= tc[:name].tr("-", "_") %>.min 0
 <% end -%>
diff --git a/cookbooks/nominatim/attributes/default.rb b/cookbooks/nominatim/attributes/default.rb
index e29fd7931..bf87600ba 100644
--- a/cookbooks/nominatim/attributes/default.rb
+++ b/cookbooks/nominatim/attributes/default.rb
@@ -13,10 +13,10 @@ default[:nominatim][:ui_repository] = "https://github.com/osm-search/nominatim-u
 default[:nominatim][:ui_revision] = "master"
 
 default[:nominatim][:fpm_pools] = {
-  :www => {
-    :port => 8000,
+  "nominatim.openstreetmap.org" => {
     :pm => "dynamic",
-    :max_children => 60
+    :max_children => 60,
+    :prometheus_port => 9253
   }
 }
 
diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb
index 285d795f1..b5496a652 100644
--- a/cookbooks/nominatim/recipes/default.rb
+++ b/cookbooks/nominatim/recipes/default.rb
@@ -380,7 +380,7 @@ end
 end
 
 node[:nominatim][:fpm_pools].each do |name, data|
-  php_fpm name.to_s do
+  php_fpm name do
     port data[:port]
     pm data[:pm]
     pm_max_children data[:max_children]
@@ -388,6 +388,7 @@ node[:nominatim][:fpm_pools].each do |name, data|
     pm_min_spare_servers 10
     pm_max_spare_servers 20
     pm_max_requests 10000
+    prometheus_port data[:prometheus_port]
   end
 end
 
diff --git a/cookbooks/nominatim/templates/default/nginx.erb b/cookbooks/nominatim/templates/default/nginx.erb
index 88bd3c909..a44e9382c 100644
--- a/cookbooks/nominatim/templates/default/nginx.erb
+++ b/cookbooks/nominatim/templates/default/nginx.erb
@@ -1,5 +1,5 @@
 upstream nominatim_service {
-  server 127.0.0.1:<%= @pools[:www][:port ]%>;
+  server unix:/run/php/nominatim.openstreetmap.org.sock;
 }
 
 map $uri $nominatim_script_name {
diff --git a/cookbooks/php/resources/fpm.rb b/cookbooks/php/resources/fpm.rb
index c461cf679..0178e64ce 100644
--- a/cookbooks/php/resources/fpm.rb
+++ b/cookbooks/php/resources/fpm.rb
@@ -45,7 +45,7 @@ action :create do
     owner "root"
     group "root"
     mode "644"
-    variables new_resource.to_hash
+    variables new_resource.to_hash.merge(:pool => new_resource.pool)
   end
 
   if new_resource.prometheus_port
diff --git a/cookbooks/planet/recipes/replication.rb b/cookbooks/planet/recipes/replication.rb
index 2a3e65cb2..165b9282a 100644
--- a/cookbooks/planet/recipes/replication.rb
+++ b/cookbooks/planet/recipes/replication.rb
@@ -137,6 +137,18 @@ directory "/store/planet/replication/test" do
   mode "755"
 end
 
+directory "/store/planet/replication/test/day" do
+  owner "planet"
+  group "planet"
+  mode "755"
+end
+
+directory "/store/planet/replication/test/hour" do
+  owner "planet"
+  group "planet"
+  mode "755"
+end
+
 directory "/store/planet/replication/test/minute" do
   owner "planet"
   group "planet"
@@ -174,6 +186,18 @@ directory "/var/run/lock/changeset-replication/" do
   mode "750"
 end
 
+directory "/var/lib/replication" do
+  owner "planet"
+  group "planet"
+  mode "755"
+end
+
+directory "/var/lib/replication/test" do
+  owner "planet"
+  group "planet"
+  mode "755"
+end
+
 template "/etc/replication/auth.conf" do
   source "replication.auth.erb"
   user "root"
@@ -223,6 +247,76 @@ systemd_timer "replication-minutely" do
   accuracy_sec 5
 end
 
+directory "/var/lib/replication/test/hour" do
+  owner "planet"
+  group "planet"
+  mode "755"
+end
+
+template "/var/lib/replication/test/hour/configuration.txt" do
+  source "replication.config.erb"
+  owner "planet"
+  group "planet"
+  mode "644"
+  variables :base => "test/minute", :interval => 3600
+end
+
+link "/var/lib/replication/test/hour/data" do
+  to "/store/planet/replication/test/hour"
+end
+
+systemd_service "replication-hourly" do
+  description "Hourly replication"
+  user "planet"
+  exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/hour"
+  private_tmp true
+  private_devices true
+  protect_system "full"
+  protect_home true
+  restrict_address_families %w[AF_INET AF_INET6]
+  no_new_privileges true
+end
+
+systemd_timer "replication-hourly" do
+  description "Daily replication"
+  on_calendar "*-*-* *:02/15:00"
+end
+
+directory "/var/lib/replication/test/day" do
+  owner "planet"
+  group "planet"
+  mode "755"
+end
+
+template "/var/lib/replication/test/day/configuration.txt" do
+  source "replication.config.erb"
+  owner "planet"
+  group "planet"
+  mode "644"
+  variables :base => "test/hour", :interval => 86400
+end
+
+link "/var/lib/replication/test/day/data" do
+  to "/store/planet/replication/test/day"
+end
+
+systemd_service "replication-daily" do
+  description "Daily replication"
+  user "planet"
+  exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/day"
+  private_tmp true
+  private_devices true
+  protect_system "full"
+  protect_home true
+  restrict_address_families %w[AF_INET AF_INET6]
+  no_new_privileges true
+end
+
+systemd_timer "replication-daily" do
+  description "Daily replication"
+  on_calendar "*-*-* *:02/15:00"
+end
+
 template "/etc/replication/changesets.conf" do
   source "changesets.conf.erb"
   user "root"
@@ -239,12 +333,6 @@ template "/etc/replication/users-agreed.conf" do
   variables :password => db_passwords["planetdiff"]
 end
 
-directory "/var/lib/replication" do
-  owner "planet"
-  group "planet"
-  mode "755"
-end
-
 directory "/var/lib/replication/minute" do
   owner "planet"
   group "planet"
@@ -314,6 +402,14 @@ if node[:planet][:replication] == "enabled"
     action [:enable, :start]
   end
 
+  service "replication-hourly.timer" do
+    action [:enable, :start]
+  end
+
+  service "replication-daily.timer" do
+    action [:enable, :start]
+  end
+
   cron_d "replication-minutely" do
     user "planet"
     command "/usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute"
@@ -353,6 +449,14 @@ else
     action [:stop, :disable]
   end
 
+  service "replication-hourly.timer" do
+    action [:stop, :disable]
+  end
+
+  service "replication-daily.timer" do
+    action [:stop, :disable]
+  end
+
   cron_d "replication-minutely" do
     action :delete
   end
diff --git a/cookbooks/postgresql/attributes/default.rb b/cookbooks/postgresql/attributes/default.rb
index 2d9fc1079..c7eeecff8 100644
--- a/cookbooks/postgresql/attributes/default.rb
+++ b/cookbooks/postgresql/attributes/default.rb
@@ -2,6 +2,7 @@ default[:postgresql][:versions] = []
 default[:postgresql][:clusters] = {}
 default[:postgresql][:settings][:defaults][:port] = "5432"
 default[:postgresql][:settings][:defaults][:max_connections] = "100"
+default[:postgresql][:settings][:defaults][:ssl] = "true"
 default[:postgresql][:settings][:defaults][:shared_buffers] = "32MB"
 default[:postgresql][:settings][:defaults][:temp_buffers] = "8MB"
 default[:postgresql][:settings][:defaults][:work_mem] = "1MB"
diff --git a/cookbooks/postgresql/templates/default/postgresql.conf.erb b/cookbooks/postgresql/templates/default/postgresql.conf.erb
index 3c84ec994..e2892f68a 100644
--- a/cookbooks/postgresql/templates/default/postgresql.conf.erb
+++ b/cookbooks/postgresql/templates/default/postgresql.conf.erb
@@ -28,7 +28,7 @@ unix_socket_directory = '/var/run/postgresql'
 
 # - Security and Authentication -
 
-ssl = true
+ssl = <%= @settings[:ssl] || @defaults[:ssl] %>
 ssl_renegotiation_limit = 0
 
 #------------------------------------------------------------------------------
@@ -86,7 +86,9 @@ archive_command = '<%= @settings[:archive_command] || @defaults[:archive_command
 # - Sending Server(s) -
 
 max_wal_senders = <%= @settings[:max_wal_senders] || @defaults[:max_wal_senders] %>
+<% if @version.to_f >= 9.4 -%>
 max_replication_slots = <%= @settings[:max_replication_slots] || @defaults[:max_replication_slots] %>
+<% end -%>
 
 # - Standby Servers -
 
diff --git a/cookbooks/prometheus/templates/default/grafana.ini.erb b/cookbooks/prometheus/templates/default/grafana.ini.erb
index 08f1d5dfd..d3ea1ce65 100644
--- a/cookbooks/prometheus/templates/default/grafana.ini.erb
+++ b/cookbooks/prometheus/templates/default/grafana.ini.erb
@@ -15,3 +15,8 @@ host = localhost:25
 skip_verify = true
 from_address = admins@openstreetmap.org
 from_name = Prometheus
+
+[auth.anonymous]
+enabled = true
+org_name = OpenStreetMap
+org_role = Viewer
diff --git a/cookbooks/tilecache/templates/default/nginx_tile.conf.erb b/cookbooks/tilecache/templates/default/nginx_tile.conf.erb
index 0ea85d755..338e0d51d 100644
--- a/cookbooks/tilecache/templates/default/nginx_tile.conf.erb
+++ b/cookbooks/tilecache/templates/default/nginx_tile.conf.erb
@@ -154,6 +154,14 @@ map $http_referer $denied_referer {
   '~^https?://[^.]*\.cellmapper\.net/'   1;
 }
 
+map $http_referer $censored_referer {
+  default                                 0; # Not denied
+  # Blocked on board instructions
+  '~^https?://schiebt-sie-ab\.de/'        1;
+  '~^https?://[^.]*\.schiebt-sie-ab\.de/' 1;
+}
+
+
 map $http_referer $osm_referer {
   default                                 '';    # False
   '~^https:\/\/www\.openstreetmap\.org\/' 'osm'; # True
@@ -405,6 +413,11 @@ server {
         return 418;
       }
 
+      if ($censored_referer) {
+        set $limit_rate 512;
+        return 451 "Unavailable at OSMF Board request";
+      }
+
       # Strip any ?query parameters from urls
       set $args '';
 
diff --git a/roles/dev.rb b/roles/dev.rb
index 565d4b70c..c192ac182 100644
--- a/roles/dev.rb
+++ b/roles/dev.rb
@@ -131,7 +131,7 @@ default_attributes(
     }
   },
   :postgresql => {
-    :versions => ["9.1", "12"],
+    :versions => ["12"],
     :settings => {
       :defaults => {
         :shared_buffers => "1GB",
@@ -140,9 +140,6 @@ default_attributes(
         :max_stack_depth => "4MB",
         :effective_cache_size => "4GB"
       },
-      "9.1" => {
-        :port => "5433"
-      },
       "12" => {
         :port => "5432",
         :wal_level => "logical",
diff --git a/roles/pummelzacken.rb b/roles/pummelzacken.rb
index add524938..05393514c 100644
--- a/roles/pummelzacken.rb
+++ b/roles/pummelzacken.rb
@@ -34,7 +34,7 @@ default_attributes(
         :random_page_cost => "1.5",
         :effective_cache_size => "60GB",
         :effective_io_concurrency => "256",
-        :fsync => "off"
+        :fsync => "on"
       }
     }
   },
@@ -45,7 +45,7 @@ default_attributes(
     }
   },
   :nominatim => {
-    :state => "off",
+    :state => "standalone",
     :dbadmins => %w[lonvia tomh],
     :dbcluster => "13/main",
     :postgis => "3",