From: Grant Date: Thu, 3 Dec 2020 08:45:08 +0000 (+0000) Subject: Merge pull request #361 from polarbearing/patch-1 X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/2eeabe9b50dc30e53e2477f7cc55a45ea5912a7a?hp=9feb2672a73e40ba91d06f7e3b1a1313b020dd96 Merge pull request #361 from polarbearing/patch-1 DMCA form: add Content-Type header to generated email --- diff --git a/Gemfile.lock b/Gemfile.lock index dceed86bd..d4dbea51a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -4,18 +4,18 @@ GEM ast (2.4.1) bcrypt_pbkdf (1.0.1) builder (3.2.4) - chef-utils (16.6.14) - cookstyle (7.2.1) - rubocop (= 1.3.1) + chef-utils (16.7.61) + cookstyle (7.3.10) + rubocop (= 1.5.0) diff-lcs (1.4.4) docker-api (2.0.0) excon (>= 0.47.0) multi_json ed25519 (1.2.4) - erubi (1.9.0) + erubi (1.10.0) excon (0.78.0) ffi (1.13.1) - gssapi (1.3.0) + gssapi (1.3.1) ffi (>= 1.0.1) gyoku (1.3.1) builder (>= 2.1.2) @@ -34,11 +34,11 @@ GEM logging (2.3.0) little-plugger (~> 1.1) multi_json (~> 1.14) - mixlib-install (3.12.3) + mixlib-install (3.12.5) mixlib-shellout mixlib-versioning thor - mixlib-shellout (3.1.6) + mixlib-shellout (3.2.2) chef-utils mixlib-versioning (1.2.12) multi_json (1.15.0) @@ -49,13 +49,13 @@ GEM net-ssh (>= 4.0.0) net-telnet (0.1.1) nori (2.6.0) - parallel (1.20.0) + parallel (1.20.1) parser (2.7.2.0) ast (~> 2.4.1) pastel (0.8.0) tty-color (~> 0.5) rainbow (3.0.0) - regexp_parser (1.8.2) + regexp_parser (2.0.0) rexml (3.2.4) rspec (3.9.0) rspec-core (~> 3.9.0) @@ -73,16 +73,16 @@ GEM diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.9.0) rspec-support (3.9.3) - rubocop (1.3.1) + rubocop (1.5.0) parallel (~> 1.10) parser (>= 2.7.1.5) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8) + regexp_parser (>= 2.0) rexml - rubocop-ast (>= 1.1.1) + rubocop-ast (>= 1.2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 2.0) - rubocop-ast (1.1.1) + rubocop-ast (1.3.0) parser (>= 2.7.1.5) ruby-progressbar (1.10.1) rubyntlm (0.6.2) @@ -103,7 +103,7 @@ GEM unicode-display_width (~> 1.5) unicode_utils (~> 1.4) strings-ansi (0.2.0) - test-kitchen (2.7.2) + test-kitchen (2.8.0) bcrypt_pbkdf (~> 1.0) ed25519 (~> 1.2) license-acceptance (>= 1.0.11, < 3.0) @@ -122,7 +122,7 @@ GEM pastel (~> 0.8) strings (~> 0.2.0) tty-cursor (~> 0.7) - tty-color (0.5.2) + tty-color (0.6.0) tty-cursor (0.7.1) tty-prompt (0.22.0) pastel (~> 0.8) @@ -143,7 +143,7 @@ GEM logging (>= 1.6.1, < 3.0) nori (~> 2.0) rubyntlm (~> 0.6.0, >= 0.6.1) - winrm-elevated (1.2.2) + winrm-elevated (1.2.3) erubi (~> 1.8) winrm (~> 2.0) winrm-fs (~> 1.0) diff --git a/cookbooks/devices/templates/default/udev.rules.erb b/cookbooks/devices/templates/default/udev.rules.erb index 2ac6e431f..8ac2886c2 100644 --- a/cookbooks/devices/templates/default/udev.rules.erb +++ b/cookbooks/devices/templates/default/udev.rules.erb @@ -85,6 +85,8 @@ SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x1563 SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x1586", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" # Ethernet controller: Intel Corporation Ethernet Controller X710 for 10GBASE-T SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x15ff", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation Ethernet Connection I354 (rev 03) +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x1f41", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" # Ethernet controller: Intel Corporation Ethernet Connection X722 for 10GBASE-T SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x37d2", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" diff --git a/cookbooks/mediawiki/resources/site.rb b/cookbooks/mediawiki/resources/site.rb index 2fc1ed115..598a7044e 100644 --- a/cookbooks/mediawiki/resources/site.rb +++ b/cookbooks/mediawiki/resources/site.rb @@ -458,6 +458,13 @@ action :create do update_site false end + mediawiki_extension "OSMCALWikiWidget" do + site new_resource.site + repository "https://github.com/thomersch/OSMCALWikiWidget.git" + tag "live" + update_site false + end + mediawiki_extension "SimpleMap" do site new_resource.site template "mw-ext-SimpleMap.inc.php.erb" diff --git a/cookbooks/munin/templates/default/munin.conf.erb b/cookbooks/munin/templates/default/munin.conf.erb index fbedbb70b..3aa31b432 100644 --- a/cookbooks/munin/templates/default/munin.conf.erb +++ b/cookbooks/munin/templates/default/munin.conf.erb @@ -519,7 +519,6 @@ unknown_limit 144 nginx_requests.graph_args --lower-limit 0 <% @tilecaches.each do |tc| -%> nginx_requests.<%= tc[:name].tr("-", "_") %>.label <%= tc[:name] %> - nginx_requests.<%= tc[:name].tr("-", "_") %>.cdef <%= tc[:name].tr("-", "_") %>,8,* nginx_requests.<%= tc[:name].tr("-", "_") %>.draw AREASTACK nginx_requests.<%= tc[:name].tr("-", "_") %>.min 0 <% end -%> diff --git a/cookbooks/nominatim/attributes/default.rb b/cookbooks/nominatim/attributes/default.rb index e29fd7931..bf87600ba 100644 --- a/cookbooks/nominatim/attributes/default.rb +++ b/cookbooks/nominatim/attributes/default.rb @@ -13,10 +13,10 @@ default[:nominatim][:ui_repository] = "https://github.com/osm-search/nominatim-u default[:nominatim][:ui_revision] = "master" default[:nominatim][:fpm_pools] = { - :www => { - :port => 8000, + "nominatim.openstreetmap.org" => { :pm => "dynamic", - :max_children => 60 + :max_children => 60, + :prometheus_port => 9253 } } diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb index 75c357902..b5496a652 100644 --- a/cookbooks/nominatim/recipes/default.rb +++ b/cookbooks/nominatim/recipes/default.rb @@ -380,7 +380,7 @@ end end node[:nominatim][:fpm_pools].each do |name, data| - php_fpm name.to_s do + php_fpm name do port data[:port] pm data[:pm] pm_max_children data[:max_children] @@ -388,6 +388,7 @@ node[:nominatim][:fpm_pools].each do |name, data| pm_min_spare_servers 10 pm_max_spare_servers 20 pm_max_requests 10000 + prometheus_port data[:prometheus_port] end end @@ -412,11 +413,13 @@ nginx_site "default" do action [:delete] end +frontends = search(:node, "recipes:web\\:\\:frontend").sort_by(&:name) + nginx_site "nominatim" do template "nginx.erb" directory build_directory variables :pools => node[:nominatim][:fpm_pools], - :frontends => search(:node, "recipes:web\\:\\:frontend"), + :frontends => frontends, :confdir => "#{basedir}/etc", :ui_directory => ui_directory end @@ -454,9 +457,12 @@ end include_recipe "fail2ban" +frontend_addresses = frontends.collect { |f| f.ipaddresses(:role => :external) } + fail2ban_jail "nominatim_limit_req" do filter "nginx-limit-req" logpath "#{node[:nominatim][:logdir]}/nominatim.openstreetmap.org-error.log" ports [80, 443] maxretry 5 + ignoreips frontend_addresses.flatten.sort end diff --git a/cookbooks/nominatim/templates/default/nginx.erb b/cookbooks/nominatim/templates/default/nginx.erb index d56d99c8a..a44e9382c 100644 --- a/cookbooks/nominatim/templates/default/nginx.erb +++ b/cookbooks/nominatim/templates/default/nginx.erb @@ -1,5 +1,5 @@ upstream nominatim_service { - server 127.0.0.1:<%= @pools[:www][:port ]%>; + server unix:/run/php/nominatim.openstreetmap.org.sock; } map $uri $nominatim_script_name { @@ -49,7 +49,7 @@ map $http_referer $missing_referer { geo $whitelisted { default 0; <% @frontends.each do |frontend| -%> -<% frontend.ipaddresses(:role => :external) do |address| -%> +<% frontend.ipaddresses(:role => :external).sort.each do |address| -%> <%= address %> 1; <% end -%> <% end -%> @@ -86,9 +86,22 @@ map $blocked_user_agent $limit_tarpit { 2 $binary_remote_addr; } +map $missing_email$missing_referer$http_user_agent $generic_mozilla { + default 0; + ~^11Mozilla/4.0 1; + ~^11Mozilla/5.0 2; +} + +map $whitelisted$generic_mozilla$uri $limit_reverse { + default ""; + ~01/reverse.* $binary_remote_addr; + ~02/reverse.* $binary_remote_addr; +} + limit_req_zone $limit_www zone=www:50m rate=2r/s; limit_req_zone $limit_tarpit zone=tarpit:10m rate=1r/s; limit_req_zone $binary_remote_addr zone=blocked:10m rate=20r/m; +limit_req_zone $limit_reverse zone=reverse:10m rate=10r/m; server { listen 80 default_server; @@ -166,6 +179,7 @@ server { limit_req zone=www burst=10; limit_req zone=tarpit burst=2; + limit_req zone=reverse burst=5; limit_req_status 429; fastcgi_pass nominatim_service; include fastcgi_params; @@ -188,6 +202,7 @@ server { limit_req zone=www burst=10; limit_req zone=tarpit burst=2; + limit_req zone=reverse burst=5; limit_req_status 429; fastcgi_pass nominatim_service; include fastcgi_params; diff --git a/cookbooks/php/resources/fpm.rb b/cookbooks/php/resources/fpm.rb index c461cf679..0178e64ce 100644 --- a/cookbooks/php/resources/fpm.rb +++ b/cookbooks/php/resources/fpm.rb @@ -45,7 +45,7 @@ action :create do owner "root" group "root" mode "644" - variables new_resource.to_hash + variables new_resource.to_hash.merge(:pool => new_resource.pool) end if new_resource.prometheus_port diff --git a/cookbooks/planet/files/default/cgi/HEADER.cgi b/cookbooks/planet/files/default/cgi/HEADER.cgi index 30bf73307..874c9c28d 100644 --- a/cookbooks/planet/files/default/cgi/HEADER.cgi +++ b/cookbooks/planet/files/default/cgi/HEADER.cgi @@ -69,7 +69,7 @@ print """ The files found here are regularly-updated, complete copies of the OpenStreetMap.org database, and those published before the 12 September 2012 are distributed under a Creative Commons Attribution-ShareAlike 2.0 license, those published after are Open Data Commons Open Database License 1.0 licensed. For more information, see the project wiki.

-

WARNING Download speeds are currently restricted to 4096 KB/s due to limited available capacity on our Internet connection. Please use a mirror if possible.

+

WARNING Download speeds are currently restricted to 4096 KB/s due to limited available capacity on our Internet connection. Please use torrents or a mirror if possible.

@@ -113,9 +113,8 @@ database, and those published before the 12 September 2012 are distributed under is a tool for importing the data into a Postgis database for rendering maps.

- Processed coastline data - derived from OSM data is also needed for rendering usable maps, and can be found in a - single shapefile (360MB). + Processed coastline data + derived from OSM data is also needed for rendering usable maps.

diff --git a/cookbooks/planet/recipes/replication.rb b/cookbooks/planet/recipes/replication.rb index 2a3e65cb2..165b9282a 100644 --- a/cookbooks/planet/recipes/replication.rb +++ b/cookbooks/planet/recipes/replication.rb @@ -137,6 +137,18 @@ directory "/store/planet/replication/test" do mode "755" end +directory "/store/planet/replication/test/day" do + owner "planet" + group "planet" + mode "755" +end + +directory "/store/planet/replication/test/hour" do + owner "planet" + group "planet" + mode "755" +end + directory "/store/planet/replication/test/minute" do owner "planet" group "planet" @@ -174,6 +186,18 @@ directory "/var/run/lock/changeset-replication/" do mode "750" end +directory "/var/lib/replication" do + owner "planet" + group "planet" + mode "755" +end + +directory "/var/lib/replication/test" do + owner "planet" + group "planet" + mode "755" +end + template "/etc/replication/auth.conf" do source "replication.auth.erb" user "root" @@ -223,6 +247,76 @@ systemd_timer "replication-minutely" do accuracy_sec 5 end +directory "/var/lib/replication/test/hour" do + owner "planet" + group "planet" + mode "755" +end + +template "/var/lib/replication/test/hour/configuration.txt" do + source "replication.config.erb" + owner "planet" + group "planet" + mode "644" + variables :base => "test/minute", :interval => 3600 +end + +link "/var/lib/replication/test/hour/data" do + to "/store/planet/replication/test/hour" +end + +systemd_service "replication-hourly" do + description "Hourly replication" + user "planet" + exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/hour" + private_tmp true + private_devices true + protect_system "full" + protect_home true + restrict_address_families %w[AF_INET AF_INET6] + no_new_privileges true +end + +systemd_timer "replication-hourly" do + description "Daily replication" + on_calendar "*-*-* *:02/15:00" +end + +directory "/var/lib/replication/test/day" do + owner "planet" + group "planet" + mode "755" +end + +template "/var/lib/replication/test/day/configuration.txt" do + source "replication.config.erb" + owner "planet" + group "planet" + mode "644" + variables :base => "test/hour", :interval => 86400 +end + +link "/var/lib/replication/test/day/data" do + to "/store/planet/replication/test/day" +end + +systemd_service "replication-daily" do + description "Daily replication" + user "planet" + exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/day" + private_tmp true + private_devices true + protect_system "full" + protect_home true + restrict_address_families %w[AF_INET AF_INET6] + no_new_privileges true +end + +systemd_timer "replication-daily" do + description "Daily replication" + on_calendar "*-*-* *:02/15:00" +end + template "/etc/replication/changesets.conf" do source "changesets.conf.erb" user "root" @@ -239,12 +333,6 @@ template "/etc/replication/users-agreed.conf" do variables :password => db_passwords["planetdiff"] end -directory "/var/lib/replication" do - owner "planet" - group "planet" - mode "755" -end - directory "/var/lib/replication/minute" do owner "planet" group "planet" @@ -314,6 +402,14 @@ if node[:planet][:replication] == "enabled" action [:enable, :start] end + service "replication-hourly.timer" do + action [:enable, :start] + end + + service "replication-daily.timer" do + action [:enable, :start] + end + cron_d "replication-minutely" do user "planet" command "/usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute" @@ -353,6 +449,14 @@ else action [:stop, :disable] end + service "replication-hourly.timer" do + action [:stop, :disable] + end + + service "replication-daily.timer" do + action [:stop, :disable] + end + cron_d "replication-minutely" do action :delete end diff --git a/cookbooks/planet/templates/default/planetdump.erb b/cookbooks/planet/templates/default/planetdump.erb index ae6302382..5b32054a6 100644 --- a/cookbooks/planet/templates/default/planetdump.erb +++ b/cookbooks/planet/templates/default/planetdump.erb @@ -97,6 +97,7 @@ function mk_torrent { -w https://ftp.spline.de/pub/openstreetmap/${web_path} \ -w https://osm.openarchive.site/${name} \ -w https://downloads.opencagedata.com/planet/${name} \ + -w https://planet.osm-hr.org/${web_path} \ -c "OpenStreetMap ${type} data export, licensed under https://opendatacommons.org/licenses/odbl/ by OpenStreetMap contributors" \ -o ${name}.torrent } diff --git a/cookbooks/postgresql/attributes/default.rb b/cookbooks/postgresql/attributes/default.rb index 2d9fc1079..c7eeecff8 100644 --- a/cookbooks/postgresql/attributes/default.rb +++ b/cookbooks/postgresql/attributes/default.rb @@ -2,6 +2,7 @@ default[:postgresql][:versions] = [] default[:postgresql][:clusters] = {} default[:postgresql][:settings][:defaults][:port] = "5432" default[:postgresql][:settings][:defaults][:max_connections] = "100" +default[:postgresql][:settings][:defaults][:ssl] = "true" default[:postgresql][:settings][:defaults][:shared_buffers] = "32MB" default[:postgresql][:settings][:defaults][:temp_buffers] = "8MB" default[:postgresql][:settings][:defaults][:work_mem] = "1MB" diff --git a/cookbooks/postgresql/templates/default/postgresql.conf.erb b/cookbooks/postgresql/templates/default/postgresql.conf.erb index 3c84ec994..e2892f68a 100644 --- a/cookbooks/postgresql/templates/default/postgresql.conf.erb +++ b/cookbooks/postgresql/templates/default/postgresql.conf.erb @@ -28,7 +28,7 @@ unix_socket_directory = '/var/run/postgresql' # - Security and Authentication - -ssl = true +ssl = <%= @settings[:ssl] || @defaults[:ssl] %> ssl_renegotiation_limit = 0 #------------------------------------------------------------------------------ @@ -86,7 +86,9 @@ archive_command = '<%= @settings[:archive_command] || @defaults[:archive_command # - Sending Server(s) - max_wal_senders = <%= @settings[:max_wal_senders] || @defaults[:max_wal_senders] %> +<% if @version.to_f >= 9.4 -%> max_replication_slots = <%= @settings[:max_replication_slots] || @defaults[:max_replication_slots] %> +<% end -%> # - Standby Servers - diff --git a/cookbooks/prometheus/templates/default/grafana.ini.erb b/cookbooks/prometheus/templates/default/grafana.ini.erb index 08f1d5dfd..d3ea1ce65 100644 --- a/cookbooks/prometheus/templates/default/grafana.ini.erb +++ b/cookbooks/prometheus/templates/default/grafana.ini.erb @@ -15,3 +15,8 @@ host = localhost:25 skip_verify = true from_address = admins@openstreetmap.org from_name = Prometheus + +[auth.anonymous] +enabled = true +org_name = OpenStreetMap +org_role = Viewer diff --git a/cookbooks/tile/files/default/html/favicon.ico b/cookbooks/tile/files/default/html/favicon.ico index 27b042b5c..975e1cb0d 100644 Binary files a/cookbooks/tile/files/default/html/favicon.ico and b/cookbooks/tile/files/default/html/favicon.ico differ diff --git a/cookbooks/tilecache/templates/default/nginx_tile.conf.erb b/cookbooks/tilecache/templates/default/nginx_tile.conf.erb index 0ea85d755..338e0d51d 100644 --- a/cookbooks/tilecache/templates/default/nginx_tile.conf.erb +++ b/cookbooks/tilecache/templates/default/nginx_tile.conf.erb @@ -154,6 +154,14 @@ map $http_referer $denied_referer { '~^https?://[^.]*\.cellmapper\.net/' 1; } +map $http_referer $censored_referer { + default 0; # Not denied + # Blocked on board instructions + '~^https?://schiebt-sie-ab\.de/' 1; + '~^https?://[^.]*\.schiebt-sie-ab\.de/' 1; +} + + map $http_referer $osm_referer { default ''; # False '~^https:\/\/www\.openstreetmap\.org\/' 'osm'; # True @@ -405,6 +413,11 @@ server { return 418; } + if ($censored_referer) { + set $limit_rate 512; + return 451 "Unavailable at OSMF Board request"; + } + # Strip any ?query parameters from urls set $args ''; diff --git a/roles/dev.rb b/roles/dev.rb index 565d4b70c..c192ac182 100644 --- a/roles/dev.rb +++ b/roles/dev.rb @@ -131,7 +131,7 @@ default_attributes( } }, :postgresql => { - :versions => ["9.1", "12"], + :versions => ["12"], :settings => { :defaults => { :shared_buffers => "1GB", @@ -140,9 +140,6 @@ default_attributes( :max_stack_depth => "4MB", :effective_cache_size => "4GB" }, - "9.1" => { - :port => "5433" - }, "12" => { :port => "5432", :wal_level => "logical", diff --git a/roles/pummelzacken.rb b/roles/pummelzacken.rb index b6d866984..05393514c 100644 --- a/roles/pummelzacken.rb +++ b/roles/pummelzacken.rb @@ -25,7 +25,7 @@ default_attributes( } }, :postgresql => { - :versions => ["12"], + :versions => ["13"], :settings => { :defaults => { :listen_addresses => "10.0.0.20", @@ -47,8 +47,8 @@ default_attributes( :nominatim => { :state => "standalone", :dbadmins => %w[lonvia tomh], - :dbcluster => "12/main", - :postgis => "2.5", + :dbcluster => "13/main", + :postgis => "3", :enable_backup => true, :flatnode_file => "/ssd/nominatim/nodes.store", :tablespaces => {