From: Tom Hughes Date: Wed, 7 May 2025 11:32:36 +0000 (+0100) Subject: Merge remote-tracking branch 'github/pull/761' X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/HEAD?ds=inline;hp=631e07aeacee422c8b79b02152f2cd9bbba12599 Merge remote-tracking branch 'github/pull/761' --- diff --git a/.github/workflows/test-kitchen.yml b/.github/workflows/test-kitchen.yml index e1c8df555..0587f1bf5 100644 --- a/.github/workflows/test-kitchen.yml +++ b/.github/workflows/test-kitchen.yml @@ -127,6 +127,10 @@ jobs: suite: osqa - os: debian-12 suite: apt-repository + - os: debian-12 + suite: blogs + - os: debian-12 + suite: community - os: debian-12 suite: dev - os: debian-12 @@ -135,16 +139,34 @@ jobs: suite: git-server - os: debian-12 suite: git-web + - os: debian-12 + suite: gps-tile - os: debian-12 suite: imagery-tiler + - os: debian-12 + suite: irc - os: debian-12 suite: letsencrypt + - os: debian-12 + suite: matomo + - os: debian-12 + suite: nominatim - os: debian-12 suite: otrs + - os: debian-12 + suite: overpass + - os: debian-12 + suite: prometheus-server - os: debian-12 suite: serverinfo + - os: debian-12 + suite: subversion - os: debian-12 suite: supybot + - os: debian-12 + suite: taginfo + - os: debian-12 + suite: trac - os: debian-12 suite: vectortile - os: debian-12 @@ -153,9 +175,15 @@ jobs: suite: web-frontend - os: debian-12 suite: web-rails + - os: debian-12 + suite: wiki exclude: - suite: apt-repository os: ubuntu-2204 + - suite: blogs + os: ubuntu-2204 + - suite: community + os: ubuntu-2204 - suite: dev os: ubuntu-2204 - suite: dns @@ -164,18 +192,36 @@ jobs: os: ubuntu-2204 - suite: git-web os: ubuntu-2204 - - suite: mailman + - suite: gps-tile + os: ubuntu-2204 + - suite: irc os: ubuntu-2204 - suite: letsencrypt os: ubuntu-2204 + - suite: mailman + os: ubuntu-2204 + - suite: matomo + os: ubuntu-2204 + - suite: nominatim + os: ubuntu-2204 - suite: osqa os: ubuntu-2204 - suite: otrs os: ubuntu-2204 + - suite: overpass + os: ubuntu-2204 + - suite: prometheus-server + os: ubuntu-2204 - suite: serverinfo os: ubuntu-2204 + - suite: subversion + os: ubuntu-2204 - suite: supybot os: ubuntu-2204 + - suite: taginfo + os: ubuntu-2204 + - suite: trac + os: ubuntu-2204 - suite: vectortile os: ubuntu-2204 - suite: web-cgimap @@ -184,6 +230,8 @@ jobs: os: ubuntu-2204 - suite: web-rails os: ubuntu-2204 + - suite: wiki + os: ubuntu-2204 fail-fast: false steps: - name: Login to GitHub Container Registry diff --git a/Gemfile.lock b/Gemfile.lock index 40032233d..3c91fa610 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,20 +1,23 @@ GEM remote: https://rubygems.org/ specs: - activesupport (7.1.4.2) + activesupport (7.1.5.1) base64 + benchmark (>= 0.3) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) + logger (>= 1.4.2) minitest (>= 5.1) mutex_m + securerandom (>= 0.3) tzinfo (~> 2.0) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) - ast (2.4.2) - aws-eventstream (1.3.0) + ast (2.4.3) + aws-eventstream (1.3.2) aws-partitions (1.863.0) aws-sdk-accessanalyzer (1.44.0) aws-sdk-core (~> 3, >= 3.188.0) @@ -257,7 +260,7 @@ GEM aws-sdk-core (~> 3, >= 3.188.0) aws-sigv4 (~> 1.1) aws-sigv2 (1.2.0) - aws-sigv4 (1.10.1) + aws-sigv4 (1.11.0) aws-eventstream (~> 1, >= 1.0.2) azure_graph_rbac (0.17.2) ms_rest_azure (~> 0.12.0) @@ -271,12 +274,12 @@ GEM ms_rest_azure (~> 0.12.0) base64 (0.2.0) bcrypt_pbkdf (1.1.1) - bigdecimal (3.1.8) - bson (4.15.0) + benchmark (0.4.0) + bigdecimal (3.1.9) builder (3.3.0) - chef-config (18.5.0) + chef-config (18.7.6) addressable - chef-utils (= 18.5.0) + chef-utils (= 18.7.6) fuzzyurl mixlib-config (>= 2.2.12, < 4.0) mixlib-shellout (>= 2.0, < 4.0) @@ -284,77 +287,43 @@ GEM chef-telemetry (1.1.1) chef-config concurrent-ruby (~> 1.0) - chef-utils (18.5.0) + chef-utils (18.7.6) concurrent-ruby coderay (1.1.3) - concurrent-ruby (1.3.4) - connection_pool (2.4.1) - cookstyle (7.32.8) - rubocop (= 1.25.1) + concurrent-ruby (1.3.5) + connection_pool (2.5.2) + cookstyle (8.1.2) + rubocop (= 1.75.5) declarative (0.0.20) - diff-lcs (1.5.1) - docker-api (2.3.0) + diff-lcs (1.6.1) + docker-api (2.4.0) excon (>= 0.64.0) multi_json domain_name (0.6.20240107) drb (2.2.1) - dry-configurable (1.2.0) - dry-core (~> 1.0, < 2) - zeitwerk (~> 2.6) - dry-core (1.0.1) - concurrent-ruby (~> 1.0) - zeitwerk (~> 2.6) - dry-inflector (1.1.0) - dry-logic (1.5.0) - concurrent-ruby (~> 1.0) - dry-core (~> 1.0, < 2) - zeitwerk (~> 2.6) - dry-struct (1.6.0) - dry-core (~> 1.0, < 2) - dry-types (>= 1.7, < 2) - ice_nine (~> 0.11) - zeitwerk (~> 2.6) - dry-types (1.7.2) - bigdecimal (~> 3.0) - concurrent-ruby (~> 1.0) - dry-core (~> 1.0) - dry-inflector (~> 1.0) - dry-logic (~> 1.4) - zeitwerk (~> 2.6) ed25519 (1.3.0) - erubi (1.13.0) - excon (0.112.0) - faraday (1.10.4) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0) - faraday-multipart (~> 1.0) + erubi (1.13.1) + excon (1.2.5) + logger + faraday (1.3.1) faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.0) - faraday-patron (~> 1.0) - faraday-rack (~> 1.0) - faraday-retry (~> 1.0) + multipart-post (>= 1.2, < 3) ruby2_keywords (>= 0.0.4) faraday-cookie_jar (0.0.7) faraday (>= 0.8.0) http-cookie (~> 1.0.0) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-follow_redirects (0.3.0) - faraday (>= 1, < 3) - faraday-httpclient (1.0.1) - faraday-multipart (1.0.4) - multipart-post (~> 2) faraday-net_http (1.0.2) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - faraday-rack (1.0.0) - faraday-retry (1.0.3) - faraday_middleware (1.2.1) + faraday_middleware (1.0.0) faraday (~> 1.0) - ffi (1.17.0) + ffi (1.17.2) + ffi (1.17.2-aarch64-linux-gnu) + ffi (1.17.2-aarch64-linux-musl) + ffi (1.17.2-arm-linux-gnu) + ffi (1.17.2-arm-linux-musl) + ffi (1.17.2-x86-linux-gnu) + ffi (1.17.2-x86-linux-musl) + ffi (1.17.2-x86_64-linux-gnu) + ffi (1.17.2-x86_64-linux-musl) fuzzyurl (0.9.0) google-apis-admin_directory_v1 (0.46.0) google-apis-core (>= 0.11.0, < 2.a) @@ -389,35 +358,26 @@ GEM gyoku (1.4.0) builder (>= 2.1.2) rexml (~> 3.0) - hashdiff (1.0.1) - hashie (5.0.0) - highline (3.1.1) - reline - http-cookie (1.0.7) + hashie (4.1.0) + http-cookie (1.0.8) domain_name (~> 0.5) - httpclient (2.8.3) - i18n (1.14.6) + httpclient (2.9.0) + mutex_m + i18n (1.14.7) concurrent-ruby (~> 1.0) - ice_nine (0.11.2) inifile (3.0.0) - inspec (5.22.58) - faraday_middleware (>= 0.12.2, < 1.3) - inspec-core (= 5.22.58) - mongo (= 2.13.2) - progress_bar (~> 1.3.3) - rake - train (~> 3.10) - train-aws (~> 0.2) + inspec (4.24.32) + faraday_middleware (>= 0.12.2, < 1.1) + inspec-core (= 4.24.32) + train (~> 3.0) + train-aws (~> 0.1) train-habitat (~> 0.1) - train-kubernetes (~> 0.1) train-winrm (~> 0.2) - inspec-core (5.22.58) + inspec-core (4.24.32) addressable (~> 2.4) - chef-telemetry (~> 1.0, >= 1.0.8) - cookstyle - faraday (>= 1, < 3) - faraday-follow_redirects (~> 0.3) - hashie (>= 3.4, < 6.0) + chef-telemetry (~> 1.0) + faraday (>= 0.9.0, < 1.4) + hashie (>= 3.4, < 5.0) license-acceptance (>= 0.2.13, < 3.0) method_source (>= 0.8, < 2.0) mixlib-log (~> 3.0) @@ -425,66 +385,55 @@ GEM parallel (~> 1.9) parslet (>= 1.5, < 3.0) pry (~> 0.13) - rspec (>= 3.9, <= 3.12) + rspec (~> 3.10) rspec-its (~> 1.2) rubyzip (>= 1.2.2, < 3.0) semverse (~> 3.0) sslshake (~> 1.2) - thor (>= 0.20, < 1.3.0) + thor (>= 0.20, < 2.0) tomlrb (>= 1.2, < 2.1) - train-core (~> 3.10) + train-core (~> 3.0) tty-prompt (~> 0.17) tty-table (~> 0.10) - io-console (0.7.2) jmespath (1.6.2) - json (2.7.3) - jsonpath (1.1.5) - multi_json - jwt (2.9.3) + json (2.11.3) + jwt (2.10.1) base64 - k8s-ruby (0.16.0) - dry-configurable - dry-struct - dry-types - excon (~> 0.71) - hashdiff (~> 1.0.0) - jsonpath (~> 1.1) - recursive-open-struct (~> 1.1.3) - yajl-ruby (~> 1.4.0) - yaml-safe_load_stream3 kitchen-dokken (2.20.7) docker-api (>= 1.33, < 3) lockfile (~> 2.1) test-kitchen (>= 1.15, < 4) - kitchen-inspec (2.6.2) + kitchen-inspec (3.0.0) hashie (>= 3.4, <= 5.0) - inspec (>= 2.2.64, < 6.0) + inspec (>= 2.2.64, < 7.0) test-kitchen (>= 2.7, < 4) + language_server-protocol (3.17.0.4) license-acceptance (2.1.13) pastel (~> 0.7) tomlrb (>= 1.2, < 3.0) tty-box (~> 0.6) tty-prompt (~> 0.20) + lint_roller (1.1.0) little-plugger (1.1.4) lockfile (2.1.3) + logger (1.7.0) logging (2.4.0) little-plugger (~> 1.1) multi_json (~> 1.14) method_source (1.1.0) mini_mime (1.1.5) - minitest (5.25.1) + minitest (5.25.5) mixlib-config (3.0.27) tomlrb mixlib-install (3.12.30) mixlib-shellout mixlib-versioning thor - mixlib-log (3.0.9) - mixlib-shellout (3.3.3) + mixlib-log (3.2.3) + ffi (>= 1.15.5) + mixlib-shellout (3.3.9) chef-utils mixlib-versioning (1.2.12) - mongo (2.13.2) - bson (>= 4.8.2, < 5.0.0) ms_rest (0.7.6) concurrent-ruby (~> 1.0) faraday (>= 0.9, < 2.0.0) @@ -496,75 +445,72 @@ GEM ms_rest (~> 0.7.6) multi_json (1.15.0) multipart-post (2.4.1) - mutex_m (0.2.0) - net-scp (4.0.0) + mutex_m (0.3.0) + net-scp (4.1.0) net-ssh (>= 2.6.5, < 8.0.0) net-ssh (7.3.0) net-ssh-gateway (2.0.0) net-ssh (>= 4.0.0) nori (2.7.1) bigdecimal - options (2.3.2) os (1.1.4) - parallel (1.26.3) - parser (3.3.5.0) + parallel (1.27.0) + parser (3.3.8.0) ast (~> 2.4.1) racc parslet (2.0.0) pastel (0.8.0) tty-color (~> 0.5) - progress_bar (1.3.4) - highline (>= 1.6) - options (~> 2.3.0) - pry (0.14.2) + prism (1.4.0) + pry (0.15.2) coderay (~> 1.1) method_source (~> 1.0) public_suffix (6.0.1) racc (1.8.1) rainbow (3.1.1) - rake (13.2.1) - recursive-open-struct (1.1.3) - regexp_parser (2.9.2) - reline (0.5.10) - io-console (~> 0.5) + regexp_parser (2.10.0) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) retriable (3.1.2) - rexml (3.3.9) - rspec (3.12.0) - rspec-core (~> 3.12.0) - rspec-expectations (~> 3.12.0) - rspec-mocks (~> 3.12.0) - rspec-core (3.12.3) - rspec-support (~> 3.12.0) - rspec-expectations (3.12.4) + rexml (3.4.1) + rspec (3.13.0) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.3) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.3) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) + rspec-support (~> 3.13.0) rspec-its (1.3.1) rspec-core (>= 3.0.0) rspec-expectations (>= 3.0.0) - rspec-mocks (3.12.7) + rspec-mocks (3.13.2) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-support (3.12.2) - rubocop (1.25.1) + rspec-support (~> 3.13.0) + rspec-support (3.13.2) + rubocop (1.75.5) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) parallel (~> 1.10) - parser (>= 3.1.0.0) + parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml - rubocop-ast (>= 1.15.1, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.44.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.32.3) - parser (>= 3.3.1.0) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.44.1) + parser (>= 3.3.7.2) + prism (~> 1.4) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) rubyntlm (0.6.5) base64 - rubyzip (2.3.2) + rubyzip (2.4.1) + securerandom (0.4.1) semverse (3.0.2) signet (0.19.0) addressable (~> 2.8) @@ -591,7 +537,7 @@ GEM winrm (~> 2.0) winrm-elevated (~> 1.0) winrm-fs (~> 1.1) - thor (1.2.2) + thor (1.3.2) timeliness (0.3.10) tomlrb (1.3.0) trailblazer-option (0.1.2) @@ -703,9 +649,6 @@ GEM net-scp (>= 1.2, < 5.0) net-ssh (>= 2.9, < 8.0) train-habitat (0.2.22) - train-kubernetes (0.2.1) - k8s-ruby (~> 0.16.0) - train (~> 3.0) train-winrm (0.2.13) winrm (>= 2.3.6, < 3.0) winrm-elevated (~> 1.2.2) @@ -753,12 +696,18 @@ GEM rubyzip (~> 2.0) winrm (~> 2.0) wisper (2.0.1) - yajl-ruby (1.4.3) - yaml-safe_load_stream3 (0.1.2) zeitwerk (2.6.18) PLATFORMS + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl ruby + x86-linux-gnu + x86-linux-musl + x86_64-linux-gnu + x86_64-linux-musl DEPENDENCIES cookstyle @@ -768,4 +717,4 @@ DEPENDENCIES zeitwerk (< 2.7) BUNDLED WITH - 2.2.16 + 2.6.2 diff --git a/cookbooks/accounts/files/default/craig/.ssh/authorized_keys b/cookbooks/accounts/files/default/craig/.ssh/authorized_keys new file mode 100644 index 000000000..101e5e13c --- /dev/null +++ b/cookbooks/accounts/files/default/craig/.ssh/authorized_keys @@ -0,0 +1,2 @@ +# DO NOT EDIT - This file is being maintained by Chef - use authorized_keys2 instead +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc26tRbrQoczW3UFfXkdt6auqFg/Ut6spGMT476fFsJFjaYp98E2lca2W9vyJq4nSn0tdxwcO4LGK1ACdhZ/81I/68d7CPv5zNjJMehgwQ1BJTM5HWaap08cEINZMQ0xt6Neyz+HIFiaJVzxmyLJCnaaCeQX/t2NmL+nQV6rJq4qS2L434Bw1qGM73zjNja4bB2IN0y5yWDRTSLg+t+DKH26DC4OJn4+pxKsyt2egB7MNj9my1MRcjPVeo/bxz3nWoxKtX9dWq9UFrd7trfSXK+7Y+9fFHl41rrrYbn3UFKcDL6Rzvp2bFytDW6FlWmuptGajWnm2HpqI69bsO7uw1 diff --git a/cookbooks/apache/recipes/default.rb b/cookbooks/apache/recipes/default.rb index cade29d31..a17e2f4cb 100644 --- a/cookbooks/apache/recipes/default.rb +++ b/cookbooks/apache/recipes/default.rb @@ -114,7 +114,7 @@ fail2ban_jail "apache-forbidden" do end fail2ban_filter "apache-evasive" do - failregex ": Blacklisting address : possible DoS attack\.$" + failregex ": Blacklisting address : possible DoS attack\\.$" end fail2ban_jail "apache-evasive" do diff --git a/cookbooks/apt/recipes/default.rb b/cookbooks/apt/recipes/default.rb index 2ead9baf2..1d876dfb0 100644 --- a/cookbooks/apt/recipes/default.rb +++ b/cookbooks/apt/recipes/default.rb @@ -49,7 +49,7 @@ if platform?("debian") archive_components = %w[main contrib non-free non-free-firmware] backport_packages = case node[:lsb][:codename] when "bookworm" then %W[amd64-microcode exim4 firmware-free firmware-nonfree intel-microcode libosmium linux-signed-#{dpkg_arch} osm2pgsql otrs2 pyosmium smartmontools systemd cgi-mapserver] - else %W[] + else %w[] end elsif intel? archive_host = if node[:country] diff --git a/cookbooks/awscli/recipes/default.rb b/cookbooks/awscli/recipes/default.rb index 22684864d..90574c3bb 100644 --- a/cookbooks/awscli/recipes/default.rb +++ b/cookbooks/awscli/recipes/default.rb @@ -77,12 +77,25 @@ ruby_block "install-awscli" do require "fileutils" awscli_version_string = shell_out("#{cache_dir}/awscli/dist/aws", "--version") awscli_version = awscli_version_string.stdout.split(" ").first.split("/").last - FileUtils.mkdir_p("/opt/awscli/v2/#{awscli_version}/bin/", :mode => 0755) - FileUtils.mv("#{cache_dir}/awscli/dist", "/opt/awscli/v2/#{awscli_version}/dist", :force => true) - FileUtils.ln_sf("/opt/awscli/v2/#{awscli_version}/dist/aws", "/opt/awscli/v2/#{awscli_version}/bin/aws") - FileUtils.ln_sf("/opt/awscli/v2/#{awscli_version}/dist/aws_completer", "/opt/awscli/v2/#{awscli_version}/bin/aws_completer") - FileUtils.rm("/opt/awscli/v2/current") if File.exist?("/opt/awscli/v2/current") - FileUtils.ln_sf("/opt/awscli/v2/#{awscli_version}", "/opt/awscli/v2/current") + + install_dir = "/opt/awscli/v2/#{awscli_version}" + + FileUtils.mkdir_p("#{install_dir}/bin/", :mode => 0755) + FileUtils.mv("#{cache_dir}/awscli/dist", "#{install_dir}/dist", :force => true) + FileUtils.ln_sf("#{install_dir}/dist/aws", "#{install_dir}/bin/aws") + FileUtils.ln_sf("#{install_dir}/dist/aws_completer", "#{install_dir}/bin/aws_completer") + + FileUtils.rm_f("/opt/awscli/v2/current") + FileUtils.ln_sf(install_dir, "/opt/awscli/v2/current") + + # Retain the last 5 versions, including the current one + versions = Dir.glob("/opt/awscli/v2/*").select { |dir| File.directory?(dir) && dir != "/opt/awscli/v2/current" } + versions.sort_by! { |dir| File.mtime(dir) }.reverse! + versions_to_delete = versions[5..] || [] + + versions_to_delete.each do |dir| + FileUtils.rm_rf(dir) + end end action :nothing subscribes :run, "archive_file[#{cache_dir}/#{awscli_zip}]", :immediately diff --git a/cookbooks/blogs/recipes/default.rb b/cookbooks/blogs/recipes/default.rb index 5c650f242..86dfd3afe 100644 --- a/cookbooks/blogs/recipes/default.rb +++ b/cookbooks/blogs/recipes/default.rb @@ -22,7 +22,7 @@ include_recipe "apache" include_recipe "git" include_recipe "ruby" -package %W[ +package %w[ make gcc g++ diff --git a/cookbooks/civicrm/recipes/default.rb b/cookbooks/civicrm/recipes/default.rb index 2aed4f64f..d10ed0788 100644 --- a/cookbooks/civicrm/recipes/default.rb +++ b/cookbooks/civicrm/recipes/default.rb @@ -200,21 +200,21 @@ node[:civicrm][:extensions].each_value do |details| end settings = edit_file "#{civicrm_directory}/civicrm/templates/CRM/common/civicrm.settings.php.template" do |line| - line.gsub!(/%%cms%%/, "WordPress") - line.gsub!(/%%CMSdbUser%%/, "civicrm") - line.gsub!(/%%CMSdbPass%%/, database_password) - line.gsub!(/%%CMSdbHost%%/, "localhost") - line.gsub!(/%%CMSdbName%%/, "civicrm") - line.gsub!(/%%dbUser%%/, "civicrm") - line.gsub!(/%%dbPass%%/, database_password) - line.gsub!(/%%dbHost%%/, "localhost") - line.gsub!(/%%dbName%%/, "civicrm") - line.gsub!(/%%crmRoot%%/, "#{civicrm_directory}/civicrm/") - line.gsub!(/%%templateCompileDir%%/, "/srv/supporting.openstreetmap.org/wp-content/uploads/civicrm/templates_c/") - line.gsub!(/%%baseURL%%/, "http://supporting.openstreetmap.org/") - line.gsub!(/%%siteKey%%/, site_key) - line.gsub!(/%%credKeys%%/, cred_keys) - line.gsub!(/%%signKeys%%/, sign_keys) + line.gsub!("%%cms%%", "WordPress") + line.gsub!("%%CMSdbUser%%", "civicrm") + line.gsub!("%%CMSdbPass%%", database_password) + line.gsub!("%%CMSdbHost%%", "localhost") + line.gsub!("%%CMSdbName%%", "civicrm") + line.gsub!("%%dbUser%%", "civicrm") + line.gsub!("%%dbPass%%", database_password) + line.gsub!("%%dbHost%%", "localhost") + line.gsub!("%%dbName%%", "civicrm") + line.gsub!("%%crmRoot%%", "#{civicrm_directory}/civicrm/") + line.gsub!("%%templateCompileDir%%", "/srv/supporting.openstreetmap.org/wp-content/uploads/civicrm/templates_c/") + line.gsub!("%%baseURL%%", "http://supporting.openstreetmap.org/") + line.gsub!("%%siteKey%%", site_key) + line.gsub!("%%credKeys%%", cred_keys) + line.gsub!("%%signKeys%%", sign_keys) line.gsub!(%r{// *define\('CIVICRM_CMSDIR', '/path/to/install/root/'\);}, "define('CIVICRM_CMSDIR', '/srv/supporting.openstreetmap.org');") # Don't recompile smarty templates on every call https://docs.civicrm.org/sysadmin/en/latest/setup/optimizations/#disable-compile-check line.gsub!(%r{// define\('CIVICRM_TEMPLATE_COMPILE_CHECK', FALSE\);}, "define('CIVICRM_TEMPLATE_COMPILE_CHECK', FALSE);") diff --git a/cookbooks/community/recipes/default.rb b/cookbooks/community/recipes/default.rb index 0628d883a..73739ac30 100644 --- a/cookbooks/community/recipes/default.rb +++ b/cookbooks/community/recipes/default.rb @@ -63,7 +63,7 @@ git "/srv/community.openstreetmap.org/docker" do action :sync repository "https://github.com/discourse/discourse_docker.git" # DANGER launch wrapper automatically updates git repo if rebuild method used: https://github.com/discourse/discourse_docker/blob/107ffb40fe8b1ea40e00814468db974a4f3f8e8f/launcher#L799 - revision "3715498fc188d60c0b579443383c4e973cf26f59" + revision "721facba644f645211571026d6677b015c15e5d6" user "root" group "root" notifies :run, "notify_group[discourse_container_new_data]" diff --git a/cookbooks/community/templates/default/web_only.yml.erb b/cookbooks/community/templates/default/web_only.yml.erb index ff9127ff1..fddb054a9 100644 --- a/cookbooks/community/templates/default/web_only.yml.erb +++ b/cookbooks/community/templates/default/web_only.yml.erb @@ -19,7 +19,7 @@ links: # any extra arguments for Docker? # docker_args: -# Latest Version v3.4.1 +# Latest Version v3.4.3 # Discourse only support tests-passed and stable branches params: version: stable diff --git a/cookbooks/devices/metadata.rb b/cookbooks/devices/metadata.rb index 2f71f6857..a25b0e16f 100644 --- a/cookbooks/devices/metadata.rb +++ b/cookbooks/devices/metadata.rb @@ -6,3 +6,4 @@ description "Configures devices" version "0.1" supports "ubuntu" +depends "chef" diff --git a/cookbooks/devices/templates/default/udev.rules.erb b/cookbooks/devices/templates/default/udev.rules.erb index be9903891..ad2451afa 100644 --- a/cookbooks/devices/templates/default/udev.rules.erb +++ b/cookbooks/devices/templates/default/udev.rules.erb @@ -95,47 +95,6 @@ SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x37d2 # Disable Firmware Based LLDP handler SUBSYSTEM=="net", ACTION=="add", ENV{INTERFACE}=="*", DRIVERS=="i40e", RUN+="/sbin/ethtool --set-priv-flags $name disable-fw-lldp on" -# Workaround unreliable Western Digital WD RE3/RE4 disks (ATA only) -# Set sufficent Linux subsystem timeout and fix severe NCQ performance issue -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD5002ABYS-02B1B0", ATTR{device/timeout}="90", ATTR{device/queue_depth}="1", ATTR{queue/nr_requests}="256" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", ATTR{device/timeout}="90", ATTR{device/queue_depth}="1", ATTR{queue/nr_requests}="256" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", ATTR{device/timeout}="90", ATTR{device/queue_depth}="1", ATTR{queue/nr_requests}="256" -# Disable Disk Write Cache, Set AAM and Power Management correctly -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", RUN+="/sbin/hdparm -q -W0 -q -M254 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", RUN+="/sbin/hdparm -q -W0 -q -M254 -q -B254 $env{DEVNAME}" - -# Set Disks TLED / SCT Error Recovery Control -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD5000AAKS-00A7B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD2000FYYZ-01UL1B2", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="TOSHIBA_DT01ACA300", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST31000340NS", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="HGST_HTS725050A7E630", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="HGST_HTE721010A9E630", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" - -# Add SSD optimisation -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="OCZ-VERTEX3", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="OCZ-VERTEX3", ATTR{queue/scheduler}="noop" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/scheduler}="noop" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/read_ahead_kb}="256" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/scheduler}="noop" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/read_ahead_kb}="256" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/scheduler}="noop" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/read_ahead_kb}="256" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST240FN0021", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST240FN0021", ATTR{queue/scheduler}="noop" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="SuperMicro_SSD", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="SuperMicro_SSD", ATTR{queue/scheduler}="noop" - # Delete failed disk in cmok ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST_M13FQBL", ENV{ID_SERIAL}=="ST_M13FQBL_QNR_BFW", ATTR{device/delete}="1" @@ -155,6 +114,8 @@ ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_MODEL}=="QEMU_HA ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_MODEL}=="QEMU_HARDDISK", ATTR{queue/scheduler}="noop" # Vendor is sometimes missing -# Increase default MD raid5/raid6 strip cache + group_thread_cnt -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{MD_LEVEL}=="raid5", ATTR{md/stripe_cache_size}="8192", ATTR{md/group_thread_cnt}="4" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{MD_LEVEL}=="raid6", ATTR{md/stripe_cache_size}="8192", ATTR{md/group_thread_cnt}="4" +# Tune md stripe cache and thread count for RAID-5 / RAID-6 arrays +<% + group_threads = [(node.cpu_cores.to_i / 2.0).round, 4].max +%> +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{MD_LEVEL}=="raid[56]", ATTR{md/stripe_cache_size}="8192", ATTR{md/group_thread_cnt}="<%= group_threads %>" diff --git a/cookbooks/dns/recipes/default.rb b/cookbooks/dns/recipes/default.rb index 0ec59c8b3..d5f0c1e60 100644 --- a/cookbooks/dns/recipes/default.rb +++ b/cookbooks/dns/recipes/default.rb @@ -60,7 +60,7 @@ end dpkg_package "dnscontrol" do source "#{cache_dir}/dnscontrol-#{dnscontrol_version}.deb" - version "#{dnscontrol_version}" + version dnscontrol_version end directory "/srv/dns.openstreetmap.org" do diff --git a/cookbooks/exim/recipes/default.rb b/cookbooks/exim/recipes/default.rb index 7354e93d3..e839d6045 100644 --- a/cookbooks/exim/recipes/default.rb +++ b/cookbooks/exim/recipes/default.rb @@ -161,7 +161,7 @@ if node[:exim][:dkim_selectors] mode "755" end - node[:exim][:dkim_selectors].each do |domain, _selector| + node[:exim][:dkim_selectors].each_key do |domain| file "/etc/exim4/dkim-keys/#{domain}" do content keys[domain].join("\n") owner "root" diff --git a/cookbooks/gps-tile/templates/default/apache.erb b/cookbooks/gps-tile/templates/default/apache.erb index 9adf707dd..a1b7ba94c 100644 --- a/cookbooks/gps-tile/templates/default/apache.erb +++ b/cookbooks/gps-tile/templates/default/apache.erb @@ -47,6 +47,7 @@ RewriteRule ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 [R=permanent,L] # Redirect to https + RewriteCond %{REQUEST_URI} !^/server-status RewriteRule (.*) https://%{SERVER_NAME}/$1 [R=permanent,L] # Setup logging diff --git a/cookbooks/hardware/recipes/default.rb b/cookbooks/hardware/recipes/default.rb index 0dafc1f91..4340bf504 100644 --- a/cookbooks/hardware/recipes/default.rb +++ b/cookbooks/hardware/recipes/default.rb @@ -426,7 +426,7 @@ if !intel_ssds.empty? || !intel_nvmes.empty? end dpkg_package "sst" do - version "#{sst_package_version}" + version sst_package_version source "#{Chef::Config[:file_cache_path]}/sst_#{sst_package_version}_amd64.deb" end diff --git a/cookbooks/imagery/recipes/tiler.rb b/cookbooks/imagery/recipes/tiler.rb index 571425bda..2f6942b66 100644 --- a/cookbooks/imagery/recipes/tiler.rb +++ b/cookbooks/imagery/recipes/tiler.rb @@ -37,19 +37,19 @@ container_image = if arm? podman_service "titiler" do description "Container service for titiler" image container_image - volume :"/store/imagery" => "/store/imagery", + volume :"/store/imagery" => "/store/imagery", :"/srv/imagery/sockets" => "/sockets" - environment :GDAL_CACHEMAX => 200, - :GDAL_BAND_BLOCK_CACHE => "HASHSET", - :GDAL_DISABLE_READDIR_ON_OPEN => "EMPTY_DIR", - :GDAL_INGESTED_BYTES_AT_OPEN => 32768, - :GDAL_HTTP_MERGE_CONSECUTIVE_RANGES => "YES", - :GDAL_HTTP_MULTIPLEX => "YES", - :GDAL_HTTP_VERSION => 2, - :VSI_CACHE => "TRUE", - :VSI_CACHE_SIZE => 5000000, - :TITILER_API_ROOT_PATH => "/api/v1/titiler", - :FORWARDED_ALLOW_IPS => "*" # https://docs.gunicorn.org/en/latest/settings.html#forwarded-allow-ips + environment :GDAL_CACHEMAX => 200, + :GDAL_BAND_BLOCK_CACHE => "HASHSET", + :GDAL_DISABLE_READDIR_ON_OPEN => "EMPTY_DIR", + :GDAL_INGESTED_BYTES_AT_OPEN => 32768, + :GDAL_HTTP_MERGE_CONSECUTIVE_RANGES => "YES", + :GDAL_HTTP_MULTIPLEX => "YES", + :GDAL_HTTP_VERSION => 2, + :VSI_CACHE => "TRUE", + :VSI_CACHE_SIZE => 5000000, + :TITILER_API_ROOT_PATH => "/api/v1/titiler", + :FORWARDED_ALLOW_IPS => "*" # https://docs.gunicorn.org/en/latest/settings.html#forwarded-allow-ips command "gunicorn -k uvicorn.workers.UvicornWorker titiler.application.main:app --bind unix:/sockets/titiler.sock --workers #{node.cpu_cores}" end diff --git a/cookbooks/imagery/resources/site.rb b/cookbooks/imagery/resources/site.rb index 7151c68c6..b4079f0ca 100644 --- a/cookbooks/imagery/resources/site.rb +++ b/cookbooks/imagery/resources/site.rb @@ -87,7 +87,7 @@ action :create do end layers = Dir.glob("/srv/imagery/layers/#{new_resource.site}/*.yml").collect do |path| - YAML.safe_load(::File.read(path), :permitted_classes => [Symbol]) + YAML.safe_load_file(path, :permitted_classes => [Symbol]) end declare_resource :template, "/srv/#{new_resource.site}/imagery.js" do diff --git a/cookbooks/kibana/recipes/default.rb b/cookbooks/kibana/recipes/default.rb index 2f2fc6f28..2085cdce2 100644 --- a/cookbooks/kibana/recipes/default.rb +++ b/cookbooks/kibana/recipes/default.rb @@ -79,7 +79,7 @@ end node[:kibana][:sites].each do |name, details| file "/etc/kibana/#{name}.yml" do - content YAML.dump(YAML.safe_load(File.read("/opt/kibana-#{version}/config/kibana.yml")).merge( + content YAML.dump(YAML.safe_load_file("/opt/kibana-#{version}/config/kibana.yml").merge( "port" => details[:port], "host" => "127.0.0.1", "elasticsearch_url" => details[:elasticsearch_url], diff --git a/cookbooks/logstash/recipes/default.rb b/cookbooks/logstash/recipes/default.rb index 46bc3fea0..a4b237c00 100644 --- a/cookbooks/logstash/recipes/default.rb +++ b/cookbooks/logstash/recipes/default.rb @@ -75,10 +75,8 @@ template "/etc/cron.daily/expire-logstash" do mode "755" end -forwarders = [] - -search(:node, "recipes:logstash\\:\\:forwarder").each do |forwarder| - forwarders.append(forwarder.ipaddresses(:role => :external)) +forwarders = search(:node, "recipes:logstash\\:\\:forwarder").map do |forwarder| + forwarder.ipaddresses(:role => :external) end search(:node, "roles:gateway").each do |forwarder| diff --git a/cookbooks/matomo/attributes/default.rb b/cookbooks/matomo/attributes/default.rb index f4eb0c061..56ae90ca7 100644 --- a/cookbooks/matomo/attributes/default.rb +++ b/cookbooks/matomo/attributes/default.rb @@ -1,4 +1,4 @@ -default[:matomo][:version] = "5.1.1" +default[:matomo][:version] = "5.3.1" default[:matomo][:plugins] = { "Actions" => nil, "Annotations" => nil, @@ -21,6 +21,7 @@ default[:matomo][:plugins] = { "Diagnostics" => nil, "Ecommerce" => nil, "Events" => nil, + "FeatureFlags" => nil, "Feedback" => nil, "GeoIp2" => nil, "Goals" => nil, diff --git a/cookbooks/matomo/templates/default/config.erb b/cookbooks/matomo/templates/default/config.erb index 457badc83..a98ec4d81 100644 --- a/cookbooks/matomo/templates/default/config.erb +++ b/cookbooks/matomo/templates/default/config.erb @@ -7,11 +7,11 @@ password = "<%= @passwords['database'] %>" dbname = "piwik" tables_prefix = "piwik_" charset = "utf8mb4" - +collation = "utf8mb4_unicode_ci" + [General] force_ssl = 1 -force_ssl_login = 1 -login_allowlist_apply_to_reporting_api_requests = "0" +login_allowlist_apply_to_reporting_api_requests = 0 proxy_client_headers[] = "HTTP_X_FORWARDED_FOR" trusted_hosts[] = "matomo.openstreetmap.org" trusted_hosts[] = "piwik.openstreetmap.org" diff --git a/cookbooks/mediawiki/resources/extension.rb b/cookbooks/mediawiki/resources/extension.rb index 6240f6e87..fea8614d2 100644 --- a/cookbooks/mediawiki/resources/extension.rb +++ b/cookbooks/mediawiki/resources/extension.rb @@ -74,7 +74,7 @@ action :create do user node[:mediawiki][:user] group node[:mediawiki][:group] mode "664" - variables new_resource.variables + variables new_resource.variables.merge(:site => new_resource.site) end else file "#{mediawiki_directory}/LocalSettings.d/Ext-#{new_resource.extension}.inc.php" do diff --git a/cookbooks/mediawiki/resources/site.rb b/cookbooks/mediawiki/resources/site.rb index 096484a99..e87c0f5ce 100644 --- a/cookbooks/mediawiki/resources/site.rb +++ b/cookbooks/mediawiki/resources/site.rb @@ -44,13 +44,17 @@ property :private_site, :kind_of => [TrueClass, FalseClass], :default => false property :hcaptcha_public_key, :kind_of => String, :default => "" property :hcaptcha_private_key, :kind_of => String, :default => "" property :extra_file_extensions, :kind_of => [String, Array], :default => [] +property :namespaces, :kind_of => Hash, :default => {} +property :force_ui_messages, :kind_of => Array, :default => [] +property :watch_category_membership, :kind_of => [TrueClass, FalseClass], :default => false property :fpm_max_children, :kind_of => Integer, :default => 5 property :fpm_start_servers, :kind_of => Integer, :default => 2 property :fpm_min_spare_servers, :kind_of => Integer, :default => 1 property :fpm_max_spare_servers, :kind_of => Integer, :default => 3 -property :fpm_request_terminate_timeout, :kind_of => Integer, :default => 300 +property :fpm_request_terminate_timeout, :kind_of => Integer, :default => 120 property :fpm_prometheus_port, :kind_of => Integer property :reload_apache, :kind_of => [TrueClass, FalseClass], :default => true +property :backup_enabled, :kind_of => [TrueClass, FalseClass], :default => true action :create do node.default[:mediawiki][:sites][new_resource.site] = { @@ -96,6 +100,7 @@ action :create do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "664" + variables :version => new_resource.version end execute "#{mediawiki_directory}/composer.json" do @@ -172,6 +177,12 @@ action :create do :directory => site_directory, :database_params => database_params only_if { ::File.exist?("#{mediawiki_directory}/LocalSettings.php") } + only_if { new_resource.backup_enabled } + end + + file "/etc/cron.daily/mediawiki-#{cron_name}-backup" do + action :delete + not_if { new_resource.backup_enabled } end # MobileFrontend extension is required by MinervaNeue skin @@ -472,7 +483,7 @@ action :create do request_terminate_timeout new_resource.fpm_request_terminate_timeout php_admin_values "open_basedir" => "#{site_directory}/:/usr/share/php/:/dev/null:/tmp/" php_values "memory_limit" => "500M", - "max_execution_time" => "240", + "max_execution_time" => "60", "upload_max_filesize" => "70M", "post_max_size" => "100M" prometheus_port new_resource.fpm_prometheus_port @@ -559,7 +570,8 @@ action_class do end def mediawiki_reference - shell_out!("git", "ls-remote", "--refs", "--sort=-version:refname", + shell_out!("git", "-c", "versionsort.suffix=-rc", + "ls-remote", "--refs", "--sort=-version:refname", "https://gerrit.wikimedia.org/r/mediawiki/core.git", "refs/tags/#{new_resource.version}.*") .stdout @@ -596,7 +608,10 @@ action_class do :site_readonly => new_resource.site_readonly, :extra_file_extensions => new_resource.extra_file_extensions, :private_accounts => new_resource.private_accounts, - :private_site => new_resource.private_site + :private_site => new_resource.private_site, + :namespaces => new_resource.namespaces, + :force_ui_messages => new_resource.force_ui_messages, + :watch_category_membership => new_resource.watch_category_membership } end diff --git a/cookbooks/mediawiki/templates/default/LocalSettings.php.erb b/cookbooks/mediawiki/templates/default/LocalSettings.php.erb index 93aab016d..9671130d8 100644 --- a/cookbooks/mediawiki/templates/default/LocalSettings.php.erb +++ b/cookbooks/mediawiki/templates/default/LocalSettings.php.erb @@ -293,95 +293,25 @@ $wgExpensiveParserFunctionLimit = 500; $wgSiteNotice = "<%= @mediawiki[:site_notice] %>"; <% end -%> <% if @mediawiki[:site_readonly] -%> -$wgReadOnly = "<%= @mediawiki[:site_readonly] %>"; +$wgReadOnly = ( PHP_SAPI === 'cli' ) ? false : "<%= @mediawiki[:site_readonly] %>"; <% end -%> -<% if @name == "wiki.openstreetmap.org" -%> -# DE -define('NS_LANG_DE', 200); -$wgExtraNamespaces[NS_LANG_DE] = 'DE'; -$wgNamespacesWithSubpages[NS_LANG_DE] = TRUE; -$wgContentNamespaces[] = NS_LANG_DE; -define('NS_LANG_DE_TALK', 201); -$wgExtraNamespaces[NS_LANG_DE_TALK] = 'DE_talk'; -$wgNamespacesWithSubpages[NS_LANG_DE_TALK] = TRUE; - -# FR -define('NS_LANG_FR', 202); -$wgExtraNamespaces[NS_LANG_FR] = 'FR'; -$wgNamespacesWithSubpages[NS_LANG_FR] = TRUE; -$wgContentNamespaces[] = NS_LANG_FR; -define('NS_LANG_FR_TALK', 203); -$wgExtraNamespaces[NS_LANG_FR_TALK] = 'FR_talk'; -$wgNamespacesWithSubpages[NS_LANG_FR_TALK] = TRUE; - -# ES -define('NS_LANG_ES', 204); -$wgExtraNamespaces[NS_LANG_ES] = 'ES'; -$wgNamespacesWithSubpages[NS_LANG_ES] = TRUE; -$wgContentNamespaces[] = NS_LANG_ES; -define('NS_LANG_ES_TALK', 205); -$wgExtraNamespaces[NS_LANG_ES_TALK] = 'ES_talk'; -$wgNamespacesWithSubpages[NS_LANG_ES_TALK] = TRUE; - -# IT -define('NS_LANG_IT', 206); -$wgExtraNamespaces[NS_LANG_IT] = 'IT'; -$wgNamespacesWithSubpages[NS_LANG_IT] = TRUE; -$wgContentNamespaces[] = NS_LANG_IT; -define('NS_LANG_IT_TALK', 207); -$wgExtraNamespaces[NS_LANG_IT_TALK] = 'IT_talk'; -$wgNamespacesWithSubpages[NS_LANG_IT_TALK] = TRUE; - -# NL -define('NS_LANG_NL', 208); -$wgExtraNamespaces[NS_LANG_NL] = 'NL'; -$wgNamespacesWithSubpages[NS_LANG_NL] = TRUE; -$wgContentNamespaces[] = NS_LANG_NL; -define('NS_LANG_NL_TALK', 209); -$wgExtraNamespaces[NS_LANG_NL_TALK] = 'NL_talk'; -$wgNamespacesWithSubpages[NS_LANG_NL_TALK] = TRUE; - -# RU -define('NS_LANG_RU', 210); -$wgExtraNamespaces[NS_LANG_RU] = 'RU'; -$wgNamespacesWithSubpages[NS_LANG_RU] = TRUE; -$wgContentNamespaces[] = NS_LANG_RU; -define('NS_LANG_RU_TALK', 211); -$wgExtraNamespaces[NS_LANG_RU_TALK] = 'RU_talk'; -$wgNamespacesWithSubpages[NS_LANG_RU_TALK] = TRUE; - -# JA -define('NS_LANG_JA', 212); -$wgExtraNamespaces[NS_LANG_JA] = 'JA'; -$wgNamespacesWithSubpages[NS_LANG_JA] = TRUE; -$wgContentNamespaces[] = NS_LANG_JA; -define('NS_LANG_JA_TALK', 213); -$wgExtraNamespaces[NS_LANG_JA_TALK] = 'JA_talk'; -$wgNamespacesWithSubpages[NS_LANG_JA_TALK] = TRUE; - -# Proposal -# namespace features a specific search weight defined at -# cookbooks/mediawiki/templates/default/mw-ext-CirrusSearch.inc.php.erb -define('NS_PROPOSAL', 3000); -$wgExtraNamespaces[NS_PROPOSAL] = 'Proposal'; -$wgNamespacesWithSubpages[NS_PROPOSAL] = TRUE; -$wgContentNamespaces[] = NS_PROPOSAL; -define('NS_PROPOSAL_TALK', 3001); -$wgExtraNamespaces[NS_PROPOSAL_TALK] = 'Proposal_talk'; -$wgNamespacesWithSubpages[NS_PROPOSAL_TALK] = TRUE; - -$wgNamespacesToBeSearchedDefault[NS_LANG_DE] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_FR] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_ES] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_IT] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_NL] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_RU] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_JA] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_PROPOSAL] = TRUE; +<% @mediawiki[:namespaces].each do |name, details| -%> +# <%= name %> +define('NS_<%= name.upcase %>', <%= details[:id] %>); +$wgExtraNamespaces[NS_<%= name.upcase %>] = '<%= name %>'; +$wgNamespacesWithSubpages[NS_<%= name.upcase %>] = TRUE; +$wgContentNamespaces[] = NS_<%= name.upcase %>; +$wgNamespacesToBeSearchedDefault[NS_<%= name.upcase %>] = TRUE; +define('NS_<%= name.upcase %>_TALK', <%= details[:talk_id] %>); +$wgExtraNamespaces[NS_<%= name.upcase %>_TALK] = '<%= name %>_talk'; +$wgNamespacesWithSubpages[NS_<%= name.upcase %>_TALK] = TRUE; +<% end -%> + +<% unless @mediawiki[:force_ui_messages].empty? -%> # defines which links of the sidebar are translatable -$wgForceUIMsgAsContentMsg = array( 'mainpage-url', 'mapfeatures-url', 'contributors-url', 'helppage', 'blogs-url', 'shop-url', 'sitesupport-url' ); +$wgForceUIMsgAsContentMsg = array( '<%= @mediawiki[:force_ui_messages].join("', '") %>' ); <% end -%> # load extensions @@ -389,8 +319,8 @@ $wgForceUIMsgAsContentMsg = array( 'mainpage-url', 'mapfeatures-url', 'contribut <%= "require_once('#{file}');" %> <% end -%> -<% if @name == "wiki.openstreetmap.org" -%> -# wiki.openstreetmap.org specific config loaded after extensions +<% if @mediawiki[:watch_category_membership] -%> +# must be after extensions are loaded $wgRCWatchCategoryMembership = true; <% end -%> @@ -432,3 +362,13 @@ unset( $wgGroupsRemoveFromSelf['autoconfirmed'] ); # Mediawiki 1.38 has fix to allow this to be set by $wgVirtualRestConfig # https://phabricator.wikimedia.org/T285478 $wgHTTPTimeout = 240; + +# Enable night mode for Minerva and Vector skins +# https://github.com/openstreetmap/operations/issues/1230 +$wgMinervaNightMode['base'] = true; +$wgVectorNightMode['logged_in'] = true; +$wgVectorNightMode['logged_out'] = true; + +# Set extremely low timeout to avoid PHP-FPM timeouts on slow connections to Wikimedia Commons (rate limiting) or similar +$wgHTTPMaxTimeout = 5; +$wgHTTPMaxConnectTimeout = 3; diff --git a/cookbooks/mediawiki/templates/default/composer.local.json.erb b/cookbooks/mediawiki/templates/default/composer.local.json.erb index 338a0de9d..37382c5f6 100644 --- a/cookbooks/mediawiki/templates/default/composer.local.json.erb +++ b/cookbooks/mediawiki/templates/default/composer.local.json.erb @@ -8,6 +8,7 @@ } }, "require": { +<% if @version.to_f < 1.43 -%> "guzzlehttp/psr7": "2.4.5", "data-values/common": "1.0.0", "data-values/data-values": "3.0.0", @@ -16,5 +17,6 @@ "data-values/number": "0.11.1", "data-values/serialization": "1.2.4", "data-values/time": "1.0.4" +<% end -%> } } diff --git a/cookbooks/mysql/recipes/default.rb b/cookbooks/mysql/recipes/default.rb index baeff798e..63f70395b 100644 --- a/cookbooks/mysql/recipes/default.rb +++ b/cookbooks/mysql/recipes/default.rb @@ -28,7 +28,7 @@ mysql_variant = if platform?("ubuntu") package "#{mysql_variant}-server" package "#{mysql_variant}-client" -service "#{mysql_variant}" do +service mysql_variant do action [:enable, :start] supports :status => true, :restart => true end diff --git a/cookbooks/networking/recipes/default.rb b/cookbooks/networking/recipes/default.rb index 831878ab4..efc79e431 100644 --- a/cookbooks/networking/recipes/default.rb +++ b/cookbooks/networking/recipes/default.rb @@ -23,8 +23,6 @@ require "ipaddr" require "yaml" -include_recipe "ruby" - keys = data_bag_item("networking", "keys") file "/etc/netplan/00-installer-config.yaml" do @@ -59,7 +57,7 @@ interfaces = node[:networking][:interfaces].collect do |name, interface| [interface[:interface], name] end.to_h -node[:networking][:interfaces].each do |_, interface| +node[:networking][:interfaces].each_value do |interface| next unless interface[:interface] =~ /^(.*)\.(\d+)$/ vlan_interface = Regexp.last_match(1) @@ -73,7 +71,7 @@ node[:networking][:interfaces].each do |_, interface| node.default[:networking][:interfaces][parent][:vlans] << vlan_id end -node[:networking][:interfaces].each do |_, interface| +node[:networking][:interfaces].each_value do |interface| if interface[:interface] =~ /^.*\.(\d+)$/ template "/etc/systemd/network/10-#{interface[:interface]}.netdev" do source "vlan.netdev.erb" @@ -318,7 +316,11 @@ link "/etc/resolv.conf" do to "../run/systemd/resolve/stub-resolv.conf" end -gem_package "dbus-systemd" +package "ruby" + +gem_package "dbus-systemd" do + gem_binary node[:ruby][:system_gem] +end prometheus_exporter "resolved" do port 10028 @@ -341,10 +343,8 @@ end package "nftables" -interfaces = [] - -node.interfaces(:role => :external).each do |interface| - interfaces << interface[:interface] +interfaces = node.interfaces(:role => :external).map do |interface| + interface[:interface] end template "/etc/nftables.conf" do diff --git a/cookbooks/overpass/recipes/default.rb b/cookbooks/overpass/recipes/default.rb index 665b6730b..ffd5f49ac 100644 --- a/cookbooks/overpass/recipes/default.rb +++ b/cookbooks/overpass/recipes/default.rb @@ -107,7 +107,7 @@ apache_site "default" do action :disable end -apache_site "#{node[:overpass][:fqdn]}" do +apache_site node[:overpass][:fqdn] do template "apache.erb" directory "#{basedir}/site" variables :script_directory => "#{basedir}/cgi-bin" @@ -152,7 +152,7 @@ systemd_service "overpass-dispatcher" do description "Overpass Main Dispatcher" wants ["overpass-area-dispatcher.service"] working_directory basedir - exec_start "#{basedir}/bin/dispatcher --osm-base #{meta_map_short[node[:overpass][:meta_mode]]} --db-dir=#{basedir}/db --rate-limit=#{node[:overpass][:rate_limit]} --space=#{node[:overpass][:dispatcher_space]}" + exec_start "#{basedir}/bin/dispatcher --allow-duplicate-queries=yes --osm-base #{meta_map_short[node[:overpass][:meta_mode]]} --db-dir=#{basedir}/db --rate-limit=#{node[:overpass][:rate_limit]} --space=#{node[:overpass][:dispatcher_space]}" exec_stop "#{basedir}/bin/dispatcher --osm-base --terminate" standard_output "append:#{logdir}/osm_base.log" user username @@ -166,7 +166,7 @@ systemd_service "overpass-area-dispatcher" do description "Overpass Area Dispatcher" after ["overpass-dispatcher.service"] working_directory basedir - exec_start "#{basedir}/bin/dispatcher --areas #{meta_map_short[node[:overpass][:meta_mode]]} --db-dir=#{basedir}/db" + exec_start "#{basedir}/bin/dispatcher --allow-duplicate-queries=yes --areas #{meta_map_short[node[:overpass][:meta_mode]]} --db-dir=#{basedir}/db" exec_stop "#{basedir}/bin/dispatcher --areas --terminate" standard_output "append:#{logdir}/areas.log" user username diff --git a/cookbooks/podman/resources/service.rb b/cookbooks/podman/resources/service.rb index d9a328192..5178980d9 100644 --- a/cookbooks/podman/resources/service.rb +++ b/cookbooks/podman/resources/service.rb @@ -36,10 +36,10 @@ action :create do notify_access "all" environment "PODMAN_SYSTEMD_UNIT" => "%n" exec_start_pre "/bin/rm --force %t/%n.ctr-id" - exec_start "/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon "\ - "--userns=auto --label=io.containers.autoupdate=registry "\ - "--pids-limit=-1 #{publish_options} #{environment_options} "\ - "#{volume_options} --rm --sdnotify=conmon --detach --replace "\ + exec_start "/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon " \ + "--userns=auto --label=io.containers.autoupdate=registry " \ + "--pids-limit=-1 #{publish_options} #{environment_options} " \ + "#{volume_options} --rm --sdnotify=conmon --detach --replace " \ "--name=%N #{new_resource.image} #{new_resource.command}" exec_stop "/usr/bin/podman stop --ignore --time=10 --cidfile=%t/%n.ctr-id" exec_stop_post "/usr/bin/podman rm --force --ignore --cidfile=%t/%n.ctr-id" diff --git a/cookbooks/podman/resources/site.rb b/cookbooks/podman/resources/site.rb index 7cab5a5d2..225021eb7 100644 --- a/cookbooks/podman/resources/site.rb +++ b/cookbooks/podman/resources/site.rb @@ -67,7 +67,7 @@ action_class do def ports @ports ||= if ::File.exist?(ports_file) - YAML.safe_load(::File.read(ports_file)) + YAML.safe_load_file(ports_file) else {} end diff --git a/cookbooks/postgresql/libraries/postgresql.rb b/cookbooks/postgresql/libraries/postgresql.rb index b2df4aed6..789120d6b 100644 --- a/cookbooks/postgresql/libraries/postgresql.rb +++ b/cookbooks/postgresql/libraries/postgresql.rb @@ -122,7 +122,7 @@ module OpenStreetMap def schemas(database) @schemas ||= {} @schemas[database] ||= query("SELECT n.nspname, pg_catalog.pg_get_userbyid(n.nspowner) AS usename, n.nspacl FROM pg_namespace AS n WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema'", :database => database).each_with_object({}) do |schema, schemas| - name = "#{schema[:nspname]}" + name = schema[:nspname] schemas[name] = { :owner => schema[:usename], @@ -163,7 +163,7 @@ module OpenStreetMap def parse_acl(acl) parse_array(acl).each_with_object({}) do |entry, permissions| - entry = entry.sub(/^"(.*)"$/) { Regexp.last_match[1].gsub(/\\"/, '"') }.sub(%r{/.*$}, "") + entry = entry.sub(/^"(.*)"$/) { Regexp.last_match[1].gsub('\"', '"') }.sub(%r{/.*$}, "") user, privileges = entry.split("=") user = user.sub(/^"(.*)"$/, "\\1") diff --git a/cookbooks/postgresql/resources/schema.rb b/cookbooks/postgresql/resources/schema.rb index a7bf0ebdb..e22324d9b 100644 --- a/cookbooks/postgresql/resources/schema.rb +++ b/cookbooks/postgresql/resources/schema.rb @@ -109,6 +109,6 @@ action_class do end def qualified_name - "#{new_resource.name}" + new_resource.name end end diff --git a/cookbooks/ruby/attributes/default.rb b/cookbooks/ruby/attributes/default.rb index 194f29957..39ac754e5 100644 --- a/cookbooks/ruby/attributes/default.rb +++ b/cookbooks/ruby/attributes/default.rb @@ -1,5 +1,16 @@ default[:ruby][:fullstaq] = true +default[:ruby][:system_version] = if platform?("debian") + "3.1" + elsif node[:lsb][:release].to_f < 22.04 + "2.7" + else + "3.0" + end +default[:ruby][:system_interpreter] = "/usr/bin/ruby#{node[:ruby][:system_version]}" +default[:ruby][:system_gem] = "/usr/bin/gem#{node[:ruby][:system_version]}" +default[:ruby][:system_bundle] = "/usr/bin/bundle#{node[:ruby][:system_version]}" + if node[:ruby][:fullstaq] default[:ruby][:version] = "3.4" @@ -9,15 +20,9 @@ if node[:ruby][:fullstaq] else - default[:ruby][:version] = if platform?("debian") - "3.1" - elsif node[:lsb][:release].to_f < 22.04 - "2.7" - else - "3.0" - end - default[:ruby][:interpreter] = "/usr/bin/ruby#{node[:ruby][:version]}" - default[:ruby][:gem] = "/usr/bin/gem#{node[:ruby][:version]}" - default[:ruby][:bundle] = "/usr/bin/bundle#{node[:ruby][:version]}" + default[:ruby][:version] = node[:ruby][:system_version] + default[:ruby][:interpreter] = node[:ruby][:system_interpreter] + default[:ruby][:gem] = node[:ruby][:system_gem] + default[:ruby][:bundle] = node[:ruby][:system_bundle] end diff --git a/cookbooks/ruby/recipes/default.rb b/cookbooks/ruby/recipes/default.rb index 7301e372c..886f18fe2 100644 --- a/cookbooks/ruby/recipes/default.rb +++ b/cookbooks/ruby/recipes/default.rb @@ -38,7 +38,7 @@ if node[:ruby][:fullstaq] else - package %W[ + package %w[ ruby ruby-dev ruby-bundler diff --git a/cookbooks/stateofthemap/recipes/container.rb b/cookbooks/stateofthemap/recipes/container.rb index 8e97accc0..af8514fa7 100644 --- a/cookbooks/stateofthemap/recipes/container.rb +++ b/cookbooks/stateofthemap/recipes/container.rb @@ -30,3 +30,8 @@ end aliases ["#{year}.stateofthemap.com", "#{year}.sotm.org"] end end + +podman_site "stateofthemap.eu" do + image "ghcr.io/openstreetmap/stateofthemap-eu-website:latest" + aliases ["www.stateofthemap.eu"] +end diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb index f969546af..782b7ad86 100644 --- a/cookbooks/tile/recipes/default.rb +++ b/cookbooks/tile/recipes/default.rb @@ -566,8 +566,8 @@ systemd_service "expire-tiles" do sandbox true restrict_address_families "AF_UNIX" read_write_paths tile_directories + [ - "/var/lib/replicate/expire-queue" - ] + "/var/lib/replicate/expire-queue" + ] end systemd_path "expire-tiles" do diff --git a/cookbooks/vectortile/attributes/default.rb b/cookbooks/vectortile/attributes/default.rb index f72621213..6ad85e7ec 100644 --- a/cookbooks/vectortile/attributes/default.rb +++ b/cookbooks/vectortile/attributes/default.rb @@ -9,6 +9,8 @@ default[:vectortile][:replication][:tileupdate] = true default[:vectortile][:replication][:threads] = node.cpu_cores default[:vectortile][:tilekiln][:version] = "0.7.1" +default[:vectortile][:spirit][:version] = "7c68ecdd82606fd64dfe6e2ba7a1f1741afcc34c" +default[:vectortile][:themepark][:version] = "beb454cc56e88533fb398ab293489c4e91f4d42b" default[:postgresql][:versions] |= [node[:vectortile][:database][:cluster].split("/").first] default[:postgresql][:monitor_database] = "tiles" diff --git a/cookbooks/vectortile/recipes/default.rb b/cookbooks/vectortile/recipes/default.rb index 0f4041565..f4de30c52 100644 --- a/cookbooks/vectortile/recipes/default.rb +++ b/cookbooks/vectortile/recipes/default.rb @@ -77,6 +77,7 @@ package %w[ style_directory = "/srv/vector.openstreetmap.org/spirit" git style_directory do repository "https://github.com/pnorman/spirit.git" + revision node[:vectortile][:spirit][:version] user "tileupdate" group "tileupdate" end @@ -86,7 +87,7 @@ shortbread_config = "#{style_directory}/shortbread.yaml" themepark_directory = "/srv/vector.openstreetmap.org/osm2pgsql-themepark" git themepark_directory do repository "https://github.com/osm2pgsql-dev/osm2pgsql-themepark.git" - revision "444bfbda82dea2899e77ac7f0e88ddf7f62c3b45" + revision node[:vectortile][:themepark][:version] user "tileupdate" group "tileupdate" end @@ -122,7 +123,7 @@ template "/usr/local/bin/import-planet" do owner "root" group "root" mode "755" - variables :node_store_options => "#{node_store_options}" + variables :node_store_options => node_store_options end template "/usr/local/bin/tilekiln-storage-init" do @@ -130,7 +131,7 @@ template "/usr/local/bin/tilekiln-storage-init" do owner "root" group "root" mode "755" - variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :storage_database => "tiles", :config_path => "#{shortbread_config}" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :storage_database => "tiles", :config_path => shortbread_config end postgresql_user "tomh" do @@ -208,10 +209,10 @@ end end %w[addresses aerialways aeroways boundaries boundary_labels bridges buildings -dam_lines dam_polygons ferries land pier_lines pier_polygons place_labels -planet_osm_nodes planet_osm_rels planet_osm_ways pois public_transport railways -road_routes roads sites street_polygons streets_labels_points -streets_polygons_labels water_area_labels water_areas water_lines water_lines_labels].each do |table| + dam_lines dam_polygons ferries land pier_lines pier_polygons place_labels + planet_osm_nodes planet_osm_rels planet_osm_ways pois public_transport railways + road_routes roads sites street_polygons streets_labels_points + streets_polygons_labels water_area_labels water_areas water_lines water_lines_labels].each do |table| postgresql_table table do cluster node[:vectortile][:database][:cluster] database "spirit" @@ -250,7 +251,7 @@ template "/usr/local/bin/vector-update" do owner "root" group "root" mode "755" - variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => "#{shortbread_config}", :diff_size => "1000", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :post_processing => "/usr/local/bin/tiles-rerender" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => shortbread_config, :diff_size => "1000", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :post_processing => "/usr/local/bin/tiles-rerender" end rerender_layers = %w[addresses boundaries bridges buildings land pois public_transport sites street_polygons streets water_lines_labels water_lines water_polygons].join(" ") @@ -260,7 +261,7 @@ template "/usr/local/bin/tiles-rerender" do owner "root" group "root" mode "755" - variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :update_threads => 4, :layers => "#{rerender_layers}" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => shortbread_config, :expiry_dir => "/srv/vector.openstreetmap.org/data/", :update_threads => 4, :layers => rerender_layers.to_s end systemd_service "replicate" do diff --git a/cookbooks/web/resources/rails_port.rb b/cookbooks/web/resources/rails_port.rb index 681690452..b6209d50b 100644 --- a/cookbooks/web/resources/rails_port.rb +++ b/cookbooks/web/resources/rails_port.rb @@ -95,7 +95,7 @@ property :doorkeeper_signing_key, String property :user_account_deletion_delay, Integer action :create do - package %W[ + package %w[ imagemagick libvips42 nodejs @@ -415,14 +415,14 @@ action :create do recursive true end - bundle_config "#{rails_directory}" do + bundle_config rails_directory do user new_resource.user group new_resource.group settings "deployment" => "true", "build.nokogiri" => "--use-system-libraries" end - bundle_install "#{rails_directory}" do + bundle_install rails_directory do action :nothing user new_resource.user group new_resource.group @@ -453,7 +453,7 @@ action :create do only_if { new_resource.build_assets } end - bundle_exec "#{rails_directory}/app/assets/javascripts/i18n" do + bundle_exec "#{rails_directory}/config/i18n-js.yml" do action :nothing directory rails_directory command "rails i18n:js:export" @@ -463,7 +463,18 @@ action :create do user new_resource.user group new_resource.group subscribes :run, "git[#{rails_directory}]" - only_if { new_resource.build_assets } + only_if { new_resource.build_assets && ::File.exist?("#{rails_directory}/config/i18n-js.yml") } + end + + bundle_exec "#{rails_directory}/config/i18n.yml" do + action :nothing + directory rails_directory + command "i18n export" + environment "HOME" => rails_directory + user new_resource.user + group new_resource.group + subscribes :run, "git[#{rails_directory}]" + only_if { new_resource.build_assets && ::File.exist?("#{rails_directory}/config/i18n.yml") } end bundle_exec "#{rails_directory}/public/assets" do @@ -480,7 +491,8 @@ action :create do subscribes :run, "file[#{rails_directory}/config/settings.local.yml]" subscribes :run, "file[#{rails_directory}/config/storage.yml]" subscribes :run, "bundle_exec[#{rails_directory}/package.json]" - subscribes :run, "bundle_exec[#{rails_directory}/app/assets/javascripts/i18n]" + subscribes :run, "bundle_exec[#{rails_directory}/config/i18n-js.yml]" + subscribes :run, "bundle_exec[#{rails_directory}/config/i18n.yml]" only_if { new_resource.build_assets } end diff --git a/cookbooks/web/templates/default/apache.frontend.erb b/cookbooks/web/templates/default/apache.frontend.erb index 98097d7d9..f05b99b5c 100644 --- a/cookbooks/web/templates/default/apache.frontend.erb +++ b/cookbooks/web/templates/default/apache.frontend.erb @@ -2,7 +2,7 @@ # # Setup logging -# +# SetEnvIfNoCase Authorization "^Basic " AUTH_METHOD=basic SetEnvIfNoCase Authorization "^OAuth " AUTH_METHOD=oauth1 SetEnvIfNoCase Authorization "^Bearer " AUTH_METHOD=oauth2 @@ -108,6 +108,13 @@ ErrorLog /var/log/apache2/error.log RewriteCond "%{QUERY_STRING}" "^q=abcde&t=20" RewriteRule "^/api/0\.6/notes/search$" - [R=429,L] + # + # Ignore GoogleAssociationService request storm + # https://en.osm.town/@osm_tech/114205363076771822 + # + RewriteCond %{HTTP_USER_AGENT} "GoogleAssociationService" + RewriteRule "^/\.well-known/assetlinks\.json$" - [R=429,L] + # # Force special MIME type for crossdomain.xml files # diff --git a/cookbooks/web/templates/default/deliver-message.erb b/cookbooks/web/templates/default/deliver-message.erb index 76538183b..dddd647f1 100644 --- a/cookbooks/web/templates/default/deliver-message.erb +++ b/cookbooks/web/templates/default/deliver-message.erb @@ -3,4 +3,4 @@ export RAILS_ENV="production" export SECRET_KEY_BASE="<%= @secret_key_base %>" -exec /usr/local/bin/passenger-ruby /srv/www.openstreetmap.org/rails/script/deliver-message "$@" +exec <%= node[:ruby][:bundle] %> exec /srv/www.openstreetmap.org/rails/script/deliver-message "$@" diff --git a/cookbooks/wiki/attributes/default.rb b/cookbooks/wiki/attributes/default.rb index 0cde6a313..71f656567 100644 --- a/cookbooks/wiki/attributes/default.rb +++ b/cookbooks/wiki/attributes/default.rb @@ -1,2 +1,9 @@ -# Force apache to listen only on localhost -# default[:apache][:listen_address] = "127.0.0.1" +default[:wiki][:site_name] = "wiki.openstreetmap.org" +default[:wiki][:site_aliases] = [ + "wiki.osm.org", "wiki.openstreetmap.com", "wiki.openstreetmaps.org", + "osm.wiki", "www.osm.wiki", "wiki.osm.wiki" +] +default[:wiki][:site_notice] = nil +default[:wiki][:site_readonly] = nil +default[:wiki][:test_mode] = false +default[:wiki][:mediawiki_version] = "1.43" diff --git a/cookbooks/wiki/recipes/default.rb b/cookbooks/wiki/recipes/default.rb index 74e96d2cc..0590346f3 100644 --- a/cookbooks/wiki/recipes/default.rb +++ b/cookbooks/wiki/recipes/default.rb @@ -19,6 +19,8 @@ include_recipe "mediawiki" +site_name = node[:wiki][:site_name] + passwords = data_bag_item("wiki", "passwords") package "lua5.1" # newer versions do not work with Scribuntu! @@ -27,14 +29,15 @@ apache_site "default" do action [:disable] end -mediawiki_site "wiki.openstreetmap.org" do - aliases ["wiki.osm.org", "wiki.openstreetmap.com", "wiki.openstreetmaps.org", - "osm.wiki", "www.osm.wiki", "wiki.osm.wiki"] +mediawiki_site site_name do + aliases node[:wiki][:site_aliases] + + version node[:wiki][:mediawiki_version] - fpm_max_children 200 - fpm_start_servers 25 - fpm_min_spare_servers 25 - fpm_max_spare_servers 50 + fpm_max_children 300 + fpm_start_servers 50 + fpm_min_spare_servers 50 + fpm_max_spare_servers 150 fpm_prometheus_port 9253 database_name "wiki" @@ -54,54 +57,71 @@ mediawiki_site "wiki.openstreetmap.org" do hcaptcha_public_key "b67a410b-955e-4049-b432-f9c00e0202c0" hcaptcha_private_key passwords["hcaptcha"] - # site_notice "MAINTENANCE: WIKI READ-ONLY UNTIL Monday 16 May 2016 - 11:00am UTC/GMT." - # site_readonly "MAINTENANCE: WIKI READ-ONLY UNTIL Monday 16 May 2016 - 11:00am UTC/GMT." + namespaces "DE" => { :id => 200, :talk_id => 201 }, + "FR" => { :id => 202, :talk_id => 203 }, + "ES" => { :id => 204, :talk_id => 205 }, + "IT" => { :id => 206, :talk_id => 207 }, + "NL" => { :id => 208, :talk_id => 209 }, + "RU" => { :id => 210, :talk_id => 211 }, + "JA" => { :id => 212, :talk_id => 213 }, + "Proposal" => { :id => 3000, :talk_id => 3001 } + + force_ui_messages %w[mainpage-url mapfeatures-url contributors-url helppage blogs-url shop-url sitesupport-url] + + watch_category_membership true + + site_notice node[:wiki][:site_notice] + site_readonly node[:wiki][:site_readonly] + + if node[:wiki][:test_mode] + backup_enabled false + end end mediawiki_extension "CodeEditor" do - site "wiki.openstreetmap.org" + site site_name end mediawiki_extension "CodeMirror" do - site "wiki.openstreetmap.org" + site site_name end mediawiki_extension "Scribunto" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Scribunto.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "Wikibase" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Wikibase.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "OsmWikibase" do - site "wiki.openstreetmap.org" + site site_name repository "https://github.com/nyurik/OsmWikibase.git" reference "master" end mediawiki_extension "Echo" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Echo.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "Thanks" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Thanks.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "TimedMediaHandler" do - site "wiki.openstreetmap.org" + site site_name end mediawiki_extension "MultiMaps" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-MultiMaps.inc.php.erb" template_cookbook "wiki" variables :thunderforest_key => passwords["thunderforest"] @@ -109,37 +129,60 @@ mediawiki_extension "MultiMaps" do end mediawiki_extension "JsonConfig" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-JsonConfig.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "Kartographer" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Kartographer.inc.php.erb" template_cookbook "wiki" end -cookbook_file "/srv/wiki.openstreetmap.org/osm_logo_wiki.png" do +mediawiki_extension "TemplateStyles" do + site site_name +end + +mediawiki_extension "DynamicPageListEngine" do + site site_name + only_if { node[:wiki][:test_mode] } +end + +mediawiki_extension "WikibaseCirrusSearch" do + site site_name + template "mw-ext-WikibaseCirrusSearch.inc.php.erb" + template_cookbook "wiki" + only_if { node[:wiki][:test_mode] } +end + +mediawiki_extension "Translate" do + site site_name + template "mw-ext-Translate.inc.php.erb" + template_cookbook "wiki" + only_if { node[:wiki][:test_mode] } +end + +cookbook_file "/srv/#{site_name}/osm_logo_wiki.png" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "644" end -template "/srv/wiki.openstreetmap.org/robots.txt" do +template "/srv/#{site_name}/robots.txt" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "644" source "robots.txt.erb" end -cookbook_file "/srv/wiki.openstreetmap.org/favicon.ico" do +cookbook_file "/srv/#{site_name}/favicon.ico" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "644" end -directory "/srv/wiki.openstreetmap.org/dump" do +directory "/srv/#{site_name}/dump" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "0775" @@ -149,13 +192,13 @@ systemd_service "wiki-dump" do description "Wiki dump" type "oneshot" exec_start "/usr/bin/php w/maintenance/dumpBackup.php --full --quiet --output=gzip:dump/dump.xml.gz" - working_directory "/srv/wiki.openstreetmap.org" + working_directory "/srv/#{site_name}" user "wiki" nice 19 sandbox :enable_network => true memory_deny_write_execute false restrict_address_families "AF_UNIX" - read_write_paths "/srv/wiki.openstreetmap.org/dump" + read_write_paths "/srv/#{site_name}/dump" end systemd_timer "wiki-dump" do @@ -173,14 +216,14 @@ systemd_service "wiki-rdf-dump" do exec_start [ "/usr/bin/php w/extensions/Wikibase/repo/maintenance/dumpRdf.php --wiki wiki --format ttl --flavor full-dump --entity-type item --entity-type property --no-cache --output /tmp/wikibase-rdf.ttl", "/bin/gzip -9 /tmp/wikibase-rdf.ttl", - "/bin/mv /tmp/wikibase-rdf.ttl.gz /srv/wiki.openstreetmap.org/dump/wikibase-rdf.ttl.gz" + "/bin/mv /tmp/wikibase-rdf.ttl.gz /srv/#{site_name}/dump/wikibase-rdf.ttl.gz" ] - working_directory "/srv/wiki.openstreetmap.org" + working_directory "/srv/#{site_name}" user "wiki" sandbox :enable_network => true memory_deny_write_execute false restrict_address_families "AF_UNIX" - read_write_paths "/srv/wiki.openstreetmap.org/dump" + read_write_paths "/srv/#{site_name}/dump" end systemd_timer "wiki-rdf-dump" do diff --git a/cookbooks/wiki/templates/default/mw-ext-Translate.inc.php.erb b/cookbooks/wiki/templates/default/mw-ext-Translate.inc.php.erb new file mode 100644 index 000000000..0ae1afa71 --- /dev/null +++ b/cookbooks/wiki/templates/default/mw-ext-Translate.inc.php.erb @@ -0,0 +1,39 @@ + 'ttmserver', + 'class' => 'ElasticSearchTTMServer', + 'cutoff' => 0.75, + /* + * See http://elastica.io/getting-started/installation.html + * See https://github.com/ruflin/Elastica/blob/8.x/src/Client.php + */ + 'config' => [ 'servers' => [ [ 'host' => '127.0.0.1', 'port' => 9114 ] ] ] +]; diff --git a/cookbooks/wiki/templates/default/mw-ext-Wikibase.inc.php.erb b/cookbooks/wiki/templates/default/mw-ext-Wikibase.inc.php.erb index 6f7e8d6eb..d609a3d57 100644 --- a/cookbooks/wiki/templates/default/mw-ext-Wikibase.inc.php.erb +++ b/cookbooks/wiki/templates/default/mw-ext-Wikibase.inc.php.erb @@ -89,7 +89,7 @@ $wgWBClientSettings['showExternalRecentChanges'] = true; // Base URL for building links to the repository. // Assumes your wiki is setup as "http://repo.example.org/wiki/" // This can be protocol relative, such as "//www.wikidata.org" -$wgWBClientSettings['repoUrl'] = "https://wiki.openstreetmap.org"; +$wgWBClientSettings['repoUrl'] = "https://<%= @site %>"; // This setting is optional if you have the same type of setup for your // repo and client. It will default to using the client's $wgArticlePath setting, diff --git a/cookbooks/wiki/templates/default/mw-ext-WikibaseCirrusSearch.inc.php.erb b/cookbooks/wiki/templates/default/mw-ext-WikibaseCirrusSearch.inc.php.erb new file mode 100644 index 000000000..e6859b785 --- /dev/null +++ b/cookbooks/wiki/templates/default/mw-ext-WikibaseCirrusSearch.inc.php.erb @@ -0,0 +1,6 @@ + +User-agent: * +Disallow: / +<% else -%> User-agent: ia_archiver Allow: / @@ -31,3 +35,4 @@ Crawl-delay: 60 Sitemap: https://wiki.openstreetmap.org/sitemap-index-wiki.xml Host: wiki.openstreetmap.org +<% end -%> diff --git a/cookbooks/wordpress/resources/site.rb b/cookbooks/wordpress/resources/site.rb index 52bba4ce5..2c07ba31e 100644 --- a/cookbooks/wordpress/resources/site.rb +++ b/cookbooks/wordpress/resources/site.rb @@ -82,10 +82,10 @@ action :create do end wp_config = edit_file "#{site_directory}/wp-config-sample.php" do |line| - line.gsub!(/database_name_here/, new_resource.database_name) - line.gsub!(/username_here/, new_resource.database_user) - line.gsub!(/password_here/, new_resource.database_password) - line.gsub!(/wp_/, new_resource.database_prefix) + line.gsub!("database_name_here", new_resource.database_name) + line.gsub!("username_here", new_resource.database_user) + line.gsub!("password_here", new_resource.database_password) + line.gsub!("wp_", new_resource.database_prefix) line.gsub!(/('AUTH_KEY', *)'put your unique phrase here'/, "\\1'#{auth_key}'") line.gsub!(/('SECURE_AUTH_KEY', *)'put your unique phrase here'/, "\\1'#{secure_auth_key}'") diff --git a/roles/angor.rb b/roles/angor.rb index be08e517c..5fba17667 100644 --- a/roles/angor.rb +++ b/roles/angor.rb @@ -26,8 +26,8 @@ default_attributes( :gmoncrieff => { :status => :user }, :zander => { :status => :user }, :"za-imagery" => { - :status => :role, - :members => [:grant, :htonl, :gmoncrieff, :zander] + :status => :role, + :members => [:grant, :htonl, :gmoncrieff, :zander] } } } diff --git a/roles/dev.rb b/roles/dev.rb index 432fe0d1c..81a7f34d2 100644 --- a/roles/dev.rb +++ b/roles/dev.rb @@ -94,8 +94,8 @@ default_attributes( :members => [:apmon, :maba] }, :"za-imagery" => { - :status => :role, - :members => [:grant, :htonl, :gmoncrieff, :zander] + :status => :role, + :members => [:grant, :htonl, :gmoncrieff, :zander] } } }, diff --git a/roles/dribble.rb b/roles/dribble.rb index b0bba8021..2110271c3 100644 --- a/roles/dribble.rb +++ b/roles/dribble.rb @@ -36,6 +36,12 @@ default_attributes( :effective_cache_size => "350GB" } } + }, + :vectortile => { + :replication => { + :enabled => false, + :tileupdate => false + } } ) diff --git a/roles/fafnir.rb b/roles/fafnir.rb index e7edc263c..dfed57f7f 100644 --- a/roles/fafnir.rb +++ b/roles/fafnir.rb @@ -7,7 +7,6 @@ default_attributes( :last_address => "10.0.79.254" }, :exim => { - :external_interface => "<;${if <{${randint:100}}{75} {184.104.226.98;2001:470:1:b3b::2}{87.252.214.98;2001:4d78:fe03:1c::2}}", :routes => { :openstreetmap => { :comment => "openstreetmap.org", diff --git a/roles/geodns.rb b/roles/geodns.rb index a1df01532..74168f4b7 100644 --- a/roles/geodns.rb +++ b/roles/geodns.rb @@ -12,7 +12,7 @@ default_attributes( :list => false, :transfer_logging => false, :hosts_allow => [ - "184.104.226.102", # idris HE + "184.104.226.102", # idris HE "2001:470:1:b3b::6", # idris HE "87.252.214.102", # idris Equinix "2001:4d78:fe03:1c::6" # idris Equinix diff --git a/roles/lockheed.rb b/roles/lockheed.rb index 82acb48b4..34cfc05da 100644 --- a/roles/lockheed.rb +++ b/roles/lockheed.rb @@ -72,10 +72,10 @@ default_attributes( :max_size => "196608M" }, :proxy => { - :enable => true, - :keys_zone => "proxy_cache_zone:2048M", - :inactive => "180d", - :max_size => "196608M" + :enable => true, + :keys_zone => "proxy_cache_zone:2048M", + :inactive => "180d", + :max_size => "196608M" } } } diff --git a/roles/muirdris.rb b/roles/muirdris.rb index a2c578567..7ec8c2de2 100644 --- a/roles/muirdris.rb +++ b/roles/muirdris.rb @@ -2,9 +2,6 @@ name "muirdris" description "Master role applied to muirdris" default_attributes( - :memcached => { - :memory_limit => 128 * 1024 - }, :networking => { :interfaces => { :internal => { @@ -32,10 +29,23 @@ default_attributes( } } } + }, + :wiki => { + :site_name => "test.wiki.openstreetmap.org", + :site_aliases => [], + :site_notice => "TEST INSTANCE: Use wiki.openstreetmap.org for real work", + :test_mode => true + } +) + +override_attributes( + :memcached => { + :memory_limit => 128 * 1024 } ) run_list( "role[equinix-dub-public]", - "role[gps-tile]" + "role[gps-tile]", + "role[wiki]" ) diff --git a/roles/wiki.rb b/roles/wiki.rb index 652726346..c6cb02000 100644 --- a/roles/wiki.rb +++ b/roles/wiki.rb @@ -14,7 +14,7 @@ default_attributes( :server_limit => 32, :max_request_workers => 800, :threads_per_child => 50, - :max_connections_per_child => 10000 + :max_connections_per_child => 100000 }, :evasive => { :page_count => 400, diff --git a/test/integration/dev/inspec/mysql_spec.rb b/test/integration/dev/inspec/mysql_spec.rb index 549f33da9..628d8bee2 100644 --- a/test/integration/dev/inspec/mysql_spec.rb +++ b/test/integration/dev/inspec/mysql_spec.rb @@ -8,7 +8,7 @@ describe package("#{mysql_variant}-server") do it { should be_installed } end -describe service("#{mysql_variant}") do +describe service(mysql_variant) do it { should be_enabled } it { should be_running } end diff --git a/test/integration/mysql/inspec/mysql_spec.rb b/test/integration/mysql/inspec/mysql_spec.rb index 549f33da9..628d8bee2 100644 --- a/test/integration/mysql/inspec/mysql_spec.rb +++ b/test/integration/mysql/inspec/mysql_spec.rb @@ -8,7 +8,7 @@ describe package("#{mysql_variant}-server") do it { should be_installed } end -describe service("#{mysql_variant}") do +describe service(mysql_variant) do it { should be_enabled } it { should be_running } end