From: Tom Hughes Date: Mon, 2 Jun 2025 21:14:28 +0000 (+0100) Subject: Update bundle X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/HEAD?hp=332e5e0843799eb42132e229ecc64e05d507059f Update bundle --- diff --git a/.github/workflows/test-kitchen.yml b/.github/workflows/test-kitchen.yml index e1c8df555..0587f1bf5 100644 --- a/.github/workflows/test-kitchen.yml +++ b/.github/workflows/test-kitchen.yml @@ -127,6 +127,10 @@ jobs: suite: osqa - os: debian-12 suite: apt-repository + - os: debian-12 + suite: blogs + - os: debian-12 + suite: community - os: debian-12 suite: dev - os: debian-12 @@ -135,16 +139,34 @@ jobs: suite: git-server - os: debian-12 suite: git-web + - os: debian-12 + suite: gps-tile - os: debian-12 suite: imagery-tiler + - os: debian-12 + suite: irc - os: debian-12 suite: letsencrypt + - os: debian-12 + suite: matomo + - os: debian-12 + suite: nominatim - os: debian-12 suite: otrs + - os: debian-12 + suite: overpass + - os: debian-12 + suite: prometheus-server - os: debian-12 suite: serverinfo + - os: debian-12 + suite: subversion - os: debian-12 suite: supybot + - os: debian-12 + suite: taginfo + - os: debian-12 + suite: trac - os: debian-12 suite: vectortile - os: debian-12 @@ -153,9 +175,15 @@ jobs: suite: web-frontend - os: debian-12 suite: web-rails + - os: debian-12 + suite: wiki exclude: - suite: apt-repository os: ubuntu-2204 + - suite: blogs + os: ubuntu-2204 + - suite: community + os: ubuntu-2204 - suite: dev os: ubuntu-2204 - suite: dns @@ -164,18 +192,36 @@ jobs: os: ubuntu-2204 - suite: git-web os: ubuntu-2204 - - suite: mailman + - suite: gps-tile + os: ubuntu-2204 + - suite: irc os: ubuntu-2204 - suite: letsencrypt os: ubuntu-2204 + - suite: mailman + os: ubuntu-2204 + - suite: matomo + os: ubuntu-2204 + - suite: nominatim + os: ubuntu-2204 - suite: osqa os: ubuntu-2204 - suite: otrs os: ubuntu-2204 + - suite: overpass + os: ubuntu-2204 + - suite: prometheus-server + os: ubuntu-2204 - suite: serverinfo os: ubuntu-2204 + - suite: subversion + os: ubuntu-2204 - suite: supybot os: ubuntu-2204 + - suite: taginfo + os: ubuntu-2204 + - suite: trac + os: ubuntu-2204 - suite: vectortile os: ubuntu-2204 - suite: web-cgimap @@ -184,6 +230,8 @@ jobs: os: ubuntu-2204 - suite: web-rails os: ubuntu-2204 + - suite: wiki + os: ubuntu-2204 fail-fast: false steps: - name: Login to GitHub Container Registry diff --git a/.kitchen.yml b/.kitchen.yml index 88cfc7459..b04297118 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -62,6 +62,18 @@ platforms: - RUN /usr/bin/apt-get install -y eatmydata - RUN echo /usr/lib/$(uname -m)-linux-gnu/libeatmydata.so >>/etc/ld.so.preload +# When using podman we have to manually start chef to workaround a volume issue +lifecycle: + pre_create: | + if command -v podman >/dev/null 2>&1; then + podman create --name chef-latest --replace docker.io/chef/chef:latest sh + podman start chef-latest + fi + post_destroy: | + if command -v podman >/dev/null 2>&1; then + podman container rm -iv chef-latest + fi + suites: - name: accounts run_list: @@ -93,6 +105,9 @@ suites: - name: blogs run_list: - recipe[blogs::default] + attributes: + ruby: + version: 3.3 - name: chef run_list: - recipe[chef::default] @@ -402,6 +417,8 @@ suites: run_list: - recipe[taginfo::default] attributes: + ruby: + version: 3.3 taginfo: sites: - name: taginfo.example.com diff --git a/Gemfile.lock b/Gemfile.lock index 40032233d..88f6d2919 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,20 +1,23 @@ GEM remote: https://rubygems.org/ specs: - activesupport (7.1.4.2) + activesupport (7.1.5.1) base64 + benchmark (>= 0.3) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) + logger (>= 1.4.2) minitest (>= 5.1) mutex_m + securerandom (>= 0.3) tzinfo (~> 2.0) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) - ast (2.4.2) - aws-eventstream (1.3.0) + ast (2.4.3) + aws-eventstream (1.4.0) aws-partitions (1.863.0) aws-sdk-accessanalyzer (1.44.0) aws-sdk-core (~> 3, >= 3.188.0) @@ -256,8 +259,8 @@ GEM aws-sdk-wafv2 (1.74.0) aws-sdk-core (~> 3, >= 3.188.0) aws-sigv4 (~> 1.1) - aws-sigv2 (1.2.0) - aws-sigv4 (1.10.1) + aws-sigv2 (1.3.0) + aws-sigv4 (1.12.0) aws-eventstream (~> 1, >= 1.0.2) azure_graph_rbac (0.17.2) ms_rest_azure (~> 0.12.0) @@ -269,14 +272,14 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_storage (0.23.0) ms_rest_azure (~> 0.12.0) - base64 (0.2.0) + base64 (0.3.0) bcrypt_pbkdf (1.1.1) - bigdecimal (3.1.8) - bson (4.15.0) + benchmark (0.4.1) + bigdecimal (3.2.1) builder (3.3.0) - chef-config (18.5.0) + chef-config (18.7.10) addressable - chef-utils (= 18.5.0) + chef-utils (= 18.7.10) fuzzyurl mixlib-config (>= 2.2.12, < 4.0) mixlib-shellout (>= 2.0, < 4.0) @@ -284,77 +287,43 @@ GEM chef-telemetry (1.1.1) chef-config concurrent-ruby (~> 1.0) - chef-utils (18.5.0) + chef-utils (18.7.10) concurrent-ruby coderay (1.1.3) - concurrent-ruby (1.3.4) - connection_pool (2.4.1) - cookstyle (7.32.8) - rubocop (= 1.25.1) + concurrent-ruby (1.3.5) + connection_pool (2.5.3) + cookstyle (8.1.4) + rubocop (= 1.75.8) declarative (0.0.20) - diff-lcs (1.5.1) - docker-api (2.3.0) + diff-lcs (1.6.2) + docker-api (2.4.0) excon (>= 0.64.0) multi_json domain_name (0.6.20240107) - drb (2.2.1) - dry-configurable (1.2.0) - dry-core (~> 1.0, < 2) - zeitwerk (~> 2.6) - dry-core (1.0.1) - concurrent-ruby (~> 1.0) - zeitwerk (~> 2.6) - dry-inflector (1.1.0) - dry-logic (1.5.0) - concurrent-ruby (~> 1.0) - dry-core (~> 1.0, < 2) - zeitwerk (~> 2.6) - dry-struct (1.6.0) - dry-core (~> 1.0, < 2) - dry-types (>= 1.7, < 2) - ice_nine (~> 0.11) - zeitwerk (~> 2.6) - dry-types (1.7.2) - bigdecimal (~> 3.0) - concurrent-ruby (~> 1.0) - dry-core (~> 1.0) - dry-inflector (~> 1.0) - dry-logic (~> 1.4) - zeitwerk (~> 2.6) - ed25519 (1.3.0) - erubi (1.13.0) - excon (0.112.0) - faraday (1.10.4) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0) - faraday-multipart (~> 1.0) + drb (2.2.3) + ed25519 (1.4.0) + erubi (1.13.1) + excon (1.2.7) + logger + faraday (1.3.1) faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.0) - faraday-patron (~> 1.0) - faraday-rack (~> 1.0) - faraday-retry (~> 1.0) + multipart-post (>= 1.2, < 3) ruby2_keywords (>= 0.0.4) faraday-cookie_jar (0.0.7) faraday (>= 0.8.0) http-cookie (~> 1.0.0) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-follow_redirects (0.3.0) - faraday (>= 1, < 3) - faraday-httpclient (1.0.1) - faraday-multipart (1.0.4) - multipart-post (~> 2) faraday-net_http (1.0.2) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - faraday-rack (1.0.0) - faraday-retry (1.0.3) - faraday_middleware (1.2.1) + faraday_middleware (1.0.0) faraday (~> 1.0) - ffi (1.17.0) + ffi (1.17.2) + ffi (1.17.2-aarch64-linux-gnu) + ffi (1.17.2-aarch64-linux-musl) + ffi (1.17.2-arm-linux-gnu) + ffi (1.17.2-arm-linux-musl) + ffi (1.17.2-x86-linux-gnu) + ffi (1.17.2-x86-linux-musl) + ffi (1.17.2-x86_64-linux-gnu) + ffi (1.17.2-x86_64-linux-musl) fuzzyurl (0.9.0) google-apis-admin_directory_v1 (0.46.0) google-apis-core (>= 0.11.0, < 2.a) @@ -389,35 +358,26 @@ GEM gyoku (1.4.0) builder (>= 2.1.2) rexml (~> 3.0) - hashdiff (1.0.1) - hashie (5.0.0) - highline (3.1.1) - reline - http-cookie (1.0.7) + hashie (4.1.0) + http-cookie (1.0.8) domain_name (~> 0.5) - httpclient (2.8.3) - i18n (1.14.6) + httpclient (2.9.0) + mutex_m + i18n (1.14.7) concurrent-ruby (~> 1.0) - ice_nine (0.11.2) inifile (3.0.0) - inspec (5.22.58) - faraday_middleware (>= 0.12.2, < 1.3) - inspec-core (= 5.22.58) - mongo (= 2.13.2) - progress_bar (~> 1.3.3) - rake - train (~> 3.10) - train-aws (~> 0.2) + inspec (4.24.32) + faraday_middleware (>= 0.12.2, < 1.1) + inspec-core (= 4.24.32) + train (~> 3.0) + train-aws (~> 0.1) train-habitat (~> 0.1) - train-kubernetes (~> 0.1) train-winrm (~> 0.2) - inspec-core (5.22.58) + inspec-core (4.24.32) addressable (~> 2.4) - chef-telemetry (~> 1.0, >= 1.0.8) - cookstyle - faraday (>= 1, < 3) - faraday-follow_redirects (~> 0.3) - hashie (>= 3.4, < 6.0) + chef-telemetry (~> 1.0) + faraday (>= 0.9.0, < 1.4) + hashie (>= 3.4, < 5.0) license-acceptance (>= 0.2.13, < 3.0) method_source (>= 0.8, < 2.0) mixlib-log (~> 3.0) @@ -425,66 +385,55 @@ GEM parallel (~> 1.9) parslet (>= 1.5, < 3.0) pry (~> 0.13) - rspec (>= 3.9, <= 3.12) + rspec (~> 3.10) rspec-its (~> 1.2) rubyzip (>= 1.2.2, < 3.0) semverse (~> 3.0) sslshake (~> 1.2) - thor (>= 0.20, < 1.3.0) + thor (>= 0.20, < 2.0) tomlrb (>= 1.2, < 2.1) - train-core (~> 3.10) + train-core (~> 3.0) tty-prompt (~> 0.17) tty-table (~> 0.10) - io-console (0.7.2) jmespath (1.6.2) - json (2.7.3) - jsonpath (1.1.5) - multi_json - jwt (2.9.3) + json (2.12.2) + jwt (2.10.1) base64 - k8s-ruby (0.16.0) - dry-configurable - dry-struct - dry-types - excon (~> 0.71) - hashdiff (~> 1.0.0) - jsonpath (~> 1.1) - recursive-open-struct (~> 1.1.3) - yajl-ruby (~> 1.4.0) - yaml-safe_load_stream3 kitchen-dokken (2.20.7) docker-api (>= 1.33, < 3) lockfile (~> 2.1) test-kitchen (>= 1.15, < 4) - kitchen-inspec (2.6.2) + kitchen-inspec (3.0.0) hashie (>= 3.4, <= 5.0) - inspec (>= 2.2.64, < 6.0) + inspec (>= 2.2.64, < 7.0) test-kitchen (>= 2.7, < 4) + language_server-protocol (3.17.0.5) license-acceptance (2.1.13) pastel (~> 0.7) tomlrb (>= 1.2, < 3.0) tty-box (~> 0.6) tty-prompt (~> 0.20) + lint_roller (1.1.0) little-plugger (1.1.4) lockfile (2.1.3) + logger (1.7.0) logging (2.4.0) little-plugger (~> 1.1) multi_json (~> 1.14) method_source (1.1.0) mini_mime (1.1.5) - minitest (5.25.1) + minitest (5.25.5) mixlib-config (3.0.27) tomlrb mixlib-install (3.12.30) mixlib-shellout mixlib-versioning thor - mixlib-log (3.0.9) - mixlib-shellout (3.3.3) + mixlib-log (3.2.3) + ffi (>= 1.15.5) + mixlib-shellout (3.3.9) chef-utils mixlib-versioning (1.2.12) - mongo (2.13.2) - bson (>= 4.8.2, < 5.0.0) ms_rest (0.7.6) concurrent-ruby (~> 1.0) faraday (>= 0.9, < 2.0.0) @@ -496,77 +445,74 @@ GEM ms_rest (~> 0.7.6) multi_json (1.15.0) multipart-post (2.4.1) - mutex_m (0.2.0) - net-scp (4.0.0) + mutex_m (0.3.0) + net-scp (4.1.0) net-ssh (>= 2.6.5, < 8.0.0) net-ssh (7.3.0) net-ssh-gateway (2.0.0) net-ssh (>= 4.0.0) nori (2.7.1) bigdecimal - options (2.3.2) os (1.1.4) - parallel (1.26.3) - parser (3.3.5.0) + parallel (1.27.0) + parser (3.3.8.0) ast (~> 2.4.1) racc parslet (2.0.0) pastel (0.8.0) tty-color (~> 0.5) - progress_bar (1.3.4) - highline (>= 1.6) - options (~> 2.3.0) - pry (0.14.2) + prism (1.4.0) + pry (0.15.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (6.0.1) + public_suffix (6.0.2) racc (1.8.1) rainbow (3.1.1) - rake (13.2.1) - recursive-open-struct (1.1.3) - regexp_parser (2.9.2) - reline (0.5.10) - io-console (~> 0.5) + regexp_parser (2.10.0) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) retriable (3.1.2) - rexml (3.3.9) - rspec (3.12.0) - rspec-core (~> 3.12.0) - rspec-expectations (~> 3.12.0) - rspec-mocks (~> 3.12.0) - rspec-core (3.12.3) - rspec-support (~> 3.12.0) - rspec-expectations (3.12.4) + rexml (3.4.1) + rspec (3.13.1) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.4) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) + rspec-support (~> 3.13.0) rspec-its (1.3.1) rspec-core (>= 3.0.0) rspec-expectations (>= 3.0.0) - rspec-mocks (3.12.7) + rspec-mocks (3.13.5) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-support (3.12.2) - rubocop (1.25.1) + rspec-support (~> 3.13.0) + rspec-support (3.13.4) + rubocop (1.75.8) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) parallel (~> 1.10) - parser (>= 3.1.0.0) + parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml - rubocop-ast (>= 1.15.1, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.44.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.32.3) - parser (>= 3.3.1.0) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.44.1) + parser (>= 3.3.7.2) + prism (~> 1.4) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) rubyntlm (0.6.5) base64 - rubyzip (2.3.2) + rubyzip (2.4.1) + securerandom (0.4.1) semverse (3.0.2) - signet (0.19.0) + signet (0.20.0) addressable (~> 2.8) faraday (>= 0.17.5, < 3.a) jwt (>= 1.5, < 3.0) @@ -591,7 +537,7 @@ GEM winrm (~> 2.0) winrm-elevated (~> 1.0) winrm-fs (~> 1.1) - thor (1.2.2) + thor (1.3.2) timeliness (0.3.10) tomlrb (1.3.0) trailblazer-option (0.1.2) @@ -703,9 +649,6 @@ GEM net-scp (>= 1.2, < 5.0) net-ssh (>= 2.9, < 8.0) train-habitat (0.2.22) - train-kubernetes (0.2.1) - k8s-ruby (~> 0.16.0) - train (~> 3.0) train-winrm (0.2.13) winrm (>= 2.3.6, < 3.0) winrm-elevated (~> 1.2.2) @@ -753,12 +696,18 @@ GEM rubyzip (~> 2.0) winrm (~> 2.0) wisper (2.0.1) - yajl-ruby (1.4.3) - yaml-safe_load_stream3 (0.1.2) zeitwerk (2.6.18) PLATFORMS + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl ruby + x86-linux-gnu + x86-linux-musl + x86_64-linux-gnu + x86_64-linux-musl DEPENDENCIES cookstyle @@ -768,4 +717,4 @@ DEPENDENCIES zeitwerk (< 2.7) BUNDLED WITH - 2.2.16 + 2.6.2 diff --git a/README.md b/README.md index 86145db17..6b6304834 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This repository manages the configuration of all the servers run by the OpenStreetMap Foundation's Operations Working Group. We use -[Chef](https://www.chef.io/) to automated the configuration of all of our +[Chef](https://www.chef.io/) to automate the configuration of all of our servers. [OSMF Operations Working Group](https://operations.osmfoundation.org/) @@ -14,20 +14,20 @@ servers. We make extensive use of roles to configure the servers. In general we have: -## Server-specific roles (e.g. [faffy.rb](roles/faffy.rb)) +## Server-specific roles (e.g., [faffy.rb](roles/faffy.rb)) These deal with particular setup or quirks of a server, such as its IP address. They also include roles representing the service they are performing, and the location they are in and any particular hardware they have that needs configuration. All our servers are [named after dragons](https://wiki.openstreetmap.org/wiki/Servers/Name_Ideas). -## Hardware-specific roles (e.g. [hp-g9.rb](roles/hp-g9.rb)) +## Hardware-specific roles (e.g., [hp-g9.rb](roles/hp-g9.rb)) Covers anything specific to a certain piece of hardware, like a motherboard, that could apply to multiple machines. -## Location-specific roles (e.g. [equinix-dub.rb](roles/equinix-dub.rb)) +## Location-specific roles (e.g., [equinix-dub.rb](roles/equinix-dub.rb)) These form a hierarchy of datacentres, organisations, and countries where our servers are located. -## Service-specific roles (e.g. [web-frontend](roles/web-frontend.rb)) +## Service-specific roles (e.g., [web-frontend](roles/web-frontend.rb)) These cover the services that the server is running, and will include the recipes required for that service along with any specific configurations and other cascading roles. diff --git a/cookbooks/accounts/files/default/craig/.ssh/authorized_keys b/cookbooks/accounts/files/default/craig/.ssh/authorized_keys new file mode 100644 index 000000000..101e5e13c --- /dev/null +++ b/cookbooks/accounts/files/default/craig/.ssh/authorized_keys @@ -0,0 +1,2 @@ +# DO NOT EDIT - This file is being maintained by Chef - use authorized_keys2 instead +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc26tRbrQoczW3UFfXkdt6auqFg/Ut6spGMT476fFsJFjaYp98E2lca2W9vyJq4nSn0tdxwcO4LGK1ACdhZ/81I/68d7CPv5zNjJMehgwQ1BJTM5HWaap08cEINZMQ0xt6Neyz+HIFiaJVzxmyLJCnaaCeQX/t2NmL+nQV6rJq4qS2L434Bw1qGM73zjNja4bB2IN0y5yWDRTSLg+t+DKH26DC4OJn4+pxKsyt2egB7MNj9my1MRcjPVeo/bxz3nWoxKtX9dWq9UFrd7trfSXK+7Y+9fFHl41rrrYbn3UFKcDL6Rzvp2bFytDW6FlWmuptGajWnm2HpqI69bsO7uw1 diff --git a/cookbooks/apache/recipes/default.rb b/cookbooks/apache/recipes/default.rb index cade29d31..a17e2f4cb 100644 --- a/cookbooks/apache/recipes/default.rb +++ b/cookbooks/apache/recipes/default.rb @@ -114,7 +114,7 @@ fail2ban_jail "apache-forbidden" do end fail2ban_filter "apache-evasive" do - failregex ": Blacklisting address : possible DoS attack\.$" + failregex ": Blacklisting address : possible DoS attack\\.$" end fail2ban_jail "apache-evasive" do diff --git a/cookbooks/apt/recipes/default.rb b/cookbooks/apt/recipes/default.rb index 2ead9baf2..1d876dfb0 100644 --- a/cookbooks/apt/recipes/default.rb +++ b/cookbooks/apt/recipes/default.rb @@ -49,7 +49,7 @@ if platform?("debian") archive_components = %w[main contrib non-free non-free-firmware] backport_packages = case node[:lsb][:codename] when "bookworm" then %W[amd64-microcode exim4 firmware-free firmware-nonfree intel-microcode libosmium linux-signed-#{dpkg_arch} osm2pgsql otrs2 pyosmium smartmontools systemd cgi-mapserver] - else %W[] + else %w[] end elsif intel? archive_host = if node[:country] diff --git a/cookbooks/apt/recipes/fullstaq-ruby.rb b/cookbooks/apt/recipes/fullstaq-ruby.rb new file mode 100644 index 000000000..d9dad7d37 --- /dev/null +++ b/cookbooks/apt/recipes/fullstaq-ruby.rb @@ -0,0 +1,27 @@ +# +# Cookbook:: apt +# Recipe:: fullstaq-ruby +# +# Copyright:: 2025, Tom Hughes +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "apt" + +apt_repository "fullstaq-ruby" do + uri "https://apt.fullstaqruby.org" + distribution "#{node[:platform]}-#{node[:platform_version]}" + components ["main"] + key "https://raw.githubusercontent.com/fullstaq-ruby/server-edition/main/fullstaq-ruby.asc" +end diff --git a/cookbooks/awscli/recipes/default.rb b/cookbooks/awscli/recipes/default.rb index 22684864d..90574c3bb 100644 --- a/cookbooks/awscli/recipes/default.rb +++ b/cookbooks/awscli/recipes/default.rb @@ -77,12 +77,25 @@ ruby_block "install-awscli" do require "fileutils" awscli_version_string = shell_out("#{cache_dir}/awscli/dist/aws", "--version") awscli_version = awscli_version_string.stdout.split(" ").first.split("/").last - FileUtils.mkdir_p("/opt/awscli/v2/#{awscli_version}/bin/", :mode => 0755) - FileUtils.mv("#{cache_dir}/awscli/dist", "/opt/awscli/v2/#{awscli_version}/dist", :force => true) - FileUtils.ln_sf("/opt/awscli/v2/#{awscli_version}/dist/aws", "/opt/awscli/v2/#{awscli_version}/bin/aws") - FileUtils.ln_sf("/opt/awscli/v2/#{awscli_version}/dist/aws_completer", "/opt/awscli/v2/#{awscli_version}/bin/aws_completer") - FileUtils.rm("/opt/awscli/v2/current") if File.exist?("/opt/awscli/v2/current") - FileUtils.ln_sf("/opt/awscli/v2/#{awscli_version}", "/opt/awscli/v2/current") + + install_dir = "/opt/awscli/v2/#{awscli_version}" + + FileUtils.mkdir_p("#{install_dir}/bin/", :mode => 0755) + FileUtils.mv("#{cache_dir}/awscli/dist", "#{install_dir}/dist", :force => true) + FileUtils.ln_sf("#{install_dir}/dist/aws", "#{install_dir}/bin/aws") + FileUtils.ln_sf("#{install_dir}/dist/aws_completer", "#{install_dir}/bin/aws_completer") + + FileUtils.rm_f("/opt/awscli/v2/current") + FileUtils.ln_sf(install_dir, "/opt/awscli/v2/current") + + # Retain the last 5 versions, including the current one + versions = Dir.glob("/opt/awscli/v2/*").select { |dir| File.directory?(dir) && dir != "/opt/awscli/v2/current" } + versions.sort_by! { |dir| File.mtime(dir) }.reverse! + versions_to_delete = versions[5..] || [] + + versions_to_delete.each do |dir| + FileUtils.rm_rf(dir) + end end action :nothing subscribes :run, "archive_file[#{cache_dir}/#{awscli_zip}]", :immediately diff --git a/cookbooks/bind/templates/default/db.10.erb b/cookbooks/bind/templates/default/db.10.erb index 06aef6442..298811c68 100644 --- a/cookbooks/bind/templates/default/db.10.erb +++ b/cookbooks/bind/templates/default/db.10.erb @@ -2,7 +2,7 @@ $TTL 604800 @ IN SOA <%= node[:fdqn] %>. root.openstreetmap.org. ( - 2021092001 ; Serial + 2025021801 ; Serial 604800 ; Refresh 86400 ; Retry 2419200 ; Expire @@ -44,6 +44,7 @@ $TTL 604800 100.48.0 IN PTR pdu1.ams.openstreetmap.org. 101.48.0 IN PTR pdu2.ams.openstreetmap.org. 102.48.0 IN PTR oob1.ams.openstreetmap.org. +103.48.0 IN PTR ats1.ams.openstreetmap.org. 3.49.0 IN PTR faffy.oob.openstreetmap.org. 4.49.0 IN PTR dribble.oob.openstreetmap.org. diff --git a/cookbooks/blogs/recipes/default.rb b/cookbooks/blogs/recipes/default.rb index 0fb2cc7aa..86dfd3afe 100644 --- a/cookbooks/blogs/recipes/default.rb +++ b/cookbooks/blogs/recipes/default.rb @@ -22,7 +22,7 @@ include_recipe "apache" include_recipe "git" include_recipe "ruby" -package %W[ +package %w[ make gcc g++ @@ -44,10 +44,18 @@ git "/srv/blogs.openstreetmap.org" do group "blogs" end +bundle_config "/srv/blogs.openstreetmap.org" do + action :nothing + user "blogs" + group "blogs" + settings "deployment" => "true", + "without" => "development:test", + "build.sqlite3" => "--enable-system-libraries" + subscribes :create, "git[/srv/blogs.openstreetmap.org]", :immediately +end + bundle_install "/srv/blogs.openstreetmap.org" do action :nothing - options "--deployment --without development test" - environment "BUNDLE_PATH" => "vendor/bundle" user "blogs" group "blogs" subscribes :run, "git[/srv/blogs.openstreetmap.org]", :immediately @@ -56,7 +64,6 @@ end bundle_exec "/srv/blogs.openstreetmap.org" do action :nothing command "pluto build -t osm -o build" - environment "BUNDLE_PATH" => "vendor/bundle" user "blogs" group "blogs" subscribes :run, "git[/srv/blogs.openstreetmap.org]", :immediately diff --git a/cookbooks/blogs/templates/default/blogs-update.erb b/cookbooks/blogs/templates/default/blogs-update.erb index a7d021495..15cc82483 100644 --- a/cookbooks/blogs/templates/default/blogs-update.erb +++ b/cookbooks/blogs/templates/default/blogs-update.erb @@ -2,8 +2,6 @@ cd /srv/blogs.openstreetmap.org -export BUNDLE_PATH="vendor/bundle" - <%= node[:ruby][:bundle] %> exec pluto \ --quieter \ --config=/srv/blogs.openstreetmap.org build \ diff --git a/cookbooks/civicrm/recipes/default.rb b/cookbooks/civicrm/recipes/default.rb index 2aed4f64f..d10ed0788 100644 --- a/cookbooks/civicrm/recipes/default.rb +++ b/cookbooks/civicrm/recipes/default.rb @@ -200,21 +200,21 @@ node[:civicrm][:extensions].each_value do |details| end settings = edit_file "#{civicrm_directory}/civicrm/templates/CRM/common/civicrm.settings.php.template" do |line| - line.gsub!(/%%cms%%/, "WordPress") - line.gsub!(/%%CMSdbUser%%/, "civicrm") - line.gsub!(/%%CMSdbPass%%/, database_password) - line.gsub!(/%%CMSdbHost%%/, "localhost") - line.gsub!(/%%CMSdbName%%/, "civicrm") - line.gsub!(/%%dbUser%%/, "civicrm") - line.gsub!(/%%dbPass%%/, database_password) - line.gsub!(/%%dbHost%%/, "localhost") - line.gsub!(/%%dbName%%/, "civicrm") - line.gsub!(/%%crmRoot%%/, "#{civicrm_directory}/civicrm/") - line.gsub!(/%%templateCompileDir%%/, "/srv/supporting.openstreetmap.org/wp-content/uploads/civicrm/templates_c/") - line.gsub!(/%%baseURL%%/, "http://supporting.openstreetmap.org/") - line.gsub!(/%%siteKey%%/, site_key) - line.gsub!(/%%credKeys%%/, cred_keys) - line.gsub!(/%%signKeys%%/, sign_keys) + line.gsub!("%%cms%%", "WordPress") + line.gsub!("%%CMSdbUser%%", "civicrm") + line.gsub!("%%CMSdbPass%%", database_password) + line.gsub!("%%CMSdbHost%%", "localhost") + line.gsub!("%%CMSdbName%%", "civicrm") + line.gsub!("%%dbUser%%", "civicrm") + line.gsub!("%%dbPass%%", database_password) + line.gsub!("%%dbHost%%", "localhost") + line.gsub!("%%dbName%%", "civicrm") + line.gsub!("%%crmRoot%%", "#{civicrm_directory}/civicrm/") + line.gsub!("%%templateCompileDir%%", "/srv/supporting.openstreetmap.org/wp-content/uploads/civicrm/templates_c/") + line.gsub!("%%baseURL%%", "http://supporting.openstreetmap.org/") + line.gsub!("%%siteKey%%", site_key) + line.gsub!("%%credKeys%%", cred_keys) + line.gsub!("%%signKeys%%", sign_keys) line.gsub!(%r{// *define\('CIVICRM_CMSDIR', '/path/to/install/root/'\);}, "define('CIVICRM_CMSDIR', '/srv/supporting.openstreetmap.org');") # Don't recompile smarty templates on every call https://docs.civicrm.org/sysadmin/en/latest/setup/optimizations/#disable-compile-check line.gsub!(%r{// define\('CIVICRM_TEMPLATE_COMPILE_CHECK', FALSE\);}, "define('CIVICRM_TEMPLATE_COMPILE_CHECK', FALSE);") diff --git a/cookbooks/community/recipes/default.rb b/cookbooks/community/recipes/default.rb index 000a57a7e..74a95a923 100644 --- a/cookbooks/community/recipes/default.rb +++ b/cookbooks/community/recipes/default.rb @@ -62,9 +62,8 @@ end git "/srv/community.openstreetmap.org/docker" do action :sync repository "https://github.com/discourse/discourse_docker.git" - # Revision pin not possible as launch wrapper automatically updates git repo. - revision "main" - depth 1 + # DANGER launch wrapper automatically updates git repo if rebuild method used: https://github.com/discourse/discourse_docker/blob/107ffb40fe8b1ea40e00814468db974a4f3f8e8f/launcher#L799 + revision "9ebce86fda3d9787d752de6311df15dcab9700f1" user "root" group "root" notifies :run, "notify_group[discourse_container_new_data]" @@ -122,18 +121,24 @@ notify_group "discourse_container_new_web_only" do notifies :run, "execute[discourse_container_data_start]", :immediately # noop if site up notifies :run, "execute[discourse_container_web_only_bootstrap]", :immediately # site up but runs in parallel. Slow notifies :run, "execute[discourse_container_web_only_destroy]", :immediately # site down - notifies :run, "execute[discourse_container_data_rebuild]", :immediately # site down + notifies :run, "execute[discourse_container_data_destroy]", :immediately # site down + notifies :run, "execute[discourse_container_data_bootstrap]", :immediately # site down + notifies :run, "execute[discourse_container_data_start]", :immediately # site down notifies :run, "execute[discourse_container_web_only_start]", :immediately # site restore end notify_group "discourse_container_new_data" do notifies :run, "execute[discourse_container_web_only_destroy]", :immediately # site down - notifies :run, "execute[discourse_container_data_rebuild]", :immediately # site down + notifies :run, "execute[discourse_container_data_destroy]", :immediately # site down + notifies :run, "execute[discourse_container_data_bootstrap]", :immediately # site down + notifies :run, "execute[discourse_container_data_start]", :immediately # site down notifies :run, "execute[discourse_container_web_only_start]", :immediately # site restore end notify_group "discourse_container_new_mail_receiver" do - notifies :run, "execute[discourse_container_mail_receiver_rebuild]", :immediately + notifies :run, "execute[discourse_container_mail_receiver_destroy]", :immediately + notifies :run, "execute[discourse_container_mail_receiver_bootstrap]", :immediately + notifies :run, "execute[discourse_container_mail_receiver_start]", :immediately end # Attempt at a failsafe to ensure all containers are running @@ -144,17 +149,26 @@ notify_group "discourse_container_ensure_all_running" do notifies :run, "execute[discourse_container_mail_receiver_start]", :delayed end -execute "discourse_container_data_start" do +execute "discourse_container_data_bootstrap" do action :nothing - command "./launcher start data" + command "./launcher bootstrap data" + cwd "/srv/community.openstreetmap.org/docker/" + user "root" + group "root" + retries 2 # Postgres upgrades required a second run +end + +execute "discourse_container_data_destroy" do + action :nothing + command "./launcher destroy data" cwd "/srv/community.openstreetmap.org/docker/" user "root" group "root" end -execute "discourse_container_data_rebuild" do +execute "discourse_container_data_start" do action :nothing - command "./launcher rebuild data" + command "./launcher start data" cwd "/srv/community.openstreetmap.org/docker/" user "root" group "root" @@ -184,10 +198,17 @@ execute "discourse_container_web_only_start" do group "root" end -# Rebuild: Stop Destroy Bootstap Start -execute "discourse_container_mail_receiver_rebuild" do +execute "discourse_container_mail_receiver_bootstrap" do + action :nothing + command "./launcher bootstrap mail-receiver" + cwd "/srv/community.openstreetmap.org/docker/" + user "root" + group "root" +end + +execute "discourse_container_mail_receiver_destroy" do action :nothing - command "./launcher rebuild mail-receiver" + command "./launcher destroy mail-receiver" cwd "/srv/community.openstreetmap.org/docker/" user "root" group "root" diff --git a/cookbooks/community/templates/default/data.yml.erb b/cookbooks/community/templates/default/data.yml.erb index c5c59d288..37573874e 100644 --- a/cookbooks/community/templates/default/data.yml.erb +++ b/cookbooks/community/templates/default/data.yml.erb @@ -3,7 +3,7 @@ # templates: - - "templates/postgres.13.template.yml" # NOTE UPDATE THE HOOK REPLACE FOR MAX CONNECTIONS BELOW + - "templates/postgres.15.template.yml" # NOTE UPDATE THE HOOK REPLACE FOR MAX CONNECTIONS BELOW - "templates/redis.template.yml" # any extra arguments for Docker? @@ -51,6 +51,6 @@ hooks: run: # Make sure this matches the postgresql version template above - replace: - filename: "/etc/postgresql/13/main/postgresql.conf" + filename: "/etc/postgresql/15/main/postgresql.conf" from: /#?max_connections *=.*/ to: "max_connections = $db_max_connections" diff --git a/cookbooks/community/templates/default/web_only.yml.erb b/cookbooks/community/templates/default/web_only.yml.erb index 5beef4802..51bcc4aaa 100644 --- a/cookbooks/community/templates/default/web_only.yml.erb +++ b/cookbooks/community/templates/default/web_only.yml.erb @@ -19,7 +19,7 @@ links: # any extra arguments for Docker? # docker_args: -# Latest Version v3.3.3 +# Latest Version v3.4.4 # Discourse only support tests-passed and stable branches params: version: stable @@ -116,20 +116,19 @@ hooks: cmd: - sudo -H -E -u discourse cp /shared/feeds/update-feeds.atom public/update-feeds.atom after_ssl: - - replace: - filename: "/etc/nginx/conf.d/discourse.conf" - from: /listen 80;/ - to: | - listen 80; - listen [::]:80; - rewrite ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 permanent; - - - replace: - filename: "/etc/nginx/conf.d/discourse.conf" - from: /add_header.+/ - to: | - add_header Strict-Transport-Security 'max-age=63072000' always; + - file: + path: "/etc/nginx/conf.d/outlets/server/25-https-osm-settings.conf" + contents: | ssl_stapling on; resolver <%= @resolvers.join(" ") %>; resolver_timeout 5s; ssl_dhparam /shared/ssl/dhparam.pem; + +run: + - replace: + filename: "/etc/nginx/conf.d/discourse.conf" + from: /listen 80;/ + to: | + listen 80; + listen [::]:80; + rewrite ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 permanent; diff --git a/cookbooks/db/files/default/monthly-reindex.sql b/cookbooks/db/files/default/monthly-reindex.sql index ab9ae105b..ffa5faadb 100644 --- a/cookbooks/db/files/default/monthly-reindex.sql +++ b/cookbooks/db/files/default/monthly-reindex.sql @@ -28,6 +28,7 @@ REINDEX (VERBOSE) TABLE CONCURRENTLY oauth_applications; REINDEX (VERBOSE) TABLE CONCURRENTLY redactions; REINDEX (VERBOSE) TABLE CONCURRENTLY reports; REINDEX (VERBOSE) TABLE CONCURRENTLY schema_migrations; +REINDEX (VERBOSE) TABLE CONCURRENTLY social_links; REINDEX (VERBOSE) TABLE CONCURRENTLY user_blocks; REINDEX (VERBOSE) TABLE CONCURRENTLY user_mutes; REINDEX (VERBOSE) TABLE CONCURRENTLY user_preferences; diff --git a/cookbooks/db/recipes/master.rb b/cookbooks/db/recipes/master.rb index 3e57941f7..f27820652 100644 --- a/cookbooks/db/recipes/master.rb +++ b/cookbooks/db/recipes/master.rb @@ -182,6 +182,7 @@ PROMETHEUS_PERMISSIONS = { relations reports schema_migrations + social_links user_blocks user_mutes user_preferences @@ -232,6 +233,7 @@ end oauth_openid_requests_id_seq redactions_id_seq reports_id_seq + social_links_id_seq user_blocks_id_seq user_mutes_id_seq user_roles_id_seq diff --git a/cookbooks/dev/README.md b/cookbooks/dev/README.md index 0253da49b..3918d375b 100644 --- a/cookbooks/dev/README.md +++ b/cookbooks/dev/README.md @@ -2,4 +2,4 @@ This cookbook configures development servers, such as dev.openstreetmap.org. It installs packages required by the users and configures apache for the various -user and api developement sites. +user and api development sites. diff --git a/cookbooks/dev/recipes/default.rb b/cookbooks/dev/recipes/default.rb index 073bb3e13..4f6a160f7 100644 --- a/cookbooks/dev/recipes/default.rb +++ b/cookbooks/dev/recipes/default.rb @@ -61,6 +61,7 @@ package %w[ gnuplot-nox golang graphviz + htop irssi jq libargon2-dev @@ -95,6 +96,7 @@ package %w[ lzip lzop mailutils + moreutils make nano ncftp @@ -103,7 +105,6 @@ package %w[ osmium-tool osmosis pandoc - pandoc pbzip2 php-apcu php-cgi @@ -145,10 +146,12 @@ package %w[ python3-venv r-base redis + siege tmux unrar unzip whois + xxd zip zlib1g-dev ] @@ -201,7 +204,7 @@ template "/srv/dev.openstreetmap.org/index.html" do end ssl_certificate "dev.openstreetmap.org" do - domains "dev.openstreetmap.org" + domains ["dev.openstreetmap.org", "dev.osm.org"] notifies :reload, "service[apache2]" end @@ -223,7 +226,7 @@ file "/etc/apache2/conf.d/phppgadmin" do end ssl_certificate "phppgadmin.dev.openstreetmap.org" do - domains "phppgadmin.dev.openstreetmap.org" + domains ["phppgadmin.dev.openstreetmap.org", "phppgadmin.dev.osm.org"] notifies :reload, "service[apache2]" end @@ -349,7 +352,7 @@ if node[:postgresql][:clusters][:"15/main"] gpx_directory = "#{site_directory}/gpx" if details[:repository] - site_aliases = details[:aliases] || [] + site_aliases = details[:aliases] || ["#{name}.apis.dev.osm.org"] secret_key_base = persistent_token("dev", "rails", name, "secret_key_base") postgresql_database database_name do @@ -570,7 +573,7 @@ if node[:postgresql][:clusters][:"15/main"] end ssl_certificate "apis.dev.openstreetmap.org" do - domains "apis.dev.openstreetmap.org" + domains ["apis.dev.openstreetmap.org", "apis.dev.osm.org"] notifies :reload, "service[apache2]" end @@ -599,7 +602,8 @@ ssl_certificate "ooc.openstreetmap.org" do domains ["ooc.openstreetmap.org", "a.ooc.openstreetmap.org", "b.ooc.openstreetmap.org", - "c.ooc.openstreetmap.org"] + "c.ooc.openstreetmap.org", + "ooc.osm.org"] notifies :reload, "service[apache2]" end diff --git a/cookbooks/dev/templates/default/apache.apis.erb b/cookbooks/dev/templates/default/apache.apis.erb index 4d78fdd6a..470a1fa5f 100644 --- a/cookbooks/dev/templates/default/apache.apis.erb +++ b/cookbooks/dev/templates/default/apache.apis.erb @@ -2,6 +2,8 @@ ServerName apis.dev.openstreetmap.org + ServerAlias apis.dev.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -16,6 +18,8 @@ ServerName apis.dev.openstreetmap.org + ServerAlias apis.dev.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/apis.dev.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.dev.erb b/cookbooks/dev/templates/default/apache.dev.erb index 2d3d9473d..2dd13c2b9 100644 --- a/cookbooks/dev/templates/default/apache.dev.erb +++ b/cookbooks/dev/templates/default/apache.dev.erb @@ -2,6 +2,8 @@ ServerName dev.openstreetmap.org + ServerAlias dev.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -21,6 +23,8 @@ ServerName dev.openstreetmap.org + ServerAlias dev.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/dev.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.ooc.erb b/cookbooks/dev/templates/default/apache.ooc.erb index 0a9b0b979..a532300e1 100644 --- a/cookbooks/dev/templates/default/apache.ooc.erb +++ b/cookbooks/dev/templates/default/apache.ooc.erb @@ -5,6 +5,8 @@ ServerAlias a.ooc.openstreetmap.org ServerAlias b.ooc.openstreetmap.org ServerAlias c.ooc.openstreetmap.org + ServerAlias ooc.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -26,6 +28,8 @@ ServerAlias a.ooc.openstreetmap.org ServerAlias b.ooc.openstreetmap.org ServerAlias c.ooc.openstreetmap.org + ServerAlias ooc.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/ooc.openstreetmap.org-access.log combined_extended @@ -37,6 +41,8 @@ ServerName npe.openstreetmap.org + ServerAlias npe.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/npe.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.phppgadmin.erb b/cookbooks/dev/templates/default/apache.phppgadmin.erb index 546a05cbe..67c37df16 100644 --- a/cookbooks/dev/templates/default/apache.phppgadmin.erb +++ b/cookbooks/dev/templates/default/apache.phppgadmin.erb @@ -2,6 +2,8 @@ ServerName phppgadmin.dev.openstreetmap.org + ServerAlias phppgadmin.dev.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -23,6 +25,8 @@ ServerName phppgadmin.dev.openstreetmap.org + ServerAlias phppgadmin.dev.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/phppgadmin.dev.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.rails.erb b/cookbooks/dev/templates/default/apache.rails.erb index 957866c5c..082931dde 100644 --- a/cookbooks/dev/templates/default/apache.rails.erb +++ b/cookbooks/dev/templates/default/apache.rails.erb @@ -48,6 +48,7 @@ RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/relations(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/node/[0-9]+/ways(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/(way|relation)/[0-9]+/full(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] + RewriteCond %{REQUEST_METHOD} ^(HEAD|GET)$ RewriteRule ^/api/0\.6/(nodes|ways|relations)(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/changeset/[0-9]+/(upload|download)(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] <% end -%> diff --git a/cookbooks/dev/templates/default/apache.user.erb b/cookbooks/dev/templates/default/apache.user.erb index 373d12258..0c51e44e0 100644 --- a/cookbooks/dev/templates/default/apache.user.erb +++ b/cookbooks/dev/templates/default/apache.user.erb @@ -4,9 +4,10 @@ WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> processes ServerName <%= @user %>.dev.openstreetmap.org - ServerAdmin webmaster@openstreetmap.org ServerAlias <%= @user %>.dev.osm.org + ServerAdmin webmaster@openstreetmap.org + SSLEngine on SSLCertificateFile /etc/ssl/certs/<%= @user %>.dev.openstreetmap.org.pem SSLCertificateKeyFile /etc/ssl/private/<%= @user %>.dev.openstreetmap.org.key @@ -36,9 +37,10 @@ WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> processes ServerName <%= @user %>.dev.openstreetmap.org - ServerAdmin webmaster@openstreetmap.org ServerAlias <%= @user %>.dev.osm.org + ServerAdmin webmaster@openstreetmap.org + CustomLog /var/log/apache2/<%= @user %>.dev.openstreetmap.org-access.log combined_extended ErrorLog /var/log/apache2/<%= @user %>.dev.openstreetmap.org-error.log diff --git a/cookbooks/devices/metadata.rb b/cookbooks/devices/metadata.rb index 2f71f6857..a25b0e16f 100644 --- a/cookbooks/devices/metadata.rb +++ b/cookbooks/devices/metadata.rb @@ -6,3 +6,4 @@ description "Configures devices" version "0.1" supports "ubuntu" +depends "chef" diff --git a/cookbooks/devices/templates/default/udev.rules.erb b/cookbooks/devices/templates/default/udev.rules.erb index be9903891..ad2451afa 100644 --- a/cookbooks/devices/templates/default/udev.rules.erb +++ b/cookbooks/devices/templates/default/udev.rules.erb @@ -95,47 +95,6 @@ SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x37d2 # Disable Firmware Based LLDP handler SUBSYSTEM=="net", ACTION=="add", ENV{INTERFACE}=="*", DRIVERS=="i40e", RUN+="/sbin/ethtool --set-priv-flags $name disable-fw-lldp on" -# Workaround unreliable Western Digital WD RE3/RE4 disks (ATA only) -# Set sufficent Linux subsystem timeout and fix severe NCQ performance issue -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD5002ABYS-02B1B0", ATTR{device/timeout}="90", ATTR{device/queue_depth}="1", ATTR{queue/nr_requests}="256" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", ATTR{device/timeout}="90", ATTR{device/queue_depth}="1", ATTR{queue/nr_requests}="256" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", ATTR{device/timeout}="90", ATTR{device/queue_depth}="1", ATTR{queue/nr_requests}="256" -# Disable Disk Write Cache, Set AAM and Power Management correctly -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", RUN+="/sbin/hdparm -q -W0 -q -M254 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", RUN+="/sbin/hdparm -q -W0 -q -M254 -q -B254 $env{DEVNAME}" - -# Set Disks TLED / SCT Error Recovery Control -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD5000AAKS-00A7B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD2000FYYZ-01UL1B2", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="TOSHIBA_DT01ACA300", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST31000340NS", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="HGST_HTS725050A7E630", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="HGST_HTE721010A9E630", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" - -# Add SSD optimisation -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="OCZ-VERTEX3", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="OCZ-VERTEX3", ATTR{queue/scheduler}="noop" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/scheduler}="noop" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/read_ahead_kb}="256" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/scheduler}="noop" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/read_ahead_kb}="256" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/scheduler}="noop" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/read_ahead_kb}="256" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST240FN0021", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST240FN0021", ATTR{queue/scheduler}="noop" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="SuperMicro_SSD", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="SuperMicro_SSD", ATTR{queue/scheduler}="noop" - # Delete failed disk in cmok ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST_M13FQBL", ENV{ID_SERIAL}=="ST_M13FQBL_QNR_BFW", ATTR{device/delete}="1" @@ -155,6 +114,8 @@ ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_MODEL}=="QEMU_HA ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_MODEL}=="QEMU_HARDDISK", ATTR{queue/scheduler}="noop" # Vendor is sometimes missing -# Increase default MD raid5/raid6 strip cache + group_thread_cnt -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{MD_LEVEL}=="raid5", ATTR{md/stripe_cache_size}="8192", ATTR{md/group_thread_cnt}="4" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{MD_LEVEL}=="raid6", ATTR{md/stripe_cache_size}="8192", ATTR{md/group_thread_cnt}="4" +# Tune md stripe cache and thread count for RAID-5 / RAID-6 arrays +<% + group_threads = [(node.cpu_cores.to_i / 2.0).round, 4].max +%> +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{MD_LEVEL}=="raid[56]", ATTR{md/stripe_cache_size}="8192", ATTR{md/group_thread_cnt}="<%= group_threads %>" diff --git a/cookbooks/dhcpd/README.md b/cookbooks/dhcpd/README.md index 3d2a00c7c..f95dbb0d0 100644 --- a/cookbooks/dhcpd/README.md +++ b/cookbooks/dhcpd/README.md @@ -1,3 +1,3 @@ # dhcpd Cookbook -Configures the dhcpd service, which used for the internal network at UCL. +Configures the dhcpd service, which is used for our internal networks. diff --git a/cookbooks/dhcpd/recipes/default.rb b/cookbooks/dhcpd/recipes/default.rb index 0e6b9ec3f..685d08c3b 100644 --- a/cookbooks/dhcpd/recipes/default.rb +++ b/cookbooks/dhcpd/recipes/default.rb @@ -53,7 +53,7 @@ remote_file "/srv/tftp/netboot.xyz.kpxe" do mode "644" end -domain = "#{node[:networking][:roles][:external][:zone]}.openstreetmap.org" +domain = node[:networking][:search].first template "/etc/dhcp/dhcpd.conf" do source "dhcpd.conf.erb" diff --git a/cookbooks/dhcpd/templates/default/dhcpd.conf.erb b/cookbooks/dhcpd/templates/default/dhcpd.conf.erb index 1581475e5..ed4fe2d03 100644 --- a/cookbooks/dhcpd/templates/default/dhcpd.conf.erb +++ b/cookbooks/dhcpd/templates/default/dhcpd.conf.erb @@ -55,6 +55,12 @@ host pdu2.ams.openstreetmap.org { fixed-address 10.0.48.101; } +host ats1.ams.openstreetmap.org { + hardware ethernet 00:c0:b7:e5:5e:f1; + server-name "ats1.ams.openstreetmap.org"; + fixed-address 10.0.48.103; +} + host oob1.dub.openstreetmap.org { hardware ethernet 62:bd:62:a6:05:25; server-name "oob1.dub.openstreetmap.org"; diff --git a/cookbooks/dns/recipes/default.rb b/cookbooks/dns/recipes/default.rb index d1b3d159e..d5f0c1e60 100644 --- a/cookbooks/dns/recipes/default.rb +++ b/cookbooks/dns/recipes/default.rb @@ -42,7 +42,7 @@ package %w[ cache_dir = Chef::Config[:file_cache_path] -dnscontrol_version = "4.15.1" +dnscontrol_version = "4.17.0" dnscontrol_arch = if arm? "arm64" @@ -60,7 +60,7 @@ end dpkg_package "dnscontrol" do source "#{cache_dir}/dnscontrol-#{dnscontrol_version}.deb" - version "#{dnscontrol_version}" + version dnscontrol_version end directory "/srv/dns.openstreetmap.org" do diff --git a/cookbooks/exim/recipes/default.rb b/cookbooks/exim/recipes/default.rb index 7354e93d3..21a076d56 100644 --- a/cookbooks/exim/recipes/default.rb +++ b/cookbooks/exim/recipes/default.rb @@ -47,6 +47,10 @@ end if node[:exim][:certificate_names] include_recipe "apache" + apache_site "default" do + action [:disable] + end + apache_site node[:exim][:certificate_names].first do template "apache.erb" variables :aliases => node[:exim][:certificate_names].drop(1) @@ -161,7 +165,7 @@ if node[:exim][:dkim_selectors] mode "755" end - node[:exim][:dkim_selectors].each do |domain, _selector| + node[:exim][:dkim_selectors].each_key do |domain| file "/etc/exim4/dkim-keys/#{domain}" do content keys[domain].join("\n") owner "root" diff --git a/cookbooks/exim/templates/default/exim4.conf.erb b/cookbooks/exim/templates/default/exim4.conf.erb index ffc8be609..3558af884 100644 --- a/cookbooks/exim/templates/default/exim4.conf.erb +++ b/cookbooks/exim/templates/default/exim4.conf.erb @@ -753,6 +753,9 @@ signed_smtp: multi_domain = false hosts_try_dane = tls_require_ciphers = <%= node[:ssl][:gnutls_ciphers] %>:%LATEST_RECORD_VERSION +<% if node[:exim][:external_interface] -%> + interface = <%= node[:exim][:external_interface] %> +<% end -%> # This transport is used for handling pipe deliveries generated by alias or diff --git a/cookbooks/git/templates/default/apache.erb b/cookbooks/git/templates/default/apache.erb index daff9de88..0d7f71f4e 100644 --- a/cookbooks/git/templates/default/apache.erb +++ b/cookbooks/git/templates/default/apache.erb @@ -44,6 +44,14 @@ CustomLog /var/log/apache2/<%= @name %>-access.log combined_extended ErrorLog /var/log/apache2/<%= @name %>-error.log + Alias /robots.txt /srv/<%= node[:git][:host] %>/robots.txt + + + # Make absolutely sure it comes out as a plain file + SetHandler none + Require all granted + + SetEnv GIT_PROJECT_ROOT /var/lib/git SetEnv GIT_HTTP_EXPORT_ALL SetEnv GIT_HTTP_MAX_REQUEST_BUFFER 100M diff --git a/cookbooks/git/templates/default/robots.txt.erb b/cookbooks/git/templates/default/robots.txt.erb index b60d5e279..61efcd142 100644 --- a/cookbooks/git/templates/default/robots.txt.erb +++ b/cookbooks/git/templates/default/robots.txt.erb @@ -1,4 +1,14 @@ # DO NOT EDIT - This file is being maintained by Chef User-agent: * -Disallow: /*/snapshot/ +Disallow: /*a=search* +Disallow: /*/search/* +Disallow: /*a=blobdiff* +Disallow: /*/blobdiff/* +Disallow: /*/blobdiff_plain/* +Disallow: /*a=commitdiff* +Disallow: /*/commitdiff/* +Disallow: /*a=snapshot* +Disallow: /*/snapshot/* +Disallow: /*a=blame* +Disallow: /*/blame/* diff --git a/cookbooks/gps-tile/templates/default/apache.erb b/cookbooks/gps-tile/templates/default/apache.erb index 9adf707dd..a1b7ba94c 100644 --- a/cookbooks/gps-tile/templates/default/apache.erb +++ b/cookbooks/gps-tile/templates/default/apache.erb @@ -47,6 +47,7 @@ RewriteRule ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 [R=permanent,L] # Redirect to https + RewriteCond %{REQUEST_URI} !^/server-status RewriteRule (.*) https://%{SERVER_NAME}/$1 [R=permanent,L] # Setup logging diff --git a/cookbooks/hardware/recipes/default.rb b/cookbooks/hardware/recipes/default.rb index 0dafc1f91..142084866 100644 --- a/cookbooks/hardware/recipes/default.rb +++ b/cookbooks/hardware/recipes/default.rb @@ -410,8 +410,8 @@ intel_nvmes = nvmes.select { |pci| pci[:vendor_name] == "Intel Corporation" } if !intel_ssds.empty? || !intel_nvmes.empty? package "unzip" - sst_tool_version = "2-0" - sst_package_version = "2.0.300-0" + sst_tool_version = "2-3" + sst_package_version = "2.3.320-0" remote_file "#{Chef::Config[:file_cache_path]}/sst-cli-linux-deb--#{sst_tool_version}.zip" do source "https://sdmsdfwdriver.blob.core.windows.net/files/kba-gcc/drivers-downloads/ka-00085/sst--#{sst_tool_version}/sst-cli-linux-deb--#{sst_tool_version}.zip" @@ -426,7 +426,7 @@ if !intel_ssds.empty? || !intel_nvmes.empty? end dpkg_package "sst" do - version "#{sst_package_version}" + version sst_package_version source "#{Chef::Config[:file_cache_path]}/sst_#{sst_package_version}_amd64.deb" end diff --git a/cookbooks/imagery/recipes/au_act_aerial.rb b/cookbooks/imagery/recipes/au_act_aerial.rb new file mode 100644 index 000000000..cc0135998 --- /dev/null +++ b/cookbooks/imagery/recipes/au_act_aerial.rb @@ -0,0 +1,65 @@ +# +# Cookbook:: imagery +# Recipe:: au_act_aerial +# +# Copyright:: 2025, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery" + +imagery_site "act-imagery.openstreetmap.org" do + title "OpenStreetMap - ACT Imagery" + aliases ["act-imagery.osm.org"] + # https://leafletjs.com/reference.html#latlngbounds format + # [[south, west], [north, east]] + bbox [[-35.942, 148.729], [-35.117, 149.430]] +end + +imagery_layer "act_aerial_imagery_202409" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202409" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202409/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202409/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + default_layer true + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +imagery_layer "act_aerial_imagery_202311" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202311" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202311/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202311/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +imagery_layer "act_aerial_imagery_202305" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202305" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202305/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202305/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end diff --git a/cookbooks/imagery/recipes/au_agri.rb b/cookbooks/imagery/recipes/au_agri.rb index 931ecf3f1..fb011c111 100644 --- a/cookbooks/imagery/recipes/au_agri.rb +++ b/cookbooks/imagery/recipes/au_agri.rb @@ -33,7 +33,7 @@ imagery_layer "au_ga_agri" do copyright "Commonwealth of Australia (Geoscience Australia) - Creative Commons Attribution 4.0 International Licence" background_colour "0 0 0" # Black projection "EPSG:3857" - source "/store/imagery/au/agri/combine.vrt" + source "/store/imagery/au/agri/combine-cutline-cog.tif" max_zoom 17 - revision 1 + revision 3 end diff --git a/cookbooks/imagery/recipes/au_vic_melbourne_aerial.rb b/cookbooks/imagery/recipes/au_vic_melbourne_aerial.rb new file mode 100644 index 000000000..cda5e80d4 --- /dev/null +++ b/cookbooks/imagery/recipes/au_vic_melbourne_aerial.rb @@ -0,0 +1,59 @@ +# +# Cookbook:: imagery +# Recipe:: au_vic_melbourne_aerial +# +# Copyright:: 2024, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery" + +imagery_site "au-vic-melbourne-imagery.openstreetmap.org" do + title "OpenStreetMap - City of Melbourne - Aerial Imagery" + aliases ["au-vic-melbourne-imagery.osm.org"] + # https://leafletjs.com/reference.html#latlngbounds format + # [[south, west], [north, east]] + bbox [[-37.850667, 144.896981], [-37.775451, 144.991351]] +end + +imagery_layer "melbourne-2020" do + site "au-vic-melbourne-imagery.openstreetmap.org" + title "City of Melbourne 2020" + source "/store/imagery/au/city-of-melbourne/CoM_May2020_2cm.cog.tiff" + copyright "(c) 2020 City of Melbourne" + max_zoom 23 + extension "jpeg" + revision 2 + default_layer true +end + +imagery_layer "melbourne-2019" do + site "au-vic-melbourne-imagery.openstreetmap.org" + title "City of Melbourne 2019" + source "/store/imagery/au/city-of-melbourne/CoM_03Feb2019.cog.tiff" + copyright "(c) 2019 City of Melbourne" + max_zoom 21 + extension "jpeg" + revision 2 +end + +imagery_layer "melbourne-2018" do + site "au-vic-melbourne-imagery.openstreetmap.org" + title "City of Melbourne 2018" + source "/store/imagery/au/city-of-melbourne/CoM_May2018_10cm.COG.tiff" + copyright "(c) 2018 City of Melbourne" + max_zoom 21 + extension "jpeg" + revision 2 +end diff --git a/cookbooks/imagery/recipes/tiler.rb b/cookbooks/imagery/recipes/tiler.rb index ec7daa835..2f6942b66 100644 --- a/cookbooks/imagery/recipes/tiler.rb +++ b/cookbooks/imagery/recipes/tiler.rb @@ -37,21 +37,20 @@ container_image = if arm? podman_service "titiler" do description "Container service for titiler" image container_image - volume :"/store/imagery" => "/store/imagery", + volume :"/store/imagery" => "/store/imagery", :"/srv/imagery/sockets" => "/sockets" - environment :BIND => "unix:/sockets/titiler.sock", - :WORKERS_PER_CORE => 1, - :GDAL_CACHEMAX => 200, - :GDAL_BAND_BLOCK_CACHE => "HASHSET", - :GDAL_DISABLE_READDIR_ON_OPEN => "EMPTY_DIR", - :GDAL_INGESTED_BYTES_AT_OPEN => 32768, - :GDAL_HTTP_MERGE_CONSECUTIVE_RANGES => "YES", - :GDAL_HTTP_MULTIPLEX => "YES", - :GDAL_HTTP_VERSION => 2, - :VSI_CACHE => "TRUE", - :VSI_CACHE_SIZE => 5000000, - :TITILER_API_ROOT_PATH => "/api/v1/titiler", - :FORWARDED_ALLOW_IPS => "*" # https://docs.gunicorn.org/en/latest/settings.html#forwarded-allow-ips + environment :GDAL_CACHEMAX => 200, + :GDAL_BAND_BLOCK_CACHE => "HASHSET", + :GDAL_DISABLE_READDIR_ON_OPEN => "EMPTY_DIR", + :GDAL_INGESTED_BYTES_AT_OPEN => 32768, + :GDAL_HTTP_MERGE_CONSECUTIVE_RANGES => "YES", + :GDAL_HTTP_MULTIPLEX => "YES", + :GDAL_HTTP_VERSION => 2, + :VSI_CACHE => "TRUE", + :VSI_CACHE_SIZE => 5000000, + :TITILER_API_ROOT_PATH => "/api/v1/titiler", + :FORWARDED_ALLOW_IPS => "*" # https://docs.gunicorn.org/en/latest/settings.html#forwarded-allow-ips + command "gunicorn -k uvicorn.workers.UvicornWorker titiler.application.main:app --bind unix:/sockets/titiler.sock --workers #{node.cpu_cores}" end systemd_service "titiler-restart" do @@ -64,7 +63,7 @@ end systemd_timer "titiler-restart" do on_boot_sec "10m" - on_unit_inactive_sec "30m" + on_unit_inactive_sec "2h" randomized_delay_sec "20m" end diff --git a/cookbooks/imagery/recipes/za_ngi_topo.rb b/cookbooks/imagery/recipes/za_ngi_topo.rb index a1fd92c61..ac5a28b37 100644 --- a/cookbooks/imagery/recipes/za_ngi_topo.rb +++ b/cookbooks/imagery/recipes/za_ngi_topo.rb @@ -32,6 +32,7 @@ imagery_layer "za_ngi_topo_250k" do source "/store/imagery/za/ngi-topo-250k/ngi-topo-250k-combined.vrt" copyright 'State Copyright © 1996–2010 Chief Directorate: National Geo-spatial Information' default_layer true + revision 2 end imagery_layer "za_ngi_topo_50k" do @@ -40,4 +41,5 @@ imagery_layer "za_ngi_topo_50k" do projection "EPSG:3857" source "/store/imagery/za/ngi-topo-50k/ngi-topo-50k-combined.vrt" copyright 'State Copyright © 1996–2013 Chief Directorate: National Geo-spatial Information' + revision 2 end diff --git a/cookbooks/imagery/resources/site.rb b/cookbooks/imagery/resources/site.rb index 5ce7d2b88..b4079f0ca 100644 --- a/cookbooks/imagery/resources/site.rb +++ b/cookbooks/imagery/resources/site.rb @@ -87,7 +87,7 @@ action :create do end layers = Dir.glob("/srv/imagery/layers/#{new_resource.site}/*.yml").collect do |path| - YAML.safe_load(::File.read(path), :permitted_classes => [Symbol]) + YAML.safe_load_file(path, :permitted_classes => [Symbol]) end declare_resource :template, "/srv/#{new_resource.site}/imagery.js" do @@ -105,14 +105,19 @@ action :create do description "Map server for #{new_resource.site} layer" environment "MS_DEBUGLEVEL" => "0", "MS_ERRORFILE" => "stderr", - "GDAL_CACHEMAX" => "128" + "GDAL_CACHEMAX" => "128", + "GDAL_HTTP_TCP_KEEPALIVE" => "YES", + "GDAL_HTTP_VERSION" => "2TLS", + "GDAL_ENABLE_WMS_CACHE" => "NO" limit_nofile 16384 - memory_max "4G" + memory_high "12G" + memory_max "12G" + limit_core 0 user "imagery" group "imagery" exec_start "/usr/bin/multiwatch -f 8 --signal=TERM -- /usr/lib/cgi-bin/mapserv" standard_input "socket" - sandbox true + sandbox :enable_network => true restrict_address_families "AF_UNIX" timeout_stop_sec 60 not_if { new_resource.uses_tiler } @@ -153,7 +158,7 @@ action :create do systemd_timer "mapserv-fcgi-#{new_resource.site}-stop" do on_boot_sec "10m" - on_unit_inactive_sec "30m" + on_unit_inactive_sec "6h" randomized_delay_sec "20m" not_if { new_resource.uses_tiler } end diff --git a/cookbooks/imagery/templates/default/mapserver.map.erb b/cookbooks/imagery/templates/default/mapserver.map.erb index c918ca9f1..728723fcc 100644 --- a/cookbooks/imagery/templates/default/mapserver.map.erb +++ b/cookbooks/imagery/templates/default/mapserver.map.erb @@ -64,6 +64,5 @@ MAP STATUS DEFAULT TYPE RASTER PROCESSING "RESAMPLE=AVERAGE" - PROCESSING "CLOSE_CONNECTION=DEFER" END # layer END diff --git a/cookbooks/imagery/templates/default/nginx_imagery.conf.erb b/cookbooks/imagery/templates/default/nginx_imagery.conf.erb index 03a439343..a232d7830 100644 --- a/cookbooks/imagery/templates/default/nginx_imagery.conf.erb +++ b/cookbooks/imagery/templates/default/nginx_imagery.conf.erb @@ -23,8 +23,11 @@ upstream <%= @name %>_fastcgi { <% end -%> server { - listen 443 ssl http2; - listen [::]:443 ssl http2; + listen 443 ssl; + listen [::]:443 ssl; + + http2 on; + server_name <%= @name %> a.<%= @name %> b.<%= @name %> c.<%= @name %><% @aliases.each do |alias_name| %> <%= alias_name %> a.<%= alias_name %> b.<%= alias_name %> c.<%= alias_name %><%- end -%>; http2_max_concurrent_streams 512; diff --git a/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb b/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb index c85dd87bb..adc2d23c0 100644 --- a/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb +++ b/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb @@ -1,11 +1,9 @@ <% require 'uri' %> # DO NOT EDIT - This file is being maintained by Chef -location ~* "^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(png|jpg|jpeg)$" { +location ~* "^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(jpg|jpeg|png|webp)$" { <% if @uses_tiler -%> set $args ""; - rewrite ^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.jpg /mosaicjson/tiles/WebMercatorQuad/$1/$2/$3@1x?url=<%= URI.encode_www_form_component(@source) %>&pixel_selection=first&tile_format=jpeg break; - rewrite ^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.jpeg /mosaicjson/tiles/WebMercatorQuad/$1/$2/$3@1x?url=<%= URI.encode_www_form_component(@source) %>&pixel_selection=first&tile_format=jpeg break; - rewrite ^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.png /mosaicjson/tiles/WebMercatorQuad/$1/$2/$3@1x?url=<%= URI.encode_www_form_component(@source) %>&pixel_selection=first&tile_format=png break; + rewrite ^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(jpg|jpeg|png|webp) /mosaicjson/tiles/WebMercatorQuad/$1/$2/$3@1x.$4?url=<%= URI.encode_www_form_component(@source) %>&pixel_selection=first break; proxy_pass http://<%= @site %>_tiler_backend; proxy_set_header Host $host; proxy_set_header Referer $http_referer; @@ -17,7 +15,7 @@ location ~* "^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(png|jpg|jpeg)$" { proxy_set_header Cache-Control ""; proxy_set_header Pragma ""; proxy_redirect off; - proxy_cache_key "<%= @layer %><%= @revision %> $request_method $1 $2 $3"; + proxy_cache_key "<%= @layer %><%= @revision %> $request_method $1 $2 $3 $4"; proxy_cache proxy_cache_zone; proxy_cache_valid 200 204 180d; proxy_cache_use_stale error timeout updating http_502 http_503 http_504; @@ -68,8 +66,8 @@ location ~* "^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(png|jpg|jpeg)$" { gzip off; # Add HTTP Cache-Control + Expires Headers - expires 7d; - add_header Cache-Control "stale-while-revalidate=604800, stale-if-error=604800"; + expires 31d; + add_header Cache-Control "stale-while-revalidate=2678400, stale-if-error=2678400"; add_header x-cache-status $upstream_cache_status; # Allow CORS requests @@ -77,8 +75,8 @@ location ~* "^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(png|jpg|jpeg)$" { } <% if @root_layer -%> -rewrite "^/(\d+)/(\d+)/(\d+)\.(png|jpg|jpeg)$" "/layer/<%= @layer %>/$1/$2/$3.$4" last; +rewrite "^/(\d+)/(\d+)/(\d+)\.(jpg|jpeg|png|webp)$" "/layer/<%= @layer %>/$1/$2/$3.$4" last; <% end -%> <% @url_aliases.each do |url| -%> -rewrite "^<%= url %>/(\d+)/(\d+)/(\d+)\.(png|jpg|jpeg)$" "/layer/<%= @layer %>/$1/$2/$3.$4" last; +rewrite "^<%= url %>/(\d+)/(\d+)/(\d+)\.(jpg|jpeg|png|webp)$" "/layer/<%= @layer %>/$1/$2/$3.$4" last; <% end -%> diff --git a/cookbooks/imagery/templates/default/nginx_titiler.conf.erb b/cookbooks/imagery/templates/default/nginx_titiler.conf.erb index 155620949..055c2f849 100644 --- a/cookbooks/imagery/templates/default/nginx_titiler.conf.erb +++ b/cookbooks/imagery/templates/default/nginx_titiler.conf.erb @@ -21,8 +21,11 @@ upstream titiler_api_backend { } server { - listen 443 ssl http2; - listen [::]:443 ssl http2; + listen 443 ssl; + listen [::]:443 ssl; + + http2 on; + server_name <%= @name %> <% @aliases.each do |alias_name| %> <%= alias_name %><%- end -%>; http2_max_concurrent_streams 512; diff --git a/cookbooks/kibana/recipes/default.rb b/cookbooks/kibana/recipes/default.rb index 2f2fc6f28..2085cdce2 100644 --- a/cookbooks/kibana/recipes/default.rb +++ b/cookbooks/kibana/recipes/default.rb @@ -79,7 +79,7 @@ end node[:kibana][:sites].each do |name, details| file "/etc/kibana/#{name}.yml" do - content YAML.dump(YAML.safe_load(File.read("/opt/kibana-#{version}/config/kibana.yml")).merge( + content YAML.dump(YAML.safe_load_file("/opt/kibana-#{version}/config/kibana.yml").merge( "port" => details[:port], "host" => "127.0.0.1", "elasticsearch_url" => details[:elasticsearch_url], diff --git a/cookbooks/letsencrypt/files/default/bin/renew-hook b/cookbooks/letsencrypt/files/default/bin/deploy-hook similarity index 100% rename from cookbooks/letsencrypt/files/default/bin/renew-hook rename to cookbooks/letsencrypt/files/default/bin/deploy-hook diff --git a/cookbooks/letsencrypt/files/default/bin/renew b/cookbooks/letsencrypt/files/default/bin/renew index 6a0482185..f3c7cd0cd 100755 --- a/cookbooks/letsencrypt/files/default/bin/renew +++ b/cookbooks/letsencrypt/files/default/bin/renew @@ -1,10 +1,9 @@ #!/bin/sh -cd /srv/acme.openstreetmap.org - /usr/bin/certbot renew \ + --key-type ecdsa \ --quiet \ --config-dir /srv/acme.openstreetmap.org/config \ --work-dir /srv/acme.openstreetmap.org/work \ --logs-dir /srv/acme.openstreetmap.org/logs \ - --renew-hook /srv/acme.openstreetmap.org/bin/renew-hook + --deploy-hook /srv/acme.openstreetmap.org/bin/deploy-hook diff --git a/cookbooks/letsencrypt/metadata.rb b/cookbooks/letsencrypt/metadata.rb index 52ef99f7c..a826ce5e4 100644 --- a/cookbooks/letsencrypt/metadata.rb +++ b/cookbooks/letsencrypt/metadata.rb @@ -9,3 +9,4 @@ supports "ubuntu" depends "accounts" depends "apache" depends "chef" +depends "ruby" diff --git a/cookbooks/letsencrypt/recipes/default.rb b/cookbooks/letsencrypt/recipes/default.rb index 382a0a58c..3e492f6af 100644 --- a/cookbooks/letsencrypt/recipes/default.rb +++ b/cookbooks/letsencrypt/recipes/default.rb @@ -20,13 +20,11 @@ include_recipe "accounts" include_recipe "apache" include_recipe "chef::knife" +include_recipe "ruby" keys = data_bag_item("chef", "keys") -package %w[ - certbot - ruby -] +package "certbot" directory "/etc/letsencrypt" do owner "letsencrypt" @@ -116,6 +114,13 @@ remote_directory "/srv/acme.openstreetmap.org/bin" do files_mode "755" end +template "/srv/acme.openstreetmap.org/bin/upload" do + source "upload.erb" + owner "root" + group "root" + mode "755" +end + directory "/srv/acme.openstreetmap.org/requests" do owner "root" group "root" @@ -168,6 +173,13 @@ Dir.glob("*", :base => "/srv/acme.openstreetmap.org/requests") do |name| end end +template "/srv/acme.openstreetmap.org/bin/check-certificate" do + source "check-certificate.erb" + owner "root" + group "root" + mode "755" +end + template "/srv/acme.openstreetmap.org/bin/check-certificates" do source "check-certificates.erb" owner "root" diff --git a/cookbooks/letsencrypt/templates/default/apache.erb b/cookbooks/letsencrypt/templates/default/apache.erb index 37d1df528..2073622d5 100644 --- a/cookbooks/letsencrypt/templates/default/apache.erb +++ b/cookbooks/letsencrypt/templates/default/apache.erb @@ -23,6 +23,8 @@ SSLCertificateFile /etc/ssl/certs/acme.openstreetmap.org.pem SSLCertificateKeyFile /etc/ssl/private/acme.openstreetmap.org.key + Timeout 5 + DocumentRoot /srv/acme.openstreetmap.org/html diff --git a/cookbooks/letsencrypt/files/default/bin/check-certificate b/cookbooks/letsencrypt/templates/default/check-certificate.erb old mode 100755 new mode 100644 similarity index 93% rename from cookbooks/letsencrypt/files/default/bin/check-certificate rename to cookbooks/letsencrypt/templates/default/check-certificate.erb index f24681589..319072b6b --- a/cookbooks/letsencrypt/files/default/bin/check-certificate +++ b/cookbooks/letsencrypt/templates/default/check-certificate.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "socket" require "openssl" @@ -33,6 +33,10 @@ if ssl puts "Certificate #{domains.first} on #{host} expires at #{certificate.not_after}" end + unless certificate.public_key.is_a?(OpenSSL::PKey::EC) + puts "Certificate #{domains.first} on #{host} does not use ECDSA key type" + end + digest = OpenSSL::Digest::SHA1.new certificate_id = OpenSSL::OCSP::CertificateId.new(certificate, issuer, digest) ocsp_request = OpenSSL::OCSP::Request.new.add_certid(certificate_id) diff --git a/cookbooks/letsencrypt/templates/default/request.erb b/cookbooks/letsencrypt/templates/default/request.erb index eaefa5bbe..65e80b303 100644 --- a/cookbooks/letsencrypt/templates/default/request.erb +++ b/cookbooks/letsencrypt/templates/default/request.erb @@ -2,6 +2,11 @@ # DO NOT EDIT - This file is being maintained by Chef +if [ "$(id -un)" != "letsencrypt" ]; then + echo "Error: This script must be run as user letsencrypt" >&2 + exit 1 +fi + /usr/bin/certbot certonly \ --non-interactive \ --config-dir /srv/acme.openstreetmap.org/config \ @@ -10,12 +15,12 @@ --email operations@osmfoundation.org \ --agree-tos \ --expand \ + --renew-with-new-domains \ + --cert-name <%= @domains.first %> \ <% @domains.each do |domain| -%> --domain <%= domain %> \ <% end -%> --webroot \ - --webroot-path /srv/acme.openstreetmap.org/html - -/srv/acme.openstreetmap.org/bin/upload \ - <%= @domains.first %> \ - /srv/acme.openstreetmap.org/config/live/<%= @domains.first %> + --webroot-path /srv/acme.openstreetmap.org/html \ + --deploy-hook /srv/acme.openstreetmap.org/bin/deploy-hook \ + "$@" diff --git a/cookbooks/letsencrypt/files/default/bin/upload b/cookbooks/letsencrypt/templates/default/upload.erb old mode 100755 new mode 100644 similarity index 67% rename from cookbooks/letsencrypt/files/default/bin/upload rename to cookbooks/letsencrypt/templates/default/upload.erb index a52f30257..e89912c4f --- a/cookbooks/letsencrypt/files/default/bin/upload +++ b/cookbooks/letsencrypt/templates/default/upload.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "json" require "tempfile" @@ -17,4 +17,4 @@ file = Tempfile.new(["letsencrypt", ".json"]) file.puts JSON.generate(bag) file.close -system("/opt/chef/embedded/bin/knife", "data", "bag", "from", "file", "letsencrypt", file.path) +system("/opt/chef/embedded/bin/knife", "data", "bag", "from", "file", "letsencrypt", file.path, "--config", "/srv/acme.openstreetmap.org/.chef/knife.rb", "--key", "/srv/acme.openstreetmap.org/.chef/client.pem") diff --git a/cookbooks/logstash/recipes/default.rb b/cookbooks/logstash/recipes/default.rb index 46bc3fea0..a4b237c00 100644 --- a/cookbooks/logstash/recipes/default.rb +++ b/cookbooks/logstash/recipes/default.rb @@ -75,10 +75,8 @@ template "/etc/cron.daily/expire-logstash" do mode "755" end -forwarders = [] - -search(:node, "recipes:logstash\\:\\:forwarder").each do |forwarder| - forwarders.append(forwarder.ipaddresses(:role => :external)) +forwarders = search(:node, "recipes:logstash\\:\\:forwarder").map do |forwarder| + forwarder.ipaddresses(:role => :external) end search(:node, "roles:gateway").each do |forwarder| diff --git a/cookbooks/matomo/attributes/default.rb b/cookbooks/matomo/attributes/default.rb index f4eb0c061..56ae90ca7 100644 --- a/cookbooks/matomo/attributes/default.rb +++ b/cookbooks/matomo/attributes/default.rb @@ -1,4 +1,4 @@ -default[:matomo][:version] = "5.1.1" +default[:matomo][:version] = "5.3.1" default[:matomo][:plugins] = { "Actions" => nil, "Annotations" => nil, @@ -21,6 +21,7 @@ default[:matomo][:plugins] = { "Diagnostics" => nil, "Ecommerce" => nil, "Events" => nil, + "FeatureFlags" => nil, "Feedback" => nil, "GeoIp2" => nil, "Goals" => nil, diff --git a/cookbooks/matomo/templates/default/config.erb b/cookbooks/matomo/templates/default/config.erb index 457badc83..a98ec4d81 100644 --- a/cookbooks/matomo/templates/default/config.erb +++ b/cookbooks/matomo/templates/default/config.erb @@ -7,11 +7,11 @@ password = "<%= @passwords['database'] %>" dbname = "piwik" tables_prefix = "piwik_" charset = "utf8mb4" - +collation = "utf8mb4_unicode_ci" + [General] force_ssl = 1 -force_ssl_login = 1 -login_allowlist_apply_to_reporting_api_requests = "0" +login_allowlist_apply_to_reporting_api_requests = 0 proxy_client_headers[] = "HTTP_X_FORWARDED_FOR" trusted_hosts[] = "matomo.openstreetmap.org" trusted_hosts[] = "piwik.openstreetmap.org" diff --git a/cookbooks/mediawiki/recipes/default.rb b/cookbooks/mediawiki/recipes/default.rb index c7249e8fa..d784e7b7f 100644 --- a/cookbooks/mediawiki/recipes/default.rb +++ b/cookbooks/mediawiki/recipes/default.rb @@ -41,6 +41,7 @@ package %w[ composer unzip ffmpeg + firejail ] # Mediawiki enhanced difference engine diff --git a/cookbooks/mediawiki/resources/extension.rb b/cookbooks/mediawiki/resources/extension.rb index 6240f6e87..fea8614d2 100644 --- a/cookbooks/mediawiki/resources/extension.rb +++ b/cookbooks/mediawiki/resources/extension.rb @@ -74,7 +74,7 @@ action :create do user node[:mediawiki][:user] group node[:mediawiki][:group] mode "664" - variables new_resource.variables + variables new_resource.variables.merge(:site => new_resource.site) end else file "#{mediawiki_directory}/LocalSettings.d/Ext-#{new_resource.extension}.inc.php" do diff --git a/cookbooks/mediawiki/resources/site.rb b/cookbooks/mediawiki/resources/site.rb index 0bbddfe81..e87c0f5ce 100644 --- a/cookbooks/mediawiki/resources/site.rb +++ b/cookbooks/mediawiki/resources/site.rb @@ -44,13 +44,17 @@ property :private_site, :kind_of => [TrueClass, FalseClass], :default => false property :hcaptcha_public_key, :kind_of => String, :default => "" property :hcaptcha_private_key, :kind_of => String, :default => "" property :extra_file_extensions, :kind_of => [String, Array], :default => [] +property :namespaces, :kind_of => Hash, :default => {} +property :force_ui_messages, :kind_of => Array, :default => [] +property :watch_category_membership, :kind_of => [TrueClass, FalseClass], :default => false property :fpm_max_children, :kind_of => Integer, :default => 5 property :fpm_start_servers, :kind_of => Integer, :default => 2 property :fpm_min_spare_servers, :kind_of => Integer, :default => 1 property :fpm_max_spare_servers, :kind_of => Integer, :default => 3 -property :fpm_request_terminate_timeout, :kind_of => Integer, :default => 300 +property :fpm_request_terminate_timeout, :kind_of => Integer, :default => 120 property :fpm_prometheus_port, :kind_of => Integer property :reload_apache, :kind_of => [TrueClass, FalseClass], :default => true +property :backup_enabled, :kind_of => [TrueClass, FalseClass], :default => true action :create do node.default[:mediawiki][:sites][new_resource.site] = { @@ -96,6 +100,7 @@ action :create do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "664" + variables :version => new_resource.version end execute "#{mediawiki_directory}/composer.json" do @@ -172,6 +177,12 @@ action :create do :directory => site_directory, :database_params => database_params only_if { ::File.exist?("#{mediawiki_directory}/LocalSettings.php") } + only_if { new_resource.backup_enabled } + end + + file "/etc/cron.daily/mediawiki-#{cron_name}-backup" do + action :delete + not_if { new_resource.backup_enabled } end # MobileFrontend extension is required by MinervaNeue skin @@ -425,6 +436,7 @@ action :create do if new_resource.commons mediawiki_extension "QuickInstantCommons" do site new_resource.site + template "mw-ext-QuickInstantCommons.inc.php.erb" update_site false end else @@ -471,7 +483,7 @@ action :create do request_terminate_timeout new_resource.fpm_request_terminate_timeout php_admin_values "open_basedir" => "#{site_directory}/:/usr/share/php/:/dev/null:/tmp/" php_values "memory_limit" => "500M", - "max_execution_time" => "240", + "max_execution_time" => "60", "upload_max_filesize" => "70M", "post_max_size" => "100M" prometheus_port new_resource.fpm_prometheus_port @@ -558,7 +570,8 @@ action_class do end def mediawiki_reference - shell_out!("git", "ls-remote", "--refs", "--sort=-version:refname", + shell_out!("git", "-c", "versionsort.suffix=-rc", + "ls-remote", "--refs", "--sort=-version:refname", "https://gerrit.wikimedia.org/r/mediawiki/core.git", "refs/tags/#{new_resource.version}.*") .stdout @@ -595,7 +608,10 @@ action_class do :site_readonly => new_resource.site_readonly, :extra_file_extensions => new_resource.extra_file_extensions, :private_accounts => new_resource.private_accounts, - :private_site => new_resource.private_site + :private_site => new_resource.private_site, + :namespaces => new_resource.namespaces, + :force_ui_messages => new_resource.force_ui_messages, + :watch_category_membership => new_resource.watch_category_membership } end diff --git a/cookbooks/mediawiki/templates/default/LocalSettings.php.erb b/cookbooks/mediawiki/templates/default/LocalSettings.php.erb index 93aab016d..9671130d8 100644 --- a/cookbooks/mediawiki/templates/default/LocalSettings.php.erb +++ b/cookbooks/mediawiki/templates/default/LocalSettings.php.erb @@ -293,95 +293,25 @@ $wgExpensiveParserFunctionLimit = 500; $wgSiteNotice = "<%= @mediawiki[:site_notice] %>"; <% end -%> <% if @mediawiki[:site_readonly] -%> -$wgReadOnly = "<%= @mediawiki[:site_readonly] %>"; +$wgReadOnly = ( PHP_SAPI === 'cli' ) ? false : "<%= @mediawiki[:site_readonly] %>"; <% end -%> -<% if @name == "wiki.openstreetmap.org" -%> -# DE -define('NS_LANG_DE', 200); -$wgExtraNamespaces[NS_LANG_DE] = 'DE'; -$wgNamespacesWithSubpages[NS_LANG_DE] = TRUE; -$wgContentNamespaces[] = NS_LANG_DE; -define('NS_LANG_DE_TALK', 201); -$wgExtraNamespaces[NS_LANG_DE_TALK] = 'DE_talk'; -$wgNamespacesWithSubpages[NS_LANG_DE_TALK] = TRUE; - -# FR -define('NS_LANG_FR', 202); -$wgExtraNamespaces[NS_LANG_FR] = 'FR'; -$wgNamespacesWithSubpages[NS_LANG_FR] = TRUE; -$wgContentNamespaces[] = NS_LANG_FR; -define('NS_LANG_FR_TALK', 203); -$wgExtraNamespaces[NS_LANG_FR_TALK] = 'FR_talk'; -$wgNamespacesWithSubpages[NS_LANG_FR_TALK] = TRUE; - -# ES -define('NS_LANG_ES', 204); -$wgExtraNamespaces[NS_LANG_ES] = 'ES'; -$wgNamespacesWithSubpages[NS_LANG_ES] = TRUE; -$wgContentNamespaces[] = NS_LANG_ES; -define('NS_LANG_ES_TALK', 205); -$wgExtraNamespaces[NS_LANG_ES_TALK] = 'ES_talk'; -$wgNamespacesWithSubpages[NS_LANG_ES_TALK] = TRUE; - -# IT -define('NS_LANG_IT', 206); -$wgExtraNamespaces[NS_LANG_IT] = 'IT'; -$wgNamespacesWithSubpages[NS_LANG_IT] = TRUE; -$wgContentNamespaces[] = NS_LANG_IT; -define('NS_LANG_IT_TALK', 207); -$wgExtraNamespaces[NS_LANG_IT_TALK] = 'IT_talk'; -$wgNamespacesWithSubpages[NS_LANG_IT_TALK] = TRUE; - -# NL -define('NS_LANG_NL', 208); -$wgExtraNamespaces[NS_LANG_NL] = 'NL'; -$wgNamespacesWithSubpages[NS_LANG_NL] = TRUE; -$wgContentNamespaces[] = NS_LANG_NL; -define('NS_LANG_NL_TALK', 209); -$wgExtraNamespaces[NS_LANG_NL_TALK] = 'NL_talk'; -$wgNamespacesWithSubpages[NS_LANG_NL_TALK] = TRUE; - -# RU -define('NS_LANG_RU', 210); -$wgExtraNamespaces[NS_LANG_RU] = 'RU'; -$wgNamespacesWithSubpages[NS_LANG_RU] = TRUE; -$wgContentNamespaces[] = NS_LANG_RU; -define('NS_LANG_RU_TALK', 211); -$wgExtraNamespaces[NS_LANG_RU_TALK] = 'RU_talk'; -$wgNamespacesWithSubpages[NS_LANG_RU_TALK] = TRUE; - -# JA -define('NS_LANG_JA', 212); -$wgExtraNamespaces[NS_LANG_JA] = 'JA'; -$wgNamespacesWithSubpages[NS_LANG_JA] = TRUE; -$wgContentNamespaces[] = NS_LANG_JA; -define('NS_LANG_JA_TALK', 213); -$wgExtraNamespaces[NS_LANG_JA_TALK] = 'JA_talk'; -$wgNamespacesWithSubpages[NS_LANG_JA_TALK] = TRUE; - -# Proposal -# namespace features a specific search weight defined at -# cookbooks/mediawiki/templates/default/mw-ext-CirrusSearch.inc.php.erb -define('NS_PROPOSAL', 3000); -$wgExtraNamespaces[NS_PROPOSAL] = 'Proposal'; -$wgNamespacesWithSubpages[NS_PROPOSAL] = TRUE; -$wgContentNamespaces[] = NS_PROPOSAL; -define('NS_PROPOSAL_TALK', 3001); -$wgExtraNamespaces[NS_PROPOSAL_TALK] = 'Proposal_talk'; -$wgNamespacesWithSubpages[NS_PROPOSAL_TALK] = TRUE; - -$wgNamespacesToBeSearchedDefault[NS_LANG_DE] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_FR] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_ES] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_IT] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_NL] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_RU] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_LANG_JA] = TRUE; -$wgNamespacesToBeSearchedDefault[NS_PROPOSAL] = TRUE; +<% @mediawiki[:namespaces].each do |name, details| -%> +# <%= name %> +define('NS_<%= name.upcase %>', <%= details[:id] %>); +$wgExtraNamespaces[NS_<%= name.upcase %>] = '<%= name %>'; +$wgNamespacesWithSubpages[NS_<%= name.upcase %>] = TRUE; +$wgContentNamespaces[] = NS_<%= name.upcase %>; +$wgNamespacesToBeSearchedDefault[NS_<%= name.upcase %>] = TRUE; +define('NS_<%= name.upcase %>_TALK', <%= details[:talk_id] %>); +$wgExtraNamespaces[NS_<%= name.upcase %>_TALK] = '<%= name %>_talk'; +$wgNamespacesWithSubpages[NS_<%= name.upcase %>_TALK] = TRUE; +<% end -%> + +<% unless @mediawiki[:force_ui_messages].empty? -%> # defines which links of the sidebar are translatable -$wgForceUIMsgAsContentMsg = array( 'mainpage-url', 'mapfeatures-url', 'contributors-url', 'helppage', 'blogs-url', 'shop-url', 'sitesupport-url' ); +$wgForceUIMsgAsContentMsg = array( '<%= @mediawiki[:force_ui_messages].join("', '") %>' ); <% end -%> # load extensions @@ -389,8 +319,8 @@ $wgForceUIMsgAsContentMsg = array( 'mainpage-url', 'mapfeatures-url', 'contribut <%= "require_once('#{file}');" %> <% end -%> -<% if @name == "wiki.openstreetmap.org" -%> -# wiki.openstreetmap.org specific config loaded after extensions +<% if @mediawiki[:watch_category_membership] -%> +# must be after extensions are loaded $wgRCWatchCategoryMembership = true; <% end -%> @@ -432,3 +362,13 @@ unset( $wgGroupsRemoveFromSelf['autoconfirmed'] ); # Mediawiki 1.38 has fix to allow this to be set by $wgVirtualRestConfig # https://phabricator.wikimedia.org/T285478 $wgHTTPTimeout = 240; + +# Enable night mode for Minerva and Vector skins +# https://github.com/openstreetmap/operations/issues/1230 +$wgMinervaNightMode['base'] = true; +$wgVectorNightMode['logged_in'] = true; +$wgVectorNightMode['logged_out'] = true; + +# Set extremely low timeout to avoid PHP-FPM timeouts on slow connections to Wikimedia Commons (rate limiting) or similar +$wgHTTPMaxTimeout = 5; +$wgHTTPMaxConnectTimeout = 3; diff --git a/cookbooks/mediawiki/templates/default/composer.local.json.erb b/cookbooks/mediawiki/templates/default/composer.local.json.erb index 73678e1cc..37382c5f6 100644 --- a/cookbooks/mediawiki/templates/default/composer.local.json.erb +++ b/cookbooks/mediawiki/templates/default/composer.local.json.erb @@ -8,6 +8,15 @@ } }, "require": { - "guzzlehttp/psr7": "2.4.5" +<% if @version.to_f < 1.43 -%> + "guzzlehttp/psr7": "2.4.5", + "data-values/common": "1.0.0", + "data-values/data-values": "3.0.0", + "data-values/geo": "4.2.3", + "data-values/interfaces": "1.0.0", + "data-values/number": "0.11.1", + "data-values/serialization": "1.2.4", + "data-values/time": "1.0.4" +<% end -%> } } diff --git a/cookbooks/mediawiki/templates/default/mw-ext-QuickInstantCommons.inc.php.erb b/cookbooks/mediawiki/templates/default/mw-ext-QuickInstantCommons.inc.php.erb new file mode 100644 index 000000000..06151ed58 --- /dev/null +++ b/cookbooks/mediawiki/templates/default/mw-ext-QuickInstantCommons.inc.php.erb @@ -0,0 +1,18 @@ + '\MediaWiki\Extension\QuickInstantCommons\Repo', + 'name' => 'wikimediacommons', + 'directory' => $wgUploadDirectory, + 'apibase' => 'https://commons.wikimedia.org/w/api.php', + 'hashLevels' => 2, + 'thumbUrl' => 'https://upload.wikimedia.org/wikipedia/commons/thumb', + 'fetchDescription' => true, + 'descriptionCacheExpiry' => 60*60*24*30, + 'transformVia404' => true, + 'abbrvThreshold' => 160, + 'apiMetadataExpiry' => 60*60*24*30, + 'disabledMediaHandlers' => [TiffHandler::class] +]; diff --git a/cookbooks/mysql/recipes/default.rb b/cookbooks/mysql/recipes/default.rb index baeff798e..63f70395b 100644 --- a/cookbooks/mysql/recipes/default.rb +++ b/cookbooks/mysql/recipes/default.rb @@ -28,7 +28,7 @@ mysql_variant = if platform?("ubuntu") package "#{mysql_variant}-server" package "#{mysql_variant}-client" -service "#{mysql_variant}" do +service mysql_variant do action [:enable, :start] supports :status => true, :restart => true end diff --git a/cookbooks/networking/attributes/default.rb b/cookbooks/networking/attributes/default.rb index 9832ce8f3..06511d462 100644 --- a/cookbooks/networking/attributes/default.rb +++ b/cookbooks/networking/attributes/default.rb @@ -6,11 +6,10 @@ default[:networking][:firewall][:outgoing] = [] default[:networking][:firewall][:http_rate_limit] = nil default[:networking][:firewall][:http_connection_limit] = nil default[:networking][:firewall][:allowlist] = [] -default[:networking][:roles] = {} default[:networking][:interfaces] = {} default[:networking][:nameservers] = %w[8.8.8.8 8.8.4.4 2001:4860:4860::8888 2001:4860:4860::8844] default[:networking][:search] = [] -default[:networking][:dnssec] = "allow-downgrade" +default[:networking][:dnssec] = "false" default[:networking][:hostname] = node.name default[:networking][:wireguard][:enabled] = true default[:networking][:wireguard][:keepalive] = 180 diff --git a/cookbooks/networking/recipes/default.rb b/cookbooks/networking/recipes/default.rb index 2b4d96761..efc79e431 100644 --- a/cookbooks/networking/recipes/default.rb +++ b/cookbooks/networking/recipes/default.rb @@ -23,8 +23,6 @@ require "ipaddr" require "yaml" -include_recipe "ruby" - keys = data_bag_item("networking", "keys") file "/etc/netplan/00-installer-config.yaml" do @@ -59,40 +57,21 @@ interfaces = node[:networking][:interfaces].collect do |name, interface| [interface[:interface], name] end.to_h -node[:networking][:interfaces].each do |name, interface| - if interface[:interface] =~ /^(.*)\.(\d+)$/ - vlan_interface = Regexp.last_match(1) - vlan_id = Regexp.last_match(2) - - parent = interfaces[vlan_interface] || "vlans_#{vlan_interface}" - - node.default_unless[:networking][:interfaces][parent][:interface] = vlan_interface - node.default_unless[:networking][:interfaces][parent][:vlans] = [] - - node.default[:networking][:interfaces][parent][:vlans] << vlan_id - end +node[:networking][:interfaces].each_value do |interface| + next unless interface[:interface] =~ /^(.*)\.(\d+)$/ - next unless interface[:role] && (role = node[:networking][:roles][interface[:role]]) + vlan_interface = Regexp.last_match(1) + vlan_id = Regexp.last_match(2) - if interface[:inet] && role[:inet] - node.default_unless[:networking][:interfaces][name][:inet][:prefix] = role[:inet][:prefix] - node.default_unless[:networking][:interfaces][name][:inet][:gateway] = role[:inet][:gateway] - node.default_unless[:networking][:interfaces][name][:inet][:routes] = role[:inet][:routes] - node.default_unless[:networking][:interfaces][name][:inet][:rules] = role[:inet][:rules] - end + parent = interfaces[vlan_interface] || "vlans_#{vlan_interface}" - if interface[:inet6] && role[:inet6] - node.default_unless[:networking][:interfaces][name][:inet6][:prefix] = role[:inet6][:prefix] - node.default_unless[:networking][:interfaces][name][:inet6][:gateway] = role[:inet6][:gateway] - node.default_unless[:networking][:interfaces][name][:inet6][:routes] = role[:inet6][:routes] - node.default_unless[:networking][:interfaces][name][:inet6][:rules] = role[:inet6][:rules] - end + node.default_unless[:networking][:interfaces][parent][:interface] = vlan_interface + node.default_unless[:networking][:interfaces][parent][:vlans] = [] - node.default_unless[:networking][:interfaces][name][:metric] = role[:metric] - node.default_unless[:networking][:interfaces][name][:zone] = role[:zone] + node.default[:networking][:interfaces][parent][:vlans] << vlan_id end -node[:networking][:interfaces].each do |_, interface| +node[:networking][:interfaces].each_value do |interface| if interface[:interface] =~ /^.*\.(\d+)$/ template "/etc/systemd/network/10-#{interface[:interface]}.netdev" do source "vlan.netdev.erb" @@ -103,6 +82,8 @@ node[:networking][:interfaces].each do |_, interface| notifies :run, "notify_group[networkctl-reload]" end elsif interface[:interface] =~ /^bond\d+$/ + next unless interface[:bond] + template "/etc/systemd/network/10-#{interface[:interface]}.netdev" do source "bond.netdev.erb" owner "root" @@ -335,8 +316,10 @@ link "/etc/resolv.conf" do to "../run/systemd/resolve/stub-resolv.conf" end +package "ruby" + gem_package "dbus-systemd" do - gem_binary node[:ruby][:gem] + gem_binary node[:ruby][:system_gem] end prometheus_exporter "resolved" do @@ -360,10 +343,8 @@ end package "nftables" -interfaces = [] - -node.interfaces(:role => :external).each do |interface| - interfaces << interface[:interface] +interfaces = node.interfaces(:role => :external).map do |interface| + interface[:interface] end template "/etc/nftables.conf" do diff --git a/cookbooks/networking/templates/default/network.erb b/cookbooks/networking/templates/default/network.erb index 7f38de3c5..f837c65a5 100644 --- a/cookbooks/networking/templates/default/network.erb +++ b/cookbooks/networking/templates/default/network.erb @@ -91,6 +91,9 @@ Destination=<%= destination %> <% if details[:metric] -%> Metric=<%= details[:metric] %> <% end -%> +<% if details[:table] -%> +Table=<%= details[:table] %> +<% end -%> <% if details[:type] -%> Type=<%= details[:type] %> <% end -%> @@ -107,6 +110,9 @@ Destination=<%= destination %> <% if details[:metric] -%> Metric=<%= details[:metric] %> <% end -%> +<% if details[:table] -%> +Table=<%= details[:table] %> +<% end -%> <% if details[:type] -%> Type=<%= details[:type] %> <% end -%> diff --git a/cookbooks/nominatim/templates/default/nginx.erb b/cookbooks/nominatim/templates/default/nginx.erb index af7ea960b..226c7599a 100644 --- a/cookbooks/nominatim/templates/default/nginx.erb +++ b/cookbooks/nominatim/templates/default/nginx.erb @@ -222,6 +222,7 @@ server { proxy_set_header Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Proxy-Connection ""; proxy_redirect off; proxy_pass http://nominatim_service; } diff --git a/cookbooks/openssh/recipes/default.rb b/cookbooks/openssh/recipes/default.rb index 8b57aaaef..98e2fa9f2 100644 --- a/cookbooks/openssh/recipes/default.rb +++ b/cookbooks/openssh/recipes/default.rb @@ -43,7 +43,7 @@ hosts = search(:node, "networking:interfaces").sort_by { |n| n[:hostname] }.coll names = [name] unless node.interfaces(:role => :internal).empty? - names.unshift("#{name}.#{node[:networking][:roles][:external][:zone]}.openstreetmap.org") + names.unshift("#{name}.#{node[:networking][:search].first}") end unless node.interfaces(:role => :external).empty? diff --git a/cookbooks/openssh/templates/default/ssh_known_hosts.erb b/cookbooks/openssh/templates/default/ssh_known_hosts.erb index c2d996a75..e0f173c06 100644 --- a/cookbooks/openssh/templates/default/ssh_known_hosts.erb +++ b/cookbooks/openssh/templates/default/ssh_known_hosts.erb @@ -20,11 +20,12 @@ yevaud.oob.openstreetmap.org,yevaud.oob,10.0.1.15 ssh-rsa AAAAB3NzaC1yc2EAAAABIw switch1.ams.openstreetmap.org,switch1.ams,184.104.179.129,2001:470:1:fa1::1 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCbUecW/591rGCYtkdVTUgAKTBc+lEhraeMCau7CHqMkkUn+/pe68toDySdTaO8jua1q1asuEWH3MZ2AGAPSF+xhrqJLLPQmm0WdEAQfZChxQUoqCPqlWAuM5u+bAc3ntKjw2v2myoGGAqjL27QkZDP8qPdxvKcbm2YpPVOHP56jsyp/9BjVhIpWVmMhkOAfBSjdKU+uP+NueHrJ/AueoqucQE0txeo5Mmw7GlRuLTsMNDU3Khlhjs/2Q0QGiMXV6hQxj5AgnaDZMdSRbp8lXlctDZmipr7jVk3TL+knzySih5wJ7wmeZimW8Dhcxk6HBo2lGgPGdtQgttCF5ZUGcfF switch1.ams.openstreetmap.org,switch1.ams,184.104.179.129,2001:470:1:fa1::1 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFPNVsMuGWMe2OP9AIg0hqZLSJP8zyNHWqYrXd5Mnfr12XdhzWEAKLTD14ZnJcz/GBaE0ad3IjT+hVoS/GVtxpQ= switch1.ams.openstreetmap.org,switch1.ams,184.104.179.129,2001:470:1:fa1::1 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA615MNtLzOv6qPfFsD0I6Pd0itshpHLAVImjt1ME4SM -oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2oeoe4olettjDauLuRiXRQVg5k6EO+PFfS9Ho3OzlOeYr1+h1SSPLkFtr20yOKcGXQcmYLjTbzvt/sbz2JUbfJHc2n7tMvQKK5ClVxLwqt2jpfSepGbG2KjG6vwxpkcDMH0bquzr/8ehpYmBwh3m8EARgcVNYoVL6J5OV4ILAtjNEllsrB71gitrarC8iqslRLVMjane9YKa7z0AGkwcT9w4qbcyuQ24rJF0Xo/IcraxVAYeX0Xtfx9rD3Zt1kHawaKYQdJM7RZvTtWcgCzHt738lFZ15Qj3at6vxxGpnduZiuHLB3PCW9yP4CFHm3Qs3TUyh3VqrNVFFdFvUXnP+DR3QK2E7TpViagZmXb+2AeCaLtrJ+1h0NEmQmliB4Zm4XKPsdS7k6Ul43pYQ5i1ipwZBNvhJrByQJ6UkpIjrwUlQBxXcEWGZaSL0sqqDQIYPizXZWOh7pQay2LxnHUVp0MZT30iVLYjptu9PcJ6CBwoKzhnVHPh7SON4uzDyMrk= -oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFLujNadDWF9OqF//izQ3/jm4zElWb2nMjfpeiUVCYh/Jrsl07c1bRJlnSmoxL2H0xeHJXQh1E4PPy4o/5N4GfE= -oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINxBm8YPJNtUIDh+0YWKY/5XKv8QKQBtF3ecqQcxIQNJ +oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDTgHga8l3mOpydchsZsvlaCJpxHoUV5dYBzK8m3iOedqd1UBS9KbHEfgFOvXcd3cmOnQT5dM1YXCQ74WB9hI5Wbkz+oHxWq+ZJeHOsXVIqxFzgxsI8e8gxJve7VyzxR52yiStBdLODhwwMmRPIH8AzzR93Fzi5AAQrGoHoqi/R0A/l6yTI8YnjJ2WaKzflId0YMyjR2KhOj+jCgCZIJR6DqfVpZXERMAtdVUY78Ub0E8iYyWg24gI176l4jCezmaTZ99tWo41mGXax9zXizQr4jx4yIxzRMhqAoJk8oDBkdhxudRUL2eWvBs3v1kcY3Ywtxi+wQIqClvQgBkWrHe4Jujjl8EEFuGHh6156Al7WqEnC9EcENA5Lk/IM0lzDbeT+WTyO/qU8cu035hmPMydYBHjqzo5L6yDwFXEAtJpHlr8QQDzEwgtK9bXc453cQDEJW4JWjP7UlsTlbflSBGQrY5FpI8D+MgnDYNld2kD4Fxzomf4ne+M0w19jrtNMJuE= +oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPkUAdFYwolSkUs7d1k+mRVBCV7WRrKsebj8RbjLqPCmEQo9ABSy2iAuYOYgkkqlgmB8F8OHsnNtmGzx7smCwAU= +oob1.ams.openstreetmap.org,oob1.ams,10.0.48.102 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKVdlLkjFUDExO7YCixEWvaKHGS0IQHn2nBrkMjGTwRx +ats1.ams.openstreetmap.org,ats1.ams,10.0.48.103 ssh-rsa AAAAB3NzaC1yc2EAAAACAQEAAAEBAKnx+hwj8Am4DuYXZLK1whHqOjW4Bbs/KrLqFokoZgJxzp1g+6sOAzXPy8w3dYyHBJaXCmr3RisIhG7fj0pITkywhU8ygiaEHDzPFyGqxVYe1oI3ayE47hC14Jhu33p7F+pY1xEVUa2H3Xu4F8toHnmuIjLAJ9phaEVvnjLl5n47dM1VfB5KB4NvuDejBthaZpgbALainUuKaqUL2Wgb0LxbDFRTQ+r6uUY4SEFnfqwveovdBL/edd+14/yRob0/r6zlubi5PhvMH1rCaE7flFNr61LItz06VAs+vZpLTNTIFXHqAtCjIA6e8VvvOn8wwmjxXKLsUOFZipvW9Uc9kTk= faffy.oob.openstreetmap.org,faffy.oob,10.0.49.3 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCuL6kBDuK+/xj40jp4ZP9HVoH1enU1W9MWO4Q0cxlD9sjHaz/mIbLhtPxElrw807QLsUdOx1IeD0jCLi9XAYYEF2xFFZ9DIP+qGp6hQ3XJrmOVgbDuckPjdvJWAxC5TVaWqwvHyp02biJCfSbOzsmjm2cOp03Cqq18HJOCfA3AxQ/zSQDSVemFoFPVeM67M9NgM64z1idPAvMVtW8Kd+unU/oE7V+Jil5TR5vwK+kVKEixxMtZfkJn0bNKVFWX5sJYmKWze+b95O+iQlsxmQW/p7UVM/3krKdBBT8+9b6UOv8TN0Yhopp86aXEgtvUOOyyimjqO3V5ogRMGj3b9G+9 -vhagar.oob.openstreetmap.org,10.0.49.5 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJaXI71RCPjbni+WxBnOf2CzkLi0H5ARG1RxevLO5smM/9LbXVAVgcZRnI3egmbaJF7r6queHtrQq5y27+Rz9OEoJ/DWMS/qaGWMqj1f7LDv+H44juKAxmayOYkLJUnXN/AYITjpNSfLy0kuSo9cGpHVKjQZVFGv92b0c1CPrQ0WJ3Q8Y7ERc77b2PBQmxWow/3RHv6SoeT8riCfoU5Mnuld2aNjlJlEglf79QB8r7Xonx3TDfcr0+bGbvJNhYoMPPDXFC96wCpjvIDQu09BEM91/zP/VRE3E+bvIp0WRugU6vUYAAIcKuyQpON5Fs1kWaY4AZ6/4+Qdql5B8z8twz +vhagar.oob.openstreetmap.org,vhagar.oob,10.0.49.5 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJaXI71RCPjbni+WxBnOf2CzkLi0H5ARG1RxevLO5smM/9LbXVAVgcZRnI3egmbaJF7r6queHtrQq5y27+Rz9OEoJ/DWMS/qaGWMqj1f7LDv+H44juKAxmayOYkLJUnXN/AYITjpNSfLy0kuSo9cGpHVKjQZVFGv92b0c1CPrQ0WJ3Q8Y7ERc77b2PBQmxWow/3RHv6SoeT8riCfoU5Mnuld2aNjlJlEglf79QB8r7Xonx3TDfcr0+bGbvJNhYoMPPDXFC96wCpjvIDQu09BEM91/zP/VRE3E+bvIp0WRugU6vUYAAIcKuyQpON5Fs1kWaY4AZ6/4+Qdql5B8z8twz dulcy.oob.openstreetmap.org,dulcy.oob,10.0.49.9 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgmlwXGPIPDqoMz4s5edr+G4iuBnCYcFXjY7Et9jG/ezI6aIKd9lEOXvnHJX5hrC21aqd06dhyMLwZN+eSjJhXNaLtYmm5P442H2ZnGca1KNSpF7yRVhn0eRpX39xWK3biVfMw64mzvV3636C1adCkgBwuaqCvz1EHm/KRrySfuETFLU= spike-06.oob.openstreetmap.org,spike-06.oob,10.0.49.6 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDwQVb/cYgKp21vw6+jKr5TLAD77oKunbFbjvwPL34iLRDqdJcEOdJBV1A63ZDUWSWjDUrTa0kyneoiYdD876EpuspZ4bPgmiHBvug5NR3usxa8PAsia1K+fONAGi3+s96H6Us273KHNH5QwyBNDyYaDDjIECkydU4bQT0FSX0j32eAHI62ZO/H82fUyL20PIK/nCUaJUd+5iXgcTfjP+eL66ghgPCgXGF99/w2+3EUxP2IN9J2yI57H/rYn3rj+NdZyQuzc0TAToT341cYh2+a8R1Sb/DEFiMZFE2hJ2NeVEaBYxPSlFfhRtlB6ArXg5QL7e7UScQh/ZHLU26A1ZaB spike-07.oob.openstreetmap.org,spike-07.oob,10.0.49.7 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCxLxAbMPZw+LgzB8JmTLLxqc6CZXYA3Wo18RM02+EnmtRX6AAf/VFb7E/VpgHfZecwnFK7u21R47+fFdHCb8EDYGeWbOYoEOYMoDh26H8/aFPHbldgG0xs9EwfTryWB7iZ2sD9nLv+nBpDkFXVq53kscmDlUKQrvUyj/zR55xGxcq0ruu3w05ZeGx5I6HRY0xeK+6H4s6sZtF24Zm3CttQ5M7ADgWffE35ZwA9tEdioM3J3c5EcV4HMdhl7wame6pID9+FpDEtcts5z5sfW9Y0yx3FHrXCxRWxHRbyFG9/+4RNwKJBRZrdb+VSLlrYVUFLEfb1AhOxr8ZfSsJU1V0V diff --git a/cookbooks/overpass/recipes/default.rb b/cookbooks/overpass/recipes/default.rb index 665b6730b..ffd5f49ac 100644 --- a/cookbooks/overpass/recipes/default.rb +++ b/cookbooks/overpass/recipes/default.rb @@ -107,7 +107,7 @@ apache_site "default" do action :disable end -apache_site "#{node[:overpass][:fqdn]}" do +apache_site node[:overpass][:fqdn] do template "apache.erb" directory "#{basedir}/site" variables :script_directory => "#{basedir}/cgi-bin" @@ -152,7 +152,7 @@ systemd_service "overpass-dispatcher" do description "Overpass Main Dispatcher" wants ["overpass-area-dispatcher.service"] working_directory basedir - exec_start "#{basedir}/bin/dispatcher --osm-base #{meta_map_short[node[:overpass][:meta_mode]]} --db-dir=#{basedir}/db --rate-limit=#{node[:overpass][:rate_limit]} --space=#{node[:overpass][:dispatcher_space]}" + exec_start "#{basedir}/bin/dispatcher --allow-duplicate-queries=yes --osm-base #{meta_map_short[node[:overpass][:meta_mode]]} --db-dir=#{basedir}/db --rate-limit=#{node[:overpass][:rate_limit]} --space=#{node[:overpass][:dispatcher_space]}" exec_stop "#{basedir}/bin/dispatcher --osm-base --terminate" standard_output "append:#{logdir}/osm_base.log" user username @@ -166,7 +166,7 @@ systemd_service "overpass-area-dispatcher" do description "Overpass Area Dispatcher" after ["overpass-dispatcher.service"] working_directory basedir - exec_start "#{basedir}/bin/dispatcher --areas #{meta_map_short[node[:overpass][:meta_mode]]} --db-dir=#{basedir}/db" + exec_start "#{basedir}/bin/dispatcher --allow-duplicate-queries=yes --areas #{meta_map_short[node[:overpass][:meta_mode]]} --db-dir=#{basedir}/db" exec_stop "#{basedir}/bin/dispatcher --areas --terminate" standard_output "append:#{logdir}/areas.log" user username diff --git a/cookbooks/overpass/templates/default/totp-filter.erb b/cookbooks/overpass/templates/default/totp-filter.erb index 8245f2ae3..78160cfe9 100644 --- a/cookbooks/overpass/templates/default/totp-filter.erb +++ b/cookbooks/overpass/templates/default/totp-filter.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "cgi" require "rotp" diff --git a/cookbooks/oxidized/recipes/default.rb b/cookbooks/oxidized/recipes/default.rb index fce3126d3..1840c2af3 100644 --- a/cookbooks/oxidized/recipes/default.rb +++ b/cookbooks/oxidized/recipes/default.rb @@ -130,9 +130,15 @@ git "/var/lib/oxidized/configs.git" do group "oxidized" end +bundle_config "/opt/oxidized/daemon" do + user "oxidized" + group "oxidized" + settings "deployment" => "true", + "build.rugged" => "--with-ssh" +end + bundle_install "/opt/oxidized/daemon" do action :nothing - options "--deployment" user "oxidized" group "oxidized" notifies :restart, "service[oxidized]" @@ -150,6 +156,7 @@ systemd_service "oxidized" do "OXIDIZED_LOGS" => "/var/log/oxidized" nice 10 sandbox :enable_network => true + restrict_address_families "AF_NETLINK" read_write_paths ["/run/oxidized", "/var/lib/oxidized", "/var/log/oxidized"] restart "on-failure" notifies :restart, "service[oxidized]" diff --git a/cookbooks/passenger/templates/default/passenger.conf.erb b/cookbooks/passenger/templates/default/passenger.conf.erb index 1212f7588..5533f8deb 100644 --- a/cookbooks/passenger/templates/default/passenger.conf.erb +++ b/cookbooks/passenger/templates/default/passenger.conf.erb @@ -2,8 +2,9 @@ PassengerRoot /usr/lib/ruby/vendor_ruby/phusion_passenger/locations.ini - PassengerDefaultRuby /usr/bin/ruby + PassengerDefaultRuby <%= node[:ruby][:interpreter] %> PassengerRuby /usr/local/bin/passenger-ruby + PassengerPreloadBundler on PassengerMaxPoolSize <%= node[:passenger][:max_pool_size] %> PassengerPoolIdleTime <%= node[:passenger][:pool_idle_time] %> PassengerFriendlyErrorPages off diff --git a/cookbooks/passenger/templates/default/ruby.erb b/cookbooks/passenger/templates/default/ruby.erb index f45fd210e..2af922032 100644 --- a/cookbooks/passenger/templates/default/ruby.erb +++ b/cookbooks/passenger/templates/default/ruby.erb @@ -4,4 +4,4 @@ export RUBY_GC_HEAP_INIT_SLOTS=500000 export RUBY_GC_HEAP_FREE_SLOTS=100000 export RUBY_GC_MALLOC_LIMIT=50000000 -exec /usr/bin/ruby<%= node[:ruby][:version] %> "$@" +exec <%= node[:ruby][:interpreter] %> "$@" diff --git a/cookbooks/planet/recipes/replication.rb b/cookbooks/planet/recipes/replication.rb index d719d11ae..f7552ddf2 100644 --- a/cookbooks/planet/recipes/replication.rb +++ b/cookbooks/planet/recipes/replication.rb @@ -32,14 +32,18 @@ db_passwords = data_bag_item("db", "passwords") package %w[ postgresql-client - ruby-libxml make gcc libc6-dev libpq-dev + libxml2-dev osmdbt ] +gem_package "libxml-ruby" do + gem_binary node[:ruby][:gem] +end + gem_package "pg" do gem_binary node[:ruby][:gem] end @@ -91,6 +95,13 @@ template "/usr/local/bin/users-deleted" do mode "755" end +template "/usr/local/bin/replicate-changesets" do + source "replicate-changesets.erb" + owner "root" + group "root" + mode "755" +end + ## Published deleted users directory remote_directory "/store/planet/users_deleted" do diff --git a/cookbooks/planet/templates/default/planet-file-cleanup.erb b/cookbooks/planet/templates/default/planet-file-cleanup.erb index 52ce68623..1df8eb595 100644 --- a/cookbooks/planet/templates/default/planet-file-cleanup.erb +++ b/cookbooks/planet/templates/default/planet-file-cleanup.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require 'date' require 'optparse' diff --git a/cookbooks/planet/files/default/replication-bin/replicate-changesets b/cookbooks/planet/templates/default/replicate-changesets.erb similarity index 90% rename from cookbooks/planet/files/default/replication-bin/replicate-changesets rename to cookbooks/planet/templates/default/replicate-changesets.erb index 0c53856d9..e2e5aca68 100755 --- a/cookbooks/planet/files/default/replication-bin/replicate-changesets +++ b/cookbooks/planet/templates/default/replicate-changesets.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "rubygems" require "pg" @@ -96,7 +96,7 @@ class ChangesetBuilder def add_comments(xml, cs) # grab the visible changeset comments as well - res = @conn.exec("select cc.author_id, u.display_name as author, cc.body, cc.created_at from changeset_comments cc join users u on cc.author_id=u.id where cc.changeset_id=#{cs.id} and cc.visible order by cc.created_at asc") + res = @conn.exec("select cc.id, cc.author_id, u.display_name as author, cc.body, (cc.created_at at time zone 'utc') as created_at from changeset_comments cc join users u on cc.author_id=u.id where cc.changeset_id=#{cs.id} and cc.visible order by cc.created_at asc") xml["comments_count"] = res.num_tuples.to_s # early return if there aren't any comments @@ -105,6 +105,7 @@ class ChangesetBuilder discussion = XML::Node.new("discussion") res.each do |row| comment = XML::Node.new("comment") + comment["id"] = row["id"] comment["uid"] = row["author_id"] comment["user"] = xml_sanitize(row["author"]) comment["date"] = Time.parse(row["created_at"]).getutc.xmlschema @@ -127,7 +128,7 @@ end # sync a directory to guarantee it's on disk. have to recurse to the root # to guarantee sync for newly created directories. def fdirsync(d) - while d != "/" + while d != "/" && d != "." fsync(d) d = File.dirname(d) end @@ -139,7 +140,7 @@ end class Replicator def initialize(config) @config = YAML.safe_load(File.read(config)) - @state = YAML.safe_load(File.read(@config["state_file"]), [Time]) + @state = YAML.safe_load(File.read(@config["state_file"]), :permitted_classes => [Time], :fallback => {}) @conn = PG::Connection.connect(@config["db"]) # get current time from the database rather than the current system @now = @conn.exec("select now() as now").map { |row| Time.parse(row["now"]) }[0] @@ -153,7 +154,7 @@ class Replicator # for us to look at anything that was closed recently, and filter from # there. changesets = @conn - .exec("select id, created_at, closed_at, num_changes from changesets where closed_at > ((now() at time zone 'utc') - '1 hour'::interval)") + .exec("select id, (created_at at time zone 'utc') as created_at, (closed_at at time zone 'utc') as closed_at, num_changes from changesets where (closed_at at time zone 'utc') > ((now() at time zone 'utc') - '1 hour'::interval)") .map { |row| Changeset.new(row) } .select { |cs| cs.activity_between?(last_run, @now) } @@ -162,13 +163,13 @@ class Replicator # but also add any changesets which have new comments new_ids = @conn - .exec("select distinct changeset_id from changeset_comments where created_at >= '#{last_run}' and created_at < '#{@now}' and visible") + .exec("select distinct changeset_id from changeset_comments where (created_at at time zone 'utc') >= '#{last_run}' and (created_at at time zone 'utc') < '#{@now}' and visible") .map { |row| row["changeset_id"].to_i } .reject { |c_id| cs_ids.include?(c_id) } new_ids.each do |id| @conn - .exec("select id, created_at, closed_at, num_changes from changesets where id=#{id}") + .exec("select id, (created_at at time zone 'utc') as created_at, (closed_at at time zone 'utc') as closed_at, num_changes from changesets where id=#{id}") .map { |row| Changeset.new(row) } .each { |cs| changesets << cs } end diff --git a/cookbooks/podman/resources/service.rb b/cookbooks/podman/resources/service.rb index 5ef0b5885..5178980d9 100644 --- a/cookbooks/podman/resources/service.rb +++ b/cookbooks/podman/resources/service.rb @@ -27,6 +27,7 @@ property :image, String, :required => true property :ports, Hash, :default => {} property :environment, Hash, :default => {} property :volume, Hash, :default => {} +property :command, String, :default => "" action :create do systemd_service new_resource.service do @@ -35,7 +36,11 @@ action :create do notify_access "all" environment "PODMAN_SYSTEMD_UNIT" => "%n" exec_start_pre "/bin/rm --force %t/%n.ctr-id" - exec_start "/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --userns=auto --label=io.containers.autoupdate=registry --pids-limit=-1 #{publish_options} #{environment_options} #{volume_options} --rm --sdnotify=conmon --detach --replace --name=%N #{new_resource.image}" + exec_start "/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon " \ + "--userns=auto --label=io.containers.autoupdate=registry " \ + "--pids-limit=-1 #{publish_options} #{environment_options} " \ + "#{volume_options} --rm --sdnotify=conmon --detach --replace " \ + "--name=%N #{new_resource.image} #{new_resource.command}" exec_stop "/usr/bin/podman stop --ignore --time=10 --cidfile=%t/%n.ctr-id" exec_stop_post "/usr/bin/podman rm --force --ignore --cidfile=%t/%n.ctr-id" timeout_start_sec 180 diff --git a/cookbooks/podman/resources/site.rb b/cookbooks/podman/resources/site.rb index 7cab5a5d2..225021eb7 100644 --- a/cookbooks/podman/resources/site.rb +++ b/cookbooks/podman/resources/site.rb @@ -67,7 +67,7 @@ action_class do def ports @ports ||= if ::File.exist?(ports_file) - YAML.safe_load(::File.read(ports_file)) + YAML.safe_load_file(ports_file) else {} end diff --git a/cookbooks/postgresql/libraries/postgresql.rb b/cookbooks/postgresql/libraries/postgresql.rb index b2df4aed6..789120d6b 100644 --- a/cookbooks/postgresql/libraries/postgresql.rb +++ b/cookbooks/postgresql/libraries/postgresql.rb @@ -122,7 +122,7 @@ module OpenStreetMap def schemas(database) @schemas ||= {} @schemas[database] ||= query("SELECT n.nspname, pg_catalog.pg_get_userbyid(n.nspowner) AS usename, n.nspacl FROM pg_namespace AS n WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema'", :database => database).each_with_object({}) do |schema, schemas| - name = "#{schema[:nspname]}" + name = schema[:nspname] schemas[name] = { :owner => schema[:usename], @@ -163,7 +163,7 @@ module OpenStreetMap def parse_acl(acl) parse_array(acl).each_with_object({}) do |entry, permissions| - entry = entry.sub(/^"(.*)"$/) { Regexp.last_match[1].gsub(/\\"/, '"') }.sub(%r{/.*$}, "") + entry = entry.sub(/^"(.*)"$/) { Regexp.last_match[1].gsub('\"', '"') }.sub(%r{/.*$}, "") user, privileges = entry.split("=") user = user.sub(/^"(.*)"$/, "\\1") diff --git a/cookbooks/postgresql/recipes/default.rb b/cookbooks/postgresql/recipes/default.rb index 742941ee9..eae492f16 100644 --- a/cookbooks/postgresql/recipes/default.rb +++ b/cookbooks/postgresql/recipes/default.rb @@ -124,24 +124,20 @@ clusters.each do |name, details| not_if { ::File.exist?("/var/lib/postgresql/#{name}/standby.signal") } end - exporter_options = %w[ - --collector.database_wraparound - --collector.long_running_transactions - --collector.process_idle - --collector.stat_activity_autovacuum - --collector.stat_wal_receiver - --collector.statio_user_indexes - ] - - exporter_options << "--no-collector.stat_bgwriter" if details[:version] >= 17 - prometheus_exporter "postgres" do port 10000 + details[:port].to_i service "postgres-#{prometheus_suffix}" labels "cluster" => name scrape_interval "1m" scrape_timeout "1m" - options exporter_options + options %w[ + --collector.database_wraparound + --collector.long_running_transactions + --collector.process_idle + --collector.stat_activity_autovacuum + --collector.stat_wal_receiver + --collector.statio_user_indexes + ] environment "DATA_SOURCE_NAME" => "postgres:///#{prometheus_database}?host=/run/postgresql&port=#{details[:port]}&user=prometheus&password=#{passwords['prometheus']}" restrict_address_families "AF_UNIX" subscribes :restart, "template[/etc/prometheus/exporters/postgres_queries.yml]" diff --git a/cookbooks/postgresql/resources/schema.rb b/cookbooks/postgresql/resources/schema.rb index a7bf0ebdb..e22324d9b 100644 --- a/cookbooks/postgresql/resources/schema.rb +++ b/cookbooks/postgresql/resources/schema.rb @@ -109,6 +109,6 @@ action_class do end def qualified_name - "#{new_resource.name}" + new_resource.name end end diff --git a/cookbooks/prometheus/metadata.rb b/cookbooks/prometheus/metadata.rb index 30f20802d..6148979b9 100644 --- a/cookbooks/prometheus/metadata.rb +++ b/cookbooks/prometheus/metadata.rb @@ -12,4 +12,3 @@ depends "awscli" depends "git" depends "hardware" depends "networking" -depends "ruby" diff --git a/cookbooks/prometheus/recipes/default.rb b/cookbooks/prometheus/recipes/default.rb index c6b609b06..df3ec2ea5 100644 --- a/cookbooks/prometheus/recipes/default.rb +++ b/cookbooks/prometheus/recipes/default.rb @@ -19,7 +19,8 @@ include_recipe "git" include_recipe "networking" -include_recipe "ruby" + +package "ruby" if node.internal_ipaddress node.default[:prometheus][:mode] = "internal" diff --git a/cookbooks/prometheus/recipes/smokeping.rb b/cookbooks/prometheus/recipes/smokeping.rb index 27b0c6dd6..c626eb37c 100644 --- a/cookbooks/prometheus/recipes/smokeping.rb +++ b/cookbooks/prometheus/recipes/smokeping.rb @@ -39,6 +39,7 @@ end prometheus_exporter "smokeping" do port 9374 + environment "GOMAXPROCS" => "1" options "--config.file=/etc/prometheus/exporters/smokeping.yml" capability_bounding_set "CAP_NET_RAW" ambient_capabilities "CAP_NET_RAW" diff --git a/cookbooks/prometheus/resources/collector.rb b/cookbooks/prometheus/resources/collector.rb index ce68a7e5c..4171b874d 100644 --- a/cookbooks/prometheus/resources/collector.rb +++ b/cookbooks/prometheus/resources/collector.rb @@ -27,6 +27,7 @@ property :user, :kind_of => String property :path, :kind_of => String property :options, :kind_of => [String, Array] property :environment, :kind_of => Hash, :default => {} +property :working_directory, String property :proc_subset, String property :capability_bounding_set, [String, Array] property :private_devices, [true, false] @@ -42,6 +43,7 @@ action :create do dynamic_user new_resource.user.nil? group "adm" environment new_resource.environment + working_directory new_resource.working_directory standard_output "file:/var/lib/prometheus/node-exporter/#{new_resource.collector}.new" standard_error "journal" exec_start "#{executable_path} #{executable_options}" diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb index 305afbd90..27cb72e47 100644 --- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb +++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb @@ -828,7 +828,7 @@ groups: - name: web rules: - alert: web error rate - expr: sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) / sum(rate(api_call_count_total[5m])) by (instance) > 0.002 and sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) > 0.01 + expr: sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) / sum(rate(api_call_count_total[5m])) by (instance) > 0.002 and sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) > 0.05 for: 5m labels: alertgroup: web diff --git a/cookbooks/ruby/attributes/default.rb b/cookbooks/ruby/attributes/default.rb index eefaf2247..39ac754e5 100644 --- a/cookbooks/ruby/attributes/default.rb +++ b/cookbooks/ruby/attributes/default.rb @@ -1,9 +1,28 @@ -default[:ruby][:version] = if platform?("debian") - "3.1" - elsif node[:lsb][:release].to_f < 22.04 - "2.7" - else - "3.0" - end -default[:ruby][:gem] = "/usr/bin/gem#{node[:ruby][:version]}" -default[:ruby][:bundle] = "/usr/bin/bundle#{node[:ruby][:version]}" +default[:ruby][:fullstaq] = true + +default[:ruby][:system_version] = if platform?("debian") + "3.1" + elsif node[:lsb][:release].to_f < 22.04 + "2.7" + else + "3.0" + end +default[:ruby][:system_interpreter] = "/usr/bin/ruby#{node[:ruby][:system_version]}" +default[:ruby][:system_gem] = "/usr/bin/gem#{node[:ruby][:system_version]}" +default[:ruby][:system_bundle] = "/usr/bin/bundle#{node[:ruby][:system_version]}" + +if node[:ruby][:fullstaq] + + default[:ruby][:version] = "3.4" + default[:ruby][:interpreter] = "/usr/lib/fullstaq-ruby/versions/#{node[:ruby][:version]}-jemalloc/bin/ruby" + default[:ruby][:gem] = "/usr/lib/fullstaq-ruby/versions/#{node[:ruby][:version]}-jemalloc/bin/gem" + default[:ruby][:bundle] = "/usr/lib/fullstaq-ruby/versions/#{node[:ruby][:version]}-jemalloc/bin/bundle" + +else + + default[:ruby][:version] = node[:ruby][:system_version] + default[:ruby][:interpreter] = node[:ruby][:system_interpreter] + default[:ruby][:gem] = node[:ruby][:system_gem] + default[:ruby][:bundle] = node[:ruby][:system_bundle] + +end diff --git a/cookbooks/ruby/metadata.rb b/cookbooks/ruby/metadata.rb index d8f2eac0e..6c014bdfe 100644 --- a/cookbooks/ruby/metadata.rb +++ b/cookbooks/ruby/metadata.rb @@ -6,3 +6,4 @@ description "Installs and configures ruby" version "1.0.0" supports "ubuntu" +depends "apt" diff --git a/cookbooks/ruby/recipes/default.rb b/cookbooks/ruby/recipes/default.rb index a2833aefa..886f18fe2 100644 --- a/cookbooks/ruby/recipes/default.rb +++ b/cookbooks/ruby/recipes/default.rb @@ -19,23 +19,29 @@ ruby_version = node[:ruby][:version] -package %W[ - ruby - ruby#{ruby_version} - ruby - ruby#{ruby_version}-dev -] - -gem_package "bundler#{ruby_version}-1" do - package_name "bundler" - version "~> 1.17.3" - gem_binary node[:ruby][:gem] - options "--format-executable" -end +if node[:ruby][:fullstaq] + + include_recipe "apt::fullstaq-ruby" + + package %W[ + fullstaq-ruby-common + fullstaq-ruby-#{ruby_version}-jemalloc + ] + + %w[bundle bundler erb gem irb racc rake rbs rdbg rdoc ri ruby syntax_suggest typeproc].each do |command| + link "/usr/local/bin/#{command}" do + to "/usr/lib/fullstaq-ruby/versions/#{ruby_version}-jemalloc/bin/#{command}" + owner "root" + group "root" + end + end + +else + + package %w[ + ruby + ruby-dev + ruby-bundler + ] -gem_package "bundler#{ruby_version}-2" do - package_name "bundler" - version "~> 2.3.16" - gem_binary node[:ruby][:gem] - options "--format-executable" end diff --git a/cookbooks/ruby/resources/bundle_config.rb b/cookbooks/ruby/resources/bundle_config.rb new file mode 100644 index 000000000..16902682f --- /dev/null +++ b/cookbooks/ruby/resources/bundle_config.rb @@ -0,0 +1,57 @@ +# +# Cookbook:: ruby +# Resource:: bundle_config +# +# Copyright:: 2025, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +resource_name :bundle_config +provides :bundle_config + +unified_mode true + +default_action :create + +property :directory, :kind_of => String, :name_property => true +property :user, :kind_of => String +property :group, :kind_of => String +property :settings, :kind_of => Hash + +load_current_value do |new_resource| + current_settings = shell_out!("#{node[:ruby][:bundle]} config list --parseable", :cwd => new_resource.directory).stdout.split("\n").map do |line| + line.split("=") + end.to_h + + settings current_settings +end + +action :create do + converge_if_changed :settings do + new_resource.settings.each do |name, value| + execute "bundle-config-set-#{name}" do + command "#{bundle_command} config set --local #{name} #{value}" + cwd new_resource.directory + user new_resource.user + group new_resource.group + end + end + end +end + +action_class do + def bundle_command + node[:ruby][:bundle] + end +end diff --git a/cookbooks/ruby/resources/bundle_install.rb b/cookbooks/ruby/resources/bundle_install.rb index 897bde5fe..9facccc27 100644 --- a/cookbooks/ruby/resources/bundle_install.rb +++ b/cookbooks/ruby/resources/bundle_install.rb @@ -47,6 +47,9 @@ action_class do end def after_created - subscribes :run, "gem_package[bundler#{node[:ruby][:version]}-1]" - subscribes :run, "gem_package[bundler#{node[:ruby][:version]}-2]" + if node[:ruby][:fullstaq] + subscribes :run, "package[fullstaq-ruby-#{node[:ruby][:version]}-jemalloc]" + else + subscribes :run, "package[ruby-bundler]" + end end diff --git a/cookbooks/serverinfo/attributes/default.rb b/cookbooks/serverinfo/attributes/default.rb new file mode 100644 index 000000000..bd45555d1 --- /dev/null +++ b/cookbooks/serverinfo/attributes/default.rb @@ -0,0 +1 @@ +default[:accounts][:users][:serverinfo][:status] = :role diff --git a/cookbooks/serverinfo/metadata.rb b/cookbooks/serverinfo/metadata.rb index 55d69f066..48a196ad6 100644 --- a/cookbooks/serverinfo/metadata.rb +++ b/cookbooks/serverinfo/metadata.rb @@ -6,6 +6,7 @@ description "Configures server-info web site" version "1.0.0" supports "ubuntu" +depends "accounts" depends "apache" depends "git" depends "ruby" diff --git a/cookbooks/serverinfo/recipes/default.rb b/cookbooks/serverinfo/recipes/default.rb index 2bbc31e5b..c455ddc12 100644 --- a/cookbooks/serverinfo/recipes/default.rb +++ b/cookbooks/serverinfo/recipes/default.rb @@ -17,6 +17,7 @@ # limitations under the License. # +include_recipe "accounts" include_recipe "apache" include_recipe "git" include_recipe "ruby" @@ -30,12 +31,18 @@ package %w[ pkg-config ] +directory "/srv/hardware.openstreetmap.org" do + owner "serverinfo" + group "serverinfo" + mode "755" +end + git "/srv/hardware.openstreetmap.org" do action :sync repository "https://github.com/osmfoundation/osmf-server-info.git" depth 1 - user "root" - group "root" + user "serverinfo" + group "serverinfo" notifies :run, "bundle_install[/srv/hardware.openstreetmap.org]" end @@ -45,52 +52,43 @@ roles = { :rows => search(:role, "*:*") } file "/srv/hardware.openstreetmap.org/_data/nodes.json" do content nodes.to_json mode "644" - owner "root" - group "root" + owner "serverinfo" + group "serverinfo" notifies :run, "bundle_exec[/srv/hardware.openstreetmap.org]" + sensitive true end file "/srv/hardware.openstreetmap.org/_data/roles.json" do content roles.to_json mode "644" - owner "root" - group "root" + owner "serverinfo" + group "serverinfo" notifies :run, "bundle_exec[/srv/hardware.openstreetmap.org]" end -directory "/srv/hardware.openstreetmap.org/_site" do - mode "755" - owner "nobody" - group "nogroup" -end - -directory "/srv/hardware.openstreetmap.org/vendor" do +bundle_config "/srv/hardware.openstreetmap.org" do action :create - owner "nobody" - group "nogroup" - notifies :run, "bundle_install[/srv/hardware.openstreetmap.org]", :immediately + user "serverinfo" + group "serverinfo" + settings "deployment" => "true", + "without" => "development:test", + "jobs" => node.cpu_cores.to_s + notifies :run, "bundle_exec[/srv/hardware.openstreetmap.org]" end bundle_install "/srv/hardware.openstreetmap.org" do action :nothing - user "nobody" - group "nogroup" - environment "BUNDLE_FROZEN" => "true", - "BUNDLE_WITHOUT" => "development:test", - "BUNDLE_PATH" => "vendor/bundle", - "BUNDLE_DEPLOYMENT" => "1", - "BUNDLE_JOBS" => node.cpu_cores.to_s + user "serverinfo" + group "serverinfo" notifies :run, "bundle_exec[/srv/hardware.openstreetmap.org]" end bundle_exec "/srv/hardware.openstreetmap.org" do action :nothing command "jekyll build --trace --disable-disk-cache --baseurl=https://hardware.openstreetmap.org" - user "nobody" - group "nogroup" - environment "LANG" => "C.UTF-8", - "BUNDLE_PATH" => "vendor/bundle", - "BUNDLE_DEPLOYMENT" => "1" + user "serverinfo" + group "serverinfo" + environment "LANG" => "C.UTF-8" end ssl_certificate "hardware.openstreetmap.org" do diff --git a/cookbooks/stateofthemap/recipes/container.rb b/cookbooks/stateofthemap/recipes/container.rb index 3b5bd38b9..af8514fa7 100644 --- a/cookbooks/stateofthemap/recipes/container.rb +++ b/cookbooks/stateofthemap/recipes/container.rb @@ -24,9 +24,14 @@ podman_site "stateofthemap.org" do aliases ["www.stateofthemap.org", "stateofthemap.com", "www.stateofthemap.com", "sotm.org", "www.sotm.org"] end -%w[2007 2008 2009 2013 2016 2017 2018 2019 2020 2021 2022 2024].each do |year| +%w[2007 2008 2009 2013 2016 2017 2018 2019 2020 2021 2022 2024 2025].each do |year| podman_site "#{year}.stateofthemap.org" do image "ghcr.io/openstreetmap/stateofthemap-#{year}:latest" aliases ["#{year}.stateofthemap.com", "#{year}.sotm.org"] end end + +podman_site "stateofthemap.eu" do + image "ghcr.io/openstreetmap/stateofthemap-eu-website:latest" + aliases ["www.stateofthemap.eu"] +end diff --git a/cookbooks/systemd/resources/service.rb b/cookbooks/systemd/resources/service.rb index 11ca0e28c..8b380c306 100644 --- a/cookbooks/systemd/resources/service.rb +++ b/cookbooks/systemd/resources/service.rb @@ -36,6 +36,7 @@ property :notify_access, String, :is => %w[none main exec all] property :limit_nofile, Integer property :limit_as, [Integer, String] property :limit_cpu, [Integer, String] +property :limit_core, [Integer, String] property :memory_low, [Integer, String] property :memory_high, [Integer, String] property :memory_max, [Integer, String] diff --git a/cookbooks/systemd/templates/default/service.erb b/cookbooks/systemd/templates/default/service.erb index 92d70de85..eee220691 100644 --- a/cookbooks/systemd/templates/default/service.erb +++ b/cookbooks/systemd/templates/default/service.erb @@ -45,6 +45,9 @@ LimitAS=<%= @limit_as %> <% if @limit_cpu -%> LimitCPU=<%= @limit_cpu %> <% end -%> +<% if @limit_core -%> +LimitCORE=<%= @limit_core %> +<% end -%> <% if @memory_low -%> MemoryLow=<%= @memory_low %> <% end -%> diff --git a/cookbooks/taginfo/recipes/default.rb b/cookbooks/taginfo/recipes/default.rb index 080d11924..ef3913c60 100644 --- a/cookbooks/taginfo/recipes/default.rb +++ b/cookbooks/taginfo/recipes/default.rb @@ -163,7 +163,7 @@ node[:taginfo][:sites].each do |site| settings["opensearch"]["contact"] = "webmaster@openstreetmap.org" settings["paths"]["bin_dir"] = "#{directory}/build/src" settings["sources"]["download"] = "" - settings["sources"]["create"] = "db languages projects wiki wikidata chronology" + settings["sources"]["create"] = "db languages projects wiki wikidata chronology sw" settings["sources"]["db"]["planetfile"] = "/var/lib/planet/planet.osh.pbf" settings["sources"]["chronology"]["osm_history_file"] = "/var/lib/planet/planet.osh.pbf" settings["tagstats"]["geodistribution"] = "DenseMmapArray" @@ -179,10 +179,17 @@ node[:taginfo][:sites].each do |site| notifies :restart, "service[apache2]" end + bundle_config "#{directory}/taginfo" do + user "taginfo" + group "taginfo" + settings "deployment" => "true", + "without" => "development:test" + end + bundle_install "#{directory}/taginfo" do action :nothing - user "root" - group "root" + user "taginfo" + group "taginfo" subscribes :run, "git[#{directory}/taginfo]" notifies :restart, "passenger_application[#{directory}/taginfo/web/public]" end @@ -225,7 +232,8 @@ node[:taginfo][:sites].each do |site| prometheus_collector "taginfo-#{site_name}" do interval "15m" user "taginfo" - path "#{directory}/taginfo/sources/metrics.rb" - options "#{directory}/data" + path node[:ruby][:bundle] + options "exec sources/metrics.rb #{directory}/data" + working_directory "#{directory}/taginfo" end end diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb index 13b897363..782b7ad86 100644 --- a/cookbooks/tile/recipes/default.rb +++ b/cookbooks/tile/recipes/default.rb @@ -159,6 +159,7 @@ end package %w[ python3-cairo python3-mapnik + python3-pil python3-pyotp python3-pyproj python3-setuptools @@ -565,8 +566,8 @@ systemd_service "expire-tiles" do sandbox true restrict_address_families "AF_UNIX" read_write_paths tile_directories + [ - "/var/lib/replicate/expire-queue" - ] + "/var/lib/replicate/expire-queue" + ] end systemd_path "expire-tiles" do diff --git a/cookbooks/tile/templates/default/apache.erb b/cookbooks/tile/templates/default/apache.erb index a1c4e1977..50f6b5a97 100644 --- a/cookbooks/tile/templates/default/apache.erb +++ b/cookbooks/tile/templates/default/apache.erb @@ -19,6 +19,39 @@ DocumentRoot /srv/tile.openstreetmap.org/html ScriptAlias /cgi-bin/ /srv/tile.openstreetmap.org/cgi-bin/ + # Set location of renderd socket + ModTileRenderdSocketName /run/renderd/renderd.sock + + # Set location of tile directory + ModTileTileDir /srv/tile.openstreetmap.org/tiles + + # Time to wait for a re-render before serving a dirty tile + ModTileRequestTimeout 2 + + # Timeout before giving up for a tile to be rendered that is otherwise missing + ModTileMissingRequestTimeout 10 + + # Always try and re-render dirty or missing tiles + ModTileMaxLoadOld 1000 + ModTileMaxLoadMissing 1000 + + # Maximum expiry to set on a tile + ModTileCacheDurationMax 604800 + + # Expiry time for dirty tiles that have been queued for re-rendering + ModTileCacheDurationDirty 900 + + # Minimum expiry time for fresh tiles + ModTileCacheDurationMinimum 10800 + ModTileCacheDurationMediumZoom 13 86400 + ModTileCacheDurationLowZoom 9 518400 + + # Factor controlling effect of last modification time on expiry + ModTileCacheLastModifiedFactor 0.20 + + # Load tile configuration + LoadTileConfigFile /etc/renderd.conf + # Get the real remote IP for requests via a trusted proxy RemoteIPHeader Fastly-Client-IP <% @fastly.sort.each do |address| -%> diff --git a/cookbooks/tile/templates/default/expire-tiles.erb b/cookbooks/tile/templates/default/expire-tiles.erb index b3a790f6a..5bc478082 100644 --- a/cookbooks/tile/templates/default/expire-tiles.erb +++ b/cookbooks/tile/templates/default/expire-tiles.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> # DO NOT EDIT - This file is being maintained by Chef diff --git a/cookbooks/tile/templates/default/export.erb b/cookbooks/tile/templates/default/export.erb index 8fa4672b9..eb35b1313 100644 --- a/cookbooks/tile/templates/default/export.erb +++ b/cookbooks/tile/templates/default/export.erb @@ -5,6 +5,7 @@ import cairo import cgi import http.cookies import mapnik +import io import os import pyotp import pyproj @@ -13,6 +14,7 @@ import shutil import signal import sys import tempfile +from PIL import Image # Limit maximum CPU time # The Postscript output format can sometimes take hours @@ -40,6 +42,10 @@ def output_file(file): def file_size(file): return os.fstat(file.fileno()).st_size +# Routine to retrieve BytesIO payload length +def bytesio_size(bio): + return bio.getbuffer().nbytes + # Routine to report an error def output_error(message, status = "400 Bad Request"): print("Status: %s" % status) @@ -54,6 +60,105 @@ def output_error(message, status = "400 Bad Request"): print("") print("") +# Add a copyright notice for raster formats (PNG, JPEG, WEBP) +def add_copyright_notice_raster(image, map_width, map_height, format): + # Convert the Mapnik image to PNG and store it in a BytesIO object + png = image.tostring("png") + png_io = io.BytesIO(png) + + # Load the PNG data from the BytesIO object into a Cairo ImageSurface + surface = cairo.ImageSurface.create_from_png(png_io) + + add_copyright_notice_vector(surface, map_width, map_height) + + # Convert the Cairo surface to PNG in a BytesIO object + output_io = io.BytesIO() + surface.write_to_png(output_io) + + if format == "png": + return output_io + else: + # Open the output PNG image for conversion to other formats + img = Image.open(output_io) + img_io = io.BytesIO() + img.save(img_io, format=format) + return img_io + +# Add a copyright notice for vector formats (SVG, PDF, PS) +def add_copyright_notice_vector(surface, map_width, map_height): + context = cairo.Context(surface) + + # Set the font for the copyright notice + context.set_font_face(cairo.ToyFontFace("DejaVu")) + context.set_font_size(14) + + # Define the copyright text + text = "© OpenStreetMap contributors" + + text_extents = context.text_extents(text) + text_width = text_extents.width + text_height = text_extents.height + + x_margin = 10 + y_margin = 10 + + # Position the text at the bottom-right corner + x_position = map_width - text_width - x_margin + y_position = map_height - text_height - y_margin + + # Draw a white box just large enough to fit the text + context.set_source_rgba(1, 1, 1, 0.5) + context.rectangle(x_position - x_margin, y_position - y_margin, + text_width + 2 * x_margin, text_height + 2 * y_margin) + context.fill_preserve() + + context.set_source_rgb(0, 0, 0) # Black color for the text + context.move_to(x_position - x_margin / 2, y_position + y_margin) + context.show_text(text) + +# Render and output map for raster formats (PNG, JPEG, WEBP) +def render_and_output_image(map, format): + image = mapnik.Image(map.width, map.height) + mapnik.render(map, image) + + bytes_io = add_copyright_notice_raster(image, map.width, map.height, format) + + if format == "png": + output_headers("image/png", "map.png", bytesio_size(bytes_io)) + elif format == "jpeg": + output_headers("image/jpeg", "map.jpg", bytesio_size(bytes_io)) + elif format == "webp": + output_headers("image/webp", "map.webp", bytesio_size(bytes_io)) + + output_file(bytes_io) + +# Render and output map for vector formats (SVG, PDF, PS) +def render_and_output_vector(map, format): + with tempfile.NamedTemporaryFile(prefix="export") as file: + if format == "svg": + surface = cairo.SVGSurface(file.name, map.width, map.height) + surface.restrict_to_version(cairo.SVG_VERSION_1_2) + elif format == "pdf": + surface = cairo.PDFSurface(file.name, map.width, map.height) + elif format == "ps": + surface = cairo.PSSurface(file.name, map.width, map.height) + + mapnik.render(map, surface) + + add_copyright_notice_vector(surface, map.width, map.height) + + surface.finish() + + if format == "svg": + output_headers("image/svg+xml", "map.svg", file_size(file)) + elif format == "pdf": + output_headers("application/pdf", "map.pdf", file_size(file)) + elif format == "ps": + output_headers("application/postscript", "map.ps", file_size(file)) + + output_file(file) + + # Create TOTP token validator totp = pyotp.TOTP('<%= @totp_key %>', interval = 3600) @@ -149,42 +254,13 @@ else: # Render the map if pid == 0: - if form.getvalue("format") == "png": - image = mapnik.Image(map.width, map.height) - mapnik.render(map, image) - png = image.tostring("png") - output_headers("image/png", "map.png", len(png)) - sys.stdout.buffer.write(png) - elif form.getvalue("format") == "jpeg": - image = mapnik.Image(map.width, map.height) - mapnik.render(map, image) - jpeg = image.tostring("jpeg") - output_headers("image/jpeg", "map.jpg", len(jpeg)) - sys.stdout.buffer.write(jpeg) - elif form.getvalue("format") == "svg": - file = tempfile.NamedTemporaryFile(prefix = "export") - surface = cairo.SVGSurface(file.name, map.width, map.height) - surface.restrict_to_version(cairo.SVG_VERSION_1_2) - mapnik.render(map, surface) - surface.finish() - output_headers("image/svg+xml", "map.svg", file_size(file)) - output_file(file) - elif form.getvalue("format") == "pdf": - file = tempfile.NamedTemporaryFile(prefix = "export") - surface = cairo.PDFSurface(file.name, map.width, map.height) - mapnik.render(map, surface) - surface.finish() - output_headers("application/pdf", "map.pdf", file_size(file)) - output_file(file) - elif form.getvalue("format") == "ps": - file = tempfile.NamedTemporaryFile(prefix = "export") - surface = cairo.PSSurface(file.name, map.width, map.height) - mapnik.render(map, surface) - surface.finish() - output_headers("application/postscript", "map.ps", file_size(file)) - output_file(file) + format = form.getvalue("format") + if format in ["png", "jpeg", "webp"]: + render_and_output_image(map, format) + elif format in ["svg", "pdf", "ps"]: + render_and_output_vector(map, format) else: - output_error("Unknown format '%s'" % form.getvalue("format")) + output_error("Unknown format") else: pid, status = os.waitpid(pid, 0) if status & 0xff == signal.SIGXCPU: diff --git a/cookbooks/tile/templates/default/render-lowzoom.erb b/cookbooks/tile/templates/default/render-lowzoom.erb index eb3e2e84e..9ed8f0fc0 100644 --- a/cookbooks/tile/templates/default/render-lowzoom.erb +++ b/cookbooks/tile/templates/default/render-lowzoom.erb @@ -15,7 +15,7 @@ function update_<%= style %> render_old \ --timestamp=${timestamp} \ --tile-dir=/srv/tile.openstreetmap.org/tiles \ - --socket=/var/run/renderd/renderd.sock \ + --socket=/run/renderd/renderd.sock \ --num-threads=<%= @threads %> \ --map="<%= style %>" \ --max-load=<%= node.cpu_cores - 1 %> \ diff --git a/cookbooks/tile/templates/default/renderd.conf.erb b/cookbooks/tile/templates/default/renderd.conf.erb index d29c58aa7..40b86ad74 100644 --- a/cookbooks/tile/templates/default/renderd.conf.erb +++ b/cookbooks/tile/templates/default/renderd.conf.erb @@ -1,10 +1,10 @@ # DO NOT EDIT - This file is being maintained by Chef [renderd] -socketname=/var/run/renderd/renderd.sock +socketname=/run/renderd/renderd.sock num_threads=<%= node.cpu_cores - 1 %> tile_dir=/srv/tile.openstreetmap.org/tiles -stats_file=/var/run/renderd/renderd.stats +stats_file=/run/renderd/renderd.stats [mapnik] plugins_dir=/usr/lib/mapnik/<%= node[:tile][:mapnik] %>/input diff --git a/cookbooks/tile/templates/default/tile.conf.erb b/cookbooks/tile/templates/default/tile.conf.erb index 784733731..81c0fa38d 100644 --- a/cookbooks/tile/templates/default/tile.conf.erb +++ b/cookbooks/tile/templates/default/tile.conf.erb @@ -1,34 +1,34 @@ # DO NOT EDIT - This file is being maintained by Chef # Set location of renderd socket -ModTileRenderdSocketName /var/run/renderd/renderd.sock +#ModTileRenderdSocketName /run/renderd/renderd.sock # Set location of tile directory -ModTileTileDir /srv/tile.openstreetmap.org/tiles +#ModTileTileDir /srv/tile.openstreetmap.org/tiles # Time to wait for a re-render before serving a dirty tile -ModTileRequestTimeout 2 +#ModTileRequestTimeout 2 # Timeout before giving up for a tile to be rendered that is otherwise missing -ModTileMissingRequestTimeout 10 +#ModTileMissingRequestTimeout 10 # Always try and re-render dirty or missing tiles -ModTileMaxLoadOld 1000 -ModTileMaxLoadMissing 1000 +#ModTileMaxLoadOld 1000 +#ModTileMaxLoadMissing 1000 # Maximum expiry to set on a tile -ModTileCacheDurationMax 604800 +#ModTileCacheDurationMax 604800 # Expiry time for dirty tiles that have been queued for re-rendering -ModTileCacheDurationDirty 900 +#ModTileCacheDurationDirty 900 # Minimum expiry time for fresh tiles -ModTileCacheDurationMinimum 10800 -ModTileCacheDurationMediumZoom 13 86400 -ModTileCacheDurationLowZoom 9 518400 +#ModTileCacheDurationMinimum 10800 +#ModTileCacheDurationMediumZoom 13 86400 +#ModTileCacheDurationLowZoom 9 518400 # Factor controlling effect of last modification time on expiry -ModTileCacheLastModifiedFactor 0.20 +#ModTileCacheLastModifiedFactor 0.20 # Load tile configuration -LoadTileConfigFile /etc/renderd.conf +#LoadTileConfigFile /etc/renderd.conf diff --git a/cookbooks/tile/templates/default/update-lowzoom.erb b/cookbooks/tile/templates/default/update-lowzoom.erb index 0455493d9..7d50f8662 100644 --- a/cookbooks/tile/templates/default/update-lowzoom.erb +++ b/cookbooks/tile/templates/default/update-lowzoom.erb @@ -11,7 +11,7 @@ function update_tiles render_old \ --timestamp=$(stat -c %Y "/srv/tile.openstreetmap.org/styles/<%= @style %>/project.xml") \ --tile-dir=/srv/tile.openstreetmap.org/tiles \ - --socket=/var/run/renderd/renderd.sock \ + --socket=/run/renderd/renderd.sock \ --num-threads=<%= @threads %> \ --map="<%= @style %>" \ --max-load=<%= node.cpu_cores - 1 %> \ diff --git a/cookbooks/vectortile/attributes/default.rb b/cookbooks/vectortile/attributes/default.rb index b5e0933a6..cd9f594eb 100644 --- a/cookbooks/vectortile/attributes/default.rb +++ b/cookbooks/vectortile/attributes/default.rb @@ -4,13 +4,21 @@ default[:vectortile][:database][:nodes_store] = :flat default[:vectortile][:serve][:threads] = node.cpu_cores default[:vectortile][:serve][:mode] = :live default[:vectortile][:replication][:url] = "https://osm-planet-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/planet/replication/minute" -default[:vectortile][:replication][:status] = :enabled -default[:vectortile][:replication][:tileupdate] = :enabled +default[:vectortile][:replication][:enabled] = true +default[:vectortile][:replication][:tileupdate] = true default[:vectortile][:replication][:threads] = node.cpu_cores +default[:vectortile][:rerender][:lowzoom][:enabled] = true +default[:vectortile][:rerender][:lowzoom][:maxzoom] = 9 + +default[:vectortile][:tilekiln][:version] = "0.7.1" +default[:vectortile][:spirit][:version] = "033a4117ec3604e28824b3f3608f5d163ef1b450" +default[:vectortile][:themepark][:version] = "beb454cc56e88533fb398ab293489c4e91f4d42b" default[:postgresql][:versions] |= [node[:vectortile][:database][:cluster].split("/").first] default[:postgresql][:monitor_database] = "tiles" # As an absolute worst case, the server might have the serving, update, and a manual generation process going on. -default[:postgresql][:settings][:defaults][:max_connections] = (node.cpu_cores * 6 + 20).to_s +# Each of these connects to two databases, then we add more connections so 20% are unused and we're +# not tripping alarms. +default[:postgresql][:settings][:defaults][:max_connections] = (node.cpu_cores * 8 + 20).to_s default[:accounts][:users][:tileupdate][:status] = :role default[:accounts][:users][:tilekiln][:status] = :role diff --git a/cookbooks/vectortile/metadata.rb b/cookbooks/vectortile/metadata.rb index d3b44e37e..5f4619dc6 100644 --- a/cookbooks/vectortile/metadata.rb +++ b/cookbooks/vectortile/metadata.rb @@ -9,6 +9,7 @@ supports "ubuntu" depends "accounts" depends "git" depends "nginx" +depends "podman" depends "postgresql" depends "prometheus" depends "python" diff --git a/cookbooks/vectortile/recipes/default.rb b/cookbooks/vectortile/recipes/default.rb index 48b60f3c8..b32fa1050 100644 --- a/cookbooks/vectortile/recipes/default.rb +++ b/cookbooks/vectortile/recipes/default.rb @@ -20,6 +20,7 @@ include_recipe "accounts" include_recipe "git" include_recipe "nginx" +include_recipe "podman" include_recipe "postgresql" include_recipe "prometheus" include_recipe "python" @@ -77,6 +78,7 @@ package %w[ style_directory = "/srv/vector.openstreetmap.org/spirit" git style_directory do repository "https://github.com/pnorman/spirit.git" + revision node[:vectortile][:spirit][:version] user "tileupdate" group "tileupdate" end @@ -86,7 +88,7 @@ shortbread_config = "#{style_directory}/shortbread.yaml" themepark_directory = "/srv/vector.openstreetmap.org/osm2pgsql-themepark" git themepark_directory do repository "https://github.com/osm2pgsql-dev/osm2pgsql-themepark.git" - revision "7efc00db433fa7d2c49db00c32ffd27d8ced35f4" + revision node[:vectortile][:themepark][:version] user "tileupdate" group "tileupdate" end @@ -100,7 +102,7 @@ end python_package "tilekiln" do python_virtualenv tilekiln_directory python_version "3" - version "0.6.5" + version node[:vectortile][:tilekiln][:version] end template "/srv/vector.openstreetmap.org/html/index.html" do @@ -122,7 +124,7 @@ template "/usr/local/bin/import-planet" do owner "root" group "root" mode "755" - variables :node_store_options => "#{node_store_options}" + variables :node_store_options => node_store_options end template "/usr/local/bin/tilekiln-storage-init" do @@ -130,7 +132,7 @@ template "/usr/local/bin/tilekiln-storage-init" do owner "root" group "root" mode "755" - variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :storage_database => "tiles", :config_path => "#{shortbread_config}" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :storage_database => "tiles", :config_path => shortbread_config end postgresql_user "tomh" do @@ -156,6 +158,11 @@ postgresql_database "tiles" do owner "tileupdate" end +postgresql_extension "pgstattuple" do + cluster node[:vectortile][:database][:cluster] + database "tiles" +end + postgresql_database "spirit" do cluster node[:vectortile][:database][:cluster] owner "tileupdate" @@ -208,10 +215,10 @@ end end %w[addresses aerialways aeroways boundaries boundary_labels bridges buildings -dam_lines dam_polygons ferries land pier_lines pier_polygons place_labels -planet_osm_nodes planet_osm_rels planet_osm_ways pois public_transport railways -road_routes roads sites street_polygons streets_labels_points -streets_polygons_labels water_area_labels water_areas water_lines water_lines_labels].each do |table| + dam_lines dam_polygons ferries land pier_lines pier_polygons place_labels + planet_osm_nodes planet_osm_rels planet_osm_ways pois public_transport railways + road_routes roads sites street_polygons street_labels_points + streets_polygons_labels water_area_labels water_areas water_lines water_lines_labels].each do |table| postgresql_table table do cluster node[:vectortile][:database][:cluster] database "spirit" @@ -230,6 +237,7 @@ systemd_service "tilekiln" do wants "postgresql.service" sandbox :enable_network => true restrict_address_families "AF_UNIX" + environment "PGAPPNAME" => "tilekiln" exec_start "#{tilekiln_directory}/bin/tilekiln serve #{tilekiln_mode} --storage-dbname tiles --num-threads #{node[:vectortile][:serve][:threads]} --base-url 'https://vector.openstreetmap.org'" end @@ -246,19 +254,21 @@ execute "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py" do end template "/usr/local/bin/vector-update" do - source node[:vectortile][:replication][:tileupdate] == :enabled ? "vector-update-tile.erb" : "vector-update-notile.erb" + source node[:vectortile][:replication][:tileupdate] ? "vector-update-tile.erb" : "vector-update-notile.erb" owner "root" group "root" mode "755" - variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => "#{shortbread_config}", :diff_size => "1000", :tiles_file => "/srv/vector.openstreetmap.org/data/tiles.txt", :post_processing => "/usr/local/bin/tiles-rerender" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => shortbread_config, :diff_size => "1000", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :post_processing => "/usr/local/bin/tiles-rerender" end +rerender_layers = %w[addresses boundaries bridges buildings land pois public_transport sites street_polygons streets water_lines_labels water_lines water_polygons].join(" ") + template "/usr/local/bin/tiles-rerender" do source "tiles-rerender.erb" owner "root" group "root" mode "755" - variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :tiles_file => "/srv/vector.openstreetmap.org/data/tiles.txt", :update_threads => 4 + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => shortbread_config, :expiry_dir => "/srv/vector.openstreetmap.org/data/", :update_threads => 4, :layers => rerender_layers.to_s end systemd_service "replicate" do @@ -279,7 +289,7 @@ systemd_timer "replicate" do accuracy_sec 5 end -if node[:vectortile][:replication][:status] == :enabled +if node[:vectortile][:replication][:enabled] service "replicate.timer" do action [:enable, :start] end @@ -289,6 +299,39 @@ else end end +template "/usr/local/bin/render-lowzoom" do + source "render-lowzoom.erb" + owner "root" + group "root" + mode "755" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => shortbread_config, :min_zoom => 0, :max_zoom => node[:vectortile][:rerender][:lowzoom][:maxzoom] +end + +systemd_service "render-lowzoom" do + description "Render low zoom tiles" + user "tileupdate" + after "postgresql.service" + wants "postgresql.service" + restrict_address_families "AF_UNIX" + sandbox true + exec_start "/usr/local/bin/render-lowzoom" +end + +systemd_timer "render-lowzoom" do + description "Render low zoom tiles" + on_calendar "23:00 #{node[:timezone]}" +end + +if node[:vectortile][:rerender][:lowzoom][:enabled] + service "render-lowzoom.timer" do + action [:enable, :start] + end +else + service "render-lowzoom.timer" do + action [:stop, :disable] + end +end + package %w[ ruby-pg ruby-webrick @@ -309,6 +352,7 @@ systemd_service "tilekiln-prometheus" do after "postgresql.service" wants "postgresql.service" sandbox :enable_network => true + environment "PGAPPNAME" => "tilekiln-prometheus" restrict_address_families "AF_UNIX" exec_start "#{tilekiln_directory}/bin/tilekiln prometheus --bind-host #{node[:prometheus][:address]} --storage-dbname tiles" end @@ -321,3 +365,9 @@ node.default[:prometheus][:exporters][10013] = { :name => "tilekiln", :address => "#{node[:prometheus][:address]}:10013", } + +podman_service "vectortile_demo" do + description "Container service for /demo pages" + image "ghcr.io/openstreetmap/vectortile-website:latest" + ports 8080 => 8080 +end diff --git a/cookbooks/vectortile/templates/default/import-planet.erb b/cookbooks/vectortile/templates/default/import-planet.erb index bffa34a3a..f0bac4058 100644 --- a/cookbooks/vectortile/templates/default/import-planet.erb +++ b/cookbooks/vectortile/templates/default/import-planet.erb @@ -7,7 +7,8 @@ set -e -export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;' +export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;;' +export PGAPPNAME='import-planet' # Import the osm2pgsql file specified as an argument, using the locations for spirit osm2pgsql \ diff --git a/cookbooks/vectortile/templates/default/index.html.erb b/cookbooks/vectortile/templates/default/index.html.erb index feb7f36fa..9fefec130 100644 --- a/cookbooks/vectortile/templates/default/index.html.erb +++ b/cookbooks/vectortile/templates/default/index.html.erb @@ -11,7 +11,7 @@ You've reached the OpenStreetMap.org vector tile server (<%= node['fqdn'] %>)
If you are a user...
-
You probably want OpenStreetMap itself.
+
You probably want OpenStreetMap or the Shortbread demo page.
If you are a developer...
Please be aware of the tile usage policy.
diff --git a/cookbooks/vectortile/templates/default/nginx.erb b/cookbooks/vectortile/templates/default/nginx.erb index 838d30a4f..2695e4c0c 100644 --- a/cookbooks/vectortile/templates/default/nginx.erb +++ b/cookbooks/vectortile/templates/default/nginx.erb @@ -2,6 +2,10 @@ upstream tilekiln { server 127.0.0.1:8000; } +upstream demo { + server 127.0.0.1:8080; +} + server { listen 80 default_server; listen [::]:80 default_server; @@ -41,6 +45,10 @@ server { } location /shortbread_v1/ { proxy_pass http://tilekiln; + add_header Cache-Control "public,max-age=300,stale-while-revalidate=3600,stale-if-error=86400"; + } + location /demo/ { + proxy_pass http://demo; } root /srv/vector.openstreetmap.org/html; } diff --git a/cookbooks/vectortile/templates/default/render-lowzoom.erb b/cookbooks/vectortile/templates/default/render-lowzoom.erb new file mode 100644 index 000000000..b14550f40 --- /dev/null +++ b/cookbooks/vectortile/templates/default/render-lowzoom.erb @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu +export PGAPPNAME='render-lowzoom' + +# The source daily vaccum is best placed before the tilegen, as it has marginal speed benefits +vacuumdb --analyze \ +-j "<%= node[:vectortile][:replication][:threads] %>" \ +-d "<%= @source_database %>" + +<%= @tilekiln_bin %> generate zooms \ +--source-dbname "<%= @source_database %>" \ +--storage-dbname "<%= @storage_database %>" \ +--num-threads "<%= node[:vectortile][:replication][:threads] %>" \ +--config <%= @config_path %> \ +--min-zoom <%= @min_zoom %> \ +--max-zoom <%= @max_zoom %> + +# the daily render can add bloat, so the vacuum is done after it on the tile DB +vacuumdb --analyze \ +-j "<%= node[:vectortile][:replication][:threads] %>" \ +-d "<%= @storage_database %>" diff --git a/cookbooks/vectortile/templates/default/tilekiln-storage-init.erb b/cookbooks/vectortile/templates/default/tilekiln-storage-init.erb index 972d0e488..24a799aed 100644 --- a/cookbooks/vectortile/templates/default/tilekiln-storage-init.erb +++ b/cookbooks/vectortile/templates/default/tilekiln-storage-init.erb @@ -3,6 +3,8 @@ # Usage # sudo -u tileupdate tilekiln-storage-init +export PGAPPNAME='tilekiln-storage-init' + <%= @tilekiln_bin %> storage init \ --storage-dbname <%= @storage_database %> \ --config <%= @config_path %> diff --git a/cookbooks/vectortile/templates/default/tiles-rerender.erb b/cookbooks/vectortile/templates/default/tiles-rerender.erb index f46b6f1f6..910560771 100644 --- a/cookbooks/vectortile/templates/default/tiles-rerender.erb +++ b/cookbooks/vectortile/templates/default/tiles-rerender.erb @@ -1,7 +1,15 @@ #!/bin/sh set -eu -<%= @tilekiln_bin %> generate tiles \ +export PGAPPNAME='tiles-rerender' + +cd "<%= @expiry_dir %>" +# Only expire z10+ tiles +rm -f z?-*.txt +wc -l z*.txt +(for layer in <%= @layers %>; do + find . -type f -name "z*-${layer}.txt" -execdir sed -e "s/\$/,${layer}/" {} \; +done) | <%= @tilekiln_bin %> generate layers \ --source-dbname "<%= @source_database %>" \ --storage-dbname "<%= @storage_database %>" \ --num-threads "<%= node[:vectortile][:replication][:threads] %>" \ ---config <%= @config_path %> < <%= @tiles_file %> +--config <%= @config_path %> diff --git a/cookbooks/vectortile/templates/default/vector-update-notile.erb b/cookbooks/vectortile/templates/default/vector-update-notile.erb index cd718789c..844d9d4fa 100644 --- a/cookbooks/vectortile/templates/default/vector-update-notile.erb +++ b/cookbooks/vectortile/templates/default/vector-update-notile.erb @@ -1,12 +1,13 @@ #!/bin/sh # Usage -# sudo -u tilekiln vector-update +# sudo -u tileupdate vector-update set -eu -export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;' +export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;;' +cd "<%= @expiry_dir %>" osm2pgsql-replication update \ -d "<%= @source_database %>" \ --max-diff-size "<%= @diff_size %>" diff --git a/cookbooks/vectortile/templates/default/vector-update-tile.erb b/cookbooks/vectortile/templates/default/vector-update-tile.erb index e59923b27..3db8220c0 100644 --- a/cookbooks/vectortile/templates/default/vector-update-tile.erb +++ b/cookbooks/vectortile/templates/default/vector-update-tile.erb @@ -5,11 +5,13 @@ set -eu -export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;' +export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;;' +export PGAPPNAME='vector-update' + +cd "<%= @expiry_dir %>" +rm -f z*.txt osm2pgsql-replication update \ -d "<%= @source_database %>" \ --max-diff-size "<%= @diff_size %>" \ - --post-processing "<%= @post_processing %>" \ - -- --expire-tiles=10-14 \ - --expire-output="<%= @tiles_file %>" + --post-processing "<%= @post_processing %>" diff --git a/cookbooks/web/recipes/statistics.rb b/cookbooks/web/recipes/statistics.rb index 84848b003..88c85a6fe 100644 --- a/cookbooks/web/recipes/statistics.rb +++ b/cookbooks/web/recipes/statistics.rb @@ -19,7 +19,6 @@ include_recipe "web::base" -ruby = "ruby#{node[:ruby][:version]}" rails_directory = "#{node[:web][:base_directory]}/rails" template "/usr/local/bin/statistics" do @@ -27,7 +26,7 @@ template "/usr/local/bin/statistics" do owner "root" group "root" mode "755" - variables :ruby => ruby, :directory => rails_directory + variables :ruby => node[:ruby][:interpreter], :directory => rails_directory end systemd_service "web-statistics" do diff --git a/cookbooks/web/resources/rails_port.rb b/cookbooks/web/resources/rails_port.rb index debc4db50..b6209d50b 100644 --- a/cookbooks/web/resources/rails_port.rb +++ b/cookbooks/web/resources/rails_port.rb @@ -95,7 +95,7 @@ property :doorkeeper_signing_key, String property :user_account_deletion_delay, Integer action :create do - package %W[ + package %w[ imagemagick libvips42 nodejs @@ -408,11 +408,25 @@ action :create do action :delete end - bundle_install "#{rails_directory}" do + directory "#{rails_directory}/vendor/bundle/ruby/#{node[:ruby][:version]}.0" do + owner new_resource.user + group new_resource.group + mode "775" + recursive true + end + + bundle_config rails_directory do + user new_resource.user + group new_resource.group + settings "deployment" => "true", + "build.nokogiri" => "--use-system-libraries" + end + + bundle_install rails_directory do action :nothing - user "root" - group "root" - environment "NOKOGIRI_USE_SYSTEM_LIBRARIES" => "yes" + user new_resource.user + group new_resource.group + subscribes :run, "directory[#{rails_directory}/vendor/bundle/ruby/#{node[:ruby][:version]}.0]" subscribes :run, "git[#{rails_directory}]" end @@ -439,7 +453,7 @@ action :create do only_if { new_resource.build_assets } end - bundle_exec "#{rails_directory}/app/assets/javascripts/i18n" do + bundle_exec "#{rails_directory}/config/i18n-js.yml" do action :nothing directory rails_directory command "rails i18n:js:export" @@ -449,7 +463,18 @@ action :create do user new_resource.user group new_resource.group subscribes :run, "git[#{rails_directory}]" - only_if { new_resource.build_assets } + only_if { new_resource.build_assets && ::File.exist?("#{rails_directory}/config/i18n-js.yml") } + end + + bundle_exec "#{rails_directory}/config/i18n.yml" do + action :nothing + directory rails_directory + command "i18n export" + environment "HOME" => rails_directory + user new_resource.user + group new_resource.group + subscribes :run, "git[#{rails_directory}]" + only_if { new_resource.build_assets && ::File.exist?("#{rails_directory}/config/i18n.yml") } end bundle_exec "#{rails_directory}/public/assets" do @@ -466,7 +491,8 @@ action :create do subscribes :run, "file[#{rails_directory}/config/settings.local.yml]" subscribes :run, "file[#{rails_directory}/config/storage.yml]" subscribes :run, "bundle_exec[#{rails_directory}/package.json]" - subscribes :run, "bundle_exec[#{rails_directory}/app/assets/javascripts/i18n]" + subscribes :run, "bundle_exec[#{rails_directory}/config/i18n-js.yml]" + subscribes :run, "bundle_exec[#{rails_directory}/config/i18n.yml]" only_if { new_resource.build_assets } end diff --git a/cookbooks/web/templates/default/apache.frontend.erb b/cookbooks/web/templates/default/apache.frontend.erb index 39f2a6007..f05b99b5c 100644 --- a/cookbooks/web/templates/default/apache.frontend.erb +++ b/cookbooks/web/templates/default/apache.frontend.erb @@ -2,7 +2,7 @@ # # Setup logging -# +# SetEnvIfNoCase Authorization "^Basic " AUTH_METHOD=basic SetEnvIfNoCase Authorization "^OAuth " AUTH_METHOD=oauth1 SetEnvIfNoCase Authorization "^Bearer " AUTH_METHOD=oauth2 @@ -108,6 +108,13 @@ ErrorLog /var/log/apache2/error.log RewriteCond "%{QUERY_STRING}" "^q=abcde&t=20" RewriteRule "^/api/0\.6/notes/search$" - [R=429,L] + # + # Ignore GoogleAssociationService request storm + # https://en.osm.town/@osm_tech/114205363076771822 + # + RewriteCond %{HTTP_USER_AGENT} "GoogleAssociationService" + RewriteRule "^/\.well-known/assetlinks\.json$" - [R=429,L] + # # Force special MIME type for crossdomain.xml files # @@ -186,6 +193,7 @@ ErrorLog /var/log/apache2/error.log RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/relations(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/node/[0-9]+/ways(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/(way|relation)/[0-9]+/full(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] + RewriteCond %{REQUEST_METHOD} ^(HEAD|GET)$ RewriteRule ^/api/0\.6/(nodes|ways|relations)(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/changeset/[0-9]+/(upload|download)(\.json|\.xml)?$ unix:/run/cgimap/socket|fcgi://127.0.0.1$0 [P] diff --git a/cookbooks/web/templates/default/api-statistics.erb b/cookbooks/web/templates/default/api-statistics.erb index ce3af0b34..b6425ec88 100644 --- a/cookbooks/web/templates/default/api-statistics.erb +++ b/cookbooks/web/templates/default/api-statistics.erb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!<%= node[:ruby][:interpreter] %> require "apache_log_regex" require "file-tail" diff --git a/cookbooks/web/templates/default/deliver-message.erb b/cookbooks/web/templates/default/deliver-message.erb index 76538183b..dddd647f1 100644 --- a/cookbooks/web/templates/default/deliver-message.erb +++ b/cookbooks/web/templates/default/deliver-message.erb @@ -3,4 +3,4 @@ export RAILS_ENV="production" export SECRET_KEY_BASE="<%= @secret_key_base %>" -exec /usr/local/bin/passenger-ruby /srv/www.openstreetmap.org/rails/script/deliver-message "$@" +exec <%= node[:ruby][:bundle] %> exec /srv/www.openstreetmap.org/rails/script/deliver-message "$@" diff --git a/cookbooks/wiki/attributes/default.rb b/cookbooks/wiki/attributes/default.rb index 0cde6a313..71f656567 100644 --- a/cookbooks/wiki/attributes/default.rb +++ b/cookbooks/wiki/attributes/default.rb @@ -1,2 +1,9 @@ -# Force apache to listen only on localhost -# default[:apache][:listen_address] = "127.0.0.1" +default[:wiki][:site_name] = "wiki.openstreetmap.org" +default[:wiki][:site_aliases] = [ + "wiki.osm.org", "wiki.openstreetmap.com", "wiki.openstreetmaps.org", + "osm.wiki", "www.osm.wiki", "wiki.osm.wiki" +] +default[:wiki][:site_notice] = nil +default[:wiki][:site_readonly] = nil +default[:wiki][:test_mode] = false +default[:wiki][:mediawiki_version] = "1.43" diff --git a/cookbooks/wiki/recipes/default.rb b/cookbooks/wiki/recipes/default.rb index e7a2b3e57..0590346f3 100644 --- a/cookbooks/wiki/recipes/default.rb +++ b/cookbooks/wiki/recipes/default.rb @@ -19,6 +19,8 @@ include_recipe "mediawiki" +site_name = node[:wiki][:site_name] + passwords = data_bag_item("wiki", "passwords") package "lua5.1" # newer versions do not work with Scribuntu! @@ -27,16 +29,15 @@ apache_site "default" do action [:disable] end -mediawiki_site "wiki.openstreetmap.org" do - aliases ["wiki.osm.org", "wiki.openstreetmap.com", "wiki.openstreetmap.net", - "wiki.openstreetmap.ca", "wiki.openstreetmap.eu", - "wiki.openstreetmap.pro", "wiki.openstreetmaps.org", - "osm.wiki", "www.osm.wiki", "wiki.osm.wiki"] +mediawiki_site site_name do + aliases node[:wiki][:site_aliases] + + version node[:wiki][:mediawiki_version] - fpm_max_children 200 - fpm_start_servers 25 - fpm_min_spare_servers 25 - fpm_max_spare_servers 50 + fpm_max_children 300 + fpm_start_servers 50 + fpm_min_spare_servers 50 + fpm_max_spare_servers 150 fpm_prometheus_port 9253 database_name "wiki" @@ -56,54 +57,71 @@ mediawiki_site "wiki.openstreetmap.org" do hcaptcha_public_key "b67a410b-955e-4049-b432-f9c00e0202c0" hcaptcha_private_key passwords["hcaptcha"] - # site_notice "MAINTENANCE: WIKI READ-ONLY UNTIL Monday 16 May 2016 - 11:00am UTC/GMT." - # site_readonly "MAINTENANCE: WIKI READ-ONLY UNTIL Monday 16 May 2016 - 11:00am UTC/GMT." + namespaces "DE" => { :id => 200, :talk_id => 201 }, + "FR" => { :id => 202, :talk_id => 203 }, + "ES" => { :id => 204, :talk_id => 205 }, + "IT" => { :id => 206, :talk_id => 207 }, + "NL" => { :id => 208, :talk_id => 209 }, + "RU" => { :id => 210, :talk_id => 211 }, + "JA" => { :id => 212, :talk_id => 213 }, + "Proposal" => { :id => 3000, :talk_id => 3001 } + + force_ui_messages %w[mainpage-url mapfeatures-url contributors-url helppage blogs-url shop-url sitesupport-url] + + watch_category_membership true + + site_notice node[:wiki][:site_notice] + site_readonly node[:wiki][:site_readonly] + + if node[:wiki][:test_mode] + backup_enabled false + end end mediawiki_extension "CodeEditor" do - site "wiki.openstreetmap.org" + site site_name end mediawiki_extension "CodeMirror" do - site "wiki.openstreetmap.org" + site site_name end mediawiki_extension "Scribunto" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Scribunto.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "Wikibase" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Wikibase.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "OsmWikibase" do - site "wiki.openstreetmap.org" + site site_name repository "https://github.com/nyurik/OsmWikibase.git" reference "master" end mediawiki_extension "Echo" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Echo.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "Thanks" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Thanks.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "TimedMediaHandler" do - site "wiki.openstreetmap.org" + site site_name end mediawiki_extension "MultiMaps" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-MultiMaps.inc.php.erb" template_cookbook "wiki" variables :thunderforest_key => passwords["thunderforest"] @@ -111,37 +129,60 @@ mediawiki_extension "MultiMaps" do end mediawiki_extension "JsonConfig" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-JsonConfig.inc.php.erb" template_cookbook "wiki" end mediawiki_extension "Kartographer" do - site "wiki.openstreetmap.org" + site site_name template "mw-ext-Kartographer.inc.php.erb" template_cookbook "wiki" end -cookbook_file "/srv/wiki.openstreetmap.org/osm_logo_wiki.png" do +mediawiki_extension "TemplateStyles" do + site site_name +end + +mediawiki_extension "DynamicPageListEngine" do + site site_name + only_if { node[:wiki][:test_mode] } +end + +mediawiki_extension "WikibaseCirrusSearch" do + site site_name + template "mw-ext-WikibaseCirrusSearch.inc.php.erb" + template_cookbook "wiki" + only_if { node[:wiki][:test_mode] } +end + +mediawiki_extension "Translate" do + site site_name + template "mw-ext-Translate.inc.php.erb" + template_cookbook "wiki" + only_if { node[:wiki][:test_mode] } +end + +cookbook_file "/srv/#{site_name}/osm_logo_wiki.png" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "644" end -template "/srv/wiki.openstreetmap.org/robots.txt" do +template "/srv/#{site_name}/robots.txt" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "644" source "robots.txt.erb" end -cookbook_file "/srv/wiki.openstreetmap.org/favicon.ico" do +cookbook_file "/srv/#{site_name}/favicon.ico" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "644" end -directory "/srv/wiki.openstreetmap.org/dump" do +directory "/srv/#{site_name}/dump" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "0775" @@ -151,17 +192,18 @@ systemd_service "wiki-dump" do description "Wiki dump" type "oneshot" exec_start "/usr/bin/php w/maintenance/dumpBackup.php --full --quiet --output=gzip:dump/dump.xml.gz" - working_directory "/srv/wiki.openstreetmap.org" + working_directory "/srv/#{site_name}" user "wiki" + nice 19 sandbox :enable_network => true memory_deny_write_execute false restrict_address_families "AF_UNIX" - read_write_paths "/srv/wiki.openstreetmap.org/dump" + read_write_paths "/srv/#{site_name}/dump" end systemd_timer "wiki-dump" do description "Wiki dump" - on_calendar "02:00" + on_calendar "Sun 02:30" end service "wiki-dump.timer" do @@ -174,14 +216,14 @@ systemd_service "wiki-rdf-dump" do exec_start [ "/usr/bin/php w/extensions/Wikibase/repo/maintenance/dumpRdf.php --wiki wiki --format ttl --flavor full-dump --entity-type item --entity-type property --no-cache --output /tmp/wikibase-rdf.ttl", "/bin/gzip -9 /tmp/wikibase-rdf.ttl", - "/bin/mv /tmp/wikibase-rdf.ttl.gz /srv/wiki.openstreetmap.org/dump/wikibase-rdf.ttl.gz" + "/bin/mv /tmp/wikibase-rdf.ttl.gz /srv/#{site_name}/dump/wikibase-rdf.ttl.gz" ] - working_directory "/srv/wiki.openstreetmap.org" + working_directory "/srv/#{site_name}" user "wiki" sandbox :enable_network => true memory_deny_write_execute false restrict_address_families "AF_UNIX" - read_write_paths "/srv/wiki.openstreetmap.org/dump" + read_write_paths "/srv/#{site_name}/dump" end systemd_timer "wiki-rdf-dump" do diff --git a/cookbooks/wiki/templates/default/mw-ext-Translate.inc.php.erb b/cookbooks/wiki/templates/default/mw-ext-Translate.inc.php.erb new file mode 100644 index 000000000..0ae1afa71 --- /dev/null +++ b/cookbooks/wiki/templates/default/mw-ext-Translate.inc.php.erb @@ -0,0 +1,39 @@ + 'ttmserver', + 'class' => 'ElasticSearchTTMServer', + 'cutoff' => 0.75, + /* + * See http://elastica.io/getting-started/installation.html + * See https://github.com/ruflin/Elastica/blob/8.x/src/Client.php + */ + 'config' => [ 'servers' => [ [ 'host' => '127.0.0.1', 'port' => 9114 ] ] ] +]; diff --git a/cookbooks/wiki/templates/default/mw-ext-Wikibase.inc.php.erb b/cookbooks/wiki/templates/default/mw-ext-Wikibase.inc.php.erb index 6f7e8d6eb..d609a3d57 100644 --- a/cookbooks/wiki/templates/default/mw-ext-Wikibase.inc.php.erb +++ b/cookbooks/wiki/templates/default/mw-ext-Wikibase.inc.php.erb @@ -89,7 +89,7 @@ $wgWBClientSettings['showExternalRecentChanges'] = true; // Base URL for building links to the repository. // Assumes your wiki is setup as "http://repo.example.org/wiki/" // This can be protocol relative, such as "//www.wikidata.org" -$wgWBClientSettings['repoUrl'] = "https://wiki.openstreetmap.org"; +$wgWBClientSettings['repoUrl'] = "https://<%= @site %>"; // This setting is optional if you have the same type of setup for your // repo and client. It will default to using the client's $wgArticlePath setting, diff --git a/cookbooks/wiki/templates/default/mw-ext-WikibaseCirrusSearch.inc.php.erb b/cookbooks/wiki/templates/default/mw-ext-WikibaseCirrusSearch.inc.php.erb new file mode 100644 index 000000000..e6859b785 --- /dev/null +++ b/cookbooks/wiki/templates/default/mw-ext-WikibaseCirrusSearch.inc.php.erb @@ -0,0 +1,6 @@ + +User-agent: * +Disallow: / +<% else -%> User-agent: ia_archiver Allow: / User-agent: 008 Disallow: / -User-agent: * -Disallow: /index.php/ -Disallow: /wiki/Spam -Allow: /w/load.php -Allow: /w/images/ -Allow: /w/skins/ -Disallow: /w/ -Disallow: /wiki/Special:Collection -Disallow: /wiki/Special:Random -Disallow: /wiki/Special%3ARandom -Disallow: /wiki/Special:Search - -User-agent: Exabot -Crawl-delay: 60 - -User-Agent: ImagesiftBot -Crawl-delay: 60 +User-agent: HTTrack +Disallow: / -User-agent: SemrushBot -Crawl-delay: 60 +User-agent: DataForSeoBot +Disallow: / User-agent: SeznamBot -Crawl-delay: 60 +Disallow: / + +User-agent: * +Allow: /w/api.php?action=mobileview& +Allow: /w/load.php? +Disallow: /w/ +Disallow: /api/ +Disallow: /wiki/Special: +Disallow: /wiki/Special%3A +Disallow: /index.php/ +Disallow: /wiki/Spam Sitemap: https://wiki.openstreetmap.org/sitemap-index-wiki.xml Host: wiki.openstreetmap.org +<% end -%> diff --git a/cookbooks/wordpress/resources/site.rb b/cookbooks/wordpress/resources/site.rb index 52bba4ce5..2c07ba31e 100644 --- a/cookbooks/wordpress/resources/site.rb +++ b/cookbooks/wordpress/resources/site.rb @@ -82,10 +82,10 @@ action :create do end wp_config = edit_file "#{site_directory}/wp-config-sample.php" do |line| - line.gsub!(/database_name_here/, new_resource.database_name) - line.gsub!(/username_here/, new_resource.database_user) - line.gsub!(/password_here/, new_resource.database_password) - line.gsub!(/wp_/, new_resource.database_prefix) + line.gsub!("database_name_here", new_resource.database_name) + line.gsub!("username_here", new_resource.database_user) + line.gsub!("password_here", new_resource.database_password) + line.gsub!("wp_", new_resource.database_prefix) line.gsub!(/('AUTH_KEY', *)'put your unique phrase here'/, "\\1'#{auth_key}'") line.gsub!(/('SECURE_AUTH_KEY', *)'put your unique phrase here'/, "\\1'#{secure_auth_key}'") diff --git a/roles/aarnet.rb b/roles/aarnet.rb deleted file mode 100644 index 8cab0ea35..000000000 --- a/roles/aarnet.rb +++ /dev/null @@ -1,27 +0,0 @@ -name "aarnet" -description "Role applied to all servers at AARNet" - -default_attributes( - :accounts => { - :users => { - :chm => { :status => :administrator }, - :bclifford => { :status => :administrator } - } - }, - :hosted_by => "AARNet", - :location => "Carlton, Victoria, Australia", - :timezone => "Australia/Melbourne" -) - -override_attributes( - :networking => { - :nameservers => ["202.158.207.1", "202.158.207.2"] - }, - :ntp => { - :servers => ["0.au.pool.ntp.org", "1.au.pool.ntp.org", "oceania.pool.ntp.org"] - } -) - -run_list( - "role[au]" -) diff --git a/roles/angor.rb b/roles/angor.rb index be08e517c..5fba17667 100644 --- a/roles/angor.rb +++ b/roles/angor.rb @@ -26,8 +26,8 @@ default_attributes( :gmoncrieff => { :status => :user }, :zander => { :status => :user }, :"za-imagery" => { - :status => :role, - :members => [:grant, :htonl, :gmoncrieff, :zander] + :status => :role, + :members => [:grant, :htonl, :gmoncrieff, :zander] } } } diff --git a/roles/balerion.rb b/roles/balerion.rb deleted file mode 100644 index 4aef19a05..000000000 --- a/roles/balerion.rb +++ /dev/null @@ -1,60 +0,0 @@ -name "balerion" -description "Master role applied to balerion" - -default_attributes( - :networking => { - :interfaces => { - :external => { - :interface => "bond0", - :role => :external, - :inet => { - :address => "138.44.68.134", - :prefix => "30", - :gateway => "138.44.68.133" - }, - :bond => { - :slaves => %w[ens14f0np0 ens14f1np1] - } - } - } - }, - :postgresql => { - :settings => { - :defaults => { - :effective_cache_size => "16GB" - } - } - }, - :sysctl => { - :postgres => { - :comment => "Increase shared memory for postgres", - :parameters => { - "kernel.shmmax" => 9 * 1024 * 1024 * 1024, - "kernel.shmall" => 9 * 1024 * 1024 * 1024 / 4096 - } - } - }, - :tile => { - :database => { - :cluster => "16/main", - :postgis => "3" - }, - :mapnik => "3.1", - :replication => { - :directory => "/store/replication" - }, - :styles => { - :default => { - :tile_directories => [ - { :name => "/store/tiles/default", :min_zoom => 0, :max_zoom => 19 } - ] - } - } - } -) - -run_list( - "role[aarnet]", - "role[geodns]", - "role[tile]" -) diff --git a/roles/base.rb b/roles/base.rb index 2a4f732a5..ee9b003c6 100644 --- a/roles/base.rb +++ b/roles/base.rb @@ -11,10 +11,6 @@ default_attributes( } }, :networking => { - :roles => { - :internal => { :metric => 200 }, - :external => { :metric => 100 } - }, :search => ["openstreetmap.org"] }, :prometheus => { diff --git a/roles/blogs.rb b/roles/blogs.rb index 3881d1396..87c71902a 100644 --- a/roles/blogs.rb +++ b/roles/blogs.rb @@ -1,6 +1,12 @@ name "blogs" description "Role applied to all blog aggregators" +default_attributes( + :ruby => { + :version => "3.3" + } +) + run_list( "recipe[blogs]" ) diff --git a/roles/bowser.rb b/roles/bowser.rb deleted file mode 100644 index eca0fa2cf..000000000 --- a/roles/bowser.rb +++ /dev/null @@ -1,59 +0,0 @@ -name "bowser" -description "Master role applied to bowser" - -default_attributes( - :networking => { - :interfaces => { - :external => { - :interface => "bond0", - :role => :external, - :inet => { - :address => "138.44.68.106", - :prefix => "30", - :gateway => "138.44.68.105" - }, - :bond => { - :slaves => %w[ens14f0np0 ens14f1np1] - } - } - } - }, - :postgresql => { - :settings => { - :defaults => { - :effective_cache_size => "16GB" - } - } - }, - :sysctl => { - :postgres => { - :comment => "Increase shared memory for postgres", - :parameters => { - "kernel.shmmax" => 9 * 1024 * 1024 * 1024, - "kernel.shmall" => 9 * 1024 * 1024 * 1024 / 4096 - } - } - }, - :tile => { - :database => { - :cluster => "16/main", - :postgis => "3" - }, - :mapnik => "3.1", - :replication => { - :directory => "/store/replication" - }, - :styles => { - :default => { - :tile_directories => [ - { :name => "/store/tiles/default", :min_zoom => 0, :max_zoom => 19 } - ] - } - } - } -) - -run_list( - "role[aarnet]", - "role[tile]" -) diff --git a/roles/bytemark.rb b/roles/bytemark.rb index 8f23bd5d0..109a17208 100644 --- a/roles/bytemark.rb +++ b/roles/bytemark.rb @@ -3,30 +3,7 @@ description "Role applied to all servers at Bytemark" default_attributes( :hosted_by => "Bytemark", - :location => "York, England", - :networking => { - :roles => { - :internal => { - :inet => { - :prefix => "20", - :gateway => "10.0.32.20", - :routes => { - "10.0.0.0/8" => { :via => "10.0.32.20" } - } - } - }, - :external => { - :inet => { - :prefix => "28", - :gateway => "89.16.162.17" - }, - :inet6 => { - :prefix => "64", - :gateway => "fe80::1" - } - } - } - } + :location => "York, England" ) override_attributes( diff --git a/roles/chef-repository.rb b/roles/chef-repository.rb index 55f62f6c6..550217d42 100644 --- a/roles/chef-repository.rb +++ b/roles/chef-repository.rb @@ -12,6 +12,10 @@ default_attributes( :status => :user, :shell => "/usr/bin/git-shell" }, + :pnorman => { + :status => :user, + :shell => "/usr/bin/git-shell" + }, :stereo => { :status => :user, :shell => "/usr/bin/git-shell" @@ -23,7 +27,7 @@ default_attributes( :chefrepo => { :status => :role, :members => [ - :tomh, :grant, :matt, :jburgess, :lonvia, :yellowbkpk, :bretth, :jochen, :stereo + :tomh, :grant, :matt, :jburgess, :lonvia, :yellowbkpk, :bretth, :jochen, :stereo, :pnorman ] } } diff --git a/roles/culebre.rb b/roles/culebre.rb index d798bddae..8f266566a 100644 --- a/roles/culebre.rb +++ b/roles/culebre.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.9" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp68s0f0 enp68s0f1 enp68s0f2 enp68s0f3] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.105", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.105" }, :inet6 => { - :address => "2001:470:1:b3b::9", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::9" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.105", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.105" }, :inet6 => { - :address => "2001:4d78:fe03:1c::9", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::9" } } } @@ -83,6 +63,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[tile]" ) diff --git a/roles/dev.rb b/roles/dev.rb index f85849e88..81a7f34d2 100644 --- a/roles/dev.rb +++ b/roles/dev.rb @@ -94,8 +94,8 @@ default_attributes( :members => [:apmon, :maba] }, :"za-imagery" => { - :status => :role, - :members => [:grant, :htonl, :gmoncrieff, :zander] + :status => :role, + :members => [:grant, :htonl, :gmoncrieff, :zander] } } }, @@ -116,7 +116,7 @@ default_attributes( :revision => "master", :cgimap_repository => "https://github.com/zerebubuth/openstreetmap-cgimap.git", :cgimap_revision => "master", - :aliases => ["api06.dev.openstreetmap.org"] + :aliases => ["master.apis.dev.osm.org", "api06.dev.openstreetmap.org", "api06.dev.osm.org"] }, :tomh => { :repository => "https://github.com/tomhughes/openstreetmap-website.git", @@ -159,6 +159,9 @@ default_attributes( } } }, + :ruby => { + :fullstaq => true + }, :sysctl => { :postgres => { :comment => "Increase shared memory for postgres", diff --git a/roles/dribble.rb b/roles/dribble.rb index 33707cfa4..b0bba8021 100644 --- a/roles/dribble.rb +++ b/roles/dribble.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.4" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.132", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.132" }, :inet6 => { - :address => "2001:470:1:fa1::4", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::4" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.100", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.100" }, :inet6 => { - :address => "2001:4d78:500:5e3::4", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::4" } } } @@ -60,6 +40,6 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[vectortile]" ) diff --git a/roles/dulcy.rb b/roles/dulcy.rb index d9d8fb79f..cc6b24fe6 100644 --- a/roles/dulcy.rb +++ b/roles/dulcy.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.9" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp1s0f0 enp1s0f1] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.137", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.137" }, :inet6 => { - :address => "2001:470:1:fa1::9", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::9" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.105", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.105" }, :inet6 => { - :address => "2001:4d78:500:5e3::9", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::9" } } } @@ -62,13 +42,13 @@ default_attributes( :nominatim => { :dbcluster => "17/main", :flatnode_file => "/srv/nominatim.openstreetmap.org/planet-project/nodes.store", - :enable_qa_tiles => true, + :enable_qa_tiles => false, :api_workers => 14, :api_pool_size => 10 } ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[nominatim]" ) diff --git a/roles/eddie.rb b/roles/eddie.rb index 61e452455..2acf524e1 100644 --- a/roles/eddie.rb +++ b/roles/eddie.rb @@ -6,7 +6,6 @@ default_attributes( :interfaces => { :internal => { :interface => "enp1s0f0.2801", - :role => :internal, :inet => { :address => "10.0.0.10" } diff --git a/roles/equinix-ams-public.rb b/roles/equinix-ams-public.rb new file mode 100644 index 000000000..b89ae21bb --- /dev/null +++ b/roles/equinix-ams-public.rb @@ -0,0 +1,44 @@ +name "equinix-ams-public" +description "Role applied to all public servers at Equinix Amsterdam" + +default_attributes( + :networking => { + :interfaces => { + :henet => { + :interface => "bond0.3", + :role => :external, + :metric => 150, + :source_route_table => 3, + :inet => { + :prefix => "27", + :gateway => "184.104.179.129" + }, + :inet6 => { + :prefix => "64", + :gateway => "2001:470:1:fa1::1", + :routes => { + "2600:9000::/28" => { :table => 3, :type => "unreachable" } + } + } + }, + :equinix => { + :interface => "bond0.103", + :role => :external, + :metric => 100, + :source_route_table => 103, + :inet => { + :prefix => "27", + :gateway => "82.199.86.97" + }, + :inet6 => { + :prefix => "64", + :gateway => "2001:4d78:500:5e3::1" + } + } + } + } +) + +run_list( + "role[equinix-ams]" +) diff --git a/roles/equinix-ams.rb b/roles/equinix-ams.rb index ac8c6a885..c63522bbc 100644 --- a/roles/equinix-ams.rb +++ b/roles/equinix-ams.rb @@ -3,18 +3,23 @@ description "Role applied to all servers at Equinix Amsterdam" default_attributes( :networking => { - :roles => { + :interfaces => { :internal => { + :interface => "bond0", + :role => :internal, + :metric => 200, :inet => { :prefix => "20", :gateway => "10.0.48.14", :routes => { "10.0.0.0/8" => { :via => "10.0.48.14" } } + }, + :bond => { + :mode => "802.3ad", + :lacprate => "fast", + :xmithashpolicy => "layer3+4" } - }, - :external => { - :zone => "ams" } } }, diff --git a/roles/equinix-dub-public.rb b/roles/equinix-dub-public.rb new file mode 100644 index 000000000..343dfb4f9 --- /dev/null +++ b/roles/equinix-dub-public.rb @@ -0,0 +1,44 @@ +name "equinix-dub-public" +description "Role applied to all public servers at Equinix Dublin" + +default_attributes( + :networking => { + :interfaces => { + :henet => { + :interface => "bond0.101", + :role => :external, + :metric => 150, + :source_route_table => 101, + :inet => { + :prefix => "27", + :gateway => "184.104.226.97" + }, + :inet6 => { + :prefix => 64, + :gateway => "2001:470:1:b3b::1", + :routes => { + "2600:9000::/28" => { :table => 101, :type => "unreachable" } + } + } + }, + :equinix => { + :interface => "bond0.203", + :role => :external, + :metric => 100, + :source_route_table => 203, + :inet => { + :prefix => "27", + :gateway => "87.252.214.97" + }, + :inet6 => { + :prefix => 64, + :gateway => "2001:4d78:fe03:1c::1" + } + } + } + } +) + +run_list( + "role[equinix-dub]" +) diff --git a/roles/equinix-dub.rb b/roles/equinix-dub.rb index 6561d3019..741c09ab7 100644 --- a/roles/equinix-dub.rb +++ b/roles/equinix-dub.rb @@ -12,22 +12,27 @@ default_attributes( } }, :networking => { - :roles => { + :interfaces => { :internal => { + :interface => "bond0", + :role => :internal, + :metric => 200, :inet => { :prefix => "20", :gateway => "10.0.64.2", :routes => { "10.0.0.0/8" => { :via => "10.0.64.2" } - } - } - }, - :external => { - :zone => "dub", - :inet6 => { + }, :rules => [ - { :to => "2600:9000::/28", :table => 150, :priority => 100 } + { :to => "10.0.0.0/8", :table => "main", :priority => 50 }, + { :to => "172.16.0.0/12", :table => "main", :priority => 50 }, + { :to => "192.168.0.0/16", :table => "main", :priority => 50 } ] + }, + :bond => { + :mode => "802.3ad", + :lacprate => "fast", + :xmithashpolicy => "layer3+4" } } } diff --git a/roles/faffy.rb b/roles/faffy.rb index 34b480823..e02510d18 100644 --- a/roles/faffy.rb +++ b/roles/faffy.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.3" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.131", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.131" }, :inet6 => { - :address => "2001:470:1:fa1::3", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::3" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.99", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.99" }, :inet6 => { - :address => "2001:4d78:500:5e3::3", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::3" } } } @@ -53,6 +33,6 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[dev]" ) diff --git a/roles/fafnir.rb b/roles/fafnir.rb index bb6b7b667..b1acd9e0d 100644 --- a/roles/fafnir.rb +++ b/roles/fafnir.rb @@ -18,47 +18,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.2" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.98", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.98" }, :inet6 => { - :address => "2001:470:1:b3b::2", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::2" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.98", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.98" }, :inet6 => { - :address => "2001:4d78:fe03:1c::2", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::2" } } } @@ -74,14 +54,14 @@ default_attributes( :metrics => { :uplink_interface => { :help => "Site uplink interface name", - :labels => { :site => "dublin", :name => "xe-[01]/2/[01]|ge-[01]/2/2" } + :labels => { :site => "dublin", :name => "xe-[01]/2/0|ge-[01]/2/2" } } } } ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[gateway]", "role[mail]", diff --git a/roles/firefishynet.rb b/roles/firefishynet.rb index 150f62d6f..bb4224f70 100644 --- a/roles/firefishynet.rb +++ b/roles/firefishynet.rb @@ -3,7 +3,7 @@ description "Role applied to all servers at Firefishy" default_attributes( :networking => { - :roles => { + :interfaces => { :internal => { :inet => { :prefix => "24", diff --git a/roles/fume.rb b/roles/fume.rb index 5cc06a4b1..af951e1f0 100644 --- a/roles/fume.rb +++ b/roles/fume.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.16" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.112", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.112" }, :inet6 => { - :address => "2001:470:1:b3b::10", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::10" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.112", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.112" }, :inet6 => { - :address => "2001:4d78:fe03:1c::10", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::10" } } } @@ -53,6 +33,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[community]" ) diff --git a/roles/geodns.rb b/roles/geodns.rb index a2ad092f3..74168f4b7 100644 --- a/roles/geodns.rb +++ b/roles/geodns.rb @@ -12,8 +12,10 @@ default_attributes( :list => false, :transfer_logging => false, :hosts_allow => [ - "184.104.226.102", # idris - "2001:470:1:b3b::6" # idris + "184.104.226.102", # idris HE + "2001:470:1:b3b::6", # idris HE + "87.252.214.102", # idris Equinix + "2001:4d78:fe03:1c::6" # idris Equinix ] } } diff --git a/roles/git.rb b/roles/git.rb index c401c865f..6dfbe6b24 100644 --- a/roles/git.rb +++ b/roles/git.rb @@ -12,7 +12,7 @@ default_attributes( :status => :user, :shell => "/usr/bin/git-shell" }, - :yellowbkpk => { + :pnorman => { :status => :user, :shell => "/usr/bin/git-shell" }, @@ -20,9 +20,13 @@ default_attributes( :status => :user, :shell => "/usr/bin/git-shell" }, + :yellowbkpk => { + :status => :user, + :shell => "/usr/bin/git-shell" + }, :git => { :status => :role, - :members => [:tomh, :grant, :matt, :lonvia, :yellowbkpk, :stereo] + :members => [:tomh, :grant, :matt, :lonvia, :yellowbkpk, :stereo, :pnorman] } } }, diff --git a/roles/grisu.rb b/roles/grisu.rb index 525e028d2..e7fd2f899 100644 --- a/roles/grisu.rb +++ b/roles/grisu.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.17" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.113", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.113" }, :inet6 => { - :address => "2001:470:1:b3b::11", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::11" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.113", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.113" }, :inet6 => { - :address => "2001:4d78:fe03:1c::11", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::11" } } } @@ -53,6 +33,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[overpass-query]" ) diff --git a/roles/horntail.rb b/roles/horntail.rb index d071856f4..bcb23ef11 100644 --- a/roles/horntail.rb +++ b/roles/horntail.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.10" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", - :slaves => %w[enp25s0f0 enp25s0f1] + :slaves => %w[enp25s0f0np0 enp25s0f1np1] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.106", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.106" }, :inet6 => { - :address => "2001:470:1:b3b::a", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::a" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.106", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.106" }, :inet6 => { - :address => "2001:4d78:fe03:1c::a", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::a" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[geodns]", "role[planet]" ) diff --git a/roles/idris.rb b/roles/idris.rb index 7467f67af..cdb5dc986 100644 --- a/roles/idris.rb +++ b/roles/idris.rb @@ -2,50 +2,35 @@ name "idris" description "Master role applied to idris" default_attributes( + :apache => { + :event => { + :async_request_worker_factor => 10 + } + }, :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.6" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.102", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.102" }, :inet6 => { - :address => "2001:470:1:b3b::6", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::6" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.102", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.102" }, :inet6 => { - :address => "2001:4d78:fe03:1c::6", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::6" } } } @@ -53,7 +38,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[chef-server]", "role[chef-repository]", diff --git a/roles/imagery.rb b/roles/imagery.rb index d6caa11e9..4a4a033a3 100644 --- a/roles/imagery.rb +++ b/roles/imagery.rb @@ -20,21 +20,13 @@ default_attributes( "net.core.somaxconn" => 10000 } } - }, - :nginx => { - :cache => { - :fastcgi => { - :enable => true, - :keys_zone => "fastcgi_cache_zone:256M", - :inactive => "45d", - :max_size => "51200M" - } - } } ) run_list( "recipe[imagery::au_agri]", + "recipe[imagery::au_act_aerial]", + "recipe[imagery::au_vic_melbourne_aerial]", "recipe[imagery::gb_ea]", "recipe[imagery::gb_hampshire_aerial]", "recipe[imagery::gb_os_sv]", diff --git a/roles/karm.rb b/roles/karm.rb index ac28c48d4..bb736d72c 100644 --- a/roles/karm.rb +++ b/roles/karm.rb @@ -5,15 +5,10 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.50" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp1s0f0 enp1s0f1 enp2s0f0 enp2s0f1] } } diff --git a/roles/konqi.rb b/roles/konqi.rb index 4b1fbcea6..2ef8ebd84 100644 --- a/roles/konqi.rb +++ b/roles/konqi.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.7" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.103", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.103" }, :inet6 => { - :address => "2001:470:1:b3b::7", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::7" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.103", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.103" }, :inet6 => { - :address => "2001:4d78:fe03:1c::7", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::7" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[wiki]" ) diff --git a/roles/lockheed.rb b/roles/lockheed.rb index 09eddcc8b..34cfc05da 100644 --- a/roles/lockheed.rb +++ b/roles/lockheed.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.16" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno49 eno50] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.144", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.144" }, :inet6 => { - :address => "2001:470:1:fa1::10", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::10" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.112", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.112" }, :inet6 => { - :address => "2001:4d78:500:5e3::10", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::10" } } } @@ -85,18 +65,24 @@ default_attributes( }, :nginx => { :cache => { + :fastcgi => { + :enable => true, + :keys_zone => "fastcgi_cache_zone:2048M", + :inactive => "180d", + :max_size => "196608M" + }, :proxy => { - :enable => true, - :keys_zone => "proxy_cache_zone:256M", - :inactive => "180d", - :max_size => "51200M" + :enable => true, + :keys_zone => "proxy_cache_zone:2048M", + :inactive => "180d", + :max_size => "196608M" } } } ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[imagery]" ) diff --git a/roles/longma.rb b/roles/longma.rb index b8bf00af1..b64af9aa4 100644 --- a/roles/longma.rb +++ b/roles/longma.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.13" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp68s0f0np0 enp68s0f1np1 enp68s0f2np2 enp68s0f3np3] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.109", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.109" }, :inet6 => { - :address => "2001:470:1:b3b::d", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::d" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.109", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.109" }, :inet6 => { - :address => "2001:4d78:fe03:1c::d", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::d" } } } @@ -70,6 +50,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[nominatim]" ) diff --git a/roles/muirdris.rb b/roles/muirdris.rb index b77e4f8df..7ec8c2de2 100644 --- a/roles/muirdris.rb +++ b/roles/muirdris.rb @@ -2,60 +2,50 @@ name "muirdris" description "Master role applied to muirdris" default_attributes( - :memcached => { - :memory_limit => 128 * 1024 - }, :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.15" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.111", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.111" }, :inet6 => { - :address => "2001:470:1:b3b::f", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::f" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.111", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.111" }, :inet6 => { - :address => "2001:4d78:fe03:1c::f", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::f" } } } + }, + :wiki => { + :site_name => "test.wiki.openstreetmap.org", + :site_aliases => [], + :site_notice => "TEST INSTANCE: Use wiki.openstreetmap.org for real work", + :test_mode => true + } +) + +override_attributes( + :memcached => { + :memory_limit => 128 * 1024 } ) run_list( - "role[equinix-dub]", - "role[gps-tile]" + "role[equinix-dub-public]", + "role[gps-tile]", + "role[wiki]" ) diff --git a/roles/naga.rb b/roles/naga.rb index d212b6680..1bcf3c176 100644 --- a/roles/naga.rb +++ b/roles/naga.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.8" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.104", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.104" }, :inet6 => { - :address => "2001:470:1:b3b::8", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::8" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.101", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.104" }, :inet6 => { - :address => "2001:4d78:fe03:1c::8", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::8" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[subversion]", "role[trac]", diff --git a/roles/norbert.rb b/roles/norbert.rb index ef3d33dbd..357dc0d5a 100644 --- a/roles/norbert.rb +++ b/roles/norbert.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.17" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp25s0f0 enp25s0f1] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.145", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.145" }, :inet6 => { - :address => "2001:470:1:fa1::11", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::11" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.113", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.113" }, :inet6 => { - :address => "2001:4d78:500:5e3::11", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::11" } } } @@ -56,7 +36,7 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[geodns]", "role[backup]", "role[planet]", diff --git a/roles/odin.rb b/roles/odin.rb index 1544fb9b3..9d5bd1cfc 100644 --- a/roles/odin.rb +++ b/roles/odin.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.15" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1np0 eno2np1] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.143", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.143" }, :inet6 => { - :address => "2001:470:1:fa1::f", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::f" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.111", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.111" }, :inet6 => { - :address => "2001:4d78:500:5e3::f", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::f" } } } @@ -145,6 +125,6 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[tile]" ) diff --git a/roles/osuosl.rb b/roles/osuosl.rb index 99a3a992f..83fd46a72 100644 --- a/roles/osuosl.rb +++ b/roles/osuosl.rb @@ -11,8 +11,10 @@ default_attributes( :location => "Corvallis, Oregon", :timezone => "US/Pacific", :networking => { - :roles => { + :interfaces => { :external => { + :interface => "bond0", + :role => :external, :inet => { :prefix => "28", :gateway => "140.211.167.97" diff --git a/roles/ovh.rb b/roles/ovh.rb deleted file mode 100644 index f8391a2aa..000000000 --- a/roles/ovh.rb +++ /dev/null @@ -1,20 +0,0 @@ -name "ovh" -description "Role applied to all servers at OVH" - -default_attributes( - :hosted_by => "OVH", - :location => "Roubaix, France" -) - -override_attributes( - :networking => { - :nameservers => ["213.186.33.99"] - }, - :ntp => { - :servers => ["0.fr.pool.ntp.org", "1.fr.pool.ntp.org", "europe.pool.ntp.org"] - } -) - -run_list( - "role[fr]" -) diff --git a/roles/piasa.rb b/roles/piasa.rb index b1e775ad4..085fdc66e 100644 --- a/roles/piasa.rb +++ b/roles/piasa.rb @@ -5,8 +5,6 @@ default_attributes( :networking => { :interfaces => { :external => { - :interface => "bond0", - :role => :external, :inet => { :address => "140.211.167.101" }, diff --git a/roles/ridley.rb b/roles/ridley.rb index 0e4a66522..2a6b166c1 100644 --- a/roles/ridley.rb +++ b/roles/ridley.rb @@ -20,14 +20,12 @@ default_attributes( :interfaces => { :external => { :interface => "eth0.2800", - :role => :external, :inet => { :address => "193.60.236.19" } }, :internal => { :interface => "eth0.2801", - :role => :internal, :inet => { :address => "10.0.0.3" } @@ -37,7 +35,7 @@ default_attributes( ) run_list( - "role[ucl]", + "role[ucl-public]", "role[hp-dl360-g6]", "role[gateway]", "role[foundation]", diff --git a/roles/scorch.rb b/roles/scorch.rb deleted file mode 100644 index 0649374f4..000000000 --- a/roles/scorch.rb +++ /dev/null @@ -1,40 +0,0 @@ -name "scorch" -description "Master role applied to scorch" - -default_attributes( - :devices => { - :ssd_system => { - :comment => "Tune scheduler for system disk", - :type => "block", - :bus => "scsi", - :serial => "3600605b009bbf5601fc3206407a43546", - :attrs => { - "queue/scheduler" => "noop", - "queue/nr_requests" => "256", - "queue/read_ahead_kb" => "2048" - } - } - }, - :networking => { - :interfaces => { - :external => { - :interface => "eth0", - :role => :external, - :inet => { - :address => "176.31.235.79", - :prefix => "24", - :gateway => "176.31.235.254" - }, - :inet6 => { - :address => "2001:41d0:2:fc4f::1", - :prefix => "64", - :gateway => "2001:41d0:2:fcff:ff:ff:ff:ff" - } - } - } - } -) - -run_list( - "role[ovh]" -) diff --git a/roles/shenron.rb b/roles/shenron.rb index f91677226..7f9f36a1c 100644 --- a/roles/shenron.rb +++ b/roles/shenron.rb @@ -19,14 +19,6 @@ default_attributes( "it87" ] }, - :prometheus => { - :metrics => { - :exim_queue_limit => { :metric => 250 } - } - } -) - -override_attributes( :networking => { :dnssec => "false", :interfaces => { @@ -45,6 +37,11 @@ override_attributes( } } } + }, + :prometheus => { + :metrics => { + :exim_queue_limit => { :metric => 250 } + } } ) diff --git a/roles/smaug.rb b/roles/smaug.rb index 4156fcb99..7d42c4f6e 100644 --- a/roles/smaug.rb +++ b/roles/smaug.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.14" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.110", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.110" }, :inet6 => { - :address => "2001:470:1:b3b::e", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::e" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.110", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.110" }, :inet6 => { - :address => "2001:4d78:fe03:1c::e", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::e" } } } @@ -53,6 +33,6 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[matomo]" ) diff --git a/roles/snap-01.rb b/roles/snap-01.rb index d7fb452a5..e741e2d51 100644 --- a/roles/snap-01.rb +++ b/roles/snap-01.rb @@ -5,15 +5,10 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.49" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4] } } diff --git a/roles/snap-02.rb b/roles/snap-02.rb index 80a2b3598..2a63385b3 100644 --- a/roles/snap-02.rb +++ b/roles/snap-02.rb @@ -6,7 +6,6 @@ default_attributes( :interfaces => { :internal => { :interface => "eno1.2801", - :role => :internal, :inet => { :address => "10.0.0.4" } diff --git a/roles/snap-03.rb b/roles/snap-03.rb index 4956bdab7..0164f8563 100644 --- a/roles/snap-03.rb +++ b/roles/snap-03.rb @@ -5,15 +5,10 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.50" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[enp25s0f0np0 enp25s0f1np1] } } diff --git a/roles/spike-01.rb b/roles/spike-01.rb index 80b56c5e1..51ff42c63 100644 --- a/roles/spike-01.rb +++ b/roles/spike-01.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.3" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.99", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.99" }, :inet6 => { - :address => "2001:470:1:b3b::3", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::3" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.99", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.99" }, :inet6 => { - :address => "2001:4d78:fe03:1c::3", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::3" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/spike-02.rb b/roles/spike-02.rb index 065be3faf..4e178bdec 100644 --- a/roles/spike-02.rb +++ b/roles/spike-02.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.64.4" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.100", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.100" }, :inet6 => { - :address => "2001:470:1:b3b::4", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::4" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.100", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.100" }, :inet6 => { - :address => "2001:4d78:fe03:1c::4", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::4" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/spike-03.rb b/roles/spike-03.rb index b620aecfc..a24512eaa 100644 --- a/roles/spike-03.rb +++ b/roles/spike-03.rb @@ -5,47 +5,28 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", :role => :internal, :inet => { :address => "10.0.64.5" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno49 eno50] } }, - :external_he => { - :interface => "bond0.101", - :role => :external, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.226.101", - :prefix => "27", - :gateway => "184.104.226.97" + :address => "184.104.226.101" }, :inet6 => { - :address => "2001:470:1:b3b::5", - :prefix => 64, - :gateway => "2001:470:1:b3b::1" + :address => "2001:470:1:b3b::5" } }, - :external => { - :interface => "bond0.203", - :role => :external, - :metric => 150, - :source_route_table => 150, + :equinix => { :inet => { - :address => "87.252.214.101", - :prefix => "27", - :gateway => "87.252.214.97" + :address => "87.252.214.101" }, :inet6 => { - :address => "2001:4d78:fe03:1c::5", - :prefix => 64, - :gateway => "2001:4d78:fe03:1c::1" + :address => "2001:4d78:fe03:1c::5" } } } @@ -53,7 +34,7 @@ default_attributes( ) run_list( - "role[equinix-dub]", + "role[equinix-dub-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/spike-06.rb b/roles/spike-06.rb index dd754a7ba..b5d84e6a6 100644 --- a/roles/spike-06.rb +++ b/roles/spike-06.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.6" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.134", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.134" }, :inet6 => { - :address => "2001:470:1:fa1::6", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::6" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.102", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.102" }, :inet6 => { - :address => "2001:4d78:500:5e3::6", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::6" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[web-frontend]", "role[web-statistics]", diff --git a/roles/spike-07.rb b/roles/spike-07.rb index f9d0d8f12..166432919 100644 --- a/roles/spike-07.rb +++ b/roles/spike-07.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.7" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.135", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.135" }, :inet6 => { - :address => "2001:470:1:fa1::7", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::7" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.103", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.103" }, :inet6 => { - :address => "2001:4d78:500:5e3::7", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::7" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/spike-08.rb b/roles/spike-08.rb index c46260340..46f0be788 100644 --- a/roles/spike-08.rb +++ b/roles/spike-08.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.8" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.136", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.136" }, :inet6 => { - :address => "2001:470:1:fa1::8", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::8" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.104", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.104" }, :inet6 => { - :address => "2001:4d78:500:5e3::8", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::8" } } } @@ -53,7 +33,7 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[web-frontend]" ) diff --git a/roles/stormfly-03.rb b/roles/stormfly-03.rb index d81b06c9f..374d2ab90 100644 --- a/roles/stormfly-03.rb +++ b/roles/stormfly-03.rb @@ -5,8 +5,6 @@ default_attributes( :networking => { :interfaces => { :external => { - :interface => "bond0", - :role => :external, :inet => { :address => "140.211.167.99" }, diff --git a/roles/stormfly-04.rb b/roles/stormfly-04.rb index 3a7d1a7c3..bcc11518a 100644 --- a/roles/stormfly-04.rb +++ b/roles/stormfly-04.rb @@ -8,8 +8,6 @@ default_attributes( :networking => { :interfaces => { :external => { - :interface => "bond0", - :role => :external, :inet => { :address => "140.211.167.100" }, diff --git a/roles/tabaluga.rb b/roles/tabaluga.rb index af8492b83..e40923637 100644 --- a/roles/tabaluga.rb +++ b/roles/tabaluga.rb @@ -9,47 +9,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.14" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.142", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.142" }, :inet6 => { - :address => "2001:470:1:fa1::e", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::e" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.110", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.110" }, :inet6 => { - :address => "2001:4d78:500:5e3::e", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::e" } } } @@ -65,14 +45,14 @@ default_attributes( :metrics => { :uplink_interface => { :help => "Site uplink interface name", - :labels => { :site => "amsterdam", :name => "xe-[01]/2/0|ge-[01]/2/[02]" } + :labels => { :site => "amsterdam", :name => "xe-[01]/2/0|ge-[01]/2/2" } } } } ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[hp-g9]", "role[blog-staging]", "role[taginfo]", diff --git a/roles/taginfo.rb b/roles/taginfo.rb index 40bffdce2..9b8227957 100644 --- a/roles/taginfo.rb +++ b/roles/taginfo.rb @@ -26,6 +26,9 @@ default_attributes( :passenger => { :max_pool_size => 50 }, + :ruby => { + :version => "3.3" + }, :taginfo => { :sites => [ { diff --git a/roles/tile.rb b/roles/tile.rb index 20bded97f..550c99f66 100644 --- a/roles/tile.rb +++ b/roles/tile.rb @@ -46,6 +46,9 @@ default_attributes( } } }, + :ruby => { + :fullstaq => false + }, :ssl => { :ct_report_uri => false }, diff --git a/roles/ucl-public.rb b/roles/ucl-public.rb new file mode 100644 index 000000000..7b7c6188a --- /dev/null +++ b/roles/ucl-public.rb @@ -0,0 +1,21 @@ +name "ucl-public" +description "Role applied to all public servers at UCL" + +default_attributes( + :networking => { + :interfaces => { + :external => { + :role => :external, + :metric => 100, + :inet => { + :prefix => "24", + :gateway => "193.60.236.254" + } + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/ucl.rb b/roles/ucl.rb index 516c925d3..1734989bf 100644 --- a/roles/ucl.rb +++ b/roles/ucl.rb @@ -4,8 +4,10 @@ description "Role applied to all servers at UCL" default_attributes( :location => "Slough, England", :networking => { - :roles => { + :interfaces => { :internal => { + :role => :internal, + :metric => 200, :inet => { :prefix => "20", :gateway => "10.0.0.3", @@ -13,13 +15,6 @@ default_attributes( "10.0.0.0/8" => { :via => "10.0.0.3" } } } - }, - :external => { - :zone => "ucl", - :inet => { - :prefix => "24", - :gateway => "193.60.236.254" - } } }, :wireguard => { diff --git a/roles/vhagar.rb b/roles/vhagar.rb index b5776ab2a..c8cd4a92f 100644 --- a/roles/vhagar.rb +++ b/roles/vhagar.rb @@ -5,47 +5,27 @@ default_attributes( :networking => { :interfaces => { :internal => { - :interface => "bond0", - :role => :internal, :inet => { :address => "10.0.48.5" }, :bond => { - :mode => "802.3ad", - :lacprate => "fast", - :xmithashpolicy => "layer3+4", :slaves => %w[eno1 eno2 eno3 eno4 eno5 eno6] } }, - :external_he => { - :interface => "bond0.3", - :role => :external, - :metric => 150, - :source_route_table => 100, + :henet => { :inet => { - :address => "184.104.179.133", - :prefix => "27", - :gateway => "184.104.179.129" + :address => "184.104.179.133" }, :inet6 => { - :address => "2001:470:1:fa1::5", - :prefix => 64, - :gateway => "2001:470:1:fa1::1" + :address => "2001:470:1:fa1::5" } }, - :external => { - :interface => "bond0.103", - :role => :external, - :source_route_table => 150, + :equinix => { :inet => { - :address => "82.199.86.101", - :prefix => "27", - :gateway => "82.199.86.97" + :address => "82.199.86.101" }, :inet6 => { - :address => "2001:4d78:500:5e3::5", - :prefix => 64, - :gateway => "2001:4d78:500:5e3::1" + :address => "2001:4d78:500:5e3::5" } } } @@ -60,6 +40,6 @@ default_attributes( ) run_list( - "role[equinix-ams]", + "role[equinix-ams-public]", "role[nominatim]" ) diff --git a/roles/web-frontend.rb b/roles/web-frontend.rb index 396cf36db..e7c0e5850 100644 --- a/roles/web-frontend.rb +++ b/roles/web-frontend.rb @@ -30,6 +30,9 @@ default_attributes( :passenger => { :max_pool_size => 50 }, + :ruby => { + :fullstaq => true + }, :exim => { :local_domains => ["messages.openstreetmap.org"], :routes => { diff --git a/roles/wiki.rb b/roles/wiki.rb index 0c83be4c5..c6cb02000 100644 --- a/roles/wiki.rb +++ b/roles/wiki.rb @@ -14,10 +14,10 @@ default_attributes( :server_limit => 32, :max_request_workers => 800, :threads_per_child => 50, - :max_connections_per_child => 10000 + :max_connections_per_child => 100000 }, :evasive => { - :page_count => 250, + :page_count => 400, :site_count => 500 } }, @@ -59,7 +59,7 @@ default_attributes( :mysqld => { :innodb_buffer_pool_size => "4G", :key_buffer_size => "64M", - :max_connections => "500", + :max_connections => "800", :sort_buffer_size => "8M", :tmp_table_size => "128M" } diff --git a/roles/ysera.rb b/roles/ysera.rb index bf256d44c..8d3c16f87 100644 --- a/roles/ysera.rb +++ b/roles/ysera.rb @@ -6,14 +6,12 @@ default_attributes( :interfaces => { :internal => { :interface => "eno1np0.2801", - :role => :internal, :inet => { :address => "10.0.0.15" } }, :external => { :interface => "eno1np0.2800", - :role => :external, :inet => { :address => "193.60.236.22" } @@ -115,6 +113,6 @@ default_attributes( ) run_list( - "role[ucl]", + "role[ucl-public]", "role[tile]" ) diff --git a/test/data_bags/accounts/serverinfo.json b/test/data_bags/accounts/serverinfo.json new file mode 100644 index 000000000..2b01af603 --- /dev/null +++ b/test/data_bags/accounts/serverinfo.json @@ -0,0 +1,6 @@ +{ + "id": "serverinfo", + "uid": "534", + "comment": "hardware.openstreetmap.org", + "manage_home": false +} diff --git a/test/integration/dev/inspec/mysql_spec.rb b/test/integration/dev/inspec/mysql_spec.rb index 549f33da9..628d8bee2 100644 --- a/test/integration/dev/inspec/mysql_spec.rb +++ b/test/integration/dev/inspec/mysql_spec.rb @@ -8,7 +8,7 @@ describe package("#{mysql_variant}-server") do it { should be_installed } end -describe service("#{mysql_variant}") do +describe service(mysql_variant) do it { should be_enabled } it { should be_running } end diff --git a/test/integration/mysql/inspec/mysql_spec.rb b/test/integration/mysql/inspec/mysql_spec.rb index 549f33da9..628d8bee2 100644 --- a/test/integration/mysql/inspec/mysql_spec.rb +++ b/test/integration/mysql/inspec/mysql_spec.rb @@ -8,7 +8,7 @@ describe package("#{mysql_variant}-server") do it { should be_installed } end -describe service("#{mysql_variant}") do +describe service(mysql_variant) do it { should be_enabled } it { should be_running } end diff --git a/test/integration/vectortile/inspec/rerender_spec.rb b/test/integration/vectortile/inspec/rerender_spec.rb new file mode 100644 index 000000000..c5da181ec --- /dev/null +++ b/test/integration/vectortile/inspec/rerender_spec.rb @@ -0,0 +1,7 @@ +describe file("/usr/local/bin/render-lowzoom") do + it { should be_executable.by_user("tileupdate") } +end + +describe service("render-lowzoom") do + it { should be_installed } +end