From: Grant Slater Date: Mon, 22 Sep 2025 09:40:29 +0000 (+0100) Subject: ntp: Use server instead of pool for backup Google NTP servers X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/HEAD?ds=sidebyside;hp=332e5e0843799eb42132e229ecc64e05d507059f ntp: Use server instead of pool for backup Google NTP servers pool re-resolves, but Google NTP servers are static. --- diff --git a/.github/workflows/cookstyle.yml b/.github/workflows/cookstyle.yml index b9332e236..e34d83301 100644 --- a/.github/workflows/cookstyle.yml +++ b/.github/workflows/cookstyle.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Setup ruby uses: ruby/setup-ruby@v1 with: diff --git a/.github/workflows/test-kitchen.yml b/.github/workflows/test-kitchen.yml index e1c8df555..9146173e1 100644 --- a/.github/workflows/test-kitchen.yml +++ b/.github/workflows/test-kitchen.yml @@ -12,7 +12,7 @@ concurrency: jobs: kitchen: name: Test Kitchen - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest permissions: packages: read strategy: @@ -22,6 +22,7 @@ jobs: - apache - apt - apt-repository + - awscli - backup - bind - blog @@ -61,11 +62,10 @@ jobs: - hardware - hot - ideditor + - imagery-tiler - irc - kibana - letsencrypt - - logstash - - logstash-forwarder - mail - mailman - matomo @@ -116,73 +116,97 @@ jobs: - web-cgimap - web-frontend - web-rails - - wordpress - wiki + - wordpress os: - - ubuntu-2204 - include: - - os: ubuntu-2004 - suite: mailman - - os: ubuntu-2004 - suite: osqa - - os: debian-12 - suite: apt-repository - - os: debian-12 - suite: dev - - os: debian-12 - suite: dns - - os: debian-12 - suite: git-server - - os: debian-12 - suite: git-web - - os: debian-12 - suite: imagery-tiler - - os: debian-12 - suite: letsencrypt - - os: debian-12 - suite: otrs - - os: debian-12 - suite: serverinfo - - os: debian-12 - suite: supybot - - os: debian-12 - suite: vectortile - - os: debian-12 - suite: web-cgimap - - os: debian-12 - suite: web-frontend - - os: debian-12 - suite: web-rails + - debian-12 exclude: - - suite: apt-repository + - suite: mailman + os: debian-12 + include: + - suite: blog + os: ubuntu-2004 + - suite: mailman + os: ubuntu-2004 + - suite: stateofthemap-wordpress + os: ubuntu-2004 + - suite: wordpress + os: ubuntu-2004 + - suite: accounts os: ubuntu-2204 - - suite: dev + - suite: apache os: ubuntu-2204 - - suite: dns + - suite: apt os: ubuntu-2204 - - suite: git-server + - suite: backup os: ubuntu-2204 - - suite: git-web + - suite: bind os: ubuntu-2204 - - suite: mailman + - suite: chef + os: ubuntu-2204 + - suite: clamav + os: ubuntu-2204 + - suite: db-backup + os: ubuntu-2204 + - suite: db-base + os: ubuntu-2204 + - suite: db-master + os: ubuntu-2204 + - suite: db-slave + os: ubuntu-2204 + - suite: devices + os: ubuntu-2204 + - suite: dhcpd + os: ubuntu-2204 + - suite: exim + os: ubuntu-2204 + - suite: fail2ban os: ubuntu-2204 - - suite: letsencrypt + - suite: geodns os: ubuntu-2204 - - suite: osqa + - suite: geoipupdate os: ubuntu-2204 - - suite: otrs + - suite: git os: ubuntu-2204 - - suite: serverinfo + - suite: hardware os: ubuntu-2204 - - suite: supybot + - suite: networking os: ubuntu-2204 - - suite: vectortile + - suite: ntp os: ubuntu-2204 - - suite: web-cgimap + - suite: openssh os: ubuntu-2204 - - suite: web-frontend + - suite: osmosis os: ubuntu-2204 - - suite: web-rails + - suite: planet + os: ubuntu-2204 + - suite: planet-aws + os: ubuntu-2204 + - suite: planet-dump + os: ubuntu-2204 + - suite: planet-notes + os: ubuntu-2204 + - suite: planet-replication + os: ubuntu-2204 + - suite: postgresql + os: ubuntu-2204 + - suite: prometheus + os: ubuntu-2204 + - suite: python + os: ubuntu-2204 + - suite: rsyncd + os: ubuntu-2204 + - suite: spamassassin + os: ubuntu-2204 + - suite: ssl + os: ubuntu-2204 + - suite: sysctl + os: ubuntu-2204 + - suite: sysfs + os: ubuntu-2204 + - suite: tilelog + os: ubuntu-2204 + - suite: tools os: ubuntu-2204 fail-fast: false steps: @@ -192,14 +216,8 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - if: github.repository == 'openstreetmap/chef' && github.event_name != 'pull_request' - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Setup ruby uses: ruby/setup-ruby@v1 with: diff --git a/.kitchen.yml b/.kitchen.yml index 88cfc7459..0d9e537fb 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -62,6 +62,18 @@ platforms: - RUN /usr/bin/apt-get install -y eatmydata - RUN echo /usr/lib/$(uname -m)-linux-gnu/libeatmydata.so >>/etc/ld.so.preload +# When using podman we have to manually start chef to workaround a volume issue +lifecycle: + pre_create: | + if command -v podman >/dev/null 2>&1; then + podman create --name chef-latest --replace ghcr.io/firefishy/chef-docker-image:latest sh + podman start chef-latest + fi + post_destroy: | + if command -v podman >/dev/null 2>&1; then + podman container rm -iv chef-latest + fi + suites: - name: accounts run_list: @@ -93,6 +105,9 @@ suites: - name: blogs run_list: - recipe[blogs::default] + attributes: + ruby: + version: 3.3 - name: chef run_list: - recipe[chef::default] @@ -123,7 +138,10 @@ suites: attributes: postgresql: versions: - - 15 + - 17 + dev: + rails: + postgresql_cluster: 17/main - name: devices run_list: - recipe[devices::default] @@ -238,23 +256,6 @@ suites: - name: letsencrypt run_list: - recipe[letsencrypt::default] - - name: logstash - run_list: - - recipe[logstash::default] - - name: logstash-forwarder - run_list: - - recipe[logstash::forwarder] - attributes: - logstash: - forwarder: - filebeat.inputs: - - type: filestream - id: apache - paths: - - /var/log/apache2/access.log - fields: - type: apache - fields_under_root: true - name: mail run_list: - role[mail] @@ -402,6 +403,8 @@ suites: run_list: - recipe[taginfo::default] attributes: + ruby: + version: 3.3 taginfo: sites: - name: taginfo.example.com diff --git a/Gemfile.lock b/Gemfile.lock index 40032233d..83bc82f56 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,20 +1,23 @@ GEM remote: https://rubygems.org/ specs: - activesupport (7.1.4.2) + activesupport (7.1.5.1) base64 + benchmark (>= 0.3) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) + logger (>= 1.4.2) minitest (>= 5.1) mutex_m + securerandom (>= 0.3) tzinfo (~> 2.0) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) - ast (2.4.2) - aws-eventstream (1.3.0) + ast (2.4.3) + aws-eventstream (1.4.0) aws-partitions (1.863.0) aws-sdk-accessanalyzer (1.44.0) aws-sdk-core (~> 3, >= 3.188.0) @@ -256,8 +259,8 @@ GEM aws-sdk-wafv2 (1.74.0) aws-sdk-core (~> 3, >= 3.188.0) aws-sigv4 (~> 1.1) - aws-sigv2 (1.2.0) - aws-sigv4 (1.10.1) + aws-sigv2 (1.3.0) + aws-sigv4 (1.12.0) aws-eventstream (~> 1, >= 1.0.2) azure_graph_rbac (0.17.2) ms_rest_azure (~> 0.12.0) @@ -269,14 +272,15 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_storage (0.23.0) ms_rest_azure (~> 0.12.0) - base64 (0.2.0) + base64 (0.3.0) bcrypt_pbkdf (1.1.1) - bigdecimal (3.1.8) - bson (4.15.0) + benchmark (0.4.1) + bigdecimal (3.2.3) builder (3.3.0) - chef-config (18.5.0) + cgi (0.5.0) + chef-config (18.8.11) addressable - chef-utils (= 18.5.0) + chef-utils (= 18.8.11) fuzzyurl mixlib-config (>= 2.2.12, < 4.0) mixlib-shellout (>= 2.0, < 4.0) @@ -284,77 +288,47 @@ GEM chef-telemetry (1.1.1) chef-config concurrent-ruby (~> 1.0) - chef-utils (18.5.0) + chef-utils (18.8.11) concurrent-ruby coderay (1.1.3) - concurrent-ruby (1.3.4) - connection_pool (2.4.1) - cookstyle (7.32.8) - rubocop (= 1.25.1) + concurrent-ruby (1.3.5) + connection_pool (2.5.3) + cookstyle (8.4.0) + rubocop (= 1.79.2) + csv (3.3.5) + date (3.4.1) declarative (0.0.20) - diff-lcs (1.5.1) - docker-api (2.3.0) + diff-lcs (1.6.2) + docker-api (2.4.0) excon (>= 0.64.0) multi_json domain_name (0.6.20240107) - drb (2.2.1) - dry-configurable (1.2.0) - dry-core (~> 1.0, < 2) - zeitwerk (~> 2.6) - dry-core (1.0.1) - concurrent-ruby (~> 1.0) - zeitwerk (~> 2.6) - dry-inflector (1.1.0) - dry-logic (1.5.0) - concurrent-ruby (~> 1.0) - dry-core (~> 1.0, < 2) - zeitwerk (~> 2.6) - dry-struct (1.6.0) - dry-core (~> 1.0, < 2) - dry-types (>= 1.7, < 2) - ice_nine (~> 0.11) - zeitwerk (~> 2.6) - dry-types (1.7.2) - bigdecimal (~> 3.0) - concurrent-ruby (~> 1.0) - dry-core (~> 1.0) - dry-inflector (~> 1.0) - dry-logic (~> 1.4) - zeitwerk (~> 2.6) - ed25519 (1.3.0) - erubi (1.13.0) - excon (0.112.0) - faraday (1.10.4) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0) - faraday-multipart (~> 1.0) + drb (2.2.3) + ed25519 (1.4.0) + erb (4.0.4) + cgi (>= 0.3.3) + erubi (1.13.1) + excon (1.2.7) + logger + faraday (1.3.1) faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.0) - faraday-patron (~> 1.0) - faraday-rack (~> 1.0) - faraday-retry (~> 1.0) + multipart-post (>= 1.2, < 3) ruby2_keywords (>= 0.0.4) faraday-cookie_jar (0.0.7) faraday (>= 0.8.0) http-cookie (~> 1.0.0) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-follow_redirects (0.3.0) - faraday (>= 1, < 3) - faraday-httpclient (1.0.1) - faraday-multipart (1.0.4) - multipart-post (~> 2) faraday-net_http (1.0.2) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - faraday-rack (1.0.0) - faraday-retry (1.0.3) - faraday_middleware (1.2.1) + faraday_middleware (1.0.0) faraday (~> 1.0) - ffi (1.17.0) + ffi (1.17.2) + ffi (1.17.2-aarch64-linux-gnu) + ffi (1.17.2-aarch64-linux-musl) + ffi (1.17.2-arm-linux-gnu) + ffi (1.17.2-arm-linux-musl) + ffi (1.17.2-x86-linux-gnu) + ffi (1.17.2-x86-linux-musl) + ffi (1.17.2-x86_64-linux-gnu) + ffi (1.17.2-x86_64-linux-musl) fuzzyurl (0.9.0) google-apis-admin_directory_v1 (0.46.0) google-apis-core (>= 0.11.0, < 2.a) @@ -389,35 +363,26 @@ GEM gyoku (1.4.0) builder (>= 2.1.2) rexml (~> 3.0) - hashdiff (1.0.1) - hashie (5.0.0) - highline (3.1.1) - reline - http-cookie (1.0.7) + hashie (4.1.0) + http-cookie (1.0.8) domain_name (~> 0.5) - httpclient (2.8.3) - i18n (1.14.6) + httpclient (2.9.0) + mutex_m + i18n (1.14.7) concurrent-ruby (~> 1.0) - ice_nine (0.11.2) inifile (3.0.0) - inspec (5.22.58) - faraday_middleware (>= 0.12.2, < 1.3) - inspec-core (= 5.22.58) - mongo (= 2.13.2) - progress_bar (~> 1.3.3) - rake - train (~> 3.10) - train-aws (~> 0.2) + inspec (4.24.32) + faraday_middleware (>= 0.12.2, < 1.1) + inspec-core (= 4.24.32) + train (~> 3.0) + train-aws (~> 0.1) train-habitat (~> 0.1) - train-kubernetes (~> 0.1) train-winrm (~> 0.2) - inspec-core (5.22.58) + inspec-core (4.24.32) addressable (~> 2.4) - chef-telemetry (~> 1.0, >= 1.0.8) - cookstyle - faraday (>= 1, < 3) - faraday-follow_redirects (~> 0.3) - hashie (>= 3.4, < 6.0) + chef-telemetry (~> 1.0) + faraday (>= 0.9.0, < 1.4) + hashie (>= 3.4, < 5.0) license-acceptance (>= 0.2.13, < 3.0) method_source (>= 0.8, < 2.0) mixlib-log (~> 3.0) @@ -425,66 +390,60 @@ GEM parallel (~> 1.9) parslet (>= 1.5, < 3.0) pry (~> 0.13) - rspec (>= 3.9, <= 3.12) + rspec (~> 3.10) rspec-its (~> 1.2) rubyzip (>= 1.2.2, < 3.0) semverse (~> 3.0) sslshake (~> 1.2) - thor (>= 0.20, < 1.3.0) + thor (>= 0.20, < 2.0) tomlrb (>= 1.2, < 2.1) - train-core (~> 3.10) + train-core (~> 3.0) tty-prompt (~> 0.17) tty-table (~> 0.10) - io-console (0.7.2) + io-console (0.8.1) + irb (1.15.2) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) jmespath (1.6.2) - json (2.7.3) - jsonpath (1.1.5) - multi_json - jwt (2.9.3) + json (2.13.2) + jwt (2.10.1) base64 - k8s-ruby (0.16.0) - dry-configurable - dry-struct - dry-types - excon (~> 0.71) - hashdiff (~> 1.0.0) - jsonpath (~> 1.1) - recursive-open-struct (~> 1.1.3) - yajl-ruby (~> 1.4.0) - yaml-safe_load_stream3 - kitchen-dokken (2.20.7) + kitchen-dokken (2.20.8) docker-api (>= 1.33, < 3) lockfile (~> 2.1) test-kitchen (>= 1.15, < 4) - kitchen-inspec (2.6.2) + kitchen-inspec (3.0.0) hashie (>= 3.4, <= 5.0) - inspec (>= 2.2.64, < 6.0) + inspec (>= 2.2.64, < 7.0) test-kitchen (>= 2.7, < 4) + language_server-protocol (3.17.0.5) license-acceptance (2.1.13) pastel (~> 0.7) tomlrb (>= 1.2, < 3.0) tty-box (~> 0.6) tty-prompt (~> 0.20) + lint_roller (1.1.0) little-plugger (1.1.4) lockfile (2.1.3) + logger (1.7.0) logging (2.4.0) little-plugger (~> 1.1) multi_json (~> 1.14) method_source (1.1.0) mini_mime (1.1.5) - minitest (5.25.1) + minitest (5.25.5) mixlib-config (3.0.27) tomlrb mixlib-install (3.12.30) mixlib-shellout mixlib-versioning thor - mixlib-log (3.0.9) - mixlib-shellout (3.3.3) + mixlib-log (3.2.3) + ffi (>= 1.15.5) + mixlib-shellout (3.3.9) chef-utils mixlib-versioning (1.2.12) - mongo (2.13.2) - bson (>= 4.8.2, < 5.0.0) ms_rest (0.7.6) concurrent-ruby (~> 1.0) faraday (>= 0.9, < 2.0.0) @@ -494,104 +453,120 @@ GEM faraday (>= 0.9, < 2.0.0) faraday-cookie_jar (~> 0.0.6) ms_rest (~> 0.7.6) - multi_json (1.15.0) + multi_json (1.17.0) multipart-post (2.4.1) - mutex_m (0.2.0) - net-scp (4.0.0) + mutex_m (0.3.0) + net-scp (4.1.0) net-ssh (>= 2.6.5, < 8.0.0) net-ssh (7.3.0) net-ssh-gateway (2.0.0) net-ssh (>= 4.0.0) nori (2.7.1) bigdecimal - options (2.3.2) os (1.1.4) - parallel (1.26.3) - parser (3.3.5.0) + ostruct (0.6.3) + parallel (1.27.0) + parser (3.3.9.0) ast (~> 2.4.1) racc parslet (2.0.0) pastel (0.8.0) tty-color (~> 0.5) - progress_bar (1.3.4) - highline (>= 1.6) - options (~> 2.3.0) - pry (0.14.2) + pp (0.6.2) + prettyprint + prettyprint (0.2.0) + prism (1.4.0) + pry (0.15.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (6.0.1) + psych (5.2.6) + date + stringio + public_suffix (6.0.2) racc (1.8.1) rainbow (3.1.1) - rake (13.2.1) - recursive-open-struct (1.1.3) - regexp_parser (2.9.2) - reline (0.5.10) + rdoc (6.14.2) + erb + psych (>= 4.0.0) + regexp_parser (2.11.0) + reline (0.6.2) io-console (~> 0.5) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) retriable (3.1.2) - rexml (3.3.9) - rspec (3.12.0) - rspec-core (~> 3.12.0) - rspec-expectations (~> 3.12.0) - rspec-mocks (~> 3.12.0) - rspec-core (3.12.3) - rspec-support (~> 3.12.0) - rspec-expectations (3.12.4) + rexml (3.4.3) + rspec (3.13.1) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.4) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) + rspec-support (~> 3.13.0) rspec-its (1.3.1) rspec-core (>= 3.0.0) rspec-expectations (>= 3.0.0) - rspec-mocks (3.12.7) + rspec-mocks (3.13.5) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-support (3.12.2) - rubocop (1.25.1) + rspec-support (~> 3.13.0) + rspec-support (3.13.4) + rubocop (1.79.2) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) parallel (~> 1.10) - parser (>= 3.1.0.0) + parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml - rubocop-ast (>= 1.15.1, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.46.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.32.3) - parser (>= 3.3.1.0) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.46.0) + parser (>= 3.3.7.2) + prism (~> 1.4) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) rubyntlm (0.6.5) base64 - rubyzip (2.3.2) + rubyzip (2.4.1) + securerandom (0.4.1) semverse (3.0.2) - signet (0.19.0) + signet (0.20.0) addressable (~> 2.8) faraday (>= 0.17.5, < 3.a) jwt (>= 1.5, < 3.0) multi_json (~> 1.10) sslshake (1.3.1) + stringio (3.1.7) strings (0.2.1) strings-ansi (~> 0.2) unicode-display_width (>= 1.5, < 3.0) unicode_utils (~> 1.4) strings-ansi (0.2.0) - test-kitchen (3.7.0) + syslog (0.3.0) + logger + test-kitchen (3.9.0) bcrypt_pbkdf (~> 1.0) chef-utils (>= 16.4.35) - ed25519 (~> 1.2) + csv (~> 3.3) + ed25519 (~> 1.3) + irb (~> 1.15) license-acceptance (>= 1.0.11, < 3.0) mixlib-install (~> 3.6) mixlib-shellout (>= 1.2, < 4.0) net-scp (>= 1.1, < 5.0) net-ssh (>= 2.9, < 8.0) net-ssh-gateway (>= 1.2, < 3.0) + ostruct (~> 0.6) + syslog (~> 0.3) thor (>= 0.19, < 2.0) winrm (~> 2.0) winrm-elevated (~> 1.0) winrm-fs (~> 1.1) - thor (1.2.2) + thor (1.4.0) timeliness (0.3.10) tomlrb (1.3.0) trailblazer-option (0.1.2) @@ -703,9 +678,6 @@ GEM net-scp (>= 1.2, < 5.0) net-ssh (>= 2.9, < 8.0) train-habitat (0.2.22) - train-kubernetes (0.2.1) - k8s-ruby (~> 0.16.0) - train (~> 3.0) train-winrm (0.2.13) winrm (>= 2.3.6, < 3.0) winrm-elevated (~> 1.2.2) @@ -753,12 +725,18 @@ GEM rubyzip (~> 2.0) winrm (~> 2.0) wisper (2.0.1) - yajl-ruby (1.4.3) - yaml-safe_load_stream3 (0.1.2) zeitwerk (2.6.18) PLATFORMS + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl ruby + x86-linux-gnu + x86-linux-musl + x86_64-linux-gnu + x86_64-linux-musl DEPENDENCIES cookstyle @@ -768,4 +746,4 @@ DEPENDENCIES zeitwerk (< 2.7) BUNDLED WITH - 2.2.16 + 2.6.2 diff --git a/README.md b/README.md index 86145db17..68745a5c3 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # OpenStreetMap chef cookbooks -[![Cookstyle](https://github.com/openstreetmap/chef/workflows/Cookstyle/badge.svg?branch=master&event=push)](https://github.com/openstreetmap/chef/actions?query=workflow%3ACookstyle%20branch%3Amaster%20event%3Apush) -[![Test Kitchen](https://github.com/openstreetmap/chef/workflows/Test%20Kitchen/badge.svg?branch=master&event=push)](https://github.com/openstreetmap/chef/actions?query=workflow%3A%22Test+Kitchen%22%20branch%3Amaster%20event%3Apush) +[![Cookstyle](https://github.com/openstreetmap/chef/actions/workflows/cookstyle.yml/badge.svg)](https://github.com/openstreetmap/chef/actions/workflows/cookstyle.yml) +[![Test Kitchen](https://github.com/openstreetmap/chef/actions/workflows/test-kitchen.yml/badge.svg)](https://github.com/openstreetmap/chef/actions/workflows/test-kitchen.yml) This repository manages the configuration of all the servers run by the OpenStreetMap Foundation's Operations Working Group. We use -[Chef](https://www.chef.io/) to automated the configuration of all of our +[Chef](https://www.chef.io/) to automate the configuration of all of our servers. [OSMF Operations Working Group](https://operations.osmfoundation.org/) @@ -14,20 +14,20 @@ servers. We make extensive use of roles to configure the servers. In general we have: -## Server-specific roles (e.g. [faffy.rb](roles/faffy.rb)) +## Server-specific roles (e.g., [faffy.rb](roles/faffy.rb)) These deal with particular setup or quirks of a server, such as its IP address. They also include roles representing the service they are performing, and the location they are in and any particular hardware they have that needs configuration. All our servers are [named after dragons](https://wiki.openstreetmap.org/wiki/Servers/Name_Ideas). -## Hardware-specific roles (e.g. [hp-g9.rb](roles/hp-g9.rb)) +## Hardware-specific roles (e.g., [hp-g9.rb](roles/hp-g9.rb)) Covers anything specific to a certain piece of hardware, like a motherboard, that could apply to multiple machines. -## Location-specific roles (e.g. [equinix-dub.rb](roles/equinix-dub.rb)) +## Location-specific roles (e.g., [equinix-dub.rb](roles/equinix-dub.rb)) These form a hierarchy of datacentres, organisations, and countries where our servers are located. -## Service-specific roles (e.g. [web-frontend](roles/web-frontend.rb)) +## Service-specific roles (e.g., [web-frontend](roles/web-frontend.rb)) These cover the services that the server is running, and will include the recipes required for that service along with any specific configurations and other cascading roles. @@ -41,6 +41,7 @@ Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for mor # Contact Us -* Twitter: [@OSM_Tech](https://twitter.com/OSM_Tech) -* Mastodon / Fediverse: [@OSM_Tech](https://en.osm.town/@osm_tech) -* IRC: [#OSM-Dev on irc.oftc.net](https://irc.openstreetmap.org/) +* Mastodon: [@osm_tech](https://en.osm.town/@osm_tech) +* IRC: [#osm-dev on irc.oftc.net](https://irc.openstreetmap.org/) or [#osmf-operations on irc.oftc.net](https://irc.openstreetmap.org/) +* Matrix: [#\_oftc_#osmf-operations](https://matrix.to/#/#_oftc_#osmf-operations:matrix.org) +* Email: [operations@osmfoundation.org](mailto:operations@osmfoundation.org) diff --git a/cookbooks/accounts/files/default/craig/.ssh/authorized_keys b/cookbooks/accounts/files/default/craig/.ssh/authorized_keys new file mode 100644 index 000000000..101e5e13c --- /dev/null +++ b/cookbooks/accounts/files/default/craig/.ssh/authorized_keys @@ -0,0 +1,2 @@ +# DO NOT EDIT - This file is being maintained by Chef - use authorized_keys2 instead +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc26tRbrQoczW3UFfXkdt6auqFg/Ut6spGMT476fFsJFjaYp98E2lca2W9vyJq4nSn0tdxwcO4LGK1ACdhZ/81I/68d7CPv5zNjJMehgwQ1BJTM5HWaap08cEINZMQ0xt6Neyz+HIFiaJVzxmyLJCnaaCeQX/t2NmL+nQV6rJq4qS2L434Bw1qGM73zjNja4bB2IN0y5yWDRTSLg+t+DKH26DC4OJn4+pxKsyt2egB7MNj9my1MRcjPVeo/bxz3nWoxKtX9dWq9UFrd7trfSXK+7Y+9fFHl41rrrYbn3UFKcDL6Rzvp2bFytDW6FlWmuptGajWnm2HpqI69bsO7uw1 diff --git a/cookbooks/accounts/files/default/ignisf/.ssh/authorized_keys b/cookbooks/accounts/files/default/ignisf/.ssh/authorized_keys new file mode 100644 index 000000000..22b72c6f4 --- /dev/null +++ b/cookbooks/accounts/files/default/ignisf/.ssh/authorized_keys @@ -0,0 +1,2 @@ +# DO NOT EDIT - This file is being maintained by Chef - use authorized_keys2 instead +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwMHHDBdzFr39OGEtXYpRHXiZiCB5eHQXnPR9qKzSGaDm5WljLAYRQbXnX58lAgNJYyNV+81QK68U9pRJWO6VqBt3LP1triJ5uxiJIrLg72AQ7iKS3R8b62bG1reF2Uc1zOSPT3HvWOl0FURhkn1zmvs6aeCeI7rO3fwF8IOIkxw7mycPXSXXE7QmqgQ6y8uG8LhF303NethPYbIWpJR3UfQjg0z1tXDMt+yH3NM4vRcRHaA/C0BMX2qrCGT1dhRve0f8Zz8hN7FK+1Xt/BnhEzEwG73kYDaOGOBva+oHNqBEhq5JYP2sCQYYHuRT20aGzbNgAX8hbSgdiwYEaalXL ignisf-key diff --git a/cookbooks/accounts/files/default/milliams/.ssh/authorized_keys b/cookbooks/accounts/files/default/milliams/.ssh/authorized_keys index a1fa1cf3c..4f21fdb8e 100644 --- a/cookbooks/accounts/files/default/milliams/.ssh/authorized_keys +++ b/cookbooks/accounts/files/default/milliams/.ssh/authorized_keys @@ -1 +1 @@ -ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEVkoOPte6R6jN5w7yny+YLtoZGl/XLQL2aSjhgyNHrh matt@HEX +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMbllYzPMjPeGJ/4EAM8h4Bfhs1H56UpKU/dbV3ljBRT matt@HEX diff --git a/cookbooks/accounts/files/default/nmoore/.ssh/authorized_keys b/cookbooks/accounts/files/default/nmoore/.ssh/authorized_keys new file mode 100644 index 000000000..d91c729cf --- /dev/null +++ b/cookbooks/accounts/files/default/nmoore/.ssh/authorized_keys @@ -0,0 +1,5 @@ +# DO NOT EDIT - This file is being maintained by Chef - use authorized_keys2 instead +sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIFjD7MYD8g5MIKTGpwNcx/EylNXSY5AS8TJGyfVQ7ZqFAAAABHNzaDo= Natalie Fedora Laptop +sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIKZ3IIQmPhzExk5CVOEQ4laIPskN6eVxoTxI2eon7DHRAAAABHNzaDo= Backup Key +sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIAKenC+6zGmDyfbfTFDbjKjBbCcVGlH0iyR5+X7u5R7VAAAABHNzaDo= Necklace +sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIJ+eCC1p7DVkTk+L435dd2fS9PvaaFPoDn/sp87KpYceAAAABHNzaDo= desktop diff --git a/cookbooks/apache/attributes/default.rb b/cookbooks/apache/attributes/default.rb index 075e4c86f..4f4880509 100644 --- a/cookbooks/apache/attributes/default.rb +++ b/cookbooks/apache/attributes/default.rb @@ -26,7 +26,7 @@ default[:apache][:event][:threads_per_child] = 25 default[:apache][:event][:max_request_workers] = 150 default[:apache][:event][:max_connections_per_child] = 0 -default[:apache][:listen_address] = "*" +default[:apache][:listen_address] = "[::]" default[:apache][:buffered_logs] = true diff --git a/cookbooks/apache/recipes/default.rb b/cookbooks/apache/recipes/default.rb index cade29d31..494bc4c3d 100644 --- a/cookbooks/apache/recipes/default.rb +++ b/cookbooks/apache/recipes/default.rb @@ -53,6 +53,7 @@ template "/etc/apache2/ports.conf" do owner "root" group "root" mode "644" + notifies :restart, "service[apache2]" end systemd_service "apache2" do @@ -72,6 +73,10 @@ apache_module "status" do variables :hosts => admins["hosts"] end +apache_conf "tokens" do + template "tokens.conf.erb" +end + if node[:apache][:evasive][:enable] apache_module "evasive" do conf "evasive.conf.erb" @@ -95,6 +100,8 @@ apache_module "ssl" apache_conf "ssl" do template "ssl.erb" + reload_apache false + restart_apache true # restart required for shared memory config changes end # Apache should only be started after modules enabled @@ -114,7 +121,7 @@ fail2ban_jail "apache-forbidden" do end fail2ban_filter "apache-evasive" do - failregex ": Blacklisting address : possible DoS attack\.$" + failregex ": Blacklisting address : possible DoS attack\\.$" end fail2ban_jail "apache-evasive" do diff --git a/cookbooks/apache/resources/conf.rb b/cookbooks/apache/resources/conf.rb index 9f1efb24f..f1e56d065 100644 --- a/cookbooks/apache/resources/conf.rb +++ b/cookbooks/apache/resources/conf.rb @@ -26,6 +26,7 @@ property :cookbook, :kind_of => String property :template, :kind_of => String, :required => [:create] property :variables, :kind_of => Hash, :default => {} property :reload_apache, :kind_of => [TrueClass, FalseClass], :default => true +property :restart_apache, :kind_of => [TrueClass, FalseClass], :default => false action :create do create_conf @@ -86,4 +87,5 @@ end def after_created notifies :reload, "service[apache2]" if reload_apache + notifies :restart, "service[apache2]" if restart_apache end diff --git a/cookbooks/apache/templates/default/httpd.conf.erb b/cookbooks/apache/templates/default/httpd.conf.erb index 3f78187f8..bb549eb8e 100644 --- a/cookbooks/apache/templates/default/httpd.conf.erb +++ b/cookbooks/apache/templates/default/httpd.conf.erb @@ -61,7 +61,7 @@ AddDefaultCharset utf-8 # Add extra mime types AddType application/x-xz .xz -# Configure logging +# Configure log buffering BufferedLogs <%= node[:apache][:buffered_logs] ? "On" : "Off" %> # Define an extended log format that includes request time and SSL details diff --git a/cookbooks/apache/templates/default/ssl.erb b/cookbooks/apache/templates/default/ssl.erb index 81afb3de5..72ac3b857 100644 --- a/cookbooks/apache/templates/default/ssl.erb +++ b/cookbooks/apache/templates/default/ssl.erb @@ -5,11 +5,4 @@ SSLProtocol All -SSLv2 -SSLv3 SSLHonorCipherOrder On SSLCipherSuite <%= node[:ssl][:openssl_ciphers] %> -SSLUseStapling On -SSLStaplingResponderTimeout 5 -SSLStaplingErrorCacheTimeout 60 -SSLStaplingReturnResponderErrors off -SSLStaplingFakeTryLater off -SSLStaplingCache shmcb:${APACHE_RUN_DIR}/ssl_ocspcache(512000) - Header always set Strict-Transport-Security "<%= node[:ssl][:strict_transport_security] %>" "expr=%{HTTPS} == 'on'" diff --git a/cookbooks/apache/templates/default/tokens.conf.erb b/cookbooks/apache/templates/default/tokens.conf.erb new file mode 100644 index 000000000..bc05c9494 --- /dev/null +++ b/cookbooks/apache/templates/default/tokens.conf.erb @@ -0,0 +1,7 @@ +# DO NOT EDIT - This file is being maintained by Chef + +# Hide server version on error pages +ServerSignature Off + +# Only return Apache in server header +ServerTokens Prod diff --git a/cookbooks/apt/recipes/default.rb b/cookbooks/apt/recipes/default.rb index 2ead9baf2..62c6f8d40 100644 --- a/cookbooks/apt/recipes/default.rb +++ b/cookbooks/apt/recipes/default.rb @@ -48,8 +48,8 @@ if platform?("debian") archive_suites = %w[main updates backports security] archive_components = %w[main contrib non-free non-free-firmware] backport_packages = case node[:lsb][:codename] - when "bookworm" then %W[amd64-microcode exim4 firmware-free firmware-nonfree intel-microcode libosmium linux-signed-#{dpkg_arch} osm2pgsql otrs2 pyosmium smartmontools systemd cgi-mapserver] - else %W[] + when "bookworm" then %W[amd64-microcode exim4 firmware-free firmware-nonfree intel-microcode libosmium linux linux-base linux-signed-#{dpkg_arch} osm2pgsql otrs2 pyosmium smartmontools systemd cgi-mapserver] + else %w[] end elsif intel? archive_host = if node[:country] diff --git a/cookbooks/apt/recipes/elasticsearch6.rb b/cookbooks/apt/recipes/elasticsearch6.rb index 74c02a324..66844f61c 100644 --- a/cookbooks/apt/recipes/elasticsearch6.rb +++ b/cookbooks/apt/recipes/elasticsearch6.rb @@ -23,5 +23,5 @@ apt_repository "elasticsearch6.x" do uri "https://artifacts.elastic.co/packages/6.x/apt" distribution "stable" components ["main"] - key "D27D666CD88E42B4" + key "https://artifacts.elastic.co/GPG-KEY-elasticsearch" end diff --git a/cookbooks/apt/recipes/elasticsearch7.rb b/cookbooks/apt/recipes/elasticsearch7.rb index 6dc824177..57737b635 100644 --- a/cookbooks/apt/recipes/elasticsearch7.rb +++ b/cookbooks/apt/recipes/elasticsearch7.rb @@ -23,7 +23,7 @@ apt_repository "elasticsearch7.x" do uri "https://artifacts.elastic.co/packages/7.x/apt" distribution "stable" components ["main"] - key "D27D666CD88E42B4" + key "https://artifacts.elastic.co/GPG-KEY-elasticsearch" end # Workaround for mediawiki 1.39.x which ONLY supports elasticsearch 7.10.2 diff --git a/cookbooks/apt/recipes/elasticsearch8.rb b/cookbooks/apt/recipes/elasticsearch8.rb index 61a94f7aa..b9991e474 100644 --- a/cookbooks/apt/recipes/elasticsearch8.rb +++ b/cookbooks/apt/recipes/elasticsearch8.rb @@ -23,5 +23,5 @@ apt_repository "elasticsearch8.x" do uri "https://artifacts.elastic.co/packages/8.x/apt" distribution "stable" components ["main"] - key "D27D666CD88E42B4" + key "https://artifacts.elastic.co/GPG-KEY-elasticsearch" end diff --git a/cookbooks/apt/recipes/fullstaq-ruby.rb b/cookbooks/apt/recipes/fullstaq-ruby.rb new file mode 100644 index 000000000..d9dad7d37 --- /dev/null +++ b/cookbooks/apt/recipes/fullstaq-ruby.rb @@ -0,0 +1,27 @@ +# +# Cookbook:: apt +# Recipe:: fullstaq-ruby +# +# Copyright:: 2025, Tom Hughes +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "apt" + +apt_repository "fullstaq-ruby" do + uri "https://apt.fullstaqruby.org" + distribution "#{node[:platform]}-#{node[:platform_version]}" + components ["main"] + key "https://raw.githubusercontent.com/fullstaq-ruby/server-edition/main/fullstaq-ruby.asc" +end diff --git a/cookbooks/apt/recipes/grafana.rb b/cookbooks/apt/recipes/grafana.rb index ef6531c1f..5738c23d4 100644 --- a/cookbooks/apt/recipes/grafana.rb +++ b/cookbooks/apt/recipes/grafana.rb @@ -19,15 +19,9 @@ include_recipe "apt" -remote_file "/etc/apt/trusted.gpg.d/grafana.asc" do - source "https://packages.grafana.com/gpg.key" - owner "root" - group "root" - mode "644" -end - apt_repository "grafana" do uri "https://packages.grafana.com/enterprise/deb" distribution "stable" components ["main"] + key "https://packages.grafana.com/gpg.key" end diff --git a/cookbooks/apt/recipes/hwraid.rb b/cookbooks/apt/recipes/hwraid.rb index fa5d9e1f9..c8d99bdef 100644 --- a/cookbooks/apt/recipes/hwraid.rb +++ b/cookbooks/apt/recipes/hwraid.rb @@ -35,5 +35,5 @@ apt_repository "hwraid" do uri "https://hwraid.le-vert.net/#{platform_name}" distribution distribution_name components ["main"] - key "6005210E23B3D3B4" + key "https://hwraid.le-vert.net/debian/hwraid.le-vert.net.gpg.key" end diff --git a/cookbooks/apt/recipes/management-component-pack.rb b/cookbooks/apt/recipes/management-component-pack.rb index 92d81f989..3a86443ad 100644 --- a/cookbooks/apt/recipes/management-component-pack.rb +++ b/cookbooks/apt/recipes/management-component-pack.rb @@ -23,43 +23,43 @@ apt_repository "management-component-pack" do action :remove end -if platform?("debian") - apt_repository "mcp" do - uri "https://downloads.linux.hpe.com/SDR/repo/mcp" - distribution "#{node[:lsb][:codename]}/current" - components ["non-free"] - key "C208ADDE26C2B797" - end - - if node[:dmi][:system][:product_name].end_with?("Gen9") - apt_repository "mcp-gen9" do - uri "https://downloads.linux.hpe.com/SDR/repo/mcp" - distribution "stretch/current-gen9" - components ["non-free"] - key "C208ADDE26C2B797" - end - end -elsif platform?("ubuntu") - if node[:dmi][:system][:product_name].end_with?("Gen10") - apt_repository "mcp-jammy" do - uri "https://downloads.linux.hpe.com/SDR/repo/mcp" - distribution "jammy/current" - components ["non-free"] - key "C208ADDE26C2B797" - end - - apt_repository "mcp-focal-gen10" do - uri "https://downloads.linux.hpe.com/SDR/repo/mcp" - distribution "focal/current-gen10" - components ["non-free"] - key "C208ADDE26C2B797" - end - else - apt_repository "mcp-bionic-gen9" do - uri "https://downloads.linux.hpe.com/SDR/repo/mcp" - distribution "bionic/current-gen9" - components ["non-free"] - key "C208ADDE26C2B797" - end - end -end +# if platform?("debian") +# apt_repository "mcp" do +# uri "https://downloads.linux.hpe.com/SDR/repo/mcp" +# distribution "#{node[:lsb][:codename]}/current" +# components ["non-free"] +# key ["https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key1.pub", "https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key2.pub"] +# end +# +# if node[:dmi][:system][:product_name].end_with?("Gen9") +# apt_repository "mcp-gen9" do +# uri "https://downloads.linux.hpe.com/SDR/repo/mcp" +# distribution "stretch/current-gen9" +# components ["non-free"] +# key ["https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key1.pub", "https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key2.pub"] +# end +# end +# elsif platform?("ubuntu") +# if node[:dmi][:system][:product_name].end_with?("Gen10") +# apt_repository "mcp-jammy" do +# uri "https://downloads.linux.hpe.com/SDR/repo/mcp" +# distribution "jammy/current" +# components ["non-free"] +# key ["https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key1.pub", "https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key2.pub"] +# end +# +# apt_repository "mcp-focal-gen10" do +# uri "https://downloads.linux.hpe.com/SDR/repo/mcp" +# distribution "focal/current-gen10" +# components ["non-free"] +# key ["https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key1.pub", "https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key2.pub"] +# end +# else +# apt_repository "mcp-bionic-gen9" do +# uri "https://downloads.linux.hpe.com/SDR/repo/mcp" +# distribution "bionic/current-gen9" +# components ["non-free"] +# key ["https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key1.pub", "https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key2.pub"] +# end +# end +# end diff --git a/cookbooks/apt/recipes/nginx.rb b/cookbooks/apt/recipes/nginx.rb index b80b8bb8b..117ef31a7 100644 --- a/cookbooks/apt/recipes/nginx.rb +++ b/cookbooks/apt/recipes/nginx.rb @@ -26,8 +26,7 @@ platform_name = if platform?("debian") end apt_repository "nginx" do - arch "amd64" uri "https://nginx.org/packages/#{platform_name}" components ["nginx"] - key "ABF5BD827BD9BF62" + key "https://nginx.org/keys/nginx_signing.key" end diff --git a/cookbooks/apt/recipes/passenger.rb b/cookbooks/apt/recipes/passenger.rb index 136175ba0..b7e208dd6 100644 --- a/cookbooks/apt/recipes/passenger.rb +++ b/cookbooks/apt/recipes/passenger.rb @@ -22,5 +22,5 @@ include_recipe "apt" apt_repository "passenger" do uri "https://oss-binaries.phusionpassenger.com/apt/passenger" components ["main"] - key "561F9B9CAC40B2F7" + key "https://oss-binaries.phusionpassenger.com/auto-software-signing-gpg-key.txt" end diff --git a/cookbooks/apt/recipes/postgresql.rb b/cookbooks/apt/recipes/postgresql.rb index 66e4c1c92..6fe6377d1 100644 --- a/cookbooks/apt/recipes/postgresql.rb +++ b/cookbooks/apt/recipes/postgresql.rb @@ -23,5 +23,5 @@ apt_repository "postgresql" do uri "https://apt.postgresql.org/pub/repos/apt" distribution "#{node[:lsb][:codename]}-pgdg" components ["main"] - key "7FCC7D46ACCC4CF8" + key "https://www.postgresql.org/media/keys/ACCC4CF8.asc" end diff --git a/cookbooks/apt/recipes/repository.rb b/cookbooks/apt/recipes/repository.rb index c4e30b57d..04942a2c4 100644 --- a/cookbooks/apt/recipes/repository.rb +++ b/cookbooks/apt/recipes/repository.rb @@ -62,7 +62,7 @@ execute "apt-generate-key" do not_if { ::Dir.exist?("/srv/apt.openstreetmap.org/.gnupg") } end -%w[focal jammy noble bookworm].each do |distribution| +%w[focal jammy noble bookworm trixie].each do |distribution| repository = "openstreetmap-#{distribution}" execute "aptly-repo-create-#{distribution}" do diff --git a/cookbooks/apt/recipes/yarn.rb b/cookbooks/apt/recipes/yarn.rb index 7451e96a9..fe5059bb3 100644 --- a/cookbooks/apt/recipes/yarn.rb +++ b/cookbooks/apt/recipes/yarn.rb @@ -23,5 +23,5 @@ apt_repository "yarn" do uri "https://dl.yarnpkg.com/debian" distribution "stable" components ["main"] - key "1646B01B86E50310" + key "https://dl.yarnpkg.com/debian/pubkey.gpg" end diff --git a/cookbooks/awscli/recipes/default.rb b/cookbooks/awscli/recipes/default.rb index 22684864d..90574c3bb 100644 --- a/cookbooks/awscli/recipes/default.rb +++ b/cookbooks/awscli/recipes/default.rb @@ -77,12 +77,25 @@ ruby_block "install-awscli" do require "fileutils" awscli_version_string = shell_out("#{cache_dir}/awscli/dist/aws", "--version") awscli_version = awscli_version_string.stdout.split(" ").first.split("/").last - FileUtils.mkdir_p("/opt/awscli/v2/#{awscli_version}/bin/", :mode => 0755) - FileUtils.mv("#{cache_dir}/awscli/dist", "/opt/awscli/v2/#{awscli_version}/dist", :force => true) - FileUtils.ln_sf("/opt/awscli/v2/#{awscli_version}/dist/aws", "/opt/awscli/v2/#{awscli_version}/bin/aws") - FileUtils.ln_sf("/opt/awscli/v2/#{awscli_version}/dist/aws_completer", "/opt/awscli/v2/#{awscli_version}/bin/aws_completer") - FileUtils.rm("/opt/awscli/v2/current") if File.exist?("/opt/awscli/v2/current") - FileUtils.ln_sf("/opt/awscli/v2/#{awscli_version}", "/opt/awscli/v2/current") + + install_dir = "/opt/awscli/v2/#{awscli_version}" + + FileUtils.mkdir_p("#{install_dir}/bin/", :mode => 0755) + FileUtils.mv("#{cache_dir}/awscli/dist", "#{install_dir}/dist", :force => true) + FileUtils.ln_sf("#{install_dir}/dist/aws", "#{install_dir}/bin/aws") + FileUtils.ln_sf("#{install_dir}/dist/aws_completer", "#{install_dir}/bin/aws_completer") + + FileUtils.rm_f("/opt/awscli/v2/current") + FileUtils.ln_sf(install_dir, "/opt/awscli/v2/current") + + # Retain the last 5 versions, including the current one + versions = Dir.glob("/opt/awscli/v2/*").select { |dir| File.directory?(dir) && dir != "/opt/awscli/v2/current" } + versions.sort_by! { |dir| File.mtime(dir) }.reverse! + versions_to_delete = versions[5..] || [] + + versions_to_delete.each do |dir| + FileUtils.rm_rf(dir) + end end action :nothing subscribes :run, "archive_file[#{cache_dir}/#{awscli_zip}]", :immediately diff --git a/cookbooks/backup/templates/default/expire.cron.erb b/cookbooks/backup/templates/default/expire.cron.erb index e1ec74a08..e4c2fdfca 100644 --- a/cookbooks/backup/templates/default/expire.cron.erb +++ b/cookbooks/backup/templates/default/expire.cron.erb @@ -2,7 +2,7 @@ # DO NOT EDIT - This file is being maintained by Chef -for prefix in blogs chef-server chef-repository chef-git community forum git lists osm-blog osmf-crm osmf-ledgersmb wiki-wiki.osmfoundation.org osqa otrs prometheus sotm svn switch2osm trac wiki-board.osmfoundation.org wiki-dwg.osmfoundation.org wiki-mwg.osmfoundation.org wiki-wiki.openstreetmap.org +for prefix in blogs chef-server chef-repository chef-git community forum git lists osm-blog osm-blog-staging osmf-crm osmf-ledgersmb wiki-osmfoundation.org osqa otrs prometheus sotm svn switch2osm trac wiki-board.osmfoundation.org wiki-dwg.osmfoundation.org wiki-mwg.osmfoundation.org wiki-wiki.openstreetmap.org do /usr/local/bin/expire-backups --days=3 --weeks=3 --months=3 /store/backup $prefix done diff --git a/cookbooks/bind/templates/default/db.10.erb b/cookbooks/bind/templates/default/db.10.erb index 06aef6442..298811c68 100644 --- a/cookbooks/bind/templates/default/db.10.erb +++ b/cookbooks/bind/templates/default/db.10.erb @@ -2,7 +2,7 @@ $TTL 604800 @ IN SOA <%= node[:fdqn] %>. root.openstreetmap.org. ( - 2021092001 ; Serial + 2025021801 ; Serial 604800 ; Refresh 86400 ; Retry 2419200 ; Expire @@ -44,6 +44,7 @@ $TTL 604800 100.48.0 IN PTR pdu1.ams.openstreetmap.org. 101.48.0 IN PTR pdu2.ams.openstreetmap.org. 102.48.0 IN PTR oob1.ams.openstreetmap.org. +103.48.0 IN PTR ats1.ams.openstreetmap.org. 3.49.0 IN PTR faffy.oob.openstreetmap.org. 4.49.0 IN PTR dribble.oob.openstreetmap.org. diff --git a/cookbooks/blog/recipes/staging.rb b/cookbooks/blog/recipes/staging.rb index 70ff3d8ad..19437de68 100644 --- a/cookbooks/blog/recipes/staging.rb +++ b/cookbooks/blog/recipes/staging.rb @@ -27,9 +27,11 @@ ssl_certificate "staging.blog.openstreetmap.org" do ] end -# passwords = data_bag_item("blog-staging", "passwords") +passwords = data_bag_item("blog-staging", "passwords") # wp2fa_encrypt_keys = data_bag_item("blog-staging", "wp2fa_encrypt_keys") +# The staging blog is under manual development by Mikel. Do not manage with Chef. + # directory "/srv/staging.blog.openstreetmap.org" do # owner "wordpress" # group "wordpress" @@ -132,10 +134,10 @@ end # group "wordpress" # end -# template "/etc/cron.daily/blog-staging-backup" do -# source "backup-staging.cron.erb" -# owner "root" -# group "root" -# mode "750" -# variables :passwords => passwords -# end +template "/etc/cron.daily/blog-staging-backup" do + source "backup-staging.cron.erb" + owner "root" + group "root" + mode "750" + variables :passwords => passwords +end diff --git a/cookbooks/blog/templates/default/backup-staging.cron.erb b/cookbooks/blog/templates/default/backup-staging.cron.erb index 35b0a77ff..1dd834a60 100644 --- a/cookbooks/blog/templates/default/backup-staging.cron.erb +++ b/cookbooks/blog/templates/default/backup-staging.cron.erb @@ -2,20 +2,21 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp osm-blog-staging.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=osm-blog-staging-$D.tar.gz - -mkdir $T/osm-blog-staging-$D -echo '[mysqldump]' > $T/mysqldump.opts -echo 'user=osm-blog-staging-user' >> $T/mysqldump.opts -echo 'password=<%= @passwords["osm-blog-staging-user"] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt --no-tablespaces osm-blog-staging > $T/osm-blog-staging-$D/osm-blog-staging.sql -ln -s /srv/staging.blog.openstreetmap.org $T/osm-blog-staging-$D/www +B="osm-blog-staging-$D.tar.zst" -export RSYNC_RSH="ssh -ax" +mkdir "$T/osm-blog-staging-$D" +echo '[mysqldump]' > "$T/mysqldump.opts" +echo 'user=osm-blog-staging-user' >> "$T/mysqldump.opts" +echo 'password=<%= @passwords["osm-blog-staging-user"] %>' >> "$T/mysqldump.opts" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces osm-blog-staging > "$T/osm-blog-staging-$D/osm-blog-staging.sql" +ln -s /srv/staging.blog.openstreetmap.org "$T/osm-blog-staging-$D/www" -nice tar --create --dereference --directory=$T --warning=no-file-changed osm-blog-staging-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" --warning=no-file-changed "osm-blog-staging-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/blog/templates/default/backup.cron.erb b/cookbooks/blog/templates/default/backup.cron.erb index 9e3bea7d8..59ff2f8cc 100644 --- a/cookbooks/blog/templates/default/backup.cron.erb +++ b/cookbooks/blog/templates/default/backup.cron.erb @@ -2,20 +2,21 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp osm-blog.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=osm-blog-$D.tar.gz - -mkdir $T/osm-blog-$D -echo '[mysqldump]' > $T/mysqldump.opts -echo 'user=osm-blog-user' >> $T/mysqldump.opts -echo 'password=<%= @passwords["osm-blog-user"] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt --no-tablespaces osm-blog > $T/osm-blog-$D/osm-blog.sql -ln -s /srv/blog.openstreetmap.org $T/osm-blog-$D/www +B="osm-blog-$D.tar.zst" -export RSYNC_RSH="ssh -ax" +mkdir "$T/osm-blog-$D" +echo '[mysqldump]' > "$T/mysqldump.opts" +echo 'user=osm-blog-user' >> "$T/mysqldump.opts" +echo 'password=<%= @passwords["osm-blog-user"] %>' >> "$T/mysqldump.opts" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces osm-blog > "$T/osm-blog-$D/osm-blog.sql" +ln -s /srv/blog.openstreetmap.org "$T/osm-blog-$D/www" -nice tar --create --dereference --directory=$T --warning=no-file-changed osm-blog-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" --warning=no-file-changed "osm-blog-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/blogs/recipes/default.rb b/cookbooks/blogs/recipes/default.rb index 0fb2cc7aa..86dfd3afe 100644 --- a/cookbooks/blogs/recipes/default.rb +++ b/cookbooks/blogs/recipes/default.rb @@ -22,7 +22,7 @@ include_recipe "apache" include_recipe "git" include_recipe "ruby" -package %W[ +package %w[ make gcc g++ @@ -44,10 +44,18 @@ git "/srv/blogs.openstreetmap.org" do group "blogs" end +bundle_config "/srv/blogs.openstreetmap.org" do + action :nothing + user "blogs" + group "blogs" + settings "deployment" => "true", + "without" => "development:test", + "build.sqlite3" => "--enable-system-libraries" + subscribes :create, "git[/srv/blogs.openstreetmap.org]", :immediately +end + bundle_install "/srv/blogs.openstreetmap.org" do action :nothing - options "--deployment --without development test" - environment "BUNDLE_PATH" => "vendor/bundle" user "blogs" group "blogs" subscribes :run, "git[/srv/blogs.openstreetmap.org]", :immediately @@ -56,7 +64,6 @@ end bundle_exec "/srv/blogs.openstreetmap.org" do action :nothing command "pluto build -t osm -o build" - environment "BUNDLE_PATH" => "vendor/bundle" user "blogs" group "blogs" subscribes :run, "git[/srv/blogs.openstreetmap.org]", :immediately diff --git a/cookbooks/blogs/templates/default/backup.cron.erb b/cookbooks/blogs/templates/default/backup.cron.erb index c020a5e35..d60e95080 100644 --- a/cookbooks/blogs/templates/default/backup.cron.erb +++ b/cookbooks/blogs/templates/default/backup.cron.erb @@ -2,16 +2,17 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp blogs.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=blogs-$D.tar.gz +B="blogs-$D.tar.zst" -mkdir $T/blogs-$D +mkdir "$T/blogs-$D" sqlite3 /srv/blogs.openstreetmap.org/planet.db ".backup $T/blogs-$D/planet.db" -export RSYNC_RSH="ssh -ax" - -nice tar --create --dereference --directory=$T blogs-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" "blogs-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/blogs/templates/default/blogs-update.erb b/cookbooks/blogs/templates/default/blogs-update.erb index a7d021495..15cc82483 100644 --- a/cookbooks/blogs/templates/default/blogs-update.erb +++ b/cookbooks/blogs/templates/default/blogs-update.erb @@ -2,8 +2,6 @@ cd /srv/blogs.openstreetmap.org -export BUNDLE_PATH="vendor/bundle" - <%= node[:ruby][:bundle] %> exec pluto \ --quieter \ --config=/srv/blogs.openstreetmap.org build \ diff --git a/cookbooks/chef/attributes/default.rb b/cookbooks/chef/attributes/default.rb index ce0c04e30..9ff0f65a5 100644 --- a/cookbooks/chef/attributes/default.rb +++ b/cookbooks/chef/attributes/default.rb @@ -2,4 +2,4 @@ default[:chef][:server][:version] = "15.9.38" # Set the default client version -default[:chef][:client][:version] = "18.5.0" +default[:chef][:client][:version] = "18.7.10" diff --git a/cookbooks/chef/templates/default/server-backup.cron.erb b/cookbooks/chef/templates/default/server-backup.cron.erb index 9b864768e..82d15d517 100644 --- a/cookbooks/chef/templates/default/server-backup.cron.erb +++ b/cookbooks/chef/templates/default/server-backup.cron.erb @@ -1,16 +1,21 @@ #!/bin/sh +# DO NOT EDIT - This file is being maintained by Chef + +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp chef-server.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=chef-server-$D.tar.gz +B="chef-server-$D.tar.zst" -mkdir $T/chef-server-$D -chgrp opscode-pgsql $T $T/chef-server-$D -chmod g+rwx $T $T/chef-server-$D -sudo -u opscode-pgsql /opt/opscode/embedded/bin/pg_dumpall --file=$T/chef-server-$D/chef.dmp --clean -ln -s /var/opt/opscode/bookshelf/data $T/chef-server-$D/bookshelf +mkdir "$T/chef-server-$D" +chgrp opscode-pgsql "$T" "$T/chef-server-$D" +chmod g+rwx "$T" "$T/chef-server-$D" +sudo -u opscode-pgsql /opt/opscode/embedded/bin/pg_dumpall --file="$T/chef-server-$D/chef.dmp" --clean +ln -s /var/opt/opscode/bookshelf/data "$T/chef-server-$D/bookshelf" -nice tar --create --dereference --directory=$T chef-server-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" "chef-server-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/civicrm/recipes/default.rb b/cookbooks/civicrm/recipes/default.rb index 2aed4f64f..f7f0908bc 100644 --- a/cookbooks/civicrm/recipes/default.rb +++ b/cookbooks/civicrm/recipes/default.rb @@ -21,12 +21,13 @@ include_recipe "wordpress" include_recipe "mysql" package %w[ - php-xml - php-curl rsync - wkhtmltopdf + php-curl php-bcmath + php-fileinfo php-intl + php-mbstring + php-xml ] apache_module "rewrite" @@ -200,21 +201,21 @@ node[:civicrm][:extensions].each_value do |details| end settings = edit_file "#{civicrm_directory}/civicrm/templates/CRM/common/civicrm.settings.php.template" do |line| - line.gsub!(/%%cms%%/, "WordPress") - line.gsub!(/%%CMSdbUser%%/, "civicrm") - line.gsub!(/%%CMSdbPass%%/, database_password) - line.gsub!(/%%CMSdbHost%%/, "localhost") - line.gsub!(/%%CMSdbName%%/, "civicrm") - line.gsub!(/%%dbUser%%/, "civicrm") - line.gsub!(/%%dbPass%%/, database_password) - line.gsub!(/%%dbHost%%/, "localhost") - line.gsub!(/%%dbName%%/, "civicrm") - line.gsub!(/%%crmRoot%%/, "#{civicrm_directory}/civicrm/") - line.gsub!(/%%templateCompileDir%%/, "/srv/supporting.openstreetmap.org/wp-content/uploads/civicrm/templates_c/") - line.gsub!(/%%baseURL%%/, "http://supporting.openstreetmap.org/") - line.gsub!(/%%siteKey%%/, site_key) - line.gsub!(/%%credKeys%%/, cred_keys) - line.gsub!(/%%signKeys%%/, sign_keys) + line.gsub!("%%cms%%", "WordPress") + line.gsub!("%%CMSdbUser%%", "civicrm") + line.gsub!("%%CMSdbPass%%", database_password) + line.gsub!("%%CMSdbHost%%", "localhost") + line.gsub!("%%CMSdbName%%", "civicrm") + line.gsub!("%%dbUser%%", "civicrm") + line.gsub!("%%dbPass%%", database_password) + line.gsub!("%%dbHost%%", "localhost") + line.gsub!("%%dbName%%", "civicrm") + line.gsub!("%%crmRoot%%", "#{civicrm_directory}/civicrm/") + line.gsub!("%%templateCompileDir%%", "/srv/supporting.openstreetmap.org/wp-content/uploads/civicrm/templates_c/") + line.gsub!("%%baseURL%%", "http://supporting.openstreetmap.org/") + line.gsub!("%%siteKey%%", site_key) + line.gsub!("%%credKeys%%", cred_keys) + line.gsub!("%%signKeys%%", sign_keys) line.gsub!(%r{// *define\('CIVICRM_CMSDIR', '/path/to/install/root/'\);}, "define('CIVICRM_CMSDIR', '/srv/supporting.openstreetmap.org');") # Don't recompile smarty templates on every call https://docs.civicrm.org/sysadmin/en/latest/setup/optimizations/#disable-compile-check line.gsub!(%r{// define\('CIVICRM_TEMPLATE_COMPILE_CHECK', FALSE\);}, "define('CIVICRM_TEMPLATE_COMPILE_CHECK', FALSE);") diff --git a/cookbooks/civicrm/templates/default/backup.cron.erb b/cookbooks/civicrm/templates/default/backup.cron.erb index 7738f75ca..bba562483 100644 --- a/cookbooks/civicrm/templates/default/backup.cron.erb +++ b/cookbooks/civicrm/templates/default/backup.cron.erb @@ -2,20 +2,21 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp osmf-crm.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=osmf-crm-$D.tar.gz - -mkdir $T/osmf-crm-$D -echo '[mysqldump]' > $T/mysqldump.opts -echo 'user=civicrm' >> $T/mysqldump.opts -echo 'password=<%= @passwords["database"] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt --skip-lock-tables --no-tablespaces civicrm > $T/osmf-crm-$D/civicrm.sql -ln -s /srv/supporting.openstreetmap.org $T/osmf-crm-$D/www +B="osmf-crm-$D.tar.zst" -export RSYNC_RSH="ssh -ax" +mkdir "$T/osmf-crm-$D" +echo '[mysqldump]' > "$T/mysqldump.opts" +echo 'user=civicrm' >> "$T/mysqldump.opts" +echo 'password=<%= @passwords["database"] %>' >> "$T/mysqldump.opts" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --no-tablespaces civicrm > "$T/osmf-crm-$D/civicrm.sql" +ln -s /srv/supporting.openstreetmap.org "$T/osmf-crm-$D/www" -nice tar --create --dereference --directory=$T osmf-crm-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" "osmf-crm-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/community/recipes/default.rb b/cookbooks/community/recipes/default.rb index 000a57a7e..3b2042161 100644 --- a/cookbooks/community/recipes/default.rb +++ b/cookbooks/community/recipes/default.rb @@ -62,9 +62,8 @@ end git "/srv/community.openstreetmap.org/docker" do action :sync repository "https://github.com/discourse/discourse_docker.git" - # Revision pin not possible as launch wrapper automatically updates git repo. - revision "main" - depth 1 + # DANGER launch wrapper automatically updates git repo if rebuild method used: https://github.com/discourse/discourse_docker/blob/107ffb40fe8b1ea40e00814468db974a4f3f8e8f/launcher#L799 + revision "bea85a5690baca2acc8ebb8b2f58bf5a49d2e766" user "root" group "root" notifies :run, "notify_group[discourse_container_new_data]" @@ -122,18 +121,24 @@ notify_group "discourse_container_new_web_only" do notifies :run, "execute[discourse_container_data_start]", :immediately # noop if site up notifies :run, "execute[discourse_container_web_only_bootstrap]", :immediately # site up but runs in parallel. Slow notifies :run, "execute[discourse_container_web_only_destroy]", :immediately # site down - notifies :run, "execute[discourse_container_data_rebuild]", :immediately # site down + notifies :run, "execute[discourse_container_data_destroy]", :immediately # site down + notifies :run, "execute[discourse_container_data_bootstrap]", :immediately # site down + notifies :run, "execute[discourse_container_data_start]", :immediately # site down notifies :run, "execute[discourse_container_web_only_start]", :immediately # site restore end notify_group "discourse_container_new_data" do notifies :run, "execute[discourse_container_web_only_destroy]", :immediately # site down - notifies :run, "execute[discourse_container_data_rebuild]", :immediately # site down + notifies :run, "execute[discourse_container_data_destroy]", :immediately # site down + notifies :run, "execute[discourse_container_data_bootstrap]", :immediately # site down + notifies :run, "execute[discourse_container_data_start]", :immediately # site down notifies :run, "execute[discourse_container_web_only_start]", :immediately # site restore end notify_group "discourse_container_new_mail_receiver" do - notifies :run, "execute[discourse_container_mail_receiver_rebuild]", :immediately + notifies :run, "execute[discourse_container_mail_receiver_destroy]", :immediately + notifies :run, "execute[discourse_container_mail_receiver_bootstrap]", :immediately + notifies :run, "execute[discourse_container_mail_receiver_start]", :immediately end # Attempt at a failsafe to ensure all containers are running @@ -144,17 +149,26 @@ notify_group "discourse_container_ensure_all_running" do notifies :run, "execute[discourse_container_mail_receiver_start]", :delayed end -execute "discourse_container_data_start" do +execute "discourse_container_data_bootstrap" do action :nothing - command "./launcher start data" + command "./launcher bootstrap data" + cwd "/srv/community.openstreetmap.org/docker/" + user "root" + group "root" + retries 2 # Postgres upgrades required a second run +end + +execute "discourse_container_data_destroy" do + action :nothing + command "./launcher destroy data" cwd "/srv/community.openstreetmap.org/docker/" user "root" group "root" end -execute "discourse_container_data_rebuild" do +execute "discourse_container_data_start" do action :nothing - command "./launcher rebuild data" + command "./launcher start data" cwd "/srv/community.openstreetmap.org/docker/" user "root" group "root" @@ -184,13 +198,22 @@ execute "discourse_container_web_only_start" do group "root" end -# Rebuild: Stop Destroy Bootstap Start -execute "discourse_container_mail_receiver_rebuild" do +execute "discourse_container_mail_receiver_bootstrap" do + action :nothing + command "./launcher bootstrap mail-receiver" + cwd "/srv/community.openstreetmap.org/docker/" + user "root" + group "root" + not_if { arm? } # Not yet supported on ARM https://github.com/discourse/mail-receiver/pull/28 +end + +execute "discourse_container_mail_receiver_destroy" do action :nothing - command "./launcher rebuild mail-receiver" + command "./launcher destroy mail-receiver" cwd "/srv/community.openstreetmap.org/docker/" user "root" group "root" + not_if { arm? } # Not yet supported on ARM https://github.com/discourse/mail-receiver/pull/28 end execute "discourse_container_mail_receiver_start" do @@ -199,6 +222,7 @@ execute "discourse_container_mail_receiver_start" do cwd "/srv/community.openstreetmap.org/docker/" user "root" group "root" + not_if { arm? } # Not yet supported on ARM https://github.com/discourse/mail-receiver/pull/28 end template "/etc/cron.daily/community-backup" do diff --git a/cookbooks/community/templates/default/backup.cron.erb b/cookbooks/community/templates/default/backup.cron.erb index d23cd47e9..72989a3d0 100644 --- a/cookbooks/community/templates/default/backup.cron.erb +++ b/cookbooks/community/templates/default/backup.cron.erb @@ -2,19 +2,20 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp community.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=community-$D.tar.gz - -mkdir $T/community-$D -ln -s /srv/community.openstreetmap.org/docker/containers $T/community-$D/containers -ln -s /srv/community.openstreetmap.org/shared/web-only $T/community-$D/shared-web-only -ln -s /srv/community.openstreetmap.org/shared/data/redis_data $T/community-$D/shared-data-redis_data -ln -s /srv/community.openstreetmap.org/shared/data/postgres_backup $T/community-$D/shared-data-postgres_backup +B="community-$D.tar.zst" -export RSYNC_RSH="ssh -ax" +mkdir "$T/community-$D" +ln -s /srv/community.openstreetmap.org/docker/containers "$T/community-$D/containers" +ln -s /srv/community.openstreetmap.org/shared/web-only "$T/community-$D/shared-web-only" +ln -s /srv/community.openstreetmap.org/shared/data/redis_data "$T/community-$D/shared-data-redis_data" +ln -s /srv/community.openstreetmap.org/shared/data/postgres_backup "$T/community-$D/shared-data-postgres_backup" -nice tar --create --numeric-owner --dereference --directory=$T --warning=no-file-changed community-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --numeric-owner --sort=name --dereference --directory="$T" --warning=no-file-changed --exclude="community-$D/shared-web-only/log/**/*.gz" "community-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/community/templates/default/data.yml.erb b/cookbooks/community/templates/default/data.yml.erb index c5c59d288..37573874e 100644 --- a/cookbooks/community/templates/default/data.yml.erb +++ b/cookbooks/community/templates/default/data.yml.erb @@ -3,7 +3,7 @@ # templates: - - "templates/postgres.13.template.yml" # NOTE UPDATE THE HOOK REPLACE FOR MAX CONNECTIONS BELOW + - "templates/postgres.15.template.yml" # NOTE UPDATE THE HOOK REPLACE FOR MAX CONNECTIONS BELOW - "templates/redis.template.yml" # any extra arguments for Docker? @@ -51,6 +51,6 @@ hooks: run: # Make sure this matches the postgresql version template above - replace: - filename: "/etc/postgresql/13/main/postgresql.conf" + filename: "/etc/postgresql/15/main/postgresql.conf" from: /#?max_connections *=.*/ to: "max_connections = $db_max_connections" diff --git a/cookbooks/community/templates/default/web_only.yml.erb b/cookbooks/community/templates/default/web_only.yml.erb index 5beef4802..e586d8df0 100644 --- a/cookbooks/community/templates/default/web_only.yml.erb +++ b/cookbooks/community/templates/default/web_only.yml.erb @@ -2,6 +2,7 @@ templates: - "templates/web.template.yml" - "templates/web.ipv6.template.yml" - "templates/web.ssl.template.yml" + - "templates/enable-ruby-yjit.yml" ## which TCP/IP ports should this container expose? ## If you want Discourse to share a port with another webserver like Apache or nginx, @@ -19,7 +20,7 @@ links: # any extra arguments for Docker? # docker_args: -# Latest Version v3.3.3 +# Latest Version v3.5.0 # Discourse only support tests-passed and stable branches params: version: stable @@ -103,33 +104,30 @@ hooks: - exec: cd: $home/plugins cmd: - - sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-oauth2-basic.git - - sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-solved.git - - sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-reactions.git - - sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-prometheus.git - - sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-translator.git - - sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-saved-searches.git - - sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-post-voting.git + - if [ ! -d discourse-oauth2-basic ]; then sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-oauth2-basic.git; fi + - if [ ! -d discourse-solved ]; then sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-solved.git; fi + - if [ ! -d discourse-reactions ]; then sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-reactions.git; fi + - if [ ! -d discourse-prometheus ]; then sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-prometheus.git; fi + - if [ ! -d discourse-translator ]; then sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-translator.git; fi + - if [ ! -d discourse-saved-searches ]; then sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-saved-searches.git; fi + - if [ ! -d discourse-post-voting ]; then sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-post-voting.git; fi - exec: # Needs to be copied in else builtin git cleanup fails cd: $home cmd: - sudo -H -E -u discourse cp /shared/feeds/update-feeds.atom public/update-feeds.atom after_ssl: - - replace: - filename: "/etc/nginx/conf.d/discourse.conf" - from: /listen 80;/ - to: | - listen 80; - listen [::]:80; - rewrite ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 permanent; - - - replace: - filename: "/etc/nginx/conf.d/discourse.conf" - from: /add_header.+/ - to: | - add_header Strict-Transport-Security 'max-age=63072000' always; - ssl_stapling on; + - file: + path: "/etc/nginx/conf.d/outlets/server/25-https-osm-settings.conf" + contents: | resolver <%= @resolvers.join(" ") %>; resolver_timeout 5s; ssl_dhparam /shared/ssl/dhparam.pem; + +run: + - replace: + filename: "/etc/nginx/conf.d/outlets/before-server/20-redirect-http-to-https.conf" + from: /listen 80;/ + to: | + listen 80; + rewrite ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 permanent; diff --git a/cookbooks/db/files/default/monthly-reindex.sql b/cookbooks/db/files/default/monthly-reindex.sql index ab9ae105b..ffa5faadb 100644 --- a/cookbooks/db/files/default/monthly-reindex.sql +++ b/cookbooks/db/files/default/monthly-reindex.sql @@ -28,6 +28,7 @@ REINDEX (VERBOSE) TABLE CONCURRENTLY oauth_applications; REINDEX (VERBOSE) TABLE CONCURRENTLY redactions; REINDEX (VERBOSE) TABLE CONCURRENTLY reports; REINDEX (VERBOSE) TABLE CONCURRENTLY schema_migrations; +REINDEX (VERBOSE) TABLE CONCURRENTLY social_links; REINDEX (VERBOSE) TABLE CONCURRENTLY user_blocks; REINDEX (VERBOSE) TABLE CONCURRENTLY user_mutes; REINDEX (VERBOSE) TABLE CONCURRENTLY user_preferences; diff --git a/cookbooks/db/recipes/base.rb b/cookbooks/db/recipes/base.rb index b25fbdbb2..0cf57cf71 100644 --- a/cookbooks/db/recipes/base.rb +++ b/cookbooks/db/recipes/base.rb @@ -43,7 +43,7 @@ package %w[ git "/opt/osmdbt" do action :sync repository "https://github.com/openstreetmap/osmdbt.git" - revision "v0.5" + revision "v0.9" depth 1 user "root" group "root" diff --git a/cookbooks/db/recipes/master.rb b/cookbooks/db/recipes/master.rb index 3e57941f7..c56c0557d 100644 --- a/cookbooks/db/recipes/master.rb +++ b/cookbooks/db/recipes/master.rb @@ -26,9 +26,8 @@ postgresql_user "tomh" do superuser true end -postgresql_user "matt" do +postgresql_user "grant" do cluster node[:db][:cluster] - superuser true end postgresql_user "openstreetmap" do @@ -182,6 +181,7 @@ PROMETHEUS_PERMISSIONS = { relations reports schema_migrations + social_links user_blocks user_mutes user_preferences @@ -201,7 +201,8 @@ PROMETHEUS_PERMISSIONS = { "planetdump" => PLANETDUMP_PERMISSIONS[table], "planetdiff" => PLANETDIFF_PERMISSIONS[table], "prometheus" => PROMETHEUS_PERMISSIONS[table], - "backup" => [:select] + "backup" => [:select], + "grant" => [:select] end end @@ -232,6 +233,7 @@ end oauth_openid_requests_id_seq redactions_id_seq reports_id_seq + social_links_id_seq user_blocks_id_seq user_mutes_id_seq user_roles_id_seq @@ -244,7 +246,8 @@ end permissions "openstreetmap" => [:all], "rails" => [:usage], "cgimap" => CGIMAP_PERMISSIONS[sequence], - "backup" => [:select] + "backup" => [:select], + "grant" => [:select] end end diff --git a/cookbooks/dev/README.md b/cookbooks/dev/README.md index 0253da49b..3918d375b 100644 --- a/cookbooks/dev/README.md +++ b/cookbooks/dev/README.md @@ -2,4 +2,4 @@ This cookbook configures development servers, such as dev.openstreetmap.org. It installs packages required by the users and configures apache for the various -user and api developement sites. +user and api development sites. diff --git a/cookbooks/dev/attributes/default.rb b/cookbooks/dev/attributes/default.rb index 4f16412c4..61887687b 100644 --- a/cookbooks/dev/attributes/default.rb +++ b/cookbooks/dev/attributes/default.rb @@ -1 +1,2 @@ -default[:dev][:rails] = {} +default[:dev][:rails][:postgresql_cluster] = "" +default[:dev][:rails][:sites] = {} diff --git a/cookbooks/dev/recipes/default.rb b/cookbooks/dev/recipes/default.rb index 073bb3e13..0cba087b9 100644 --- a/cookbooks/dev/recipes/default.rb +++ b/cookbooks/dev/recipes/default.rb @@ -61,6 +61,7 @@ package %w[ gnuplot-nox golang graphviz + htop irssi jq libargon2-dev @@ -95,6 +96,7 @@ package %w[ lzip lzop mailutils + moreutils make nano ncftp @@ -103,7 +105,6 @@ package %w[ osmium-tool osmosis pandoc - pandoc pbzip2 php-apcu php-cgi @@ -145,10 +146,13 @@ package %w[ python3-venv r-base redis + siege + time tmux unrar unzip whois + xxd zip zlib1g-dev ] @@ -201,7 +205,7 @@ template "/srv/dev.openstreetmap.org/index.html" do end ssl_certificate "dev.openstreetmap.org" do - domains "dev.openstreetmap.org" + domains ["dev.openstreetmap.org", "dev.osm.org"] notifies :reload, "service[apache2]" end @@ -223,7 +227,7 @@ file "/etc/apache2/conf.d/phppgadmin" do end ssl_certificate "phppgadmin.dev.openstreetmap.org" do - domains "phppgadmin.dev.openstreetmap.org" + domains ["phppgadmin.dev.openstreetmap.org", "phppgadmin.dev.osm.org"] notifies :reload, "service[apache2]" end @@ -288,9 +292,11 @@ node[:postgresql][:versions].each do |version| package "postgresql-#{version}-postgis-3" end -if node[:postgresql][:clusters][:"15/main"] +rails_cluster = node[:dev][:rails][:postgresql_cluster] + +if node[:postgresql][:clusters][rails_cluster.to_sym] postgresql_user "apis" do - cluster "15/main" + cluster rails_cluster end template "/usr/local/bin/cleanup-rails-assets" do @@ -336,10 +342,10 @@ if node[:postgresql][:clusters][:"15/main"] end Dir.glob("/srv/*.apis.dev.openstreetmap.org").each do |dir| - node.default_unless[:dev][:rails][File.basename(dir).split(".").first] = {} + node.default_unless[:dev][:rails][:sites][File.basename(dir).split(".").first] = {} end - node[:dev][:rails].each do |name, details| + node[:dev][:rails][:sites].each do |name, details| database_name = details[:database] || "apis_#{name}" site_name = "#{name}.apis.dev.openstreetmap.org" site_directory = "/srv/#{name}.apis.dev.openstreetmap.org" @@ -349,16 +355,16 @@ if node[:postgresql][:clusters][:"15/main"] gpx_directory = "#{site_directory}/gpx" if details[:repository] - site_aliases = details[:aliases] || [] + site_aliases = details[:aliases] || ["#{name}.apis.dev.osm.org"] secret_key_base = persistent_token("dev", "rails", name, "secret_key_base") postgresql_database database_name do - cluster "15/main" + cluster rails_cluster owner "apis" end postgresql_extension "#{database_name}_btree_gist" do - cluster "15/main" + cluster rails_cluster database database_name extension "btree_gist" end @@ -405,7 +411,7 @@ if node[:postgresql][:clusters][:"15/main"] group "apis" repository details[:repository] revision details[:revision] - database_port node[:postgresql][:clusters][:"15/main"][:port] + database_port node[:postgresql][:clusters][rails_cluster.to_sym][:port] database_name database_name database_username "apis" email_from "OpenStreetMap " @@ -482,7 +488,7 @@ if node[:postgresql][:clusters][:"15/main"] group "root" mode "640" variables :cgimap_socket => "/run/cgimap-#{name}/socket", - :database_port => node[:postgresql][:clusters][:"15/main"][:port], + :database_port => node[:postgresql][:clusters][rails_cluster.to_sym][:port], :database_name => database_name, :log_directory => log_directory, :options => details[:cgimap_options] @@ -551,7 +557,7 @@ if node[:postgresql][:clusters][:"15/main"] postgresql_database database_name do action :drop - cluster "15/main" + cluster rails_cluster end end end @@ -570,7 +576,7 @@ if node[:postgresql][:clusters][:"15/main"] end ssl_certificate "apis.dev.openstreetmap.org" do - domains "apis.dev.openstreetmap.org" + domains ["apis.dev.openstreetmap.org", "apis.dev.osm.org"] notifies :reload, "service[apache2]" end @@ -599,7 +605,8 @@ ssl_certificate "ooc.openstreetmap.org" do domains ["ooc.openstreetmap.org", "a.ooc.openstreetmap.org", "b.ooc.openstreetmap.org", - "c.ooc.openstreetmap.org"] + "c.ooc.openstreetmap.org", + "ooc.osm.org"] notifies :reload, "service[apache2]" end diff --git a/cookbooks/dev/templates/default/apache.apis.erb b/cookbooks/dev/templates/default/apache.apis.erb index 4d78fdd6a..470a1fa5f 100644 --- a/cookbooks/dev/templates/default/apache.apis.erb +++ b/cookbooks/dev/templates/default/apache.apis.erb @@ -2,6 +2,8 @@ ServerName apis.dev.openstreetmap.org + ServerAlias apis.dev.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -16,6 +18,8 @@ ServerName apis.dev.openstreetmap.org + ServerAlias apis.dev.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/apis.dev.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.dev.erb b/cookbooks/dev/templates/default/apache.dev.erb index 2d3d9473d..2dd13c2b9 100644 --- a/cookbooks/dev/templates/default/apache.dev.erb +++ b/cookbooks/dev/templates/default/apache.dev.erb @@ -2,6 +2,8 @@ ServerName dev.openstreetmap.org + ServerAlias dev.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -21,6 +23,8 @@ ServerName dev.openstreetmap.org + ServerAlias dev.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/dev.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.ooc.erb b/cookbooks/dev/templates/default/apache.ooc.erb index 0a9b0b979..a532300e1 100644 --- a/cookbooks/dev/templates/default/apache.ooc.erb +++ b/cookbooks/dev/templates/default/apache.ooc.erb @@ -5,6 +5,8 @@ ServerAlias a.ooc.openstreetmap.org ServerAlias b.ooc.openstreetmap.org ServerAlias c.ooc.openstreetmap.org + ServerAlias ooc.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -26,6 +28,8 @@ ServerAlias a.ooc.openstreetmap.org ServerAlias b.ooc.openstreetmap.org ServerAlias c.ooc.openstreetmap.org + ServerAlias ooc.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/ooc.openstreetmap.org-access.log combined_extended @@ -37,6 +41,8 @@ ServerName npe.openstreetmap.org + ServerAlias npe.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/npe.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.phppgadmin.erb b/cookbooks/dev/templates/default/apache.phppgadmin.erb index 546a05cbe..67c37df16 100644 --- a/cookbooks/dev/templates/default/apache.phppgadmin.erb +++ b/cookbooks/dev/templates/default/apache.phppgadmin.erb @@ -2,6 +2,8 @@ ServerName phppgadmin.dev.openstreetmap.org + ServerAlias phppgadmin.dev.osm.org + ServerAdmin webmaster@openstreetmap.org SSLEngine on @@ -23,6 +25,8 @@ ServerName phppgadmin.dev.openstreetmap.org + ServerAlias phppgadmin.dev.osm.org + ServerAdmin webmaster@openstreetmap.org CustomLog /var/log/apache2/phppgadmin.dev.openstreetmap.org-access.log combined_extended diff --git a/cookbooks/dev/templates/default/apache.rails.erb b/cookbooks/dev/templates/default/apache.rails.erb index 957866c5c..082931dde 100644 --- a/cookbooks/dev/templates/default/apache.rails.erb +++ b/cookbooks/dev/templates/default/apache.rails.erb @@ -48,6 +48,7 @@ RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/relations(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/node/[0-9]+/ways(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/(way|relation)/[0-9]+/full(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] + RewriteCond %{REQUEST_METHOD} ^(HEAD|GET)$ RewriteRule ^/api/0\.6/(nodes|ways|relations)(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] RewriteRule ^/api/0\.6/changeset/[0-9]+/(upload|download)(\.json|\.xml)?$ unix:<%= @cgimap_socket %>|fcgi://127.0.0.1$0 [P] <% end -%> diff --git a/cookbooks/dev/templates/default/apache.user.erb b/cookbooks/dev/templates/default/apache.user.erb index 373d12258..d93aa9490 100644 --- a/cookbooks/dev/templates/default/apache.user.erb +++ b/cookbooks/dev/templates/default/apache.user.erb @@ -4,9 +4,10 @@ WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> processes ServerName <%= @user %>.dev.openstreetmap.org - ServerAdmin webmaster@openstreetmap.org ServerAlias <%= @user %>.dev.osm.org + ServerAdmin webmaster@openstreetmap.org + SSLEngine on SSLCertificateFile /etc/ssl/certs/<%= @user %>.dev.openstreetmap.org.pem SSLCertificateKeyFile /etc/ssl/private/<%= @user %>.dev.openstreetmap.org.key @@ -26,6 +27,12 @@ WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> processes CustomLog /var/log/apache2/<%= @user %>.dev.openstreetmap.org-access.log combined_extended ErrorLog /var/log/apache2/<%= @user %>.dev.openstreetmap.org-error.log + # Prevent abuse by an anonymous AI bot + RewriteCond %{REQUEST_METHOD} ^(GET|HEAD)$ + RewriteCond %{HTTP_REFERER} ^-?$ + RewriteCond %{HTTP_USER_AGENT} ((CriOS|Chrome)/[1-9][0-9]?\.0\.|Chrome/100\.0\.|Chrome/122\.0\.0\.0|(Firefox|FxiOS)/[1-6]?[0-9]\.|MSIE\ [5-9]\.0|Opera/[8-9]\.|Windows\ NT\ [3-5]\.|Version/[3-5]\.[0-1]) [NC] + RewriteRule ^ - [R=429,L] + RewriteCond <%= @directory %>%{REQUEST_FILENAME} -f RewriteRule ^/cgi-bin/(.*)$ /~<%= @user %>/cgi-bin/$1 [PT,L] @@ -36,9 +43,10 @@ WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> processes ServerName <%= @user %>.dev.openstreetmap.org - ServerAdmin webmaster@openstreetmap.org ServerAlias <%= @user %>.dev.osm.org + ServerAdmin webmaster@openstreetmap.org + CustomLog /var/log/apache2/<%= @user %>.dev.openstreetmap.org-access.log combined_extended ErrorLog /var/log/apache2/<%= @user %>.dev.openstreetmap.org-error.log @@ -47,7 +55,7 @@ WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> processes > - AllowOverride AuthConfig FileInfo Indexes Options=RailsBaseURI + AllowOverride AuthConfig FileInfo Indexes Options SymLinksIfOwnerMatch Indexes Includes Require all granted diff --git a/cookbooks/dev/templates/default/apis.html.erb b/cookbooks/dev/templates/default/apis.html.erb index d630014cb..22dfbbec8 100644 --- a/cookbooks/dev/templates/default/apis.html.erb +++ b/cookbooks/dev/templates/default/apis.html.erb @@ -12,7 +12,7 @@ for testing clients against or as a data sandbox.

Repository Revision -<% node[:dev][:rails].each do |name,details| -%> +<% node[:dev][:rails][:sites].each do |name,details| -%> <%= name %> <%= details[:repository] %> diff --git a/cookbooks/devices/metadata.rb b/cookbooks/devices/metadata.rb index 2f71f6857..a25b0e16f 100644 --- a/cookbooks/devices/metadata.rb +++ b/cookbooks/devices/metadata.rb @@ -6,3 +6,4 @@ description "Configures devices" version "0.1" supports "ubuntu" +depends "chef" diff --git a/cookbooks/devices/templates/default/udev.rules.erb b/cookbooks/devices/templates/default/udev.rules.erb index be9903891..f3b28f7f7 100644 --- a/cookbooks/devices/templates/default/udev.rules.erb +++ b/cookbooks/devices/templates/default/udev.rules.erb @@ -32,6 +32,10 @@ ACTION=="add", SUBSYSTEM=="block", ENV{ID_BUS}=="<%= device[:bus] %>", ENV{ID_SE <% end -%> <% end -%> +# Tune read ahead for ancient laptop disks in shenron +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", KERNEL=="sd?", ENV{ID_MODEL}=="HGST_HTE721010A9E630", ATTR{queue/read_ahead_kb}="512" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", KERNEL=="sd?", ENV{ID_MODEL}=="HGST_HTS725050A7E630", ATTR{queue/read_ahead_kb}="512" + # Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express Gigabit Ethernet Controller SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x10ec", ATTRS{device}=="0x8168", RUN+="/sbin/ethtool -K $name tso off gso off" @@ -95,47 +99,6 @@ SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x37d2 # Disable Firmware Based LLDP handler SUBSYSTEM=="net", ACTION=="add", ENV{INTERFACE}=="*", DRIVERS=="i40e", RUN+="/sbin/ethtool --set-priv-flags $name disable-fw-lldp on" -# Workaround unreliable Western Digital WD RE3/RE4 disks (ATA only) -# Set sufficent Linux subsystem timeout and fix severe NCQ performance issue -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD5002ABYS-02B1B0", ATTR{device/timeout}="90", ATTR{device/queue_depth}="1", ATTR{queue/nr_requests}="256" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", ATTR{device/timeout}="90", ATTR{device/queue_depth}="1", ATTR{queue/nr_requests}="256" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", ATTR{device/timeout}="90", ATTR{device/queue_depth}="1", ATTR{queue/nr_requests}="256" -# Disable Disk Write Cache, Set AAM and Power Management correctly -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", RUN+="/sbin/hdparm -q -W0 -q -M254 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", RUN+="/sbin/hdparm -q -W0 -q -M254 -q -B254 $env{DEVNAME}" - -# Set Disks TLED / SCT Error Recovery Control -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD5000AAKS-00A7B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD2000FYYZ-01UL1B2", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="TOSHIBA_DT01ACA300", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST31000340NS", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="HGST_HTS725050A7E630", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="HGST_HTE721010A9E630", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" - -# Add SSD optimisation -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="OCZ-VERTEX3", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="OCZ-VERTEX3", ATTR{queue/scheduler}="noop" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/scheduler}="noop" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/read_ahead_kb}="256" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/scheduler}="noop" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/read_ahead_kb}="256" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/scheduler}="noop" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/read_ahead_kb}="256" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST240FN0021", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST240FN0021", ATTR{queue/scheduler}="noop" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="SuperMicro_SSD", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="SuperMicro_SSD", ATTR{queue/scheduler}="noop" - # Delete failed disk in cmok ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST_M13FQBL", ENV{ID_SERIAL}=="ST_M13FQBL_QNR_BFW", ATTR{device/delete}="1" @@ -155,6 +118,8 @@ ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_MODEL}=="QEMU_HA ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_MODEL}=="QEMU_HARDDISK", ATTR{queue/scheduler}="noop" # Vendor is sometimes missing -# Increase default MD raid5/raid6 strip cache + group_thread_cnt -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{MD_LEVEL}=="raid5", ATTR{md/stripe_cache_size}="8192", ATTR{md/group_thread_cnt}="4" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{MD_LEVEL}=="raid6", ATTR{md/stripe_cache_size}="8192", ATTR{md/group_thread_cnt}="4" +# Tune md stripe cache and thread count for RAID-5 / RAID-6 arrays +<% + group_threads = [(node.cpu_cores.to_i / 2.0).round, 4].max +%> +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{MD_LEVEL}=="raid[56]", ATTR{md/stripe_cache_size}="8192", ATTR{md/group_thread_cnt}="<%= group_threads %>" diff --git a/cookbooks/dhcpd/README.md b/cookbooks/dhcpd/README.md index 3d2a00c7c..f95dbb0d0 100644 --- a/cookbooks/dhcpd/README.md +++ b/cookbooks/dhcpd/README.md @@ -1,3 +1,3 @@ # dhcpd Cookbook -Configures the dhcpd service, which used for the internal network at UCL. +Configures the dhcpd service, which is used for our internal networks. diff --git a/cookbooks/dhcpd/recipes/default.rb b/cookbooks/dhcpd/recipes/default.rb index 0e6b9ec3f..685d08c3b 100644 --- a/cookbooks/dhcpd/recipes/default.rb +++ b/cookbooks/dhcpd/recipes/default.rb @@ -53,7 +53,7 @@ remote_file "/srv/tftp/netboot.xyz.kpxe" do mode "644" end -domain = "#{node[:networking][:roles][:external][:zone]}.openstreetmap.org" +domain = node[:networking][:search].first template "/etc/dhcp/dhcpd.conf" do source "dhcpd.conf.erb" diff --git a/cookbooks/dhcpd/templates/default/dhcpd.conf.erb b/cookbooks/dhcpd/templates/default/dhcpd.conf.erb index 1581475e5..ed4fe2d03 100644 --- a/cookbooks/dhcpd/templates/default/dhcpd.conf.erb +++ b/cookbooks/dhcpd/templates/default/dhcpd.conf.erb @@ -55,6 +55,12 @@ host pdu2.ams.openstreetmap.org { fixed-address 10.0.48.101; } +host ats1.ams.openstreetmap.org { + hardware ethernet 00:c0:b7:e5:5e:f1; + server-name "ats1.ams.openstreetmap.org"; + fixed-address 10.0.48.103; +} + host oob1.dub.openstreetmap.org { hardware ethernet 62:bd:62:a6:05:25; server-name "oob1.dub.openstreetmap.org"; diff --git a/cookbooks/dns/recipes/default.rb b/cookbooks/dns/recipes/default.rb index d1b3d159e..26c004c05 100644 --- a/cookbooks/dns/recipes/default.rb +++ b/cookbooks/dns/recipes/default.rb @@ -42,7 +42,7 @@ package %w[ cache_dir = Chef::Config[:file_cache_path] -dnscontrol_version = "4.15.1" +dnscontrol_version = "4.23.0" dnscontrol_arch = if arm? "arm64" @@ -60,7 +60,7 @@ end dpkg_package "dnscontrol" do source "#{cache_dir}/dnscontrol-#{dnscontrol_version}.deb" - version "#{dnscontrol_version}" + version dnscontrol_version end directory "/srv/dns.openstreetmap.org" do @@ -79,6 +79,18 @@ remote_directory "/srv/dns.openstreetmap.org/html" do files_mode "644" end +link "/srv/dns.openstreetmap.org/html/ipv4.json" do + to "/var/lib/dns/src/ipv4.json" + owner "root" + group "root" +end + +link "/srv/dns.openstreetmap.org/html/ipv6.json" do + to "/var/lib/dns/src/ipv6.json" + owner "root" + group "root" +end + zones = [] Dir.glob("/var/lib/dns/json/*.json").each do |kmlfile| diff --git a/cookbooks/elasticsearch/recipes/default.rb b/cookbooks/elasticsearch/recipes/default.rb index a48ee9987..63ed3fd45 100644 --- a/cookbooks/elasticsearch/recipes/default.rb +++ b/cookbooks/elasticsearch/recipes/default.rb @@ -36,6 +36,14 @@ template "/etc/elasticsearch/elasticsearch.yml" do notifies :restart, "service[elasticsearch]" end +systemd_service "elasticsearch-override" do + service "elasticsearch" + dropin "override" + timeout_start_sec 180 + timeout_stop_sec 180 + notifies :restart, "service[elasticsearch]" +end + service "elasticsearch" do action [:enable, :start] supports :status => true, :restart => true diff --git a/cookbooks/exim/attributes/default.rb b/cookbooks/exim/attributes/default.rb index 638ce5765..77c0907a0 100644 --- a/cookbooks/exim/attributes/default.rb +++ b/cookbooks/exim/attributes/default.rb @@ -8,5 +8,5 @@ default[:exim][:smtp_accept_max] = 20 default[:exim][:smarthost_name] = nil default[:exim][:smarthost_via] = "mail.openstreetmap.org:26" default[:exim][:routes] = {} -default[:exim][:aliases][:root] = "tomh" +default[:exim][:aliases][:root] = "tomh, grant" default[:exim][:rewrites] = [] diff --git a/cookbooks/exim/recipes/default.rb b/cookbooks/exim/recipes/default.rb index 7354e93d3..21a076d56 100644 --- a/cookbooks/exim/recipes/default.rb +++ b/cookbooks/exim/recipes/default.rb @@ -47,6 +47,10 @@ end if node[:exim][:certificate_names] include_recipe "apache" + apache_site "default" do + action [:disable] + end + apache_site node[:exim][:certificate_names].first do template "apache.erb" variables :aliases => node[:exim][:certificate_names].drop(1) @@ -161,7 +165,7 @@ if node[:exim][:dkim_selectors] mode "755" end - node[:exim][:dkim_selectors].each do |domain, _selector| + node[:exim][:dkim_selectors].each_key do |domain| file "/etc/exim4/dkim-keys/#{domain}" do content keys[domain].join("\n") owner "root" diff --git a/cookbooks/exim/templates/default/exim4.conf.erb b/cookbooks/exim/templates/default/exim4.conf.erb index ffc8be609..71996f148 100644 --- a/cookbooks/exim/templates/default/exim4.conf.erb +++ b/cookbooks/exim/templates/default/exim4.conf.erb @@ -638,7 +638,7 @@ mailman: local_part_suffix = -bounces : -bounces+* : \ -confirm+* : -join : -leave : \ -subscribe : -unsubscribe : \ - -owner : -request : -admin + -owner : -request : -admin local_part_suffix_optional transport = mailman @@ -737,6 +737,9 @@ begin transports remote_smtp: driver = smtp multi_domain = false +<% if node.platform?("debian") || (node.platform?("ubuntu") && node[:lsb][:release].to_f >= 22.04) -%> + message_linelength_limit = 1G +<% end -%> tls_require_ciphers = <%= node[:ssl][:gnutls_ciphers] %>:%LATEST_RECORD_VERSION @@ -750,9 +753,16 @@ signed_smtp: dkim_private_key = /etc/exim4/dkim-keys/${dkim_domain} dkim_identity = ${lc:${address:$h_from:}} dkim_timestamps = 1209600 +<% if node.platform?("debian") || (node.platform?("ubuntu") && node[:lsb][:release].to_f >= 22.04) -%> + message_linelength_limit = 1G +<% end -%> multi_domain = false + max_rcpt = 20 hosts_try_dane = tls_require_ciphers = <%= node[:ssl][:gnutls_ciphers] %>:%LATEST_RECORD_VERSION +<% if node[:exim][:external_interface] -%> + interface = <%= node[:exim][:external_interface] %> +<% end -%> # This transport is used for handling pipe deliveries generated by alias or diff --git a/cookbooks/foundation/recipes/wiki.rb b/cookbooks/foundation/recipes/wiki.rb index ddbe4705d..ead5694cd 100644 --- a/cookbooks/foundation/recipes/wiki.rb +++ b/cookbooks/foundation/recipes/wiki.rb @@ -40,7 +40,7 @@ mediawiki_site "osmfoundation.org" do email_sender "wiki@noreply.openstreetmap.org" email_sender_name "OSMF Wiki" private_accounts true - extra_file_extensions %w[mp3 pptx] + extra_file_extensions %w[mp3 pptx txt] version "1.39" end diff --git a/cookbooks/git/templates/default/apache.erb b/cookbooks/git/templates/default/apache.erb index daff9de88..966374067 100644 --- a/cookbooks/git/templates/default/apache.erb +++ b/cookbooks/git/templates/default/apache.erb @@ -44,6 +44,14 @@ CustomLog /var/log/apache2/<%= @name %>-access.log combined_extended ErrorLog /var/log/apache2/<%= @name %>-error.log + Alias /robots.txt /srv/<%= node[:git][:host] %>/robots.txt + + + # Make absolutely sure it comes out as a plain file + SetHandler none + Require all granted + + SetEnv GIT_PROJECT_ROOT /var/lib/git SetEnv GIT_HTTP_EXPORT_ALL SetEnv GIT_HTTP_MAX_REQUEST_BUFFER 100M @@ -59,6 +67,20 @@ RewriteRule ^/gpx-import\.git.* https://github.com/openstreetmap/gpx-import [QSD,L,R=permanent] RewriteRule ^/potlatch2\.git.* https://github.com/openstreetmap/potlatch2 [QSD,L,R=permanent] + # Prevent abuse by an anonymous AI bot + RewriteCond %{REQUEST_METHOD} ^(GET|HEAD)$ + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/blob [OR] + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/commitdiff [OR] + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/history [OR] + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/log [OR] + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/patch [OR] + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/search [OR] + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/shortlog [OR] + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/tree + RewriteCond %{HTTP_REFERER} ^-?$ + RewriteCond %{HTTP_USER_AGENT} ((CriOS|Chrome)/[1-9][0-9]?\.0\.|Chrome/100\.0\.|Chrome/122\.0\.0\.0|(Firefox|FxiOS)/[1-6]?[0-9]\.|MSIE\ [5-9]\.0|Opera/[8-9]\.|Windows\ NT\ [3-5]\.|Version/[3-5]\.[0-1]) [NC] + RewriteRule ^ - [R=429,L] + ScriptAlias /public /usr/lib/git-core/git-http-backend/public ScriptAlias /private /usr/lib/git-core/git-http-backend/private Alias /gitweb /usr/share/gitweb diff --git a/cookbooks/git/templates/default/backup.cron.erb b/cookbooks/git/templates/default/backup.cron.erb index b3404c7d5..fcfe6829f 100644 --- a/cookbooks/git/templates/default/backup.cron.erb +++ b/cookbooks/git/templates/default/backup.cron.erb @@ -2,15 +2,16 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp git.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=git-$D.tar.gz - -ln -s /var/lib/git $T/git-$D +B="git-$D.tar.zst" -export RSYNC_RSH="ssh -ax" +ln -s /var/lib/git "$T/git-$D" -nice tar --create --dereference --directory=$T git-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" "git-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/git/templates/default/robots.txt.erb b/cookbooks/git/templates/default/robots.txt.erb index b60d5e279..61efcd142 100644 --- a/cookbooks/git/templates/default/robots.txt.erb +++ b/cookbooks/git/templates/default/robots.txt.erb @@ -1,4 +1,14 @@ # DO NOT EDIT - This file is being maintained by Chef User-agent: * -Disallow: /*/snapshot/ +Disallow: /*a=search* +Disallow: /*/search/* +Disallow: /*a=blobdiff* +Disallow: /*/blobdiff/* +Disallow: /*/blobdiff_plain/* +Disallow: /*a=commitdiff* +Disallow: /*/commitdiff/* +Disallow: /*a=snapshot* +Disallow: /*/snapshot/* +Disallow: /*a=blame* +Disallow: /*/blame/* diff --git a/cookbooks/gps-tile/templates/default/apache.erb b/cookbooks/gps-tile/templates/default/apache.erb index 9adf707dd..65ef9f2b4 100644 --- a/cookbooks/gps-tile/templates/default/apache.erb +++ b/cookbooks/gps-tile/templates/default/apache.erb @@ -25,7 +25,6 @@ # Setup logging CustomLog /var/log/apache2/access.log combined_extended ErrorLog /var/log/apache2/error.log - BufferedLogs on # Always set Access-Control-Allow-Origin so that simple CORS requests # will always work and can be cached @@ -47,12 +46,12 @@ RewriteRule ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 [R=permanent,L] # Redirect to https + RewriteCond %{REQUEST_URI} !^/server-status RewriteRule (.*) https://%{SERVER_NAME}/$1 [R=permanent,L] # Setup logging CustomLog /var/log/apache2/access.log combined_extended ErrorLog /var/log/apache2/error.log - BufferedLogs on diff --git a/cookbooks/hardware/attributes/default.rb b/cookbooks/hardware/attributes/default.rb index 218e6228e..303b1efd0 100644 --- a/cookbooks/hardware/attributes/default.rb +++ b/cookbooks/hardware/attributes/default.rb @@ -1,4 +1,4 @@ -default[:hardware][:modules] = %w[lp] +default[:hardware][:modules] = %w[] default[:hardware][:blacklisted_modules] = %w[] default[:hardware][:grub][:cmdline] = %w[nomodeset] default[:hardware][:sensors] = {} diff --git a/cookbooks/hardware/recipes/default.rb b/cookbooks/hardware/recipes/default.rb index 0dafc1f91..04ead0b00 100644 --- a/cookbooks/hardware/recipes/default.rb +++ b/cookbooks/hardware/recipes/default.rb @@ -410,8 +410,8 @@ intel_nvmes = nvmes.select { |pci| pci[:vendor_name] == "Intel Corporation" } if !intel_ssds.empty? || !intel_nvmes.empty? package "unzip" - sst_tool_version = "2-0" - sst_package_version = "2.0.300-0" + sst_tool_version = "2-4" + sst_package_version = "2.4.323-0" remote_file "#{Chef::Config[:file_cache_path]}/sst-cli-linux-deb--#{sst_tool_version}.zip" do source "https://sdmsdfwdriver.blob.core.windows.net/files/kba-gcc/drivers-downloads/ka-00085/sst--#{sst_tool_version}/sst-cli-linux-deb--#{sst_tool_version}.zip" @@ -426,7 +426,7 @@ if !intel_ssds.empty? || !intel_nvmes.empty? end dpkg_package "sst" do - version "#{sst_package_version}" + version sst_package_version source "#{Chef::Config[:file_cache_path]}/sst_#{sst_package_version}_amd64.deb" end @@ -464,7 +464,7 @@ end disks = disks.compact.uniq -if disks.count.positive? +if disks.any? package "smartmontools" template "/etc/cron.daily/update-smart-drivedb" do diff --git a/cookbooks/imagery/recipes/au_act_aerial.rb b/cookbooks/imagery/recipes/au_act_aerial.rb new file mode 100644 index 000000000..f50fb55da --- /dev/null +++ b/cookbooks/imagery/recipes/au_act_aerial.rb @@ -0,0 +1,127 @@ +# +# Cookbook:: imagery +# Recipe:: au_act_aerial +# +# Copyright:: 2025, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery" + +imagery_site "act-imagery.openstreetmap.org" do + title "OpenStreetMap - ACT Imagery" + aliases ["act-imagery.osm.org"] + # https://leafletjs.com/reference.html#latlngbounds format + # [[south, west], [north, east]] + bbox [[-35.942, 148.729], [-35.117, 149.430]] +end + +imagery_layer "act_aerial_imagery_latest" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery latest" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_Current/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_Current/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 + default_layer true +end + +imagery_layer "act_aerial_imagery_202505" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202505" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2025_05_urban_75mm/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202411/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +imagery_layer "act_aerial_imagery_202503" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202503" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2025_03_urban_75mm/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202411/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +imagery_layer "act_aerial_imagery_202411" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202411" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2024_11_full_75mm/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202411/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +imagery_layer "act_aerial_imagery_202409" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202409" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2024_09_urban_75mm/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2024_09_urban_75mm/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +imagery_layer "act_aerial_imagery_202402" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202402" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2024_02_urban_75mm/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2024_09_urban_75mm/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +imagery_layer "act_aerial_imagery_202311" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202311" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2023_11_full_75mm/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202311/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +# 2025 August - No longer available - appears password protected +imagery_layer "act_aerial_imagery_202305" do + action :delete + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202305" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202305/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202305/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end diff --git a/cookbooks/imagery/recipes/au_agri.rb b/cookbooks/imagery/recipes/au_agri.rb index 931ecf3f1..fb011c111 100644 --- a/cookbooks/imagery/recipes/au_agri.rb +++ b/cookbooks/imagery/recipes/au_agri.rb @@ -33,7 +33,7 @@ imagery_layer "au_ga_agri" do copyright "Commonwealth of Australia (Geoscience Australia) - Creative Commons Attribution 4.0 International Licence" background_colour "0 0 0" # Black projection "EPSG:3857" - source "/store/imagery/au/agri/combine.vrt" + source "/store/imagery/au/agri/combine-cutline-cog.tif" max_zoom 17 - revision 1 + revision 3 end diff --git a/cookbooks/imagery/recipes/au_vic_melbourne_aerial.rb b/cookbooks/imagery/recipes/au_vic_melbourne_aerial.rb new file mode 100644 index 000000000..cda5e80d4 --- /dev/null +++ b/cookbooks/imagery/recipes/au_vic_melbourne_aerial.rb @@ -0,0 +1,59 @@ +# +# Cookbook:: imagery +# Recipe:: au_vic_melbourne_aerial +# +# Copyright:: 2024, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery" + +imagery_site "au-vic-melbourne-imagery.openstreetmap.org" do + title "OpenStreetMap - City of Melbourne - Aerial Imagery" + aliases ["au-vic-melbourne-imagery.osm.org"] + # https://leafletjs.com/reference.html#latlngbounds format + # [[south, west], [north, east]] + bbox [[-37.850667, 144.896981], [-37.775451, 144.991351]] +end + +imagery_layer "melbourne-2020" do + site "au-vic-melbourne-imagery.openstreetmap.org" + title "City of Melbourne 2020" + source "/store/imagery/au/city-of-melbourne/CoM_May2020_2cm.cog.tiff" + copyright "(c) 2020 City of Melbourne" + max_zoom 23 + extension "jpeg" + revision 2 + default_layer true +end + +imagery_layer "melbourne-2019" do + site "au-vic-melbourne-imagery.openstreetmap.org" + title "City of Melbourne 2019" + source "/store/imagery/au/city-of-melbourne/CoM_03Feb2019.cog.tiff" + copyright "(c) 2019 City of Melbourne" + max_zoom 21 + extension "jpeg" + revision 2 +end + +imagery_layer "melbourne-2018" do + site "au-vic-melbourne-imagery.openstreetmap.org" + title "City of Melbourne 2018" + source "/store/imagery/au/city-of-melbourne/CoM_May2018_10cm.COG.tiff" + copyright "(c) 2018 City of Melbourne" + max_zoom 21 + extension "jpeg" + revision 2 +end diff --git a/cookbooks/imagery/recipes/bg_imagery.rb b/cookbooks/imagery/recipes/bg_imagery.rb new file mode 100644 index 000000000..5602f83e9 --- /dev/null +++ b/cookbooks/imagery/recipes/bg_imagery.rb @@ -0,0 +1,37 @@ +# +# Cookbook:: imagery +# Recipe:: bg_imagery +# +# Copyright:: 2025, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery" + +imagery_site "bg-imagery.openstreetmap.org" do + title "OpenStreetMap - Bulgaria - Aerial Imagery" + aliases ["bg-imagery.osm.org"] + bbox [[41.235, 22.357], [44.215, 28.608]] +end + +imagery_layer "maf-orthophoto-latest" do + site "bg-imagery.openstreetmap.org" + title "Bulgaria MAF Orthophoto Latest" + source "/store/imagery/bg/maf-orthophoto-map/maf-orthophoto.vrt" + copyright "(c) Ministry of Agriculture and Food of Bulgaria" + projection "EPSG:32635" + max_zoom 20 + default_layer true + revision 2 +end diff --git a/cookbooks/imagery/recipes/br_imagery.rb b/cookbooks/imagery/recipes/br_imagery.rb new file mode 100644 index 000000000..d276593a1 --- /dev/null +++ b/cookbooks/imagery/recipes/br_imagery.rb @@ -0,0 +1,37 @@ +# +# Cookbook:: imagery +# Recipe:: br_imagery +# +# Copyright:: 2025, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery" + +imagery_site "br-imagery.openstreetmap.org" do + title "OpenStreetMap - Brazil - Aerial Imagery" + aliases ["br-imagery.osm.org"] + bbox [[-23.9813, -46.6594], [-23.6398, -46.4042]] +end + +imagery_layer "ibge-aerial-2021" do + site "br-imagery.openstreetmap.org" + title "Brazil IBGE Aerial Imagery 2021" + source "/store/imagery/br/ibge-aerial-2021/ibge-aerial-2021.webp.google.r_bilinear.bs_256.aligned.cog.tif" + copyright '(c) IBGE' + projection "EPSG:3857" + max_zoom 21 + default_layer true + revision 1 +end diff --git a/cookbooks/imagery/recipes/default.rb b/cookbooks/imagery/recipes/default.rb index 7c1a1f88c..edc860965 100644 --- a/cookbooks/imagery/recipes/default.rb +++ b/cookbooks/imagery/recipes/default.rb @@ -33,6 +33,7 @@ package %w[ cgi-mapserver mapcache-cgi mapcache-tools + libtcmalloc-minimal4 ] # Mapserver via nginx requires as fastcgi spawner @@ -44,10 +45,11 @@ package %w[ # Imagery processing Requirements package "imagemagick" -# Imagery misc compression +# Imagery misc utilities package %w[ xz-utils unzip + aria2 ] template "/etc/mapserver.conf" do @@ -96,3 +98,7 @@ systemd_tmpfile "/run/mapserver-fastcgi" do mode "0755" not_if { kitchen? } end + +service "systemd-coredump.socket" do + action [ :stop, :disable ] +end diff --git a/cookbooks/imagery/recipes/gb_os_sv.rb b/cookbooks/imagery/recipes/gb_os_sv.rb index 2345cc1e8..a883076c4 100644 --- a/cookbooks/imagery/recipes/gb_os_sv.rb +++ b/cookbooks/imagery/recipes/gb_os_sv.rb @@ -1215,6 +1215,17 @@ imagery_layer "gb_os_om_local_2024_10" do copyright "Contains OS data © Crown copyright and database right 2024" background_colour "213 244 248" # OS OpenMap Local Water Blue extension "os_om_local_png" - url_aliases ["/om-local-2024-10", "/om-local", "/sv"] +end + +imagery_layer "gb_os_om_local_2025_04" do + site "os.openstreetmap.org" + title "OS OpenMap Local - April 2025" + projection "EPSG:27700" + source "/store/imagery/gb/openmap-local/2025-04/os-openmap-local-2025-04.vrt" + copyright "Contains OS data © Crown copyright and database right 2025" + background_colour "213 244 248" # OS OpenMap Local Water Blue + extension "os_om_local_png" + url_aliases ["/om-local-2025-04", "/om-local", "/sv"] default_layer true + revision 2 end diff --git a/cookbooks/imagery/recipes/na_sgswa_topo.rb b/cookbooks/imagery/recipes/na_sgswa_topo.rb index 686e10dda..f4641af2a 100644 --- a/cookbooks/imagery/recipes/na_sgswa_topo.rb +++ b/cookbooks/imagery/recipes/na_sgswa_topo.rb @@ -29,12 +29,21 @@ imagery_layer "na_sgswa_topo_50k" do site "namibia-topo.openstreetmap.org.za" title "Namibia Topo 50k" projection "EPSG:4326" - source "/store/imagery/na/topo-50k/namibia-50k-topo-v2.vrt" - copyright "State Copyright © 1958 - 1991; Surveyor-General, Windhoek, SWA; CDSM: Chief Directorate Surveys & Mapping, Mowbray, RSA" + source "/store/imagery/na/topo-50k/namibia-50k-topo-v4-alpha-hidenodata.vrt" + copyright "State Copyright © 1958 - 1991; Surveyor-General, Windhoek, Namibia" default_layer true - background_colour "0 0 0" - extension "jpeg" max_zoom 16 + revision 4 +end + +imagery_layer "na_sgswa_topo_250k" do + site "namibia-topo.openstreetmap.org.za" + title "Namibia Topo 250k" + projection "EPSG:4326" + source "/store/imagery/na/topo-250k/new/combined.vrt" + copyright "State Copyright © 1972 - 1989; Surveyor-General, Windhoek, Namibia" + max_zoom 16 + revision 3 end imagery_layer "na_aerial" do diff --git a/cookbooks/imagery/recipes/tiler.rb b/cookbooks/imagery/recipes/tiler.rb index ec7daa835..2f6942b66 100644 --- a/cookbooks/imagery/recipes/tiler.rb +++ b/cookbooks/imagery/recipes/tiler.rb @@ -37,21 +37,20 @@ container_image = if arm? podman_service "titiler" do description "Container service for titiler" image container_image - volume :"/store/imagery" => "/store/imagery", + volume :"/store/imagery" => "/store/imagery", :"/srv/imagery/sockets" => "/sockets" - environment :BIND => "unix:/sockets/titiler.sock", - :WORKERS_PER_CORE => 1, - :GDAL_CACHEMAX => 200, - :GDAL_BAND_BLOCK_CACHE => "HASHSET", - :GDAL_DISABLE_READDIR_ON_OPEN => "EMPTY_DIR", - :GDAL_INGESTED_BYTES_AT_OPEN => 32768, - :GDAL_HTTP_MERGE_CONSECUTIVE_RANGES => "YES", - :GDAL_HTTP_MULTIPLEX => "YES", - :GDAL_HTTP_VERSION => 2, - :VSI_CACHE => "TRUE", - :VSI_CACHE_SIZE => 5000000, - :TITILER_API_ROOT_PATH => "/api/v1/titiler", - :FORWARDED_ALLOW_IPS => "*" # https://docs.gunicorn.org/en/latest/settings.html#forwarded-allow-ips + environment :GDAL_CACHEMAX => 200, + :GDAL_BAND_BLOCK_CACHE => "HASHSET", + :GDAL_DISABLE_READDIR_ON_OPEN => "EMPTY_DIR", + :GDAL_INGESTED_BYTES_AT_OPEN => 32768, + :GDAL_HTTP_MERGE_CONSECUTIVE_RANGES => "YES", + :GDAL_HTTP_MULTIPLEX => "YES", + :GDAL_HTTP_VERSION => 2, + :VSI_CACHE => "TRUE", + :VSI_CACHE_SIZE => 5000000, + :TITILER_API_ROOT_PATH => "/api/v1/titiler", + :FORWARDED_ALLOW_IPS => "*" # https://docs.gunicorn.org/en/latest/settings.html#forwarded-allow-ips + command "gunicorn -k uvicorn.workers.UvicornWorker titiler.application.main:app --bind unix:/sockets/titiler.sock --workers #{node.cpu_cores}" end systemd_service "titiler-restart" do @@ -64,7 +63,7 @@ end systemd_timer "titiler-restart" do on_boot_sec "10m" - on_unit_inactive_sec "30m" + on_unit_inactive_sec "2h" randomized_delay_sec "20m" end diff --git a/cookbooks/imagery/recipes/za_ngi_topo.rb b/cookbooks/imagery/recipes/za_ngi_topo.rb index a1fd92c61..ac5a28b37 100644 --- a/cookbooks/imagery/recipes/za_ngi_topo.rb +++ b/cookbooks/imagery/recipes/za_ngi_topo.rb @@ -32,6 +32,7 @@ imagery_layer "za_ngi_topo_250k" do source "/store/imagery/za/ngi-topo-250k/ngi-topo-250k-combined.vrt" copyright 'State Copyright © 1996–2010 Chief Directorate: National Geo-spatial Information' default_layer true + revision 2 end imagery_layer "za_ngi_topo_50k" do @@ -40,4 +41,5 @@ imagery_layer "za_ngi_topo_50k" do projection "EPSG:3857" source "/store/imagery/za/ngi-topo-50k/ngi-topo-50k-combined.vrt" copyright 'State Copyright © 1996–2013 Chief Directorate: National Geo-spatial Information' + revision 2 end diff --git a/cookbooks/imagery/resources/site.rb b/cookbooks/imagery/resources/site.rb index 5ce7d2b88..d7c549587 100644 --- a/cookbooks/imagery/resources/site.rb +++ b/cookbooks/imagery/resources/site.rb @@ -87,7 +87,7 @@ action :create do end layers = Dir.glob("/srv/imagery/layers/#{new_resource.site}/*.yml").collect do |path| - YAML.safe_load(::File.read(path), :permitted_classes => [Symbol]) + YAML.safe_load_file(path, :permitted_classes => [Symbol]) end declare_resource :template, "/srv/#{new_resource.site}/imagery.js" do @@ -105,14 +105,18 @@ action :create do description "Map server for #{new_resource.site} layer" environment "MS_DEBUGLEVEL" => "0", "MS_ERRORFILE" => "stderr", - "GDAL_CACHEMAX" => "128" + "GDAL_CACHEMAX" => "128", + "GDAL_HTTP_TCP_KEEPALIVE" => "YES", + "GDAL_HTTP_VERSION" => "2TLS", + "GDAL_ENABLE_WMS_CACHE" => "NO", + "LD_PRELOAD" => "libtcmalloc_minimal.so.4" limit_nofile 16384 - memory_max "4G" + limit_core 0 user "imagery" group "imagery" exec_start "/usr/bin/multiwatch -f 8 --signal=TERM -- /usr/lib/cgi-bin/mapserv" standard_input "socket" - sandbox true + sandbox :enable_network => true restrict_address_families "AF_UNIX" timeout_stop_sec 60 not_if { new_resource.uses_tiler } @@ -153,7 +157,7 @@ action :create do systemd_timer "mapserv-fcgi-#{new_resource.site}-stop" do on_boot_sec "10m" - on_unit_inactive_sec "30m" + on_unit_inactive_sec "6h" randomized_delay_sec "20m" not_if { new_resource.uses_tiler } end diff --git a/cookbooks/imagery/templates/default/index.html.erb b/cookbooks/imagery/templates/default/index.html.erb index c21d95e04..2fe8d786d 100644 --- a/cookbooks/imagery/templates/default/index.html.erb +++ b/cookbooks/imagery/templates/default/index.html.erb @@ -5,15 +5,15 @@ <%= @title %> - - + + - - + +