From: Tom Hughes Date: Tue, 13 Jan 2026 11:59:57 +0000 (+0000) Subject: Merge remote-tracking branch 'github/pull/824' X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/HEAD?hp=969fa2fbc12465cf76d9e699a21da0ef6b035256 Merge remote-tracking branch 'github/pull/824' --- diff --git a/.github/workflows/cookstyle.yml b/.github/workflows/cookstyle.yml index b9332e236..8447d97f1 100644 --- a/.github/workflows/cookstyle.yml +++ b/.github/workflows/cookstyle.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup ruby uses: ruby/setup-ruby@v1 with: diff --git a/.github/workflows/test-kitchen.yml b/.github/workflows/test-kitchen.yml index 201c69b46..ad5d7ca58 100644 --- a/.github/workflows/test-kitchen.yml +++ b/.github/workflows/test-kitchen.yml @@ -12,7 +12,7 @@ concurrency: jobs: kitchen: name: Test Kitchen - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest permissions: packages: read strategy: @@ -22,6 +22,7 @@ jobs: - apache - apt - apt-repository + - awscli - backup - bind - blog @@ -61,11 +62,10 @@ jobs: - hardware - hot - ideditor + - imagery-tiler - irc - kibana - letsencrypt - - logstash - - logstash-forwarder - mail - mailman - matomo @@ -116,83 +116,97 @@ jobs: - web-cgimap - web-frontend - web-rails - - wordpress - wiki + - wordpress os: - - ubuntu-2204 - include: - - os: ubuntu-2004 - suite: mailman - - os: ubuntu-2004 - suite: osqa - - os: debian-12 - suite: apt-repository - - os: debian-12 - suite: dev - - os: debian-12 - suite: dns - - os: debian-12 - suite: git-server - - os: debian-12 - suite: git-web - - os: debian-12 - suite: gps-tile - - os: debian-12 - suite: imagery-tiler - - os: debian-12 - suite: letsencrypt - - os: debian-12 - suite: matomo - - os: debian-12 - suite: otrs - - os: debian-12 - suite: serverinfo - - os: debian-12 - suite: supybot - - os: debian-12 - suite: vectortile - - os: debian-12 - suite: web-cgimap - - os: debian-12 - suite: web-frontend - - os: debian-12 - suite: web-rails - - os: debian-12 - suite: wiki + - debian-12 exclude: - - suite: apt-repository + - suite: mailman + os: debian-12 + include: + - suite: blog + os: ubuntu-2004 + - suite: mailman + os: ubuntu-2004 + - suite: stateofthemap-wordpress + os: ubuntu-2004 + - suite: wordpress + os: ubuntu-2004 + - suite: accounts os: ubuntu-2204 - - suite: dev + - suite: apache os: ubuntu-2204 - - suite: dns + - suite: apt os: ubuntu-2204 - - suite: git-server + - suite: backup os: ubuntu-2204 - - suite: git-web + - suite: bind os: ubuntu-2204 - - suite: gps-tile + - suite: chef os: ubuntu-2204 - - suite: letsencrypt + - suite: clamav os: ubuntu-2204 - - suite: mailman + - suite: db-backup + os: ubuntu-2204 + - suite: db-base + os: ubuntu-2204 + - suite: db-master + os: ubuntu-2204 + - suite: db-slave + os: ubuntu-2204 + - suite: devices + os: ubuntu-2204 + - suite: dhcpd + os: ubuntu-2204 + - suite: exim + os: ubuntu-2204 + - suite: fail2ban + os: ubuntu-2204 + - suite: geodns os: ubuntu-2204 - - suite: matomo + - suite: geoipupdate os: ubuntu-2204 - - suite: osqa + - suite: git os: ubuntu-2204 - - suite: otrs + - suite: hardware os: ubuntu-2204 - - suite: serverinfo + - suite: networking os: ubuntu-2204 - - suite: supybot + - suite: ntp os: ubuntu-2204 - - suite: vectortile + - suite: openssh os: ubuntu-2204 - - suite: web-cgimap + - suite: osmosis os: ubuntu-2204 - - suite: web-frontend + - suite: planet os: ubuntu-2204 - - suite: web-rails + - suite: planet-aws + os: ubuntu-2204 + - suite: planet-dump + os: ubuntu-2204 + - suite: planet-notes + os: ubuntu-2204 + - suite: planet-replication + os: ubuntu-2204 + - suite: postgresql + os: ubuntu-2204 + - suite: prometheus + os: ubuntu-2204 + - suite: python + os: ubuntu-2204 + - suite: rsyncd + os: ubuntu-2204 + - suite: spamassassin + os: ubuntu-2204 + - suite: ssl + os: ubuntu-2204 + - suite: sysctl + os: ubuntu-2204 + - suite: sysfs + os: ubuntu-2204 + - suite: tilelog + os: ubuntu-2204 + - suite: tools os: ubuntu-2204 fail-fast: false steps: @@ -202,18 +216,11 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - if: github.repository == 'openstreetmap/chef' && github.event_name != 'pull_request' - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup ruby uses: ruby/setup-ruby@v1 with: - ruby-version: 3.1 bundler-cache: true - name: Run kitchen test ${{ matrix.suite }}-${{ matrix.os }} run: bundle exec kitchen test ${{ matrix.suite }}-${{ matrix.os }} diff --git a/.kitchen.yml b/.kitchen.yml index 6f4eb6bb6..de96273fe 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -52,6 +52,15 @@ platforms: - RUN /usr/bin/apt-get update -y - RUN /usr/bin/apt-get install -y eatmydata - RUN echo /usr/lib/$(uname -m)-linux-gnu/libeatmydata.so >>/etc/ld.so.preload + - name: ubuntu-24.04 + driver: + image: ghcr.io/test-kitchen/dokken/ubuntu-24.04 + privileged: true + pid_one_command: /bin/systemd + intermediate_instructions: + - RUN /usr/bin/apt-get update -y + - RUN /usr/bin/apt-get install -y eatmydata + - RUN echo /usr/lib/$(uname -m)-linux-gnu/libeatmydata.so >>/etc/ld.so.preload - name: debian-12 driver: image: ghcr.io/test-kitchen/dokken/debian-12 @@ -61,6 +70,27 @@ platforms: - RUN /usr/bin/apt-get update -y - RUN /usr/bin/apt-get install -y eatmydata - RUN echo /usr/lib/$(uname -m)-linux-gnu/libeatmydata.so >>/etc/ld.so.preload + - name: debian-13 + driver: + image: ghcr.io/test-kitchen/dokken/debian-13 + privileged: true + pid_one_command: /usr/lib/systemd/systemd + intermediate_instructions: + - RUN /usr/bin/apt-get update -y + - RUN /usr/bin/apt-get install -y eatmydata + - RUN echo /usr/lib/$(uname -m)-linux-gnu/libeatmydata.so >>/etc/ld.so.preload + +# When using podman we have to manually start chef to workaround a volume issue +lifecycle: + pre_create: | + if command -v podman >/dev/null 2>&1; then + podman create --name chef-latest --replace ghcr.io/firefishy/chef-docker-image:latest sh + podman start chef-latest + fi + post_destroy: | + if command -v podman >/dev/null 2>&1; then + podman container rm -iv chef-latest + fi suites: - name: accounts @@ -72,6 +102,7 @@ suites: - name: apt run_list: - recipe[apt::default] + - recipe[apt::management-component-pack] - name: apt-repository run_list: - recipe[apt::repository] @@ -126,7 +157,10 @@ suites: attributes: postgresql: versions: - - 15 + - 17 + dev: + rails: + postgresql_cluster: 17/main - name: devices run_list: - recipe[devices::default] @@ -241,23 +275,6 @@ suites: - name: letsencrypt run_list: - recipe[letsencrypt::default] - - name: logstash - run_list: - - recipe[logstash::default] - - name: logstash-forwarder - run_list: - - recipe[logstash::forwarder] - attributes: - logstash: - forwarder: - filebeat.inputs: - - type: filestream - id: apache - paths: - - /var/log/apache2/access.log - fields: - type: apache - fields_under_root: true - name: mail run_list: - role[mail] diff --git a/.ruby-version b/.ruby-version index 0aec50e6e..5f6fc5edc 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.1.4 +3.3.10 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5345a0786..50f5b9079 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,77 +1,194 @@ -# Contribution Guidelines +# Contributing + +This repository contains the Chef code used to configure OpenStreetMap's infrastructure. +This guide walks you through running the same checks as CI locally before opening a pull request. + +## What you should run before a pull request + +At a minimum, please run: + +- **Cookstyle** (lint/style): `bundle exec cookstyle` +- **Test Kitchen** for the cookbooks/roles you changed or added (integration): `bundle exec kitchen test -` + +CI runs a large matrix; contributors are not expected to run every Kitchen instance locally. + +## Quick start (if you already have Ruby + Docker) + +From the repository root: + +```bash +bundle install +bundle exec cookstyle +bundle exec kitchen list +bundle exec kitchen test dns-ubuntu-2204 +``` + +Replace `dns-ubuntu-2204` with a Kitchen instance relevant to the cookbooks/roles you changed. ## Workflow -We operate the "Fork & Pull" model explained at +We use the "fork and pull request" workflow: + +- Fork the repository +- Create a topic branch +- Open a pull request back to `openstreetmap/chef` + +GitHub has an overview at https://help.github.com/articles/using-pull-requests + +## Local test setup + +### Prerequisites -https://help.github.com/articles/using-pull-requests +- **Docker** + - macOS: Docker Desktop + - Linux: Docker Engine + - Windows: WSL2 + Docker may work, but is not used by the operations team and may not work in your environment (see below). +- **Ruby** (see [.ruby-version](.ruby-version)) and **Bundler** +- `git` -You should fork the project into your own repo, create a topic branch -there and then make one or more pull requests back to the openstreetmap/chef repository. -Your pull requests will then be reviewed and discussed. +We recommend using `rbenv` to manage Ruby versions. On macOS, Homebrew is often the easiest way to install it. -## Running the Infrastructure Tests locally +### Recommended setup (rbenv + Bundler) -- **[Cookstyle](https://docs.chef.io/workstation/cookstyle/)** is used for linting, ensuring that our Chef recipes follow style guidelines and best practices. -- **[Test Kitchen](https://kitchen.ci/)** combined with **InSpec** and [Dokken](https://github.com/test-kitchen/kitchen-dokken) is used to verify the functionality of our Chef code, ensuring it behaves as expected. +The steps below work on macOS and Linux. The macOS-only steps are called out explicitly. -The following guidelines are to help set up and run these checks locally: +1) Install Docker -#### **1. Install Docker** -- Visit [Docker's official site](https://www.docker.com/products/docker-desktop) to download and install Docker. +- macOS: Docker Desktop - https://www.docker.com/products/docker-desktop +- Linux: Docker Engine - https://docs.docker.com/engine/install/ -#### **2. Install Homebrew (Apple MacOS only)** -- Install Homebrew by following the instructions [here](https://brew.sh/). +2) macOS only: install Homebrew (if you do not already have it) -#### **3. Install rbenv (recommended)** -- Install rbenv by following the instructions [here](https://github.com/rbenv/rbenv#installation). +- https://brew.sh/ -rbenv is a ruby version manager. rbenv allows projects to use a different version of ruby than the version of install with your operating system. +3) Install `rbenv` -> *Note on rbenv: While we recommend using rbenv for managing Ruby versions, it's not strictly necessary. If you have Ruby already installed feel free to use that. If you're not using rbenv, simply omit the `rbenv exec` prefix from the commands below.* +- https://github.com/rbenv/rbenv#installation -#### **4. Increase File Limit (Important for MacOS)** +`rbenv` is a Ruby version manager. It lets you use the Ruby version this repository expects without changing your system Ruby. -To avoid errors when running tests on MacOS, you might need to increase the number of files your system can open at once. Here's how: +If you do not use `rbenv`, omit the `rbenv exec` prefix in the commands below. + +4) macOS only: increase the open file limit (often needed) + +If you see errors like `Too many open files (Errno::EMFILE)` while running tests: -1. Run the command: ```bash ulimit -n 1024 ``` -2. To make the change permanent, add the above line to either `~/.zshrc` or `~/.bash_profile`, depending on your shell. -**Note:** MacOS has a low default limit of just 256 open files. If you exceed this while testing, you'll see an error like: `Too many open files - getcwd (Errno::EMFILE)`. This step helps prevent that. +To make it permanent, add the line above to your shell profile (for example `~/.zshrc`). + +5) Install the Ruby version for this repo + +From the repository root: -#### **5. Install Required Ruby Version (recommended)** -Navigate to the git checkout of the OpenStreetMap chef repo and run: ```bash -rbenv install +rbenv install --skip-existing ``` -This will install the recommended version of ruby for running the tests. The recommended version of ruby is defined in the [.ruby-version](.ruby-version) file. -#### **6. Install Dependencies with Bundler** +6) Install Ruby gems + ```bash rbenv exec gem install bundler -rbenv exec bundler install +rbenv exec bundle install ``` -This will install the [bundler](https://bundler.io/), the ruby gem packages manager, and then uses `bundler`` to install the required gem packages for the tests. -#### **7. Run Cookstyle for Linting and Style Checks** +## Running the checks + +### 1) Cookstyle (lint and style) + +Cookstyle checks Chef/Ruby style and common issues. + ```bash rbenv exec bundle exec cookstyle ``` -This will run [cookstyle](https://docs.chef.io/workstation/cookstyle/) a linting tool which reports on any linting issues. -> *Automatically run cookstyle lint: We have a sample [git pre-commit hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) in the `hooks/pre-commit` file which can be copied to the local checkout of this repo to the file `.git/hooks/pre-commit` to ensure the lint passes when running a git commit.* +If you prefer not to install Ruby locally, you can run Cookstyle via Docker: + +```bash +docker compose run --rm cookstyle +``` + +If you only want to lint the files you are committing, there is a sample pre-commit hook at [hooks/pre-commit](hooks/pre-commit). +Copy it to `.git/hooks/pre-commit` and make it executable. + +### 2) Test Kitchen (integration tests) + +We use Test Kitchen with InSpec and the Dokken driver to converge recipes and verify behaviour inside containers. + +1) List available instances: + +```bash +rbenv exec bundle exec kitchen list +``` + +Kitchen instances are named `-`. +The suite is usually the cookbook/role under test (see [.kitchen.yml](.kitchen.yml)), and the platform is the OS image. + +The available Kitchen suites and platforms are defined in [.kitchen.yml](.kitchen.yml). +The InSpec integration tests themselves live under [test/integration/](test/integration/). +If your change adds a new cookbook, please add a corresponding suite to [.kitchen.yml](.kitchen.yml) and include appropriate integration tests. + +2) Run a specific instance: + +```bash +rbenv exec bundle exec kitchen test dns-ubuntu-2204 +``` + +`kitchen test` will create the container, converge, verify, then destroy it. + +For faster iteration, you can split the steps: + +```bash +rbenv exec bundle exec kitchen converge dns-ubuntu-2204 +rbenv exec bundle exec kitchen verify dns-ubuntu-2204 +rbenv exec bundle exec kitchen destroy dns-ubuntu-2204 +``` + +#### Debugging a failed converge + +If `kitchen converge` fails, it can be useful to inspect the running test container. + +1) Find the container name by running `docker ps` (example output): + +```text +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +3b83c7f96575 9a7e6edc5b-dns-ubuntu-2204:latest "/bin/systemd" 14 minutes ago Up 14 minutes 9a7e6edc5b-dns-ubuntu-2204 +``` + +2) Enter the container: -#### **8. List Available Tests** ```bash -rbenv exec bundler exec kitchen list +docker exec -it 9a7e6edc5b-dns-ubuntu-2204 bash -l ``` -This lists the [Test Kitchen](https://kitchen.ci/) tests which are available. The list of tests is generated from the definitions in the [.kitchen.yml](.kitchen.yml) file. The individual tests are written in [InSpec](https://docs.chef.io/inspec/) and are stored in the `test/integration/` directory. -#### **9. Run an Example Test** +Once you are finished debugging, remember to clean up the instance: + ```bash -rbenv exec bundler exec kitchen test dns-ubuntu-2204 +rbenv exec bundle exec kitchen destroy dns-ubuntu-2204 ``` -This runs the [Test Kitchen](https://kitchen.ci/) [InSpec](https://docs.chef.io/inspec/) `dns` tests using the `Ubuntu 22.04` platform. The tests are run inside a Docker container using the Test Kitchen [Dokken driver](https://github.com/test-kitchen/kitchen-dokken). + +## Windows (WSL2) + +It may be possible to run the toolchain under WSL2 (for example Ubuntu on WSL2) with Docker. +However, the operations team do not use WSL2, and we cannot guarantee it will work or be able to troubleshoot WSL2-specific issues. + +If you try this route, aim to match the Linux instructions as closely as possible and ensure Docker is reachable from within WSL2. + +## Troubleshooting + +- **Docker is not running**: start Docker Desktop (macOS) or the Docker service (Linux). +- **Open file limit errors on macOS**: increase `ulimit -n` as described above. +- **Kitchen networking issues**: if Docker has IPv6 disabled, Dokken may fail to create its network; enabling IPv6 in Docker's settings can help. + +## Pull request checklist + +- Run `bundle exec cookstyle` locally +- Run relevant `kitchen test -` instances for the cookbooks/roles you changed or added +- Keep changes focused and include context in the pull request description + +## Need help? + +If you get stuck, the operations team are available in `#osmf-operations` on `irc.oftc.net`. +You can access `#osmf-operations` via https://irc.openstreetmap.org/ or via the Matrix IRC bridge in [#\_oftc_#osmf-operations](https://matrix.to/#/#_oftc_#osmf-operations:matrix.org). diff --git a/Dockerfile b/Dockerfile index fa0ad5960..7c037e6ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,24 @@ -# Basic Dockerfile to run cookstyle linting -# run: docker build -t chef-test . -FROM ruby:3.1-alpine as build +# Docker image used for running repo tooling (e.g. Cookstyle) in local containers. +FROM ruby:3.3-trixie AS build # Add Gem build requirements -RUN apk add --no-cache build-base +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + pkg-config \ + libxml2-dev \ + libxslt1-dev \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* # Create app directory -WORKDIR /app +WORKDIR /usr/src/app # Add Gemfile and Gemfile.lock ADD Gemfile* ./ # Install Gems -RUN gem install bundler \ - && bundle config build.nokogiri --use-system-libraries \ - && bundle config --global jobs $(nproc) \ +RUN gem install bundler -v 2.6.9 \ + && bundle config set build.nokogiri --use-system-libraries \ + && bundle config set --global jobs $(nproc) \ && bundle install - -# Add repo -ADD . . - -# Run linting -RUN bundle exec cookstyle -f fuubar diff --git a/Gemfile b/Gemfile index baf310d5a..7f934a878 100644 --- a/Gemfile +++ b/Gemfile @@ -1,9 +1,6 @@ source "https://rubygems.org" gem "cookstyle" -gem "kitchen-dokken" +gem "kitchen-dokken", ">= 2.21.4" gem "kitchen-inspec" gem "test-kitchen" - -# Lock some modules to old versions for ruby 3.1 support -gem "zeitwerk", "< 2.7" diff --git a/Gemfile.lock b/Gemfile.lock index 48cfe23a6..7c31b5860 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GEM remote: https://rubygems.org/ specs: - activesupport (7.1.5.1) + activesupport (7.1.6) base64 benchmark (>= 0.3) bigdecimal @@ -14,254 +14,9 @@ GEM mutex_m securerandom (>= 0.3) tzinfo (~> 2.0) - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) + addressable (2.8.8) + public_suffix (>= 2.0.2, < 8.0) ast (2.4.3) - aws-eventstream (1.3.2) - aws-partitions (1.863.0) - aws-sdk-accessanalyzer (1.44.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-account (1.20.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-alexaforbusiness (1.67.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-amplify (1.54.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-apigateway (1.90.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-apigatewayv2 (1.53.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-applicationautoscaling (1.79.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-athena (1.79.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-autoscaling (1.102.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-batch (1.79.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-budgets (1.62.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-cloudformation (1.97.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-cloudfront (1.86.1) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-cloudhsm (1.50.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-cloudhsmv2 (1.53.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-cloudtrail (1.74.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-cloudwatch (1.83.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-cloudwatchevents (1.69.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-cloudwatchlogs (1.77.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-codecommit (1.62.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-codedeploy (1.62.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-codepipeline (1.67.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-cognitoidentity (1.51.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-cognitoidentityprovider (1.85.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-configservice (1.103.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-core (3.190.3) - aws-eventstream (~> 1, >= 1.3.0) - aws-partitions (~> 1, >= 1.651.0) - aws-sigv4 (~> 1.8) - jmespath (~> 1, >= 1.6.1) - aws-sdk-costandusagereportservice (1.53.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-databasemigrationservice (1.91.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-dynamodb (1.98.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-ec2 (1.429.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-ecr (1.68.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-ecrpublic (1.25.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-ecs (1.135.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-efs (1.71.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-eks (1.95.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-elasticache (1.95.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-elasticbeanstalk (1.63.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-elasticloadbalancing (1.51.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-elasticloadbalancingv2 (1.96.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-elasticsearchservice (1.79.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-emr (1.81.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-eventbridge (1.54.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-firehose (1.60.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-glue (1.165.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-guardduty (1.85.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-iam (1.92.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-kafka (1.67.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-kinesis (1.54.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-kms (1.76.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-lambda (1.113.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-macie2 (1.64.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-mq (1.58.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-networkfirewall (1.39.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-networkmanager (1.40.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-organizations (1.83.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-ram (1.52.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-rds (1.208.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-redshift (1.107.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-route53 (1.83.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-route53domains (1.54.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-route53resolver (1.51.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.141.0) - aws-sdk-core (~> 3, >= 3.189.0) - aws-sdk-kms (~> 1) - aws-sigv4 (~> 1.8) - aws-sdk-s3control (1.74.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-secretsmanager (1.87.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-securityhub (1.98.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-servicecatalog (1.90.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-ses (1.58.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-shield (1.60.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-signer (1.50.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-simpledb (1.42.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv2 (~> 1.0) - aws-sdk-sms (1.52.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-sns (1.70.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-sqs (1.69.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-ssm (1.162.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-states (1.63.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-synthetics (1.39.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-transfer (1.86.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-waf (1.58.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sdk-wafv2 (1.74.0) - aws-sdk-core (~> 3, >= 3.188.0) - aws-sigv4 (~> 1.1) - aws-sigv2 (1.2.0) - aws-sigv4 (1.11.0) - aws-eventstream (~> 1, >= 1.0.2) azure_graph_rbac (0.17.2) ms_rest_azure (~> 0.12.0) azure_mgmt_key_vault (0.17.7) @@ -272,14 +27,16 @@ GEM ms_rest_azure (~> 0.12.0) azure_mgmt_storage (0.23.0) ms_rest_azure (~> 0.12.0) - base64 (0.2.0) - bcrypt_pbkdf (1.1.1) - benchmark (0.4.0) - bigdecimal (3.1.9) + base64 (0.3.0) + bcrypt_pbkdf (1.1.2) + bcrypt_pbkdf (1.1.2-arm64-darwin) + bcrypt_pbkdf (1.1.2-x86_64-darwin) + benchmark (0.5.0) + bigdecimal (4.0.1) builder (3.3.0) - chef-config (18.7.6) + chef-config (18.8.54) addressable - chef-utils (= 18.7.6) + chef-utils (= 18.8.54) fuzzyurl mixlib-config (>= 2.2.12, < 4.0) mixlib-shellout (>= 2.0, < 4.0) @@ -287,43 +44,46 @@ GEM chef-telemetry (1.1.1) chef-config concurrent-ruby (~> 1.0) - chef-utils (18.7.6) + chef-utils (18.8.54) concurrent-ruby coderay (1.1.3) - concurrent-ruby (1.3.5) - connection_pool (2.5.2) - cookstyle (8.1.1) - rubocop (= 1.75.3) + concurrent-ruby (1.3.6) + connection_pool (3.0.2) + cookstyle (8.5.3) + rubocop (= 1.81.7) + csv (3.3.5) + date (3.5.1) declarative (0.0.20) - diff-lcs (1.6.1) + diff-lcs (1.6.2) docker-api (2.4.0) excon (>= 0.64.0) multi_json domain_name (0.6.20240107) - drb (2.2.1) - ed25519 (1.3.0) + drb (2.2.3) + ed25519 (1.4.0) + erb (6.0.1) erubi (1.13.1) - excon (1.2.5) + excon (1.3.2) logger faraday (1.3.1) faraday-net_http (~> 1.0) multipart-post (>= 1.2, < 3) ruby2_keywords (>= 0.0.4) - faraday-cookie_jar (0.0.7) + faraday-cookie_jar (0.0.8) faraday (>= 0.8.0) - http-cookie (~> 1.0.0) + http-cookie (>= 1.0.0) faraday-net_http (1.0.2) - faraday_middleware (1.0.0) - faraday (~> 1.0) - ffi (1.17.2) - ffi (1.17.2-aarch64-linux-gnu) - ffi (1.17.2-aarch64-linux-musl) - ffi (1.17.2-arm-linux-gnu) - ffi (1.17.2-arm-linux-musl) - ffi (1.17.2-x86-linux-gnu) - ffi (1.17.2-x86-linux-musl) - ffi (1.17.2-x86_64-linux-gnu) - ffi (1.17.2-x86_64-linux-musl) + ffi (1.17.3) + ffi (1.17.3-aarch64-linux-gnu) + ffi (1.17.3-aarch64-linux-musl) + ffi (1.17.3-arm-linux-gnu) + ffi (1.17.3-arm-linux-musl) + ffi (1.17.3-arm64-darwin) + ffi (1.17.3-x86-linux-gnu) + ffi (1.17.3-x86-linux-musl) + ffi (1.17.3-x86_64-darwin) + ffi (1.17.3-x86_64-linux-gnu) + ffi (1.17.3-x86_64-linux-musl) fuzzyurl (0.9.0) google-apis-admin_directory_v1 (0.46.0) google-apis-core (>= 0.11.0, < 2.a) @@ -359,20 +119,13 @@ GEM builder (>= 2.1.2) rexml (~> 3.0) hashie (4.1.0) - http-cookie (1.0.8) + http-cookie (1.1.0) domain_name (~> 0.5) httpclient (2.9.0) mutex_m - i18n (1.14.7) + i18n (1.14.8) concurrent-ruby (~> 1.0) inifile (3.0.0) - inspec (4.24.32) - faraday_middleware (>= 0.12.2, < 1.1) - inspec-core (= 4.24.32) - train (~> 3.0) - train-aws (~> 0.1) - train-habitat (~> 0.1) - train-winrm (~> 0.2) inspec-core (4.24.32) addressable (~> 2.4) chef-telemetry (~> 1.0) @@ -395,19 +148,24 @@ GEM train-core (~> 3.0) tty-prompt (~> 0.17) tty-table (~> 0.10) - jmespath (1.6.2) - json (2.11.3) - jwt (2.10.1) + io-console (0.8.2) + irb (1.16.0) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + json (2.18.0) + jwt (2.10.2) base64 - kitchen-dokken (2.20.7) + kitchen-dokken (2.22.0) docker-api (>= 1.33, < 3) lockfile (~> 2.1) test-kitchen (>= 1.15, < 4) - kitchen-inspec (3.0.0) - hashie (>= 3.4, <= 5.0) - inspec (>= 2.2.64, < 7.0) - test-kitchen (>= 2.7, < 4) - language_server-protocol (3.17.0.4) + kitchen-inspec (3.1.0) + hashie (>= 3.4, < 6.0) + inspec-core (>= 2.2.64, < 8.0) + test-kitchen (>= 2.7, < 5) + train + language_server-protocol (3.17.0.5) license-acceptance (2.1.13) pastel (~> 0.7) tomlrb (>= 1.2, < 3.0) @@ -422,10 +180,11 @@ GEM multi_json (~> 1.14) method_source (1.1.0) mini_mime (1.1.5) - minitest (5.25.5) + minitest (6.0.1) + prism (~> 1.5) mixlib-config (3.0.27) tomlrb - mixlib-install (3.12.30) + mixlib-install (3.13.0) mixlib-shellout mixlib-versioning thor @@ -443,7 +202,7 @@ GEM faraday (>= 0.9, < 2.0.0) faraday-cookie_jar (~> 0.0.6) ms_rest (~> 0.7.6) - multi_json (1.15.0) + multi_json (1.19.1) multipart-post (2.4.1) mutex_m (0.3.0) net-scp (4.1.0) @@ -454,44 +213,58 @@ GEM nori (2.7.1) bigdecimal os (1.1.4) + ostruct (0.6.3) parallel (1.27.0) - parser (3.3.8.0) + parser (3.3.10.0) ast (~> 2.4.1) racc parslet (2.0.0) pastel (0.8.0) tty-color (~> 0.5) - prism (1.4.0) - pry (0.15.2) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.6.0) + pry (0.16.0) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (6.0.1) + reline (>= 0.6.0) + psych (5.3.1) + date + stringio + public_suffix (7.0.2) racc (1.8.1) rainbow (3.1.1) - regexp_parser (2.10.0) + rdoc (7.0.3) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.11.3) + reline (0.6.3) + io-console (~> 0.5) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) retriable (3.1.2) - rexml (3.4.1) - rspec (3.13.0) + rexml (3.4.4) + rspec (3.13.2) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) rspec-mocks (~> 3.13.0) - rspec-core (3.13.3) + rspec-core (3.13.6) rspec-support (~> 3.13.0) - rspec-expectations (3.13.3) + rspec-expectations (3.13.5) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) rspec-its (1.3.1) rspec-core (>= 3.0.0) rspec-expectations (>= 3.0.0) - rspec-mocks (3.13.2) + rspec-mocks (3.13.7) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-support (3.13.2) - rubocop (1.75.3) + rspec-support (3.13.6) + rubocop (1.81.7) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) @@ -499,10 +272,10 @@ GEM parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.44.0, < 2.0) + rubocop-ast (>= 1.47.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.44.1) + rubocop-ast (1.48.0) parser (>= 3.3.7.2) prism (~> 1.4) ruby-progressbar (1.13.0) @@ -512,32 +285,39 @@ GEM rubyzip (2.4.1) securerandom (0.4.1) semverse (3.0.2) - signet (0.19.0) + signet (0.21.0) addressable (~> 2.8) faraday (>= 0.17.5, < 3.a) - jwt (>= 1.5, < 3.0) + jwt (>= 1.5, < 4.0) multi_json (~> 1.10) sslshake (1.3.1) + stringio (3.2.0) strings (0.2.1) strings-ansi (~> 0.2) unicode-display_width (>= 1.5, < 3.0) unicode_utils (~> 1.4) strings-ansi (0.2.0) - test-kitchen (3.7.0) + syslog (0.3.0) + logger + test-kitchen (3.9.1) bcrypt_pbkdf (~> 1.0) chef-utils (>= 16.4.35) - ed25519 (~> 1.2) + csv (~> 3.3) + ed25519 (~> 1.3) + irb (~> 1.15) license-acceptance (>= 1.0.11, < 3.0) mixlib-install (~> 3.6) mixlib-shellout (>= 1.2, < 4.0) net-scp (>= 1.1, < 5.0) net-ssh (>= 2.9, < 8.0) net-ssh-gateway (>= 1.2, < 3.0) + ostruct (~> 0.6) + syslog (~> 0.3) thor (>= 0.19, < 2.0) winrm (~> 2.0) winrm-elevated (~> 1.0) winrm-fs (~> 1.1) - thor (1.3.2) + thor (1.5.0) timeliness (0.3.10) tomlrb (1.3.0) trailblazer-option (0.1.2) @@ -560,87 +340,6 @@ GEM inifile (~> 3.0) train-core (= 3.12.7) train-winrm (~> 0.2) - train-aws (0.2.41) - aws-partitions (~> 1.863.0) - aws-sdk-accessanalyzer (~> 1.44.0) - aws-sdk-account (~> 1.20.0) - aws-sdk-alexaforbusiness (~> 1.67.0) - aws-sdk-amplify (~> 1.54.0) - aws-sdk-apigateway (~> 1.90.0) - aws-sdk-apigatewayv2 (~> 1.53.0) - aws-sdk-applicationautoscaling (~> 1.79.0) - aws-sdk-athena (>= 1.78, < 1.80) - aws-sdk-autoscaling (= 1.102.0) - aws-sdk-batch (~> 1.79.0) - aws-sdk-budgets (~> 1.62.0) - aws-sdk-cloudformation (>= 1.96, < 1.98) - aws-sdk-cloudfront (~> 1.86.0) - aws-sdk-cloudhsm (~> 1.50.0) - aws-sdk-cloudhsmv2 (~> 1.53.0) - aws-sdk-cloudtrail (~> 1.74.0) - aws-sdk-cloudwatch (~> 1.83.0) - aws-sdk-cloudwatchevents (~> 1.69.0) - aws-sdk-cloudwatchlogs (~> 1.75) - aws-sdk-codecommit (~> 1.62.0) - aws-sdk-codedeploy (~> 1.62.0) - aws-sdk-codepipeline (~> 1.67.0) - aws-sdk-cognitoidentity (~> 1.51.0) - aws-sdk-cognitoidentityprovider (~> 1.84) - aws-sdk-configservice (~> 1.103.0) - aws-sdk-core (~> 3.190.0) - aws-sdk-costandusagereportservice (~> 1.53.0) - aws-sdk-databasemigrationservice (~> 1.91.0) - aws-sdk-dynamodb (~> 1.98.0) - aws-sdk-ec2 (>= 1.427, < 1.430) - aws-sdk-ecr (~> 1.68.0) - aws-sdk-ecrpublic (~> 1.25.0) - aws-sdk-ecs (~> 1.135.0) - aws-sdk-efs (~> 1.71.0) - aws-sdk-eks (~> 1.95.0) - aws-sdk-elasticache (~> 1.95.0) - aws-sdk-elasticbeanstalk (~> 1.63.0) - aws-sdk-elasticloadbalancing (~> 1.51.0) - aws-sdk-elasticloadbalancingv2 (~> 1.96.0) - aws-sdk-elasticsearchservice (~> 1.79.0) - aws-sdk-emr (~> 1.81.0) - aws-sdk-eventbridge (~> 1.54.0) - aws-sdk-firehose (~> 1.60.0) - aws-sdk-glue (~> 1.164) - aws-sdk-guardduty (~> 1.85.0) - aws-sdk-iam (~> 1.92.0) - aws-sdk-kafka (~> 1.67.0) - aws-sdk-kinesis (~> 1.54.0) - aws-sdk-kms (~> 1.74) - aws-sdk-lambda (~> 1.113.0) - aws-sdk-macie2 (~> 1.64.0) - aws-sdk-mq (~> 1.58.0) - aws-sdk-networkfirewall (~> 1.39.0) - aws-sdk-networkmanager (~> 1.40.0) - aws-sdk-organizations (~> 1.83.0) - aws-sdk-ram (~> 1.52.0) - aws-sdk-rds (~> 1.208.0) - aws-sdk-redshift (~> 1.107.0) - aws-sdk-route53 (~> 1.83.0) - aws-sdk-route53domains (~> 1.54.0) - aws-sdk-route53resolver (~> 1.51.0) - aws-sdk-s3 (~> 1.141.0) - aws-sdk-s3control (~> 1.74.0) - aws-sdk-secretsmanager (~> 1.87.0) - aws-sdk-securityhub (~> 1.98.0) - aws-sdk-servicecatalog (~> 1.90.0) - aws-sdk-ses (~> 1.58.0) - aws-sdk-shield (~> 1.60.0) - aws-sdk-signer (~> 1.50.0) - aws-sdk-simpledb (~> 1.42.0) - aws-sdk-sms (~> 1.52.0) - aws-sdk-sns (~> 1.70.0) - aws-sdk-sqs (~> 1.69.0) - aws-sdk-ssm (~> 1.162.0) - aws-sdk-states (~> 1.63.0) - aws-sdk-synthetics (~> 1.39.0) - aws-sdk-transfer (~> 1.86.0) - aws-sdk-waf (~> 1.58.0) - aws-sdk-wafv2 (~> 1.74.0) train-core (3.12.7) addressable (~> 2.5) ffi (!= 1.13.0) @@ -648,11 +347,11 @@ GEM mixlib-shellout (>= 2.0, < 4.0) net-scp (>= 1.2, < 5.0) net-ssh (>= 2.9, < 8.0) - train-habitat (0.2.22) train-winrm (0.2.13) winrm (>= 2.3.6, < 3.0) winrm-elevated (~> 1.2.2) winrm-fs (~> 1.0) + tsort (0.2.0) tty-box (0.7.0) pastel (~> 0.8) strings (~> 0.2.0) @@ -696,25 +395,25 @@ GEM rubyzip (~> 2.0) winrm (~> 2.0) wisper (2.0.1) - zeitwerk (2.6.18) PLATFORMS aarch64-linux-gnu aarch64-linux-musl arm-linux-gnu arm-linux-musl + arm64-darwin ruby x86-linux-gnu x86-linux-musl + x86_64-darwin x86_64-linux-gnu x86_64-linux-musl DEPENDENCIES cookstyle - kitchen-dokken + kitchen-dokken (>= 2.21.4) kitchen-inspec test-kitchen - zeitwerk (< 2.7) BUNDLED WITH - 2.6.2 + 2.6.9 diff --git a/README.md b/README.md index 6b6304834..65782f3f6 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # OpenStreetMap chef cookbooks -[![Cookstyle](https://github.com/openstreetmap/chef/workflows/Cookstyle/badge.svg?branch=master&event=push)](https://github.com/openstreetmap/chef/actions?query=workflow%3ACookstyle%20branch%3Amaster%20event%3Apush) -[![Test Kitchen](https://github.com/openstreetmap/chef/workflows/Test%20Kitchen/badge.svg?branch=master&event=push)](https://github.com/openstreetmap/chef/actions?query=workflow%3A%22Test+Kitchen%22%20branch%3Amaster%20event%3Apush) +[![Cookstyle](https://github.com/openstreetmap/chef/actions/workflows/cookstyle.yml/badge.svg)](https://github.com/openstreetmap/chef/actions/workflows/cookstyle.yml) +[![Test Kitchen](https://github.com/openstreetmap/chef/actions/workflows/test-kitchen.yml/badge.svg)](https://github.com/openstreetmap/chef/actions/workflows/test-kitchen.yml) This repository manages the configuration of all the servers run by the OpenStreetMap Foundation's Operations Working Group. We use @@ -37,10 +37,12 @@ We use the 'Organization Repository' approach, where we have all our cookbooks i # Contributing -Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for more details. The guide also includes details on how to run the tests locally. +Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for more details, including how to run the tests locally. +If you need help, the operations team are available in [#osmf-operations on irc.oftc.net](https://irc.openstreetmap.org/) or via the Matrix IRC bridge in [#\_oftc_#osmf-operations](https://matrix.to/#/#_oftc_#osmf-operations:matrix.org). # Contact Us -* Twitter: [@OSM_Tech](https://twitter.com/OSM_Tech) -* Mastodon / Fediverse: [@OSM_Tech](https://en.osm.town/@osm_tech) -* IRC: [#OSM-Dev on irc.oftc.net](https://irc.openstreetmap.org/) +* Mastodon: [@osm_tech](https://en.osm.town/@osm_tech) +* IRC: [#osm-dev on irc.oftc.net](https://irc.openstreetmap.org/) or [#osmf-operations on irc.oftc.net](https://irc.openstreetmap.org/) +* Matrix (IRC bridge): [#\_oftc_#osmf-operations](https://matrix.to/#/#_oftc_#osmf-operations:matrix.org) +* Email: [operations@osmfoundation.org](mailto:operations@osmfoundation.org) diff --git a/cookbooks/accounts/files/default/craig/.ssh/authorized_keys b/cookbooks/accounts/files/default/craig/.ssh/authorized_keys new file mode 100644 index 000000000..101e5e13c --- /dev/null +++ b/cookbooks/accounts/files/default/craig/.ssh/authorized_keys @@ -0,0 +1,2 @@ +# DO NOT EDIT - This file is being maintained by Chef - use authorized_keys2 instead +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc26tRbrQoczW3UFfXkdt6auqFg/Ut6spGMT476fFsJFjaYp98E2lca2W9vyJq4nSn0tdxwcO4LGK1ACdhZ/81I/68d7CPv5zNjJMehgwQ1BJTM5HWaap08cEINZMQ0xt6Neyz+HIFiaJVzxmyLJCnaaCeQX/t2NmL+nQV6rJq4qS2L434Bw1qGM73zjNja4bB2IN0y5yWDRTSLg+t+DKH26DC4OJn4+pxKsyt2egB7MNj9my1MRcjPVeo/bxz3nWoxKtX9dWq9UFrd7trfSXK+7Y+9fFHl41rrrYbn3UFKcDL6Rzvp2bFytDW6FlWmuptGajWnm2HpqI69bsO7uw1 diff --git a/cookbooks/accounts/files/default/ignisf/.ssh/authorized_keys b/cookbooks/accounts/files/default/ignisf/.ssh/authorized_keys new file mode 100644 index 000000000..22b72c6f4 --- /dev/null +++ b/cookbooks/accounts/files/default/ignisf/.ssh/authorized_keys @@ -0,0 +1,2 @@ +# DO NOT EDIT - This file is being maintained by Chef - use authorized_keys2 instead +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwMHHDBdzFr39OGEtXYpRHXiZiCB5eHQXnPR9qKzSGaDm5WljLAYRQbXnX58lAgNJYyNV+81QK68U9pRJWO6VqBt3LP1triJ5uxiJIrLg72AQ7iKS3R8b62bG1reF2Uc1zOSPT3HvWOl0FURhkn1zmvs6aeCeI7rO3fwF8IOIkxw7mycPXSXXE7QmqgQ6y8uG8LhF303NethPYbIWpJR3UfQjg0z1tXDMt+yH3NM4vRcRHaA/C0BMX2qrCGT1dhRve0f8Zz8hN7FK+1Xt/BnhEzEwG73kYDaOGOBva+oHNqBEhq5JYP2sCQYYHuRT20aGzbNgAX8hbSgdiwYEaalXL ignisf-key diff --git a/cookbooks/accounts/files/default/milliams/.ssh/authorized_keys b/cookbooks/accounts/files/default/milliams/.ssh/authorized_keys index a1fa1cf3c..4f21fdb8e 100644 --- a/cookbooks/accounts/files/default/milliams/.ssh/authorized_keys +++ b/cookbooks/accounts/files/default/milliams/.ssh/authorized_keys @@ -1 +1 @@ -ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEVkoOPte6R6jN5w7yny+YLtoZGl/XLQL2aSjhgyNHrh matt@HEX +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMbllYzPMjPeGJ/4EAM8h4Bfhs1H56UpKU/dbV3ljBRT matt@HEX diff --git a/cookbooks/accounts/files/default/nmoore/.ssh/authorized_keys b/cookbooks/accounts/files/default/nmoore/.ssh/authorized_keys new file mode 100644 index 000000000..d91c729cf --- /dev/null +++ b/cookbooks/accounts/files/default/nmoore/.ssh/authorized_keys @@ -0,0 +1,5 @@ +# DO NOT EDIT - This file is being maintained by Chef - use authorized_keys2 instead +sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIFjD7MYD8g5MIKTGpwNcx/EylNXSY5AS8TJGyfVQ7ZqFAAAABHNzaDo= Natalie Fedora Laptop +sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIKZ3IIQmPhzExk5CVOEQ4laIPskN6eVxoTxI2eon7DHRAAAABHNzaDo= Backup Key +sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIAKenC+6zGmDyfbfTFDbjKjBbCcVGlH0iyR5+X7u5R7VAAAABHNzaDo= Necklace +sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIJ+eCC1p7DVkTk+L435dd2fS9PvaaFPoDn/sp87KpYceAAAABHNzaDo= desktop diff --git a/cookbooks/accounts/files/default/pablobm/.ssh/authorized_keys b/cookbooks/accounts/files/default/pablobm/.ssh/authorized_keys new file mode 100644 index 000000000..76bebad63 --- /dev/null +++ b/cookbooks/accounts/files/default/pablobm/.ssh/authorized_keys @@ -0,0 +1,4 @@ +# DO NOT EDIT - This file is being maintained by Chef - use authorized_keys2 instead +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDVeKfZvIPmNCskkO+5g2BeJMCdXrI/DbEQvoF8aqUvMyQXt0eOW+A1MMx041i1u+kU8mRYw2WYGU5yqGBOW42LRMEk/SlKMANfND6fvS3bSVgZn4WkUrGpbA66KeTH6qUuVV3/4zT5Y6CUsxy86ZGQWqvTepqzAUus9IHgvsM1f/F6dGIFtw599IEcX2WLeSgEcSY1cn67w6S0AHhQwdiqxPjohEmLMHiCmXGJPgYQP3vJLwn/EAKzOenVNIELg8nESYlgCgz63yKcecr+n18Kxw45ZYYZxpeH1yLMXJBdBpuHg+BhsV9YIAzE3I7IEabZgmCJFjmd8rUOypFaCatMPmr9mxOOYgYmkS4LdV/dU1RP24DLkN0MwBrOEYyRfcApqShItWw/Rx0k63pF6uiSBte4AMhR+YKD7l/xGGvjTA0F27+kO4/uVmaHG57p9kOMSqvPP109VJTQDNpzczYelBAZ7QfSkzPJW8n2qFPgioEHJRG0+Hk5a6pq+Wp3Iqc= pablobm@mbp +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCjZe6/hMqxN6oRRc5yWPPLR6mXWkNWLk1KljSTQarOaUFDW1NhWaeO3BCaymWPYfc+2F7GY2+ziWFXcNPBAbu1ibXpssdfxr7I+qFeThH8/OVkFnrA4lWjiCfor5SHtNt1W5yNMZe98P89KOgb6FYhoKgdCTo2Hl4/5vdUVUpekTge09zDlt+JlSOeJ3vJhPrQX8sEcrcJ706IDvDv1YCN1CHgF9uwv8XbOkUoxywIlPTqs3K5NQQ1EOqX0Y+PbjJuIqSKAECA2Zbv/KVS4yjhgdWNcqRrznekUtcNcEhmL4DoqIq+axTxeeFXEdbQRBkrUIDdiXTY+L3OAjcFDP9/AZyYpltpL13512lh8GsvcmBSPNqpcvpUUlhEE6xnhvar6mR7FATrjJjmTaU1gAoDF0plzKnaqM5WKmIpOw89PnW5cVGmBLhgtIkTm4WF028so7tdCnm8HFzG23vTIFyD2E0D66+yiz9TXxw77QppCahFebK/7a6SLnAV63KNwX2uo1AisyG8kNNlTLXdTIa9KIa0bGfUOvaQPO1rtrDDrGTyL4zE+wJt+Ga+LiQH4qGCjr+kfU46FIPdvpylWVRuESEna/Jvz3CdUqHiZlzh9uRMgC4JyrVtG3FnnMk9ynK0cwsODeufr6TNwiNtUDUO16MIj32k+mh/TdtYwTAhYQ== pablobm@justice +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICup2tp1MUoisn90x6EsAxcCMU4/fP0Pz1ibp6zMooUw pablobm@radch diff --git a/cookbooks/apache/attributes/default.rb b/cookbooks/apache/attributes/default.rb index 075e4c86f..4f4880509 100644 --- a/cookbooks/apache/attributes/default.rb +++ b/cookbooks/apache/attributes/default.rb @@ -26,7 +26,7 @@ default[:apache][:event][:threads_per_child] = 25 default[:apache][:event][:max_request_workers] = 150 default[:apache][:event][:max_connections_per_child] = 0 -default[:apache][:listen_address] = "*" +default[:apache][:listen_address] = "[::]" default[:apache][:buffered_logs] = true diff --git a/cookbooks/apache/recipes/default.rb b/cookbooks/apache/recipes/default.rb index a17e2f4cb..494bc4c3d 100644 --- a/cookbooks/apache/recipes/default.rb +++ b/cookbooks/apache/recipes/default.rb @@ -53,6 +53,7 @@ template "/etc/apache2/ports.conf" do owner "root" group "root" mode "644" + notifies :restart, "service[apache2]" end systemd_service "apache2" do @@ -72,6 +73,10 @@ apache_module "status" do variables :hosts => admins["hosts"] end +apache_conf "tokens" do + template "tokens.conf.erb" +end + if node[:apache][:evasive][:enable] apache_module "evasive" do conf "evasive.conf.erb" @@ -95,6 +100,8 @@ apache_module "ssl" apache_conf "ssl" do template "ssl.erb" + reload_apache false + restart_apache true # restart required for shared memory config changes end # Apache should only be started after modules enabled diff --git a/cookbooks/apache/resources/conf.rb b/cookbooks/apache/resources/conf.rb index 9f1efb24f..f1e56d065 100644 --- a/cookbooks/apache/resources/conf.rb +++ b/cookbooks/apache/resources/conf.rb @@ -26,6 +26,7 @@ property :cookbook, :kind_of => String property :template, :kind_of => String, :required => [:create] property :variables, :kind_of => Hash, :default => {} property :reload_apache, :kind_of => [TrueClass, FalseClass], :default => true +property :restart_apache, :kind_of => [TrueClass, FalseClass], :default => false action :create do create_conf @@ -86,4 +87,5 @@ end def after_created notifies :reload, "service[apache2]" if reload_apache + notifies :restart, "service[apache2]" if restart_apache end diff --git a/cookbooks/apache/templates/default/httpd.conf.erb b/cookbooks/apache/templates/default/httpd.conf.erb index 3f78187f8..bb549eb8e 100644 --- a/cookbooks/apache/templates/default/httpd.conf.erb +++ b/cookbooks/apache/templates/default/httpd.conf.erb @@ -61,7 +61,7 @@ AddDefaultCharset utf-8 # Add extra mime types AddType application/x-xz .xz -# Configure logging +# Configure log buffering BufferedLogs <%= node[:apache][:buffered_logs] ? "On" : "Off" %> # Define an extended log format that includes request time and SSL details diff --git a/cookbooks/apache/templates/default/ssl.erb b/cookbooks/apache/templates/default/ssl.erb index 81afb3de5..72ac3b857 100644 --- a/cookbooks/apache/templates/default/ssl.erb +++ b/cookbooks/apache/templates/default/ssl.erb @@ -5,11 +5,4 @@ SSLProtocol All -SSLv2 -SSLv3 SSLHonorCipherOrder On SSLCipherSuite <%= node[:ssl][:openssl_ciphers] %> -SSLUseStapling On -SSLStaplingResponderTimeout 5 -SSLStaplingErrorCacheTimeout 60 -SSLStaplingReturnResponderErrors off -SSLStaplingFakeTryLater off -SSLStaplingCache shmcb:${APACHE_RUN_DIR}/ssl_ocspcache(512000) - Header always set Strict-Transport-Security "<%= node[:ssl][:strict_transport_security] %>" "expr=%{HTTPS} == 'on'" diff --git a/cookbooks/apache/templates/default/tokens.conf.erb b/cookbooks/apache/templates/default/tokens.conf.erb new file mode 100644 index 000000000..bc05c9494 --- /dev/null +++ b/cookbooks/apache/templates/default/tokens.conf.erb @@ -0,0 +1,7 @@ +# DO NOT EDIT - This file is being maintained by Chef + +# Hide server version on error pages +ServerSignature Off + +# Only return Apache in server header +ServerTokens Prod diff --git a/cookbooks/apt/recipes/default.rb b/cookbooks/apt/recipes/default.rb index 1d876dfb0..62c6f8d40 100644 --- a/cookbooks/apt/recipes/default.rb +++ b/cookbooks/apt/recipes/default.rb @@ -48,7 +48,7 @@ if platform?("debian") archive_suites = %w[main updates backports security] archive_components = %w[main contrib non-free non-free-firmware] backport_packages = case node[:lsb][:codename] - when "bookworm" then %W[amd64-microcode exim4 firmware-free firmware-nonfree intel-microcode libosmium linux-signed-#{dpkg_arch} osm2pgsql otrs2 pyosmium smartmontools systemd cgi-mapserver] + when "bookworm" then %W[amd64-microcode exim4 firmware-free firmware-nonfree intel-microcode libosmium linux linux-base linux-signed-#{dpkg_arch} osm2pgsql otrs2 pyosmium smartmontools systemd cgi-mapserver] else %w[] end elsif intel? diff --git a/cookbooks/apt/recipes/elasticsearch6.rb b/cookbooks/apt/recipes/elasticsearch6.rb index 74c02a324..66844f61c 100644 --- a/cookbooks/apt/recipes/elasticsearch6.rb +++ b/cookbooks/apt/recipes/elasticsearch6.rb @@ -23,5 +23,5 @@ apt_repository "elasticsearch6.x" do uri "https://artifacts.elastic.co/packages/6.x/apt" distribution "stable" components ["main"] - key "D27D666CD88E42B4" + key "https://artifacts.elastic.co/GPG-KEY-elasticsearch" end diff --git a/cookbooks/apt/recipes/elasticsearch7.rb b/cookbooks/apt/recipes/elasticsearch7.rb index 6dc824177..57737b635 100644 --- a/cookbooks/apt/recipes/elasticsearch7.rb +++ b/cookbooks/apt/recipes/elasticsearch7.rb @@ -23,7 +23,7 @@ apt_repository "elasticsearch7.x" do uri "https://artifacts.elastic.co/packages/7.x/apt" distribution "stable" components ["main"] - key "D27D666CD88E42B4" + key "https://artifacts.elastic.co/GPG-KEY-elasticsearch" end # Workaround for mediawiki 1.39.x which ONLY supports elasticsearch 7.10.2 diff --git a/cookbooks/apt/recipes/elasticsearch8.rb b/cookbooks/apt/recipes/elasticsearch8.rb index 61a94f7aa..b9991e474 100644 --- a/cookbooks/apt/recipes/elasticsearch8.rb +++ b/cookbooks/apt/recipes/elasticsearch8.rb @@ -23,5 +23,5 @@ apt_repository "elasticsearch8.x" do uri "https://artifacts.elastic.co/packages/8.x/apt" distribution "stable" components ["main"] - key "D27D666CD88E42B4" + key "https://artifacts.elastic.co/GPG-KEY-elasticsearch" end diff --git a/cookbooks/apt/recipes/fullstaq-ruby.rb b/cookbooks/apt/recipes/fullstaq-ruby.rb index d9dad7d37..ddcf1a433 100644 --- a/cookbooks/apt/recipes/fullstaq-ruby.rb +++ b/cookbooks/apt/recipes/fullstaq-ruby.rb @@ -23,5 +23,5 @@ apt_repository "fullstaq-ruby" do uri "https://apt.fullstaqruby.org" distribution "#{node[:platform]}-#{node[:platform_version]}" components ["main"] - key "https://raw.githubusercontent.com/fullstaq-ruby/server-edition/main/fullstaq-ruby.asc" + key "394F883E0C43569450FDFB92A9AF1C7C2ED65CC0" # https://raw.githubusercontent.com/fullstaq-ruby/server-edition/main/fullstaq-ruby.asc end diff --git a/cookbooks/apt/recipes/grafana.rb b/cookbooks/apt/recipes/grafana.rb index ef6531c1f..27f202339 100644 --- a/cookbooks/apt/recipes/grafana.rb +++ b/cookbooks/apt/recipes/grafana.rb @@ -19,15 +19,9 @@ include_recipe "apt" -remote_file "/etc/apt/trusted.gpg.d/grafana.asc" do - source "https://packages.grafana.com/gpg.key" - owner "root" - group "root" - mode "644" -end - apt_repository "grafana" do - uri "https://packages.grafana.com/enterprise/deb" + uri "https://apt.grafana.com" distribution "stable" components ["main"] + key "https://apt.grafana.com/gpg.key" end diff --git a/cookbooks/apt/recipes/hwraid.rb b/cookbooks/apt/recipes/hwraid.rb index fa5d9e1f9..c8d99bdef 100644 --- a/cookbooks/apt/recipes/hwraid.rb +++ b/cookbooks/apt/recipes/hwraid.rb @@ -35,5 +35,5 @@ apt_repository "hwraid" do uri "https://hwraid.le-vert.net/#{platform_name}" distribution distribution_name components ["main"] - key "6005210E23B3D3B4" + key "https://hwraid.le-vert.net/debian/hwraid.le-vert.net.gpg.key" end diff --git a/cookbooks/apt/recipes/management-component-pack.rb b/cookbooks/apt/recipes/management-component-pack.rb index 92d81f989..01cf5cb92 100644 --- a/cookbooks/apt/recipes/management-component-pack.rb +++ b/cookbooks/apt/recipes/management-component-pack.rb @@ -28,38 +28,38 @@ if platform?("debian") uri "https://downloads.linux.hpe.com/SDR/repo/mcp" distribution "#{node[:lsb][:codename]}/current" components ["non-free"] - key "C208ADDE26C2B797" + key ["https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key1.pub", "https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key2.pub"] end - if node[:dmi][:system][:product_name].end_with?("Gen9") + if node.dig(:dmi, :system, :product_name).to_s.end_with?("Gen9") apt_repository "mcp-gen9" do uri "https://downloads.linux.hpe.com/SDR/repo/mcp" distribution "stretch/current-gen9" components ["non-free"] - key "C208ADDE26C2B797" + key ["https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key1.pub", "https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key2.pub"] end end elsif platform?("ubuntu") - if node[:dmi][:system][:product_name].end_with?("Gen10") + if node.dig(:dmi, :system, :product_name).to_s.end_with?("Gen10") apt_repository "mcp-jammy" do uri "https://downloads.linux.hpe.com/SDR/repo/mcp" distribution "jammy/current" components ["non-free"] - key "C208ADDE26C2B797" + key ["https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key1.pub", "https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key2.pub"] end apt_repository "mcp-focal-gen10" do uri "https://downloads.linux.hpe.com/SDR/repo/mcp" distribution "focal/current-gen10" components ["non-free"] - key "C208ADDE26C2B797" + key ["https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key1.pub", "https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key2.pub"] end else apt_repository "mcp-bionic-gen9" do uri "https://downloads.linux.hpe.com/SDR/repo/mcp" distribution "bionic/current-gen9" components ["non-free"] - key "C208ADDE26C2B797" + key ["https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key1.pub", "https://downloads.linux.hpe.com/SDR/hpePublicKey2048_key2.pub"] end end end diff --git a/cookbooks/apt/recipes/nginx.rb b/cookbooks/apt/recipes/nginx.rb index b80b8bb8b..117ef31a7 100644 --- a/cookbooks/apt/recipes/nginx.rb +++ b/cookbooks/apt/recipes/nginx.rb @@ -26,8 +26,7 @@ platform_name = if platform?("debian") end apt_repository "nginx" do - arch "amd64" uri "https://nginx.org/packages/#{platform_name}" components ["nginx"] - key "ABF5BD827BD9BF62" + key "https://nginx.org/keys/nginx_signing.key" end diff --git a/cookbooks/apt/recipes/nodesource.rb b/cookbooks/apt/recipes/nodesource.rb index 18d5d8be7..7a6278413 100644 --- a/cookbooks/apt/recipes/nodesource.rb +++ b/cookbooks/apt/recipes/nodesource.rb @@ -20,7 +20,7 @@ include_recipe "apt" apt_repository "nodesource" do - uri "https://deb.nodesource.com/node_20.x" + uri "https://deb.nodesource.com/node_22.x" distribution "nodistro" components ["main"] key "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" diff --git a/cookbooks/apt/recipes/passenger.rb b/cookbooks/apt/recipes/passenger.rb index 136175ba0..b7e208dd6 100644 --- a/cookbooks/apt/recipes/passenger.rb +++ b/cookbooks/apt/recipes/passenger.rb @@ -22,5 +22,5 @@ include_recipe "apt" apt_repository "passenger" do uri "https://oss-binaries.phusionpassenger.com/apt/passenger" components ["main"] - key "561F9B9CAC40B2F7" + key "https://oss-binaries.phusionpassenger.com/auto-software-signing-gpg-key.txt" end diff --git a/cookbooks/apt/recipes/postgresql.rb b/cookbooks/apt/recipes/postgresql.rb index 66e4c1c92..6fe6377d1 100644 --- a/cookbooks/apt/recipes/postgresql.rb +++ b/cookbooks/apt/recipes/postgresql.rb @@ -23,5 +23,5 @@ apt_repository "postgresql" do uri "https://apt.postgresql.org/pub/repos/apt" distribution "#{node[:lsb][:codename]}-pgdg" components ["main"] - key "7FCC7D46ACCC4CF8" + key "https://www.postgresql.org/media/keys/ACCC4CF8.asc" end diff --git a/cookbooks/apt/recipes/repository.rb b/cookbooks/apt/recipes/repository.rb index c4e30b57d..04942a2c4 100644 --- a/cookbooks/apt/recipes/repository.rb +++ b/cookbooks/apt/recipes/repository.rb @@ -62,7 +62,7 @@ execute "apt-generate-key" do not_if { ::Dir.exist?("/srv/apt.openstreetmap.org/.gnupg") } end -%w[focal jammy noble bookworm].each do |distribution| +%w[focal jammy noble bookworm trixie].each do |distribution| repository = "openstreetmap-#{distribution}" execute "aptly-repo-create-#{distribution}" do diff --git a/cookbooks/apt/recipes/yarn.rb b/cookbooks/apt/recipes/yarn.rb index 7451e96a9..fe5059bb3 100644 --- a/cookbooks/apt/recipes/yarn.rb +++ b/cookbooks/apt/recipes/yarn.rb @@ -23,5 +23,5 @@ apt_repository "yarn" do uri "https://dl.yarnpkg.com/debian" distribution "stable" components ["main"] - key "1646B01B86E50310" + key "https://dl.yarnpkg.com/debian/pubkey.gpg" end diff --git a/cookbooks/backup/templates/default/expire.cron.erb b/cookbooks/backup/templates/default/expire.cron.erb index e1ec74a08..e4c2fdfca 100644 --- a/cookbooks/backup/templates/default/expire.cron.erb +++ b/cookbooks/backup/templates/default/expire.cron.erb @@ -2,7 +2,7 @@ # DO NOT EDIT - This file is being maintained by Chef -for prefix in blogs chef-server chef-repository chef-git community forum git lists osm-blog osmf-crm osmf-ledgersmb wiki-wiki.osmfoundation.org osqa otrs prometheus sotm svn switch2osm trac wiki-board.osmfoundation.org wiki-dwg.osmfoundation.org wiki-mwg.osmfoundation.org wiki-wiki.openstreetmap.org +for prefix in blogs chef-server chef-repository chef-git community forum git lists osm-blog osm-blog-staging osmf-crm osmf-ledgersmb wiki-osmfoundation.org osqa otrs prometheus sotm svn switch2osm trac wiki-board.osmfoundation.org wiki-dwg.osmfoundation.org wiki-mwg.osmfoundation.org wiki-wiki.openstreetmap.org do /usr/local/bin/expire-backups --days=3 --weeks=3 --months=3 /store/backup $prefix done diff --git a/cookbooks/blog/recipes/staging.rb b/cookbooks/blog/recipes/staging.rb index 70ff3d8ad..19437de68 100644 --- a/cookbooks/blog/recipes/staging.rb +++ b/cookbooks/blog/recipes/staging.rb @@ -27,9 +27,11 @@ ssl_certificate "staging.blog.openstreetmap.org" do ] end -# passwords = data_bag_item("blog-staging", "passwords") +passwords = data_bag_item("blog-staging", "passwords") # wp2fa_encrypt_keys = data_bag_item("blog-staging", "wp2fa_encrypt_keys") +# The staging blog is under manual development by Mikel. Do not manage with Chef. + # directory "/srv/staging.blog.openstreetmap.org" do # owner "wordpress" # group "wordpress" @@ -132,10 +134,10 @@ end # group "wordpress" # end -# template "/etc/cron.daily/blog-staging-backup" do -# source "backup-staging.cron.erb" -# owner "root" -# group "root" -# mode "750" -# variables :passwords => passwords -# end +template "/etc/cron.daily/blog-staging-backup" do + source "backup-staging.cron.erb" + owner "root" + group "root" + mode "750" + variables :passwords => passwords +end diff --git a/cookbooks/blog/templates/default/backup-staging.cron.erb b/cookbooks/blog/templates/default/backup-staging.cron.erb index 35b0a77ff..7746b5ba2 100644 --- a/cookbooks/blog/templates/default/backup-staging.cron.erb +++ b/cookbooks/blog/templates/default/backup-staging.cron.erb @@ -1,21 +1,29 @@ -#!/bin/sh +#!/bin/bash # DO NOT EDIT - This file is being maintained by Chef +set -euo pipefail + +export ZSTD_CLEVEL=16 +export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp osm-blog-staging.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=osm-blog-staging-$D.tar.gz +B="osm-blog-staging-$D.tar.zst" -mkdir $T/osm-blog-staging-$D -echo '[mysqldump]' > $T/mysqldump.opts -echo 'user=osm-blog-staging-user' >> $T/mysqldump.opts -echo 'password=<%= @passwords["osm-blog-staging-user"] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt --no-tablespaces osm-blog-staging > $T/osm-blog-staging-$D/osm-blog-staging.sql -ln -s /srv/staging.blog.openstreetmap.org $T/osm-blog-staging-$D/www +mkdir "$T/osm-blog-staging-$D" +echo '[mysqldump]' > "$T/mysqldump.opts" +echo 'user=osm-blog-staging-user' >> "$T/mysqldump.opts" +echo 'password=<%= @passwords["osm-blog-staging-user"] %>' >> "$T/mysqldump.opts" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces --max-allowed-packet=1G osm-blog-staging > "$T/osm-blog-staging-$D/osm-blog-staging.sql" +ln -s /srv/staging.blog.openstreetmap.org "$T/osm-blog-staging-$D/www" -export RSYNC_RSH="ssh -ax" +set +e +nice tar --create --dereference --directory="$T" \ + --sort=name \ + --warning=no-file-changed \ + --warning=no-file-removed \ + "osm-blog-staging-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" -nice tar --create --dereference --directory=$T --warning=no-file-changed osm-blog-staging-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/blog/templates/default/backup.cron.erb b/cookbooks/blog/templates/default/backup.cron.erb index 9e3bea7d8..7561966e7 100644 --- a/cookbooks/blog/templates/default/backup.cron.erb +++ b/cookbooks/blog/templates/default/backup.cron.erb @@ -1,21 +1,29 @@ -#!/bin/sh +#!/bin/bash # DO NOT EDIT - This file is being maintained by Chef +set -euo pipefail + +export ZSTD_CLEVEL=16 +export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp osm-blog.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=osm-blog-$D.tar.gz +B="osm-blog-$D.tar.zst" -mkdir $T/osm-blog-$D -echo '[mysqldump]' > $T/mysqldump.opts -echo 'user=osm-blog-user' >> $T/mysqldump.opts -echo 'password=<%= @passwords["osm-blog-user"] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt --no-tablespaces osm-blog > $T/osm-blog-$D/osm-blog.sql -ln -s /srv/blog.openstreetmap.org $T/osm-blog-$D/www +mkdir "$T/osm-blog-$D" +echo '[mysqldump]' > "$T/mysqldump.opts" +echo 'user=osm-blog-user' >> "$T/mysqldump.opts" +echo 'password=<%= @passwords["osm-blog-user"] %>' >> "$T/mysqldump.opts" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces --max-allowed-packet=1G osm-blog > "$T/osm-blog-$D/osm-blog.sql" +ln -s /srv/blog.openstreetmap.org "$T/osm-blog-$D/www" -export RSYNC_RSH="ssh -ax" +set +e +nice tar --create --dereference --directory="$T" \ + --sort=name \ + --warning=no-file-changed \ + --warning=no-file-removed \ + "osm-blog-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" -nice tar --create --dereference --directory=$T --warning=no-file-changed osm-blog-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/blogs/recipes/default.rb b/cookbooks/blogs/recipes/default.rb index 86dfd3afe..026c1821f 100644 --- a/cookbooks/blogs/recipes/default.rb +++ b/cookbooks/blogs/recipes/default.rb @@ -67,6 +67,7 @@ bundle_exec "/srv/blogs.openstreetmap.org" do user "blogs" group "blogs" subscribes :run, "git[/srv/blogs.openstreetmap.org]", :immediately + retries 2 # May fail on first run due to faulty blogs end ssl_certificate "blogs.openstreetmap.org" do diff --git a/cookbooks/blogs/templates/default/backup.cron.erb b/cookbooks/blogs/templates/default/backup.cron.erb index c020a5e35..3c92c2fab 100644 --- a/cookbooks/blogs/templates/default/backup.cron.erb +++ b/cookbooks/blogs/templates/default/backup.cron.erb @@ -1,17 +1,22 @@ -#!/bin/sh +#!/bin/bash # DO NOT EDIT - This file is being maintained by Chef +set -euo pipefail + +export ZSTD_CLEVEL=16 +export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp blogs.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=blogs-$D.tar.gz +B="blogs-$D.tar.zst" -mkdir $T/blogs-$D +mkdir "$T/blogs-$D" sqlite3 /srv/blogs.openstreetmap.org/planet.db ".backup $T/blogs-$D/planet.db" -export RSYNC_RSH="ssh -ax" +nice tar --create --dereference --directory="$T" \ + --sort=name \ + "blogs-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" -nice tar --create --dereference --directory=$T blogs-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/chef/attributes/default.rb b/cookbooks/chef/attributes/default.rb index ce0c04e30..9b1ed6787 100644 --- a/cookbooks/chef/attributes/default.rb +++ b/cookbooks/chef/attributes/default.rb @@ -2,4 +2,4 @@ default[:chef][:server][:version] = "15.9.38" # Set the default client version -default[:chef][:client][:version] = "18.5.0" +default[:chef][:client][:version] = "18.8.54" diff --git a/cookbooks/chef/templates/default/logrotate.erb b/cookbooks/chef/templates/default/logrotate.erb index ecaf25219..3cfef03d9 100644 --- a/cookbooks/chef/templates/default/logrotate.erb +++ b/cookbooks/chef/templates/default/logrotate.erb @@ -4,6 +4,10 @@ rotate 12 weekly compress + compresscmd /usr/bin/zstd + compressext .zst + compressoptions -T0 -11 --quiet --long + uncompresscmd /usr/bin/unzstd postrotate systemctl try-restart chef-client.service endscript diff --git a/cookbooks/chef/templates/default/server-backup.cron.erb b/cookbooks/chef/templates/default/server-backup.cron.erb index 9b864768e..aabfc6c70 100644 --- a/cookbooks/chef/templates/default/server-backup.cron.erb +++ b/cookbooks/chef/templates/default/server-backup.cron.erb @@ -1,16 +1,25 @@ -#!/bin/sh +#!/bin/bash + +# DO NOT EDIT - This file is being maintained by Chef +set -euo pipefail + +export ZSTD_CLEVEL=16 +export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp chef-server.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=chef-server-$D.tar.gz +B="chef-server-$D.tar.zst" + +mkdir "$T/chef-server-$D" +chgrp opscode-pgsql "$T" "$T/chef-server-$D" +chmod g+rwx "$T" "$T/chef-server-$D" +sudo -u opscode-pgsql /opt/opscode/embedded/bin/pg_dumpall --file="$T/chef-server-$D/chef.dmp" --clean +ln -s /var/opt/opscode/bookshelf/data "$T/chef-server-$D/bookshelf" -mkdir $T/chef-server-$D -chgrp opscode-pgsql $T $T/chef-server-$D -chmod g+rwx $T $T/chef-server-$D -sudo -u opscode-pgsql /opt/opscode/embedded/bin/pg_dumpall --file=$T/chef-server-$D/chef.dmp --clean -ln -s /var/opt/opscode/bookshelf/data $T/chef-server-$D/bookshelf +nice tar --create --dereference --directory="$T" \ + --sort=name \ + "chef-server-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" -nice tar --create --dereference --directory=$T chef-server-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/civicrm/recipes/default.rb b/cookbooks/civicrm/recipes/default.rb index d10ed0788..9e7a3b02e 100644 --- a/cookbooks/civicrm/recipes/default.rb +++ b/cookbooks/civicrm/recipes/default.rb @@ -21,12 +21,13 @@ include_recipe "wordpress" include_recipe "mysql" package %w[ - php-xml - php-curl rsync - wkhtmltopdf + php-curl php-bcmath + php-fileinfo php-intl + php-mbstring + php-xml ] apache_module "rewrite" @@ -148,7 +149,7 @@ end execute "/opt/civicrm-#{civicrm_version}/civicrm" do action :nothing - command "rsync --archive --delete --delete-delay --delay-updates /opt/civicrm-#{civicrm_version}/civicrm/ #{civicrm_directory}" + command "rsync --archive --delete --delete-delay --delay-updates --preallocate /opt/civicrm-#{civicrm_version}/civicrm/ #{civicrm_directory}" user "wordpress" group "wordpress" subscribes :run, "archive_file[#{cache_dir}/civicrm-#{civicrm_version}-wordpress.zip]", :immediately diff --git a/cookbooks/civicrm/templates/default/backup.cron.erb b/cookbooks/civicrm/templates/default/backup.cron.erb index 7738f75ca..52678c1f7 100644 --- a/cookbooks/civicrm/templates/default/backup.cron.erb +++ b/cookbooks/civicrm/templates/default/backup.cron.erb @@ -1,21 +1,26 @@ -#!/bin/sh +#!/bin/bash # DO NOT EDIT - This file is being maintained by Chef +set -euo pipefail + +export ZSTD_CLEVEL=16 +export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp osmf-crm.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=osmf-crm-$D.tar.gz +B="osmf-crm-$D.tar.zst" -mkdir $T/osmf-crm-$D -echo '[mysqldump]' > $T/mysqldump.opts -echo 'user=civicrm' >> $T/mysqldump.opts -echo 'password=<%= @passwords["database"] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt --skip-lock-tables --no-tablespaces civicrm > $T/osmf-crm-$D/civicrm.sql -ln -s /srv/supporting.openstreetmap.org $T/osmf-crm-$D/www +mkdir "$T/osmf-crm-$D" +echo '[mysqldump]' > "$T/mysqldump.opts" +echo 'user=civicrm' >> "$T/mysqldump.opts" +echo 'password=<%= @passwords["database"] %>' >> "$T/mysqldump.opts" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --no-tablespaces --max-allowed-packet=1G civicrm > "$T/osmf-crm-$D/civicrm.sql" +ln -s /srv/supporting.openstreetmap.org "$T/osmf-crm-$D/www" -export RSYNC_RSH="ssh -ax" +nice tar --create --dereference --directory="$T" \ + --sort=name \ + "osmf-crm-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" -nice tar --create --dereference --directory=$T osmf-crm-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/clamav/recipes/default.rb b/cookbooks/clamav/recipes/default.rb index 938b2f5ab..34e25ddb6 100644 --- a/cookbooks/clamav/recipes/default.rb +++ b/cookbooks/clamav/recipes/default.rb @@ -22,14 +22,56 @@ include_recipe "accounts" package %w[ clamav-daemon clamav-freshclam - clamav-unofficial-sigs ] -template "/etc/clamav-unofficial-sigs.conf.d/50-chef.conf" do - source "clamav-unofficial-sigs.conf.erb" - owner "root" - group "root" - mode "644" +if platform?("debian") && node[:platform_version].to_i >= 13 + package "clamav-unofficial-sigs" do + action :remove + end + + package %w[ + fangfrisch + clamdscan + ] + + directory "/var/lib/fangfrisch" do + owner "clamav" + group "clamav" + mode "775" + end + + template "/etc/fangfrisch.conf" do + source "fangfrisch.conf.erb" + owner "root" + group "root" + mode "644" + end + + execute "fangfrisch-initdb" do + command "/usr/bin/fangfrisch --conf /etc/fangfrisch.conf initdb" + user "clamav" + group "clamav" + not_if do + ::File.exist?("/var/lib/fangfrisch/db.sqlite") + end + end + + service "fangfrisch.timer" do + action [:enable, :start] + end + + file "/etc/clamav-unofficial-sigs.conf.d/50-chef.conf" do + action :delete + end +else + package "clamav-unofficial-sigs" + + template "/etc/clamav-unofficial-sigs.conf.d/50-chef.conf" do + source "clamav-unofficial-sigs.conf.erb" + owner "root" + group "root" + mode "644" + end end execute "freshclam" do diff --git a/cookbooks/clamav/templates/default/fangfrisch.conf.erb b/cookbooks/clamav/templates/default/fangfrisch.conf.erb new file mode 100644 index 000000000..8015182b5 --- /dev/null +++ b/cookbooks/clamav/templates/default/fangfrisch.conf.erb @@ -0,0 +1,17 @@ +# DO NOT EDIT - This file is being maintained by Chef + +[DEFAULT] +db_url = sqlite:////var/lib/fangfrisch/db.sqlite + +local_directory = /var/lib/clamav +on_update_exec = clamdscan --reload +on_update_timeout = 42 + +[interserver] +enabled = yes + +[sanesecurity] +enabled = yes + +[urlhaus] +enabled = yes diff --git a/cookbooks/community/recipes/default.rb b/cookbooks/community/recipes/default.rb index 55238e824..e4a46f9fd 100644 --- a/cookbooks/community/recipes/default.rb +++ b/cookbooks/community/recipes/default.rb @@ -63,7 +63,7 @@ git "/srv/community.openstreetmap.org/docker" do action :sync repository "https://github.com/discourse/discourse_docker.git" # DANGER launch wrapper automatically updates git repo if rebuild method used: https://github.com/discourse/discourse_docker/blob/107ffb40fe8b1ea40e00814468db974a4f3f8e8f/launcher#L799 - revision "e42fa9711e9a8b27e9618342b5b456d3ba5b8025" + revision "be223ac03aa41e07ea1814038f20db0c5450e8d8" user "root" group "root" notifies :run, "notify_group[discourse_container_new_data]" @@ -204,6 +204,7 @@ execute "discourse_container_mail_receiver_bootstrap" do cwd "/srv/community.openstreetmap.org/docker/" user "root" group "root" + not_if { arm? } # Not yet supported on ARM https://github.com/discourse/mail-receiver/pull/28 end execute "discourse_container_mail_receiver_destroy" do @@ -212,6 +213,7 @@ execute "discourse_container_mail_receiver_destroy" do cwd "/srv/community.openstreetmap.org/docker/" user "root" group "root" + not_if { arm? } # Not yet supported on ARM https://github.com/discourse/mail-receiver/pull/28 end execute "discourse_container_mail_receiver_start" do @@ -220,6 +222,7 @@ execute "discourse_container_mail_receiver_start" do cwd "/srv/community.openstreetmap.org/docker/" user "root" group "root" + not_if { arm? } # Not yet supported on ARM https://github.com/discourse/mail-receiver/pull/28 end template "/etc/cron.daily/community-backup" do diff --git a/cookbooks/community/templates/default/backup.cron.erb b/cookbooks/community/templates/default/backup.cron.erb index d23cd47e9..c6e22550b 100644 --- a/cookbooks/community/templates/default/backup.cron.erb +++ b/cookbooks/community/templates/default/backup.cron.erb @@ -1,20 +1,29 @@ -#!/bin/sh +#!/bin/bash # DO NOT EDIT - This file is being maintained by Chef +set -euo pipefail + +export ZSTD_CLEVEL=16 +export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp community.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=community-$D.tar.gz +B="community-$D.tar.zst" -mkdir $T/community-$D -ln -s /srv/community.openstreetmap.org/docker/containers $T/community-$D/containers -ln -s /srv/community.openstreetmap.org/shared/web-only $T/community-$D/shared-web-only -ln -s /srv/community.openstreetmap.org/shared/data/redis_data $T/community-$D/shared-data-redis_data -ln -s /srv/community.openstreetmap.org/shared/data/postgres_backup $T/community-$D/shared-data-postgres_backup +mkdir "$T/community-$D" +ln -s /srv/community.openstreetmap.org/docker/containers "$T/community-$D/containers" +ln -s /srv/community.openstreetmap.org/shared/web-only "$T/community-$D/shared-web-only" +ln -s /srv/community.openstreetmap.org/shared/data/redis_data "$T/community-$D/shared-data-redis_data" +ln -s /srv/community.openstreetmap.org/shared/data/postgres_backup "$T/community-$D/shared-data-postgres_backup" -export RSYNC_RSH="ssh -ax" +set +e +nice tar --create --numeric-owner --dereference --directory="$T" \ + --sort=name \ + --warning=no-file-changed \ + --warning=no-file-removed \ + --exclude="community-$D/shared-web-only/log" \ + "community-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" -nice tar --create --numeric-owner --dereference --directory=$T --warning=no-file-changed community-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/community/templates/default/web_only.yml.erb b/cookbooks/community/templates/default/web_only.yml.erb index 587507a43..4d39d6ec2 100644 --- a/cookbooks/community/templates/default/web_only.yml.erb +++ b/cookbooks/community/templates/default/web_only.yml.erb @@ -1,7 +1,7 @@ templates: - "templates/web.template.yml" - - "templates/web.ipv6.template.yml" - "templates/web.ssl.template.yml" + - "templates/enable-ruby-yjit.yml" ## which TCP/IP ports should this container expose? ## If you want Discourse to share a port with another webserver like Apache or nginx, @@ -19,7 +19,7 @@ links: # any extra arguments for Docker? # docker_args: -# Latest Version v3.4.2 +# Latest Version v3.5.3 # Discourse only support tests-passed and stable branches params: version: stable @@ -96,41 +96,36 @@ volumes: host: /srv/community.openstreetmap.org/files/update-feeds.atom guest: /shared/feeds/update-feeds.atom -## Plugins go here -## see https://meta.discourse.org/t/19157 for details hooks: after_code: - exec: cd: $home/plugins cmd: - - sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-oauth2-basic.git - - sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-solved.git - - sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-reactions.git - - sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-prometheus.git - - sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-translator.git - - sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-saved-searches.git - - sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-post-voting.git + - if [ ! -d discourse-oauth2-basic ]; then sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-oauth2-basic.git; fi + - if [ ! -d discourse-solved ]; then sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-solved.git; fi + - if [ ! -d discourse-reactions ]; then sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-reactions.git; fi + - if [ ! -d discourse-prometheus ]; then sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-prometheus.git; fi + - if [ ! -d discourse-translator ]; then sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-translator.git; fi + - if [ ! -d discourse-saved-searches ]; then sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-saved-searches.git; fi + - if [ ! -d discourse-post-voting ]; then sudo -H -E -u discourse git clone --depth 1 --branch main https://github.com/discourse/discourse-post-voting.git; fi - exec: # Needs to be copied in else builtin git cleanup fails cd: $home cmd: - sudo -H -E -u discourse cp /shared/feeds/update-feeds.atom public/update-feeds.atom after_ssl: - - replace: - filename: "/etc/nginx/conf.d/discourse.conf" - from: /add_header.+/ - to: | - add_header Strict-Transport-Security 'max-age=63072000' always; - ssl_stapling on; + - file: + path: "/etc/nginx/conf.d/outlets/server/25-https-osm-settings.conf" + contents: | resolver <%= @resolvers.join(" ") %>; resolver_timeout 5s; ssl_dhparam /shared/ssl/dhparam.pem; +# hooks: and run: use own levels run: - replace: - filename: "/etc/nginx/conf.d/discourse.conf" + filename: "/usr/local/bin/configure-ssl" from: /listen 80;/ to: | listen 80; - listen [::]:80; - rewrite ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 permanent; + rewrite ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/\$1 permanent; diff --git a/cookbooks/db/attributes/default.rb b/cookbooks/db/attributes/default.rb index cbc5635d8..ef0410c1e 100644 --- a/cookbooks/db/attributes/default.rb +++ b/cookbooks/db/attributes/default.rb @@ -1,4 +1,4 @@ default[:db][:cluster] = "15/main" -default[:postgresql][:versions] |= ["15"] +default[:postgresql][:versions] |= %w[15 17] default[:postgresql][:monitor_database] = "openstreetmap" diff --git a/cookbooks/db/files/default/monthly-reindex.sql b/cookbooks/db/files/default/monthly-reindex.sql index ab9ae105b..fa8faff76 100644 --- a/cookbooks/db/files/default/monthly-reindex.sql +++ b/cookbooks/db/files/default/monthly-reindex.sql @@ -28,6 +28,8 @@ REINDEX (VERBOSE) TABLE CONCURRENTLY oauth_applications; REINDEX (VERBOSE) TABLE CONCURRENTLY redactions; REINDEX (VERBOSE) TABLE CONCURRENTLY reports; REINDEX (VERBOSE) TABLE CONCURRENTLY schema_migrations; +REINDEX (VERBOSE) TABLE CONCURRENTLY social_links; +REINDEX (VERBOSE) TABLE CONCURRENTLY spammy_phrases; REINDEX (VERBOSE) TABLE CONCURRENTLY user_blocks; REINDEX (VERBOSE) TABLE CONCURRENTLY user_mutes; REINDEX (VERBOSE) TABLE CONCURRENTLY user_preferences; diff --git a/cookbooks/db/recipes/base.rb b/cookbooks/db/recipes/base.rb index b25fbdbb2..aa1e29b70 100644 --- a/cookbooks/db/recipes/base.rb +++ b/cookbooks/db/recipes/base.rb @@ -23,7 +23,7 @@ include_recipe "postgresql" include_recipe "python" include_recipe "ruby" -wal_secrets = data_bag_item("db", "wal-secrets") +aws_credentials = data_bag_item("db", "aws") package %w[ cmake @@ -43,7 +43,7 @@ package %w[ git "/opt/osmdbt" do action :sync repository "https://github.com/openstreetmap/osmdbt.git" - revision "v0.5" + revision "v0.9" depth 1 user "root" group "root" @@ -97,5 +97,6 @@ template "/usr/local/bin/openstreetmap-wal-g" do owner "root" group "postgres" mode "750" - variables :s3_key => wal_secrets["s3_key"] + variables :aws_credentials => aws_credentials + sensitive true end diff --git a/cookbooks/db/recipes/master.rb b/cookbooks/db/recipes/master.rb index 3e57941f7..eb15a5b67 100644 --- a/cookbooks/db/recipes/master.rb +++ b/cookbooks/db/recipes/master.rb @@ -26,9 +26,8 @@ postgresql_user "tomh" do superuser true end -postgresql_user "matt" do +postgresql_user "grant" do cluster node[:db][:cluster] - superuser true end postgresql_user "openstreetmap" do @@ -182,6 +181,8 @@ PROMETHEUS_PERMISSIONS = { relations reports schema_migrations + social_links + spammy_phrases user_blocks user_mutes user_preferences @@ -201,7 +202,8 @@ PROMETHEUS_PERMISSIONS = { "planetdump" => PLANETDUMP_PERMISSIONS[table], "planetdiff" => PLANETDIFF_PERMISSIONS[table], "prometheus" => PROMETHEUS_PERMISSIONS[table], - "backup" => [:select] + "backup" => [:select], + "grant" => [:select] end end @@ -232,6 +234,8 @@ end oauth_openid_requests_id_seq redactions_id_seq reports_id_seq + social_links_id_seq + spammy_phrases_id_seq user_blocks_id_seq user_mutes_id_seq user_roles_id_seq @@ -244,7 +248,8 @@ end permissions "openstreetmap" => [:all], "rails" => [:usage], "cgimap" => CGIMAP_PERMISSIONS[sequence], - "backup" => [:select] + "backup" => [:select], + "grant" => [:select] end end diff --git a/cookbooks/db/templates/default/backup-db.erb b/cookbooks/db/templates/default/backup-db.erb index 573f31a5e..b3609f304 100644 --- a/cookbooks/db/templates/default/backup-db.erb +++ b/cookbooks/db/templates/default/backup-db.erb @@ -6,6 +6,6 @@ D=`date +%Y-%m-%d` F=/store/backup/osm-${D}.dmp pg_dump --user=backup --format=custom --file=$F openstreetmap && \ - rsync $F backup.openstreetmap.org::backup + rsync --preallocate $F backup.openstreetmap.org::backup rm -f $F diff --git a/cookbooks/db/templates/default/wal-e.erb b/cookbooks/db/templates/default/wal-e.erb deleted file mode 100644 index b4c13bd1d..000000000 --- a/cookbooks/db/templates/default/wal-e.erb +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -# DO NOT EDIT - This file is being maintained by Chef - -export WALE_S3_PREFIX="s3://openstreetmap-wal/" -export AWS_ACCESS_KEY_ID="AKIAIQX2LTDOBIW4CZUQ" -export AWS_SECRET_ACCESS_KEY="<%= @s3_key %>" -export AWS_REGION="eu-west-2" - -exec /usr/local/bin/wal-e "$@" < /dev/null diff --git a/cookbooks/db/templates/default/wal-g.erb b/cookbooks/db/templates/default/wal-g.erb index 2297fd4ab..434fba791 100644 --- a/cookbooks/db/templates/default/wal-g.erb +++ b/cookbooks/db/templates/default/wal-g.erb @@ -4,8 +4,8 @@ export WALG_S3_PREFIX="s3://openstreetmap-wal/" export WALG_COMPRESSION_METHOD="lz4" -export AWS_ACCESS_KEY_ID="AKIAIQX2LTDOBIW4CZUQ" -export AWS_SECRET_ACCESS_KEY="<%= @s3_key %>" +export AWS_ACCESS_KEY_ID="<%= @aws_credentials["wal_access_key_id"] %>" +export AWS_SECRET_ACCESS_KEY="<%= @aws_credentials["wal_secret_access_key"] %>" export AWS_REGION="eu-west-2" exec /usr/local/bin/wal-g "$@" < /dev/null diff --git a/cookbooks/dev/attributes/default.rb b/cookbooks/dev/attributes/default.rb index 4f16412c4..61887687b 100644 --- a/cookbooks/dev/attributes/default.rb +++ b/cookbooks/dev/attributes/default.rb @@ -1 +1,2 @@ -default[:dev][:rails] = {} +default[:dev][:rails][:postgresql_cluster] = "" +default[:dev][:rails][:sites] = {} diff --git a/cookbooks/dev/recipes/default.rb b/cookbooks/dev/recipes/default.rb index f42c22ee3..98e9ff068 100644 --- a/cookbooks/dev/recipes/default.rb +++ b/cookbooks/dev/recipes/default.rb @@ -146,6 +146,8 @@ package %w[ python3-venv r-base redis + siege + time tmux unrar unzip @@ -290,9 +292,11 @@ node[:postgresql][:versions].each do |version| package "postgresql-#{version}-postgis-3" end -if node[:postgresql][:clusters][:"15/main"] +rails_cluster = node[:dev][:rails][:postgresql_cluster] + +if node[:postgresql][:clusters][rails_cluster.to_sym] postgresql_user "apis" do - cluster "15/main" + cluster rails_cluster end template "/usr/local/bin/cleanup-rails-assets" do @@ -338,10 +342,10 @@ if node[:postgresql][:clusters][:"15/main"] end Dir.glob("/srv/*.apis.dev.openstreetmap.org").each do |dir| - node.default_unless[:dev][:rails][File.basename(dir).split(".").first] = {} + node.default_unless[:dev][:rails][:sites][File.basename(dir).split(".").first] = {} end - node[:dev][:rails].each do |name, details| + node[:dev][:rails][:sites].each do |name, details| database_name = details[:database] || "apis_#{name}" site_name = "#{name}.apis.dev.openstreetmap.org" site_directory = "/srv/#{name}.apis.dev.openstreetmap.org" @@ -355,12 +359,12 @@ if node[:postgresql][:clusters][:"15/main"] secret_key_base = persistent_token("dev", "rails", name, "secret_key_base") postgresql_database database_name do - cluster "15/main" + cluster rails_cluster owner "apis" end postgresql_extension "#{database_name}_btree_gist" do - cluster "15/main" + cluster rails_cluster database database_name extension "btree_gist" end @@ -407,7 +411,7 @@ if node[:postgresql][:clusters][:"15/main"] group "apis" repository details[:repository] revision details[:revision] - database_port node[:postgresql][:clusters][:"15/main"][:port] + database_port node[:postgresql][:clusters][rails_cluster.to_sym][:port] database_name database_name database_username "apis" email_from "OpenStreetMap " @@ -454,16 +458,10 @@ if node[:postgresql][:clusters][:"15/main"] group "apis" end - directory "#{cgimap_directory}/build" do - user "apis" - group "apis" - mode "0755" - end - execute "#{cgimap_directory}/CMakeLists.txt" do action :nothing - command "cmake .." - cwd "#{cgimap_directory}/build" + command "cmake -B build" + cwd cgimap_directory user "apis" group "apis" subscribes :run, "git[#{cgimap_directory}]", :immediately @@ -484,7 +482,7 @@ if node[:postgresql][:clusters][:"15/main"] group "root" mode "640" variables :cgimap_socket => "/run/cgimap-#{name}/socket", - :database_port => node[:postgresql][:clusters][:"15/main"][:port], + :database_port => node[:postgresql][:clusters][rails_cluster.to_sym][:port], :database_name => database_name, :log_directory => log_directory, :options => details[:cgimap_options] @@ -553,7 +551,7 @@ if node[:postgresql][:clusters][:"15/main"] postgresql_database database_name do action :drop - cluster "15/main" + cluster rails_cluster end end end diff --git a/cookbooks/dev/templates/default/apache.user.erb b/cookbooks/dev/templates/default/apache.user.erb index 0c51e44e0..d93aa9490 100644 --- a/cookbooks/dev/templates/default/apache.user.erb +++ b/cookbooks/dev/templates/default/apache.user.erb @@ -27,6 +27,12 @@ WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> processes CustomLog /var/log/apache2/<%= @user %>.dev.openstreetmap.org-access.log combined_extended ErrorLog /var/log/apache2/<%= @user %>.dev.openstreetmap.org-error.log + # Prevent abuse by an anonymous AI bot + RewriteCond %{REQUEST_METHOD} ^(GET|HEAD)$ + RewriteCond %{HTTP_REFERER} ^-?$ + RewriteCond %{HTTP_USER_AGENT} ((CriOS|Chrome)/[1-9][0-9]?\.0\.|Chrome/100\.0\.|Chrome/122\.0\.0\.0|(Firefox|FxiOS)/[1-6]?[0-9]\.|MSIE\ [5-9]\.0|Opera/[8-9]\.|Windows\ NT\ [3-5]\.|Version/[3-5]\.[0-1]) [NC] + RewriteRule ^ - [R=429,L] + RewriteCond <%= @directory %>%{REQUEST_FILENAME} -f RewriteRule ^/cgi-bin/(.*)$ /~<%= @user %>/cgi-bin/$1 [PT,L] @@ -49,7 +55,7 @@ WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> processes > - AllowOverride AuthConfig FileInfo Indexes Options=RailsBaseURI + AllowOverride AuthConfig FileInfo Indexes Options SymLinksIfOwnerMatch Indexes Includes Require all granted diff --git a/cookbooks/dev/templates/default/apis.html.erb b/cookbooks/dev/templates/default/apis.html.erb index d630014cb..22dfbbec8 100644 --- a/cookbooks/dev/templates/default/apis.html.erb +++ b/cookbooks/dev/templates/default/apis.html.erb @@ -12,7 +12,7 @@ for testing clients against or as a data sandbox.

Repository Revision -<% node[:dev][:rails].each do |name,details| -%> +<% node[:dev][:rails][:sites].each do |name,details| -%> <%= name %> <%= details[:repository] %> diff --git a/cookbooks/dev/templates/default/logrotate.apis.erb b/cookbooks/dev/templates/default/logrotate.apis.erb index bd984cdd0..8b6f4d162 100644 --- a/cookbooks/dev/templates/default/logrotate.apis.erb +++ b/cookbooks/dev/templates/default/logrotate.apis.erb @@ -6,11 +6,15 @@ rotate 7 compress delaycompress + compresscmd /usr/bin/zstd + compressext .zst + compressoptions -T0 -11 --quiet --long + uncompresscmd /usr/bin/unzstd notifempty create 0660 apis apis sharedscripts postrotate PASSENGER_INSTANCE_REGISTRY_DIR=<%= node[:passenger][:instance_registry_dir] %> /usr/bin/passenger-config restart-app --ignore-app-not-running <%= @rails_directory %> > /dev/null /bin/systemctl try-reload-or-restart cgimap@<%= @name %> - endscript + endscript } diff --git a/cookbooks/devices/metadata.rb b/cookbooks/devices/metadata.rb index 2f71f6857..a25b0e16f 100644 --- a/cookbooks/devices/metadata.rb +++ b/cookbooks/devices/metadata.rb @@ -6,3 +6,4 @@ description "Configures devices" version "0.1" supports "ubuntu" +depends "chef" diff --git a/cookbooks/devices/templates/default/udev.rules.erb b/cookbooks/devices/templates/default/udev.rules.erb index be9903891..f3b28f7f7 100644 --- a/cookbooks/devices/templates/default/udev.rules.erb +++ b/cookbooks/devices/templates/default/udev.rules.erb @@ -32,6 +32,10 @@ ACTION=="add", SUBSYSTEM=="block", ENV{ID_BUS}=="<%= device[:bus] %>", ENV{ID_SE <% end -%> <% end -%> +# Tune read ahead for ancient laptop disks in shenron +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", KERNEL=="sd?", ENV{ID_MODEL}=="HGST_HTE721010A9E630", ATTR{queue/read_ahead_kb}="512" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", KERNEL=="sd?", ENV{ID_MODEL}=="HGST_HTS725050A7E630", ATTR{queue/read_ahead_kb}="512" + # Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express Gigabit Ethernet Controller SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x10ec", ATTRS{device}=="0x8168", RUN+="/sbin/ethtool -K $name tso off gso off" @@ -95,47 +99,6 @@ SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x37d2 # Disable Firmware Based LLDP handler SUBSYSTEM=="net", ACTION=="add", ENV{INTERFACE}=="*", DRIVERS=="i40e", RUN+="/sbin/ethtool --set-priv-flags $name disable-fw-lldp on" -# Workaround unreliable Western Digital WD RE3/RE4 disks (ATA only) -# Set sufficent Linux subsystem timeout and fix severe NCQ performance issue -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD5002ABYS-02B1B0", ATTR{device/timeout}="90", ATTR{device/queue_depth}="1", ATTR{queue/nr_requests}="256" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", ATTR{device/timeout}="90", ATTR{device/queue_depth}="1", ATTR{queue/nr_requests}="256" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", ATTR{device/timeout}="90", ATTR{device/queue_depth}="1", ATTR{queue/nr_requests}="256" -# Disable Disk Write Cache, Set AAM and Power Management correctly -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", RUN+="/sbin/hdparm -q -W0 -q -M254 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", RUN+="/sbin/hdparm -q -W0 -q -M254 -q -B254 $env{DEVNAME}" - -# Set Disks TLED / SCT Error Recovery Control -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1002FBYS-02A6B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD1003FBYX-01Y7B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD5000AAKS-00A7B0", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD2000FYYZ-01UL1B2", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="TOSHIBA_DT01ACA300", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,70,70 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST31000340NS", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="HGST_HTS725050A7E630", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="HGST_HTE721010A9E630", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" - -# Add SSD optimisation -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="OCZ-VERTEX3", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="OCZ-VERTEX3", ATTR{queue/scheduler}="noop" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/scheduler}="noop" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/read_ahead_kb}="256" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/scheduler}="noop" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/read_ahead_kb}="256" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/scheduler}="noop" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/read_ahead_kb}="256" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST240FN0021", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST240FN0021", ATTR{queue/scheduler}="noop" - -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="SuperMicro_SSD", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="SuperMicro_SSD", ATTR{queue/scheduler}="noop" - # Delete failed disk in cmok ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST_M13FQBL", ENV{ID_SERIAL}=="ST_M13FQBL_QNR_BFW", ATTR{device/delete}="1" @@ -155,6 +118,8 @@ ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_MODEL}=="QEMU_HA ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_MODEL}=="QEMU_HARDDISK", ATTR{queue/scheduler}="noop" # Vendor is sometimes missing -# Increase default MD raid5/raid6 strip cache + group_thread_cnt -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{MD_LEVEL}=="raid5", ATTR{md/stripe_cache_size}="8192", ATTR{md/group_thread_cnt}="4" -ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{MD_LEVEL}=="raid6", ATTR{md/stripe_cache_size}="8192", ATTR{md/group_thread_cnt}="4" +# Tune md stripe cache and thread count for RAID-5 / RAID-6 arrays +<% + group_threads = [(node.cpu_cores.to_i / 2.0).round, 4].max +%> +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{MD_LEVEL}=="raid[56]", ATTR{md/stripe_cache_size}="8192", ATTR{md/group_thread_cnt}="<%= group_threads %>" diff --git a/cookbooks/dns/recipes/default.rb b/cookbooks/dns/recipes/default.rb index d5f0c1e60..e7ca4207a 100644 --- a/cookbooks/dns/recipes/default.rb +++ b/cookbooks/dns/recipes/default.rb @@ -42,7 +42,7 @@ package %w[ cache_dir = Chef::Config[:file_cache_path] -dnscontrol_version = "4.17.0" +dnscontrol_version = "4.28.2" dnscontrol_arch = if arm? "arm64" @@ -79,6 +79,18 @@ remote_directory "/srv/dns.openstreetmap.org/html" do files_mode "644" end +link "/srv/dns.openstreetmap.org/html/ipv4.json" do + to "/var/lib/dns/src/ipv4.json" + owner "root" + group "root" +end + +link "/srv/dns.openstreetmap.org/html/ipv6.json" do + to "/var/lib/dns/src/ipv6.json" + owner "root" + group "root" +end + zones = [] Dir.glob("/var/lib/dns/json/*.json").each do |kmlfile| diff --git a/cookbooks/elasticsearch/recipes/default.rb b/cookbooks/elasticsearch/recipes/default.rb index a48ee9987..63ed3fd45 100644 --- a/cookbooks/elasticsearch/recipes/default.rb +++ b/cookbooks/elasticsearch/recipes/default.rb @@ -36,6 +36,14 @@ template "/etc/elasticsearch/elasticsearch.yml" do notifies :restart, "service[elasticsearch]" end +systemd_service "elasticsearch-override" do + service "elasticsearch" + dropin "override" + timeout_start_sec 180 + timeout_stop_sec 180 + notifies :restart, "service[elasticsearch]" +end + service "elasticsearch" do action [:enable, :start] supports :status => true, :restart => true diff --git a/cookbooks/exim/attributes/default.rb b/cookbooks/exim/attributes/default.rb index 638ce5765..77c0907a0 100644 --- a/cookbooks/exim/attributes/default.rb +++ b/cookbooks/exim/attributes/default.rb @@ -8,5 +8,5 @@ default[:exim][:smtp_accept_max] = 20 default[:exim][:smarthost_name] = nil default[:exim][:smarthost_via] = "mail.openstreetmap.org:26" default[:exim][:routes] = {} -default[:exim][:aliases][:root] = "tomh" +default[:exim][:aliases][:root] = "tomh, grant" default[:exim][:rewrites] = [] diff --git a/cookbooks/exim/recipes/default.rb b/cookbooks/exim/recipes/default.rb index e839d6045..21a076d56 100644 --- a/cookbooks/exim/recipes/default.rb +++ b/cookbooks/exim/recipes/default.rb @@ -47,6 +47,10 @@ end if node[:exim][:certificate_names] include_recipe "apache" + apache_site "default" do + action [:disable] + end + apache_site node[:exim][:certificate_names].first do template "apache.erb" variables :aliases => node[:exim][:certificate_names].drop(1) diff --git a/cookbooks/exim/templates/default/exim4.conf.erb b/cookbooks/exim/templates/default/exim4.conf.erb index 3558af884..71996f148 100644 --- a/cookbooks/exim/templates/default/exim4.conf.erb +++ b/cookbooks/exim/templates/default/exim4.conf.erb @@ -638,7 +638,7 @@ mailman: local_part_suffix = -bounces : -bounces+* : \ -confirm+* : -join : -leave : \ -subscribe : -unsubscribe : \ - -owner : -request : -admin + -owner : -request : -admin local_part_suffix_optional transport = mailman @@ -737,6 +737,9 @@ begin transports remote_smtp: driver = smtp multi_domain = false +<% if node.platform?("debian") || (node.platform?("ubuntu") && node[:lsb][:release].to_f >= 22.04) -%> + message_linelength_limit = 1G +<% end -%> tls_require_ciphers = <%= node[:ssl][:gnutls_ciphers] %>:%LATEST_RECORD_VERSION @@ -750,7 +753,11 @@ signed_smtp: dkim_private_key = /etc/exim4/dkim-keys/${dkim_domain} dkim_identity = ${lc:${address:$h_from:}} dkim_timestamps = 1209600 +<% if node.platform?("debian") || (node.platform?("ubuntu") && node[:lsb][:release].to_f >= 22.04) -%> + message_linelength_limit = 1G +<% end -%> multi_domain = false + max_rcpt = 20 hosts_try_dane = tls_require_ciphers = <%= node[:ssl][:gnutls_ciphers] %>:%LATEST_RECORD_VERSION <% if node[:exim][:external_interface] -%> diff --git a/cookbooks/fail2ban/recipes/default.rb b/cookbooks/fail2ban/recipes/default.rb index a218eb580..f473c47f1 100644 --- a/cookbooks/fail2ban/recipes/default.rb +++ b/cookbooks/fail2ban/recipes/default.rb @@ -26,7 +26,7 @@ package %w[ logrotate ] -if platform?("debian") +if platform?("debian") || (platform?("ubuntu") && node[:lsb][:release].to_f >= 24.04) package "python3-inotify" else package "gamin" diff --git a/cookbooks/foundation/files/default/.well-known/funding-manifest-urls b/cookbooks/foundation/files/default/.well-known/funding-manifest-urls new file mode 100644 index 000000000..a51eb6de3 --- /dev/null +++ b/cookbooks/foundation/files/default/.well-known/funding-manifest-urls @@ -0,0 +1 @@ +https://www.openstreetmap.org/funding.json \ No newline at end of file diff --git a/cookbooks/foundation/files/default/osm_logo.svg b/cookbooks/foundation/files/default/osm_logo.svg new file mode 100644 index 000000000..4189d70e7 --- /dev/null +++ b/cookbooks/foundation/files/default/osm_logo.svg @@ -0,0 +1,323 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cookbooks/foundation/recipes/board.rb b/cookbooks/foundation/recipes/board.rb index 597733e42..1d8d052e8 100644 --- a/cookbooks/foundation/recipes/board.rb +++ b/cookbooks/foundation/recipes/board.rb @@ -34,7 +34,7 @@ mediawiki_site "board.osmfoundation.org" do email_sender "wiki@noreply.openstreetmap.org" email_sender_name "OSMF Board Wiki" private_site true - version "1.39" + version "1.43" end cookbook_file "/srv/board.osmfoundation.org/Wiki.png" do diff --git a/cookbooks/foundation/recipes/dwg.rb b/cookbooks/foundation/recipes/dwg.rb index b8fb93f61..f7d64dbc3 100644 --- a/cookbooks/foundation/recipes/dwg.rb +++ b/cookbooks/foundation/recipes/dwg.rb @@ -35,7 +35,7 @@ mediawiki_site "dwg.osmfoundation.org" do email_sender_name "OSMF Board Wiki" private_site true extra_file_extensions %w[pptx] - version "1.39" + version "1.43" end cookbook_file "/srv/dwg.osmfoundation.org/Wiki.png" do diff --git a/cookbooks/foundation/recipes/mwg.rb b/cookbooks/foundation/recipes/mwg.rb index e9263ba0a..580c732e6 100644 --- a/cookbooks/foundation/recipes/mwg.rb +++ b/cookbooks/foundation/recipes/mwg.rb @@ -34,7 +34,7 @@ mediawiki_site "mwg.osmfoundation.org" do email_sender "wiki@noreply.openstreetmap.org" email_sender_name "OSMF Board Wiki" private_site true - version "1.39" + version "1.43" end cookbook_file "/srv/mwg.osmfoundation.org/Wiki.png" do diff --git a/cookbooks/foundation/recipes/wiki.rb b/cookbooks/foundation/recipes/wiki.rb index ddbe4705d..23a6686da 100644 --- a/cookbooks/foundation/recipes/wiki.rb +++ b/cookbooks/foundation/recipes/wiki.rb @@ -21,6 +21,8 @@ include_recipe "mediawiki" passwords = data_bag_item("foundation", "passwords") +package "lua5.1" # newer versions do not work with Scribuntu! + mediawiki_site "osmfoundation.org" do aliases ["wiki.osmfoundation.org", "www.osmfoundation.org", "foundation.openstreetmap.org", "foundation.osm.org"] @@ -34,24 +36,30 @@ mediawiki_site "osmfoundation.org" do database_user "osmf-wikiuser" database_password passwords["wiki"]["database"] admin_password passwords["wiki"]["admin"] - skin "OSMFoundation" - logo "/w/skins/OSMFoundation/img/logo.png" + skin "vector-2022" + logo "/osm_logo.svg" email_contact "webmaster@openstreetmap.org" email_sender "wiki@noreply.openstreetmap.org" email_sender_name "OSMF Wiki" private_accounts true - extra_file_extensions %w[mp3 pptx] - version "1.39" + extra_file_extensions %w[mp3 pptx txt] + version "1.43" end -mediawiki_skin "OSMFoundation" do +# mediawiki_skin "OSMFoundation" do +# site "osmfoundation.org" +# repository "https://github.com/osmfoundation/osmf-mediawiki-skin.git" +# revision "master" +# legacy false +# end + +mediawiki_extension "Scribunto" do site "osmfoundation.org" - repository "https://github.com/osmfoundation/osmf-mediawiki-skin.git" - revision "master" - legacy false + template "mw-ext-Scribunto.inc.php.erb" + template_cookbook "foundation" end -cookbook_file "/srv/osmfoundation.org/Wiki.png" do +cookbook_file "/srv/osmfoundation.org/osm_logo.svg" do owner node[:mediawiki][:user] group node[:mediawiki][:group] mode "644" @@ -63,3 +71,16 @@ template "/srv/osmfoundation.org/robots.txt" do mode "644" source "robots.txt.erb" end + +directory "/srv/osmfoundation.org/.well-known" do + owner node[:mediawiki][:user] + group node[:mediawiki][:group] + mode "755" +end + +cookbook_file "/srv/osmfoundation.org/.well-known/funding-manifest-urls" do + owner node[:mediawiki][:user] + group node[:mediawiki][:group] + mode "644" + source ".well-known/funding-manifest-urls" +end diff --git a/cookbooks/foundation/templates/default/mw-ext-Scribunto.inc.php.erb b/cookbooks/foundation/templates/default/mw-ext-Scribunto.inc.php.erb new file mode 100644 index 000000000..589817a08 --- /dev/null +++ b/cookbooks/foundation/templates/default/mw-ext-Scribunto.inc.php.erb @@ -0,0 +1,21 @@ +-access.log combined_extended ErrorLog /var/log/apache2/<%= @name %>-error.log + Alias /robots.txt /srv/<%= node[:git][:host] %>/robots.txt + + + # Make absolutely sure it comes out as a plain file + SetHandler none + Require all granted + + SetEnv GIT_PROJECT_ROOT /var/lib/git SetEnv GIT_HTTP_EXPORT_ALL SetEnv GIT_HTTP_MAX_REQUEST_BUFFER 100M @@ -59,6 +67,20 @@ RewriteRule ^/gpx-import\.git.* https://github.com/openstreetmap/gpx-import [QSD,L,R=permanent] RewriteRule ^/potlatch2\.git.* https://github.com/openstreetmap/potlatch2 [QSD,L,R=permanent] + # Prevent abuse by an anonymous AI bot + RewriteCond %{REQUEST_METHOD} ^(GET|HEAD)$ + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/blob [OR] + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/commitdiff [OR] + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/history [OR] + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/log [OR] + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/patch [OR] + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/search [OR] + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/shortlog [OR] + RewriteCond %{REQUEST_URI} ^/[^/]+\.git/tree + RewriteCond %{HTTP_REFERER} ^-?$ + RewriteCond %{HTTP_USER_AGENT} ((CriOS|Chrome)/[1-9][0-9]?\.0\.|Chrome/100\.0\.|Chrome/122\.0\.0\.0|(Firefox|FxiOS)/[1-6]?[0-9]\.|MSIE\ [5-9]\.0|Opera/[8-9]\.|Windows\ NT\ [3-5]\.|Version/[3-5]\.[0-1]) [NC] + RewriteRule ^ - [R=429,L] + ScriptAlias /public /usr/lib/git-core/git-http-backend/public ScriptAlias /private /usr/lib/git-core/git-http-backend/private Alias /gitweb /usr/share/gitweb diff --git a/cookbooks/git/templates/default/backup.cron.erb b/cookbooks/git/templates/default/backup.cron.erb index b3404c7d5..7de75fd93 100644 --- a/cookbooks/git/templates/default/backup.cron.erb +++ b/cookbooks/git/templates/default/backup.cron.erb @@ -1,16 +1,21 @@ -#!/bin/sh +#!/bin/bash # DO NOT EDIT - This file is being maintained by Chef +set -euo pipefail + +export ZSTD_CLEVEL=16 +export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp git.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=git-$D.tar.gz +B="git-$D.tar.zst" -ln -s /var/lib/git $T/git-$D +ln -s /var/lib/git "$T/git-$D" -export RSYNC_RSH="ssh -ax" +nice tar --create --dereference --directory="$T" \ + --sort=name \ + "git-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" -nice tar --create --dereference --directory=$T git-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/git/templates/default/robots.txt.erb b/cookbooks/git/templates/default/robots.txt.erb index b60d5e279..61efcd142 100644 --- a/cookbooks/git/templates/default/robots.txt.erb +++ b/cookbooks/git/templates/default/robots.txt.erb @@ -1,4 +1,14 @@ # DO NOT EDIT - This file is being maintained by Chef User-agent: * -Disallow: /*/snapshot/ +Disallow: /*a=search* +Disallow: /*/search/* +Disallow: /*a=blobdiff* +Disallow: /*/blobdiff/* +Disallow: /*/blobdiff_plain/* +Disallow: /*a=commitdiff* +Disallow: /*/commitdiff/* +Disallow: /*a=snapshot* +Disallow: /*/snapshot/* +Disallow: /*a=blame* +Disallow: /*/blame/* diff --git a/cookbooks/gps-tile/templates/default/apache.erb b/cookbooks/gps-tile/templates/default/apache.erb index a1b7ba94c..65ef9f2b4 100644 --- a/cookbooks/gps-tile/templates/default/apache.erb +++ b/cookbooks/gps-tile/templates/default/apache.erb @@ -25,7 +25,6 @@ # Setup logging CustomLog /var/log/apache2/access.log combined_extended ErrorLog /var/log/apache2/error.log - BufferedLogs on # Always set Access-Control-Allow-Origin so that simple CORS requests # will always work and can be cached @@ -53,7 +52,6 @@ # Setup logging CustomLog /var/log/apache2/access.log combined_extended ErrorLog /var/log/apache2/error.log - BufferedLogs on diff --git a/cookbooks/hardware/attributes/default.rb b/cookbooks/hardware/attributes/default.rb index 218e6228e..303b1efd0 100644 --- a/cookbooks/hardware/attributes/default.rb +++ b/cookbooks/hardware/attributes/default.rb @@ -1,4 +1,4 @@ -default[:hardware][:modules] = %w[lp] +default[:hardware][:modules] = %w[] default[:hardware][:blacklisted_modules] = %w[] default[:hardware][:grub][:cmdline] = %w[nomodeset] default[:hardware][:sensors] = {} diff --git a/cookbooks/hardware/recipes/default.rb b/cookbooks/hardware/recipes/default.rb index 4340bf504..73fc91bf6 100644 --- a/cookbooks/hardware/recipes/default.rb +++ b/cookbooks/hardware/recipes/default.rb @@ -410,8 +410,8 @@ intel_nvmes = nvmes.select { |pci| pci[:vendor_name] == "Intel Corporation" } if !intel_ssds.empty? || !intel_nvmes.empty? package "unzip" - sst_tool_version = "2-0" - sst_package_version = "2.0.300-0" + sst_tool_version = "2-4" + sst_package_version = "2.4.323-0" remote_file "#{Chef::Config[:file_cache_path]}/sst-cli-linux-deb--#{sst_tool_version}.zip" do source "https://sdmsdfwdriver.blob.core.windows.net/files/kba-gcc/drivers-downloads/ka-00085/sst--#{sst_tool_version}/sst-cli-linux-deb--#{sst_tool_version}.zip" @@ -464,7 +464,7 @@ end disks = disks.compact.uniq -if disks.count.positive? +if disks.any? package "smartmontools" template "/etc/cron.daily/update-smart-drivedb" do @@ -566,13 +566,6 @@ if watchdog_module action :install end - execute "systemctl-reload" do - action :nothing - command "systemctl daemon-reload" - user "root" - group "root" - end - directory "/etc/systemd/system.conf.d" do owner "root" group "root" @@ -584,7 +577,14 @@ if watchdog_module owner "root" group "root" mode "644" - notifies :run, "execute[systemctl-reload]" + end + + execute "systemctl-reload" do + action :nothing + command "systemctl daemon-reload" + user "root" + group "root" + subscribes :run, "template[/etc/systemd/system.conf.d/watchdog.conf]" end end diff --git a/cookbooks/imagery/files/default/transparent.png b/cookbooks/imagery/files/default/transparent.png new file mode 100644 index 000000000..26134c4c1 Binary files /dev/null and b/cookbooks/imagery/files/default/transparent.png differ diff --git a/cookbooks/imagery/recipes/au_act_aerial.rb b/cookbooks/imagery/recipes/au_act_aerial.rb index cc0135998..f50fb55da 100644 --- a/cookbooks/imagery/recipes/au_act_aerial.rb +++ b/cookbooks/imagery/recipes/au_act_aerial.rb @@ -27,14 +27,74 @@ imagery_site "act-imagery.openstreetmap.org" do bbox [[-35.942, 148.729], [-35.117, 149.430]] end +imagery_layer "act_aerial_imagery_latest" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery latest" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_Current/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_Current/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 + default_layer true +end + +imagery_layer "act_aerial_imagery_202505" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202505" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2025_05_urban_75mm/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202411/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +imagery_layer "act_aerial_imagery_202503" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202503" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2025_03_urban_75mm/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202411/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +imagery_layer "act_aerial_imagery_202411" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202411" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2024_11_full_75mm/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202411/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + imagery_layer "act_aerial_imagery_202409" do site "act-imagery.openstreetmap.org" title "ACT Aerial Imagery 202409" projection "EPSG:7855" - source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202409/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" - # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202409/MapServer/ + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2024_09_urban_75mm/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2024_09_urban_75mm/MapServer/ + copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " + background_colour "0 0 0" + extension "jpeg" + max_zoom 22 +end + +imagery_layer "act_aerial_imagery_202402" do + site "act-imagery.openstreetmap.org" + title "ACT Aerial Imagery 202402" + projection "EPSG:7855" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2024_02_urban_75mm/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2024_09_urban_75mm/MapServer/ copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " - default_layer true background_colour "0 0 0" extension "jpeg" max_zoom 22 @@ -44,7 +104,7 @@ imagery_layer "act_aerial_imagery_202311" do site "act-imagery.openstreetmap.org" title "ACT Aerial Imagery 202311" projection "EPSG:7855" - source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202311/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" + source "https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/2023_11_full_75mm/MapServer/WMTS/1.0.0/WMTSCapabilities.xml" # attribution per https://www.actmapi.act.gov.au/terms-and-conditions and https://tiles.arcgis.com/tiles/E5n4f1VY84i0xSjy/arcgis/rest/services/ACT_Aerial_Imagery_202311/MapServer/ copyright "ACT Imagery from ACTmapi (c) Australian Capital Territory and MetroMap. " background_colour "0 0 0" @@ -52,7 +112,9 @@ imagery_layer "act_aerial_imagery_202311" do max_zoom 22 end +# 2025 August - No longer available - appears password protected imagery_layer "act_aerial_imagery_202305" do + action :delete site "act-imagery.openstreetmap.org" title "ACT Aerial Imagery 202305" projection "EPSG:7855" diff --git a/cookbooks/imagery/recipes/bg_imagery.rb b/cookbooks/imagery/recipes/bg_imagery.rb new file mode 100644 index 000000000..5602f83e9 --- /dev/null +++ b/cookbooks/imagery/recipes/bg_imagery.rb @@ -0,0 +1,37 @@ +# +# Cookbook:: imagery +# Recipe:: bg_imagery +# +# Copyright:: 2025, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery" + +imagery_site "bg-imagery.openstreetmap.org" do + title "OpenStreetMap - Bulgaria - Aerial Imagery" + aliases ["bg-imagery.osm.org"] + bbox [[41.235, 22.357], [44.215, 28.608]] +end + +imagery_layer "maf-orthophoto-latest" do + site "bg-imagery.openstreetmap.org" + title "Bulgaria MAF Orthophoto Latest" + source "/store/imagery/bg/maf-orthophoto-map/maf-orthophoto.vrt" + copyright "(c) Ministry of Agriculture and Food of Bulgaria" + projection "EPSG:32635" + max_zoom 20 + default_layer true + revision 2 +end diff --git a/cookbooks/imagery/recipes/br_imagery.rb b/cookbooks/imagery/recipes/br_imagery.rb new file mode 100644 index 000000000..d276593a1 --- /dev/null +++ b/cookbooks/imagery/recipes/br_imagery.rb @@ -0,0 +1,37 @@ +# +# Cookbook:: imagery +# Recipe:: br_imagery +# +# Copyright:: 2025, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery" + +imagery_site "br-imagery.openstreetmap.org" do + title "OpenStreetMap - Brazil - Aerial Imagery" + aliases ["br-imagery.osm.org"] + bbox [[-23.9813, -46.6594], [-23.6398, -46.4042]] +end + +imagery_layer "ibge-aerial-2021" do + site "br-imagery.openstreetmap.org" + title "Brazil IBGE Aerial Imagery 2021" + source "/store/imagery/br/ibge-aerial-2021/ibge-aerial-2021.webp.google.r_bilinear.bs_256.aligned.cog.tif" + copyright '(c) IBGE' + projection "EPSG:3857" + max_zoom 21 + default_layer true + revision 1 +end diff --git a/cookbooks/imagery/recipes/default.rb b/cookbooks/imagery/recipes/default.rb index 7c1a1f88c..edc860965 100644 --- a/cookbooks/imagery/recipes/default.rb +++ b/cookbooks/imagery/recipes/default.rb @@ -33,6 +33,7 @@ package %w[ cgi-mapserver mapcache-cgi mapcache-tools + libtcmalloc-minimal4 ] # Mapserver via nginx requires as fastcgi spawner @@ -44,10 +45,11 @@ package %w[ # Imagery processing Requirements package "imagemagick" -# Imagery misc compression +# Imagery misc utilities package %w[ xz-utils unzip + aria2 ] template "/etc/mapserver.conf" do @@ -96,3 +98,7 @@ systemd_tmpfile "/run/mapserver-fastcgi" do mode "0755" not_if { kitchen? } end + +service "systemd-coredump.socket" do + action [ :stop, :disable ] +end diff --git a/cookbooks/imagery/recipes/gb_os_sv.rb b/cookbooks/imagery/recipes/gb_os_sv.rb index 2345cc1e8..43e73a87f 100644 --- a/cookbooks/imagery/recipes/gb_os_sv.rb +++ b/cookbooks/imagery/recipes/gb_os_sv.rb @@ -1215,6 +1215,28 @@ imagery_layer "gb_os_om_local_2024_10" do copyright "Contains OS data © Crown copyright and database right 2024" background_colour "213 244 248" # OS OpenMap Local Water Blue extension "os_om_local_png" - url_aliases ["/om-local-2024-10", "/om-local", "/sv"] +end + +imagery_layer "gb_os_om_local_2025_04" do + site "os.openstreetmap.org" + title "OS OpenMap Local - April 2025" + projection "EPSG:27700" + source "/store/imagery/gb/openmap-local/2025-04/os-openmap-local-2025-04.vrt" + copyright "Contains OS data © Crown copyright and database right 2025" + background_colour "213 244 248" # OS OpenMap Local Water Blue + extension "os_om_local_png" + revision 2 +end + +imagery_layer "gb_os_om_local_2025_10" do + site "os.openstreetmap.org" + title "OS OpenMap Local - October 2025" + projection "EPSG:27700" + source "/store/imagery/gb/openmap-local/2025-10/os-openmap-local-2025-10.vrt" + copyright "Contains OS data © Crown copyright and database right 2025" + background_colour "213 244 248" # OS OpenMap Local Water Blue + extension "os_om_local_png" + url_aliases ["/om-local-2025-10", "/om-local", "/sv"] default_layer true + revision 2 end diff --git a/cookbooks/imagery/recipes/na_sgswa_topo.rb b/cookbooks/imagery/recipes/na_sgswa_topo.rb index 686e10dda..af6692f5a 100644 --- a/cookbooks/imagery/recipes/na_sgswa_topo.rb +++ b/cookbooks/imagery/recipes/na_sgswa_topo.rb @@ -29,12 +29,21 @@ imagery_layer "na_sgswa_topo_50k" do site "namibia-topo.openstreetmap.org.za" title "Namibia Topo 50k" projection "EPSG:4326" - source "/store/imagery/na/topo-50k/namibia-50k-topo-v2.vrt" - copyright "State Copyright © 1958 - 1991; Surveyor-General, Windhoek, SWA; CDSM: Chief Directorate Surveys & Mapping, Mowbray, RSA" + source "/store/imagery/na/topo-50k/namibia-50k-topo-v4-alpha-hidenodata.vrt" + copyright "State Copyright © 1958 - 1991; Surveyor-General, Windhoek, Namibia" default_layer true - background_colour "0 0 0" - extension "jpeg" max_zoom 16 + revision 4 +end + +imagery_layer "na_sgswa_topo_250k" do + site "namibia-topo.openstreetmap.org.za" + title "Namibia Topo 250k" + projection "EPSG:4326" + source "/store/imagery/na/topo-250k/new/combined.vrt" + copyright "State Copyright © 1972 - 1989; Surveyor-General, Windhoek, Namibia" + max_zoom 16 + revision 3 end imagery_layer "na_aerial" do @@ -47,3 +56,13 @@ imagery_layer "na_aerial" do extension "jpeg" max_zoom 21 end + +imagery_layer "african_topo_50k" do + site "namibia-topo.openstreetmap.org.za" + title "African Topo 50k" + projection "EPSG:4326" + source "/store/imagery/african-topo/african-topo-v10.vrt" + copyright "Copyright ©" + max_zoom 16 + revision 10 +end diff --git a/cookbooks/imagery/recipes/tiler.rb b/cookbooks/imagery/recipes/tiler.rb index 2f6942b66..802a3d527 100644 --- a/cookbooks/imagery/recipes/tiler.rb +++ b/cookbooks/imagery/recipes/tiler.rb @@ -40,6 +40,7 @@ podman_service "titiler" do volume :"/store/imagery" => "/store/imagery", :"/srv/imagery/sockets" => "/sockets" environment :GDAL_CACHEMAX => 200, + :CPL_VSIL_CURL_CACHE_SIZE => 200000000, :GDAL_BAND_BLOCK_CACHE => "HASHSET", :GDAL_DISABLE_READDIR_ON_OPEN => "EMPTY_DIR", :GDAL_INGESTED_BYTES_AT_OPEN => 32768, diff --git a/cookbooks/imagery/resources/site.rb b/cookbooks/imagery/resources/site.rb index b4079f0ca..d57f74d34 100644 --- a/cookbooks/imagery/resources/site.rb +++ b/cookbooks/imagery/resources/site.rb @@ -86,6 +86,13 @@ action :create do mode "644" end + cookbook_file "/srv/#{new_resource.site}/transparent.png" do + source "transparent.png" + user "root" + group "root" + mode "644" + end + layers = Dir.glob("/srv/imagery/layers/#{new_resource.site}/*.yml").collect do |path| YAML.safe_load_file(path, :permitted_classes => [Symbol]) end @@ -105,13 +112,20 @@ action :create do description "Map server for #{new_resource.site} layer" environment "MS_DEBUGLEVEL" => "0", "MS_ERRORFILE" => "stderr", - "GDAL_CACHEMAX" => "128", + "GDAL_CACHEMAX" => "200", + "CPL_VSIL_CURL_CACHE_SIZE" => "200000000", + "GDAL_BAND_BLOCK_CACHE" => "HASHSET", + "GDAL_DISABLE_READDIR_ON_OPEN" => "EMPTY_DIR", + "GDAL_INGESTED_BYTES_AT_OPEN" => "32768", + "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES" => "YES", + "GDAL_HTTP_MULTIPLEX" => "YES", + "VSI_CACHE" => "TRUE", + "VSI_CACHE_SIZE" => "5000000", "GDAL_HTTP_TCP_KEEPALIVE" => "YES", "GDAL_HTTP_VERSION" => "2TLS", - "GDAL_ENABLE_WMS_CACHE" => "NO" + "GDAL_ENABLE_WMS_CACHE" => "NO", + "LD_PRELOAD" => "libtcmalloc_minimal.so.4" limit_nofile 16384 - memory_high "12G" - memory_max "12G" limit_core 0 user "imagery" group "imagery" diff --git a/cookbooks/imagery/templates/default/index.html.erb b/cookbooks/imagery/templates/default/index.html.erb index c21d95e04..2fe8d786d 100644 --- a/cookbooks/imagery/templates/default/index.html.erb +++ b/cookbooks/imagery/templates/default/index.html.erb @@ -5,15 +5,15 @@ <%= @title %> - - + + - - + +