From e708ed92700c61fa6ecebf4835e13de22624da23 Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Tue, 7 Oct 2025 12:53:36 +0100 Subject: [PATCH 01/16] Fix web test data_bag typo --- test/data_bags/web/passwords.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/data_bags/web/passwords.json b/test/data_bags/web/passwords.json index eb1e423a4..0b5b456cc 100644 --- a/test/data_bags/web/passwords.json +++ b/test/data_bags/web/passwords.json @@ -23,7 +23,7 @@ "totp_key": "totp", "aws_keys": { "AKIASQUXHPE7AMJQRFOS": "aws" - } + }, "openid_connect_key": [ "-----BEGIN PRIVATE KEY-----", "MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC1yJqM4c0bJNVN", -- 2.39.5 From 53e18d347b9051484aaf0762669455002e6e08d5 Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Tue, 7 Oct 2025 13:23:06 +0100 Subject: [PATCH 02/16] web: Add funding-manifest-urls for github repo in funding.json --- cookbooks/web/files/default/static/funding.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cookbooks/web/files/default/static/funding.json b/cookbooks/web/files/default/static/funding.json index a906f302e..f65284cbd 100644 --- a/cookbooks/web/files/default/static/funding.json +++ b/cookbooks/web/files/default/static/funding.json @@ -20,7 +20,8 @@ "wellKnown": "https://www.openstreetmap.org/.well-known/funding-manifest-urls" }, "repositoryUrl": { - "url": "https://github.com/openstreetmap" + "url": "https://github.com/openstreetmap", + "wellKnown": "https://github.com/openstreetmap/chef/raw/refs/heads/master/cookbooks/web/files/default/static/.well-known/funding-manifest-urls" }, "licenses": [ "spdx:ODbL-1.0" -- 2.39.5 From 865ed20e48ab5fb6929ae34711383b585c2cb0aa Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Tue, 7 Oct 2025 13:30:13 +0100 Subject: [PATCH 03/16] foundation: add funding-manifest-urls to osmfoundation.org --- .../files/default/.well-known/funding-manifest-urls | 1 + cookbooks/foundation/recipes/wiki.rb | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 cookbooks/foundation/files/default/.well-known/funding-manifest-urls diff --git a/cookbooks/foundation/files/default/.well-known/funding-manifest-urls b/cookbooks/foundation/files/default/.well-known/funding-manifest-urls new file mode 100644 index 000000000..a51eb6de3 --- /dev/null +++ b/cookbooks/foundation/files/default/.well-known/funding-manifest-urls @@ -0,0 +1 @@ +https://www.openstreetmap.org/funding.json \ No newline at end of file diff --git a/cookbooks/foundation/recipes/wiki.rb b/cookbooks/foundation/recipes/wiki.rb index ead5694cd..b6c993bb5 100644 --- a/cookbooks/foundation/recipes/wiki.rb +++ b/cookbooks/foundation/recipes/wiki.rb @@ -63,3 +63,10 @@ template "/srv/osmfoundation.org/robots.txt" do mode "644" source "robots.txt.erb" end + +cookbook_file "/srv/osmfoundation.org/.well-known/funding-manifest-urls" do + owner node[:mediawiki][:user] + group node[:mediawiki][:group] + mode "644" + source ".well-known/funding-manifest-urls" +end -- 2.39.5 From 1b3147768cde4c11a7920d7b20c57c13b7f194eb Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Tue, 7 Oct 2025 13:34:21 +0100 Subject: [PATCH 04/16] foundation: ensure .well-known created --- cookbooks/foundation/recipes/wiki.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cookbooks/foundation/recipes/wiki.rb b/cookbooks/foundation/recipes/wiki.rb index b6c993bb5..adb2c11fa 100644 --- a/cookbooks/foundation/recipes/wiki.rb +++ b/cookbooks/foundation/recipes/wiki.rb @@ -64,6 +64,12 @@ template "/srv/osmfoundation.org/robots.txt" do source "robots.txt.erb" end +directory "/srv/osmfoundation.org/.well-known" do + owner node[:mediawiki][:user] + group node[:mediawiki][:group] + mode "755" +end + cookbook_file "/srv/osmfoundation.org/.well-known/funding-manifest-urls" do owner node[:mediawiki][:user] group node[:mediawiki][:group] -- 2.39.5 From 912506c158d21692b041375fb3b0610f8776de1e Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Tue, 7 Oct 2025 13:38:48 +0100 Subject: [PATCH 05/16] web: add osmf funding-manifest-urls to funding.json --- cookbooks/web/files/default/static/funding.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cookbooks/web/files/default/static/funding.json b/cookbooks/web/files/default/static/funding.json index f65284cbd..001b9cb91 100644 --- a/cookbooks/web/files/default/static/funding.json +++ b/cookbooks/web/files/default/static/funding.json @@ -7,7 +7,8 @@ "email": "secretary@osmfoundation.org", "description": "The OpenStreetMap Foundation is a not-for-profit organization that supports the OpenStreetMap Project. The OpenStreetMap Project based at OpenStreetMap.org, is the worldwide mapping effort that includes over two million volunteers around the globe.\n\nThe foundation consists of members from around the world who elect its board of directors.\n\nMuch of the foundation’s extensive work is done in the background, yet is critical to the ongoing functioning and growth of the OpenStreetMap Project.\n\nThe foundation supports the project with several key functions:\n\n* As a UK-registered limited company, the foundation acts as a legal entity for the OpenStreetMap Project.\n* The foundation is the custodian for the computer servers and services necessary to host the OpenStreetMap project.\n* Provides a vehicle for fund-raising to support the project. The foundation can accept donations for the OpenStreetMap Project and allocate those donations to OpenStreetMap Project activities.\n* Organizes the annual conference, State-of-the-map\n* Supports and communicates with the working groups, such as communications, licensing, etc. The foundation board at times will delegate various tasks to working groups, such as public relations, Web site development, etc.\n\nOpenStreetMap is an open initiative to create and provide free map data to anyone who wants them.", "webpageUrl": { - "url": "https://osmfoundation.org" + "url": "https://osmfoundation.org", + "wellKnown": "https://osmfoundation.org/.well-known/funding-manifest-urls" } }, "projects": [ -- 2.39.5 From 45208a51540304bf3809b8f2a5392933547c972d Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Tue, 7 Oct 2025 13:26:47 -0700 Subject: [PATCH 06/16] web: funding: Don't set wellKnown for webpageUrl wellKnown is optional and only has use if the hostname that funding.json is served from differs from the webpageUrl url. Since both are served from www.openstreetmap.org, it is not necessary here. --- cookbooks/web/files/default/static/funding.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cookbooks/web/files/default/static/funding.json b/cookbooks/web/files/default/static/funding.json index 001b9cb91..401059819 100644 --- a/cookbooks/web/files/default/static/funding.json +++ b/cookbooks/web/files/default/static/funding.json @@ -17,8 +17,7 @@ "name": "OpenStreetMap", "description": "OpenStreetMap is an initiative to create and provide free geographic data, such as street maps, to anyone.\nOpenStreetMap is built by a community of mappers that contribute and maintain data about roads, trails, cafés, railway stations, and much more, all over the world.", "webpageUrl": { - "url": "https://www.openstreetmap.org", - "wellKnown": "https://www.openstreetmap.org/.well-known/funding-manifest-urls" + "url": "https://www.openstreetmap.org" }, "repositoryUrl": { "url": "https://github.com/openstreetmap", -- 2.39.5 From da62df4dabcf0f01b07e693b32565813fe509f18 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Tue, 7 Oct 2025 13:28:57 -0700 Subject: [PATCH 07/16] web: funding: Don't include empty financial history history is optional but the text says to only include if one of expenses or income is being communicated. This means it shouldn't be included if it's an empty list. --- cookbooks/web/files/default/static/funding.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cookbooks/web/files/default/static/funding.json b/cookbooks/web/files/default/static/funding.json index 401059819..33c82e1bf 100644 --- a/cookbooks/web/files/default/static/funding.json +++ b/cookbooks/web/files/default/static/funding.json @@ -187,7 +187,6 @@ "funding-website" ] } - ], - "history": [] + ] } } -- 2.39.5 From f56448a975b79aa16a00f91302dd44ac9c266f01 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Tue, 7 Oct 2025 13:33:55 -0700 Subject: [PATCH 08/16] web: funding: Fix currency mismatches Each entry in the plan array must have exactly one ISO 4217 currency code. As we not all funding channels accept all currencies, splitting the one-time donation into multiple plans allows for the donation of multiple currencies. Some channels have no ISO 4217 currency listed and cannot be part of any plan. --- .../web/files/default/static/funding.json | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/cookbooks/web/files/default/static/funding.json b/cookbooks/web/files/default/static/funding.json index 33c82e1bf..1f098eda6 100644 --- a/cookbooks/web/files/default/static/funding.json +++ b/cookbooks/web/files/default/static/funding.json @@ -75,22 +75,45 @@ ], "plans": [ { - "guid": "one-time-donation", + "guid": "one-time-donation-usd", "status": "active", - "name": "One-time donation", + "name": "One-time USD donation", "amount": 0, "currency": "USD", "frequency": "one-time", "channels": [ "funding-website", - "bitcoin", - "eur-account", "usd-account", - "gbp-account", "paypal", "gh-sponsors" ] }, + { + "guid": "one-time-donation-gbp", + "status": "active", + "name": "One-time GBP donation", + "amount": 0, + "currency": "USD", + "frequency": "one-time", + "channels": [ + "funding-website", + "gbp-account", + "paypal" + ] + }, + { + "guid": "one-time-donation-eur", + "status": "active", + "name": "One-time EUR donation", + "amount": 0, + "currency": "USD", + "frequency": "one-time", + "channels": [ + "funding-website", + "eur-account", + "paypal" + ] + }, { "guid": "normal-membership", "status": "active", -- 2.39.5 From 672e2c86381b6393c7c43f836d24ccc5372ddb74 Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Tue, 7 Oct 2025 22:28:27 +0100 Subject: [PATCH 09/16] web: funding: fix currency mismatches --- cookbooks/web/files/default/static/funding.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cookbooks/web/files/default/static/funding.json b/cookbooks/web/files/default/static/funding.json index 1f098eda6..a9519edb6 100644 --- a/cookbooks/web/files/default/static/funding.json +++ b/cookbooks/web/files/default/static/funding.json @@ -93,7 +93,7 @@ "status": "active", "name": "One-time GBP donation", "amount": 0, - "currency": "USD", + "currency": "GBP", "frequency": "one-time", "channels": [ "funding-website", @@ -106,7 +106,7 @@ "status": "active", "name": "One-time EUR donation", "amount": 0, - "currency": "USD", + "currency": "EUR", "frequency": "one-time", "channels": [ "funding-website", -- 2.39.5 From 3a0ffa23a19710b213c81fff83bb308b04153e89 Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Tue, 7 Oct 2025 22:29:09 +0100 Subject: [PATCH 10/16] spamassassin: blacklist_from repeated spammer --- cookbooks/spamassassin/templates/default/local.cf.erb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cookbooks/spamassassin/templates/default/local.cf.erb b/cookbooks/spamassassin/templates/default/local.cf.erb index bc86b14c8..dc851baf4 100644 --- a/cookbooks/spamassassin/templates/default/local.cf.erb +++ b/cookbooks/spamassassin/templates/default/local.cf.erb @@ -12,3 +12,6 @@ use_auto_whitelist 1 # Set the location for our state files bayes_path /var/spool/spamassassin/bayes auto_whitelist_path /var/spool/spamassassin/auto_whitelist + +blacklist_from *@bewertungskanzlei.de +blacklist_from *@revieweraser-sales.com -- 2.39.5 From 2aadee228ae456cf993edcca65d213c86579edaf Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Wed, 8 Oct 2025 12:18:49 +0100 Subject: [PATCH 11/16] web: remove funding.json active-contributor due to plans limit --- cookbooks/web/files/default/static/funding.json | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/cookbooks/web/files/default/static/funding.json b/cookbooks/web/files/default/static/funding.json index a9519edb6..f85542cda 100644 --- a/cookbooks/web/files/default/static/funding.json +++ b/cookbooks/web/files/default/static/funding.json @@ -138,18 +138,6 @@ "funding-website" ] }, - { - "guid": "active-contributor-membership", - "status": "active", - "name": "Active Contributor Membership", - "description": "If you are an active contributor with at least 42 mapping days in the last 365 days, or if you are equally active in OpenStreetMap with non-mapping activities, we would like you to become a member of the OSMF regardless of if and how you could pay the membership fee.\nYou can apply for an active contributor membership or become an associate or normal member, and help decide the direction and future of the OSMF.", - "amount": 0, - "currency": "USD", - "frequency": "yearly", - "channels": [ - "funding-website" - ] - }, { "guid": "corporate-supporter", "status": "active", -- 2.39.5 From d8e14faa4c5cde9d870dd84a30106bf898e0051b Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Wed, 8 Oct 2025 20:08:30 +0100 Subject: [PATCH 12/16] blogs: Retry to ignore faulty blog feeds --- cookbooks/blogs/recipes/default.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/cookbooks/blogs/recipes/default.rb b/cookbooks/blogs/recipes/default.rb index 86dfd3afe..026c1821f 100644 --- a/cookbooks/blogs/recipes/default.rb +++ b/cookbooks/blogs/recipes/default.rb @@ -67,6 +67,7 @@ bundle_exec "/srv/blogs.openstreetmap.org" do user "blogs" group "blogs" subscribes :run, "git[/srv/blogs.openstreetmap.org]", :immediately + retries 2 # May fail on first run due to faulty blogs end ssl_certificate "blogs.openstreetmap.org" do -- 2.39.5 From 9620f5263c793517e077121ac90b0fa5e657aaed Mon Sep 17 00:00:00 2001 From: Pablo Brasero Date: Thu, 9 Oct 2025 16:28:04 +0100 Subject: [PATCH 13/16] Use more conventional `bundle` over `bundler` --- CONTRIBUTING.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5345a0786..f025e7f85 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,9 +52,9 @@ This will install the recommended version of ruby for running the tests. The rec #### **6. Install Dependencies with Bundler** ```bash rbenv exec gem install bundler -rbenv exec bundler install +rbenv exec bundle install ``` -This will install the [bundler](https://bundler.io/), the ruby gem packages manager, and then uses `bundler`` to install the required gem packages for the tests. +This will install the [bundler](https://bundler.io/), the ruby gem packages manager, and then uses `bundle` to install the required gem packages for the tests. #### **7. Run Cookstyle for Linting and Style Checks** ```bash @@ -66,12 +66,12 @@ This will run [cookstyle](https://docs.chef.io/workstation/cookstyle/) a linting #### **8. List Available Tests** ```bash -rbenv exec bundler exec kitchen list +rbenv exec bundle exec kitchen list ``` This lists the [Test Kitchen](https://kitchen.ci/) tests which are available. The list of tests is generated from the definitions in the [.kitchen.yml](.kitchen.yml) file. The individual tests are written in [InSpec](https://docs.chef.io/inspec/) and are stored in the `test/integration/` directory. #### **9. Run an Example Test** ```bash -rbenv exec bundler exec kitchen test dns-ubuntu-2204 +rbenv exec bundle exec kitchen test dns-ubuntu-2204 ``` This runs the [Test Kitchen](https://kitchen.ci/) [InSpec](https://docs.chef.io/inspec/) `dns` tests using the `Ubuntu 22.04` platform. The tests are run inside a Docker container using the Test Kitchen [Dokken driver](https://github.com/test-kitchen/kitchen-dokken). -- 2.39.5 From 2530cc6d89bfb6535e9dd7b71bb11f6adc91d400 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Thu, 9 Oct 2025 20:48:26 +0100 Subject: [PATCH 14/16] Update chef client to 18.8.11 --- cookbooks/chef/attributes/default.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/chef/attributes/default.rb b/cookbooks/chef/attributes/default.rb index 9ff0f65a5..6f0077ab3 100644 --- a/cookbooks/chef/attributes/default.rb +++ b/cookbooks/chef/attributes/default.rb @@ -2,4 +2,4 @@ default[:chef][:server][:version] = "15.9.38" # Set the default client version -default[:chef][:client][:version] = "18.7.10" +default[:chef][:client][:version] = "18.8.11" -- 2.39.5 From 389ef5a720d3af5033b5dd0397d14280cfb62986 Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Fri, 10 Oct 2025 16:36:40 +0100 Subject: [PATCH 15/16] mediawiki: Fix failing backup --- cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb index d90f7f4a5..0cd5f2976 100755 --- a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb +++ b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb @@ -13,7 +13,7 @@ mkdir "$T/wiki-<%= @name %>-$D" echo '[mysqldump]' > "$T/mysqldump.opts" echo 'user=<%= @database_params[:username] %>' >> "$T/mysqldump.opts" echo 'password=<%= @database_params[:password] %>' >> "$T/mysqldump.opts" -mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --single-transaction --no-tablespaces "<%= @database_params[:name] %>" | zstd --quiet --adapt --rsyncable -o "$T/wiki-<%= @name %>-$D/wiki.sql.zst" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --single-transaction --no-tablespaces --max-allowed-packet=1G "<%= @database_params[:name] %>" | zstd --quiet --adapt --rsyncable -o "$T/wiki-<%= @name %>-$D/wiki.sql.zst" ln -s "<%= @directory %>" "$T/wiki-<%= @name %>-$D/www" nice tar --create --dereference --directory="$T" --warning=no-file-changed --warning=no-file-removed \ -- 2.39.5 From 540dd4c46a1acede3e92f826204d7faceca317fe Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Fri, 10 Oct 2025 16:41:20 +0100 Subject: [PATCH 16/16] Consistent zstd, tar and mysqldump options. Drop: zstd adapt to improve rsynable output and speed consistency Add: zstd compression level to 11 Add: zstd quiet Add: zstd long reduce output size at expense higher memory usage Add: mysqldump max-allowed-packet=1G to improve reliability with large datasets Add: tar --sort=name to improve rsynable output --- cookbooks/blog/templates/default/backup-staging.cron.erb | 4 ++-- cookbooks/blog/templates/default/backup.cron.erb | 4 ++-- cookbooks/blogs/templates/default/backup.cron.erb | 2 +- cookbooks/chef/templates/default/logrotate.erb | 2 +- cookbooks/chef/templates/default/server-backup.cron.erb | 2 +- cookbooks/civicrm/templates/default/backup.cron.erb | 4 ++-- cookbooks/community/templates/default/backup.cron.erb | 2 +- cookbooks/dev/templates/default/logrotate.apis.erb | 2 +- cookbooks/git/templates/default/backup.cron.erb | 2 +- cookbooks/letsencrypt/templates/default/logrotate.erb | 2 +- cookbooks/mailman/templates/default/backup.cron.erb | 3 ++- .../mediawiki/templates/default/mediawiki-backup.cron.erb | 5 +++-- cookbooks/nominatim/templates/default/logrotate.nginx.erb | 4 ++-- .../nominatim/templates/default/logrotate.nominatim.erb | 2 +- cookbooks/otrs/templates/default/backup.cron.erb | 2 +- cookbooks/overpass/templates/default/logrotate.erb | 2 +- cookbooks/oxidized/templates/default/logrotate.erb | 2 +- cookbooks/planet/templates/default/logrotate.apache.erb | 2 +- cookbooks/prometheus/templates/default/backup.cron.erb | 2 +- cookbooks/stateofthemap/templates/default/backup.cron.erb | 4 ++-- cookbooks/tile/templates/default/logrotate.apache.erb | 2 +- cookbooks/web/templates/default/logrotate.apache.erb | 2 +- cookbooks/web/templates/default/logrotate.web.erb | 2 +- 23 files changed, 31 insertions(+), 29 deletions(-) diff --git a/cookbooks/blog/templates/default/backup-staging.cron.erb b/cookbooks/blog/templates/default/backup-staging.cron.erb index 1dd834a60..071b85e0e 100644 --- a/cookbooks/blog/templates/default/backup-staging.cron.erb +++ b/cookbooks/blog/templates/default/backup-staging.cron.erb @@ -13,10 +13,10 @@ mkdir "$T/osm-blog-staging-$D" echo '[mysqldump]' > "$T/mysqldump.opts" echo 'user=osm-blog-staging-user' >> "$T/mysqldump.opts" echo 'password=<%= @passwords["osm-blog-staging-user"] %>' >> "$T/mysqldump.opts" -mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces osm-blog-staging > "$T/osm-blog-staging-$D/osm-blog-staging.sql" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces --max-allowed-packet=1G osm-blog-staging > "$T/osm-blog-staging-$D/osm-blog-staging.sql" ln -s /srv/staging.blog.openstreetmap.org "$T/osm-blog-staging-$D/www" -nice tar --create --dereference --directory="$T" --warning=no-file-changed "osm-blog-staging-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name --warning=no-file-changed "osm-blog-staging-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/blog/templates/default/backup.cron.erb b/cookbooks/blog/templates/default/backup.cron.erb index 59ff2f8cc..2b9f31fd8 100644 --- a/cookbooks/blog/templates/default/backup.cron.erb +++ b/cookbooks/blog/templates/default/backup.cron.erb @@ -13,10 +13,10 @@ mkdir "$T/osm-blog-$D" echo '[mysqldump]' > "$T/mysqldump.opts" echo 'user=osm-blog-user' >> "$T/mysqldump.opts" echo 'password=<%= @passwords["osm-blog-user"] %>' >> "$T/mysqldump.opts" -mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces osm-blog > "$T/osm-blog-$D/osm-blog.sql" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces --max-allowed-packet=1G osm-blog > "$T/osm-blog-$D/osm-blog.sql" ln -s /srv/blog.openstreetmap.org "$T/osm-blog-$D/www" -nice tar --create --dereference --directory="$T" --warning=no-file-changed "osm-blog-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name --warning=no-file-changed "osm-blog-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/blogs/templates/default/backup.cron.erb b/cookbooks/blogs/templates/default/backup.cron.erb index d60e95080..d0a8c13d3 100644 --- a/cookbooks/blogs/templates/default/backup.cron.erb +++ b/cookbooks/blogs/templates/default/backup.cron.erb @@ -12,7 +12,7 @@ B="blogs-$D.tar.zst" mkdir "$T/blogs-$D" sqlite3 /srv/blogs.openstreetmap.org/planet.db ".backup $T/blogs-$D/planet.db" -nice tar --create --dereference --directory="$T" "blogs-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name "blogs-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/chef/templates/default/logrotate.erb b/cookbooks/chef/templates/default/logrotate.erb index 451a1c18b..f02dbffbe 100644 --- a/cookbooks/chef/templates/default/logrotate.erb +++ b/cookbooks/chef/templates/default/logrotate.erb @@ -6,7 +6,7 @@ compress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd postrotate systemctl try-restart chef-client.service diff --git a/cookbooks/chef/templates/default/server-backup.cron.erb b/cookbooks/chef/templates/default/server-backup.cron.erb index 82d15d517..afb6db5b1 100644 --- a/cookbooks/chef/templates/default/server-backup.cron.erb +++ b/cookbooks/chef/templates/default/server-backup.cron.erb @@ -15,7 +15,7 @@ chmod g+rwx "$T" "$T/chef-server-$D" sudo -u opscode-pgsql /opt/opscode/embedded/bin/pg_dumpall --file="$T/chef-server-$D/chef.dmp" --clean ln -s /var/opt/opscode/bookshelf/data "$T/chef-server-$D/bookshelf" -nice tar --create --dereference --directory="$T" "chef-server-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name "chef-server-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/civicrm/templates/default/backup.cron.erb b/cookbooks/civicrm/templates/default/backup.cron.erb index bba562483..5c667ec6d 100644 --- a/cookbooks/civicrm/templates/default/backup.cron.erb +++ b/cookbooks/civicrm/templates/default/backup.cron.erb @@ -13,10 +13,10 @@ mkdir "$T/osmf-crm-$D" echo '[mysqldump]' > "$T/mysqldump.opts" echo 'user=civicrm' >> "$T/mysqldump.opts" echo 'password=<%= @passwords["database"] %>' >> "$T/mysqldump.opts" -mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --no-tablespaces civicrm > "$T/osmf-crm-$D/civicrm.sql" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --no-tablespaces --max-allowed-packet=1G civicrm > "$T/osmf-crm-$D/civicrm.sql" ln -s /srv/supporting.openstreetmap.org "$T/osmf-crm-$D/www" -nice tar --create --dereference --directory="$T" "osmf-crm-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name "osmf-crm-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/community/templates/default/backup.cron.erb b/cookbooks/community/templates/default/backup.cron.erb index 72989a3d0..b64abac00 100644 --- a/cookbooks/community/templates/default/backup.cron.erb +++ b/cookbooks/community/templates/default/backup.cron.erb @@ -15,7 +15,7 @@ ln -s /srv/community.openstreetmap.org/shared/web-only "$T/community-$D/shared-w ln -s /srv/community.openstreetmap.org/shared/data/redis_data "$T/community-$D/shared-data-redis_data" ln -s /srv/community.openstreetmap.org/shared/data/postgres_backup "$T/community-$D/shared-data-postgres_backup" -nice tar --create --numeric-owner --sort=name --dereference --directory="$T" --warning=no-file-changed --exclude="community-$D/shared-web-only/log/**/*.gz" "community-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --numeric-owner --dereference --directory="$T" --sort=name --warning=no-file-changed --exclude="community-$D/shared-web-only/log/**/*.gz" "community-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/dev/templates/default/logrotate.apis.erb b/cookbooks/dev/templates/default/logrotate.apis.erb index d2d1dc1e0..d00d5a06f 100644 --- a/cookbooks/dev/templates/default/logrotate.apis.erb +++ b/cookbooks/dev/templates/default/logrotate.apis.erb @@ -8,7 +8,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty create 0660 apis apis diff --git a/cookbooks/git/templates/default/backup.cron.erb b/cookbooks/git/templates/default/backup.cron.erb index fcfe6829f..2582b886e 100644 --- a/cookbooks/git/templates/default/backup.cron.erb +++ b/cookbooks/git/templates/default/backup.cron.erb @@ -11,7 +11,7 @@ B="git-$D.tar.zst" ln -s /var/lib/git "$T/git-$D" -nice tar --create --dereference --directory="$T" "git-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name "git-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/letsencrypt/templates/default/logrotate.erb b/cookbooks/letsencrypt/templates/default/logrotate.erb index 3a80f1b48..3d59e9d4a 100644 --- a/cookbooks/letsencrypt/templates/default/logrotate.erb +++ b/cookbooks/letsencrypt/templates/default/logrotate.erb @@ -5,6 +5,6 @@ compress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd } diff --git a/cookbooks/mailman/templates/default/backup.cron.erb b/cookbooks/mailman/templates/default/backup.cron.erb index a3da0832b..6d79de4b9 100644 --- a/cookbooks/mailman/templates/default/backup.cron.erb +++ b/cookbooks/mailman/templates/default/backup.cron.erb @@ -17,7 +17,8 @@ nice tar --create --dereference \ --warning=no-file-removed \ --exclude='*/archives/*.txt.gz' \ --directory="$T" \ - "lists-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" + --sort=name \ + "lists-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup diff --git a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb index 0cd5f2976..6cd775d9a 100755 --- a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb +++ b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb @@ -13,7 +13,7 @@ mkdir "$T/wiki-<%= @name %>-$D" echo '[mysqldump]' > "$T/mysqldump.opts" echo 'user=<%= @database_params[:username] %>' >> "$T/mysqldump.opts" echo 'password=<%= @database_params[:password] %>' >> "$T/mysqldump.opts" -mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --single-transaction --no-tablespaces --max-allowed-packet=1G "<%= @database_params[:name] %>" | zstd --quiet --adapt --rsyncable -o "$T/wiki-<%= @name %>-$D/wiki.sql.zst" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --single-transaction --no-tablespaces --max-allowed-packet=1G "<%= @database_params[:name] %>" | zstd --quiet --long --rsyncable -o "$T/wiki-<%= @name %>-$D/wiki.sql.zst" ln -s "<%= @directory %>" "$T/wiki-<%= @name %>-$D/www" nice tar --create --dereference --directory="$T" --warning=no-file-changed --warning=no-file-removed \ @@ -21,7 +21,8 @@ nice tar --create --dereference --directory="$T" --warning=no-file-changed --war --exclude="wiki-<%= @name %>-$D/www/w/.git" \ --exclude="wiki-<%= @name %>-$D/www/w/extensions/*/.git" \ --exclude="wiki-<%= @name %>-$D/www/dump" \ - "wiki-<%= @name %>-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" + --sort=name \ + "wiki-<%= @name %>-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/nominatim/templates/default/logrotate.nginx.erb b/cookbooks/nominatim/templates/default/logrotate.nginx.erb index 3cda39725..4b2ec18b4 100644 --- a/cookbooks/nominatim/templates/default/logrotate.nginx.erb +++ b/cookbooks/nominatim/templates/default/logrotate.nginx.erb @@ -8,7 +8,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty create 640 www-data adm @@ -26,7 +26,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty create 640 www-data adm diff --git a/cookbooks/nominatim/templates/default/logrotate.nominatim.erb b/cookbooks/nominatim/templates/default/logrotate.nominatim.erb index 838e3b2e9..0acea0b06 100644 --- a/cookbooks/nominatim/templates/default/logrotate.nominatim.erb +++ b/cookbooks/nominatim/templates/default/logrotate.nominatim.erb @@ -8,7 +8,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty postrotate diff --git a/cookbooks/otrs/templates/default/backup.cron.erb b/cookbooks/otrs/templates/default/backup.cron.erb index 2fa11f6e8..2bea81908 100644 --- a/cookbooks/otrs/templates/default/backup.cron.erb +++ b/cookbooks/otrs/templates/default/backup.cron.erb @@ -16,7 +16,7 @@ sudo -u otrs pg_dump --file="$T/otrs-$D/otrs.dmp" otrs ln -s /var/lib/otrs "$T/otrs-$D/otrs-var" ln -s /etc/apache2/sites-available/otrs.openstreetmap.org.conf "$T/otrs-$D/apache2-otrs.openstreetmap.org.conf" -nice tar --create --dereference --directory="$T" "otrs-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name "otrs-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/overpass/templates/default/logrotate.erb b/cookbooks/overpass/templates/default/logrotate.erb index 21421684e..a4e6359de 100644 --- a/cookbooks/overpass/templates/default/logrotate.erb +++ b/cookbooks/overpass/templates/default/logrotate.erb @@ -5,6 +5,6 @@ compress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd } diff --git a/cookbooks/oxidized/templates/default/logrotate.erb b/cookbooks/oxidized/templates/default/logrotate.erb index 66f2158eb..676f8761f 100644 --- a/cookbooks/oxidized/templates/default/logrotate.erb +++ b/cookbooks/oxidized/templates/default/logrotate.erb @@ -8,7 +8,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd missingok } diff --git a/cookbooks/planet/templates/default/logrotate.apache.erb b/cookbooks/planet/templates/default/logrotate.apache.erb index 87761ba56..1d6123611 100644 --- a/cookbooks/planet/templates/default/logrotate.apache.erb +++ b/cookbooks/planet/templates/default/logrotate.apache.erb @@ -8,7 +8,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty create 640 root adm diff --git a/cookbooks/prometheus/templates/default/backup.cron.erb b/cookbooks/prometheus/templates/default/backup.cron.erb index ed581d09e..5c5a8dc8d 100644 --- a/cookbooks/prometheus/templates/default/backup.cron.erb +++ b/cookbooks/prometheus/templates/default/backup.cron.erb @@ -14,7 +14,7 @@ mkdir "$T/prometheus-$D" ln -s /var/lib/prometheus/alertmanager "$T/prometheus-$D/alertmanager" ln -s /var/lib/grafana "$T/prometheus-$D/grafana" -nice tar --create --dereference --directory="$T" "prometheus-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name "prometheus-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/stateofthemap/templates/default/backup.cron.erb b/cookbooks/stateofthemap/templates/default/backup.cron.erb index e221f9694..92295fd1a 100644 --- a/cookbooks/stateofthemap/templates/default/backup.cron.erb +++ b/cookbooks/stateofthemap/templates/default/backup.cron.erb @@ -15,14 +15,14 @@ mkdir "$T/sotm-$D" echo '[mysqldump]' > "$T/mysqldump.opts" echo 'user=sotm<%= year %>' >> "$T/mysqldump.opts" echo 'password=<%= @passwords["sotm#{year}"] %>' >> "$T/mysqldump.opts" -mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces "sotm<%= year %>" > "$T/sotm-$D/sotm<%= year %>.sql" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces --max-allowed-packet=1G "sotm<%= year %>" > "$T/sotm-$D/sotm<%= year %>.sql" <% end -%> ln -s /srv/2010.stateofthemap.org "$T/sotm-$D/www2010" ln -s /srv/2011.stateofthemap.org "$T/sotm-$D/www2011" ln -s /srv/2012.stateofthemap.org "$T/sotm-$D/www2012" -nice tar --create --dereference --directory="$T" "sotm-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name "sotm-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/tile/templates/default/logrotate.apache.erb b/cookbooks/tile/templates/default/logrotate.apache.erb index 1f164db1e..5a57681b2 100644 --- a/cookbooks/tile/templates/default/logrotate.apache.erb +++ b/cookbooks/tile/templates/default/logrotate.apache.erb @@ -9,7 +9,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty create 640 root adm diff --git a/cookbooks/web/templates/default/logrotate.apache.erb b/cookbooks/web/templates/default/logrotate.apache.erb index aad47ce01..db47df2db 100644 --- a/cookbooks/web/templates/default/logrotate.apache.erb +++ b/cookbooks/web/templates/default/logrotate.apache.erb @@ -8,7 +8,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty create 640 root adm diff --git a/cookbooks/web/templates/default/logrotate.web.erb b/cookbooks/web/templates/default/logrotate.web.erb index 0f4004eee..7f30ed458 100644 --- a/cookbooks/web/templates/default/logrotate.web.erb +++ b/cookbooks/web/templates/default/logrotate.web.erb @@ -8,7 +8,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty su rails rails -- 2.39.5