From 540dd4c46a1acede3e92f826204d7faceca317fe Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Fri, 10 Oct 2025 16:41:20 +0100 Subject: [PATCH] Consistent zstd, tar and mysqldump options. Drop: zstd adapt to improve rsynable output and speed consistency Add: zstd compression level to 11 Add: zstd quiet Add: zstd long reduce output size at expense higher memory usage Add: mysqldump max-allowed-packet=1G to improve reliability with large datasets Add: tar --sort=name to improve rsynable output --- cookbooks/blog/templates/default/backup-staging.cron.erb | 4 ++-- cookbooks/blog/templates/default/backup.cron.erb | 4 ++-- cookbooks/blogs/templates/default/backup.cron.erb | 2 +- cookbooks/chef/templates/default/logrotate.erb | 2 +- cookbooks/chef/templates/default/server-backup.cron.erb | 2 +- cookbooks/civicrm/templates/default/backup.cron.erb | 4 ++-- cookbooks/community/templates/default/backup.cron.erb | 2 +- cookbooks/dev/templates/default/logrotate.apis.erb | 2 +- cookbooks/git/templates/default/backup.cron.erb | 2 +- cookbooks/letsencrypt/templates/default/logrotate.erb | 2 +- cookbooks/mailman/templates/default/backup.cron.erb | 3 ++- .../mediawiki/templates/default/mediawiki-backup.cron.erb | 5 +++-- cookbooks/nominatim/templates/default/logrotate.nginx.erb | 4 ++-- .../nominatim/templates/default/logrotate.nominatim.erb | 2 +- cookbooks/otrs/templates/default/backup.cron.erb | 2 +- cookbooks/overpass/templates/default/logrotate.erb | 2 +- cookbooks/oxidized/templates/default/logrotate.erb | 2 +- cookbooks/planet/templates/default/logrotate.apache.erb | 2 +- cookbooks/prometheus/templates/default/backup.cron.erb | 2 +- cookbooks/stateofthemap/templates/default/backup.cron.erb | 4 ++-- cookbooks/tile/templates/default/logrotate.apache.erb | 2 +- cookbooks/web/templates/default/logrotate.apache.erb | 2 +- cookbooks/web/templates/default/logrotate.web.erb | 2 +- 23 files changed, 31 insertions(+), 29 deletions(-) diff --git a/cookbooks/blog/templates/default/backup-staging.cron.erb b/cookbooks/blog/templates/default/backup-staging.cron.erb index 1dd834a60..071b85e0e 100644 --- a/cookbooks/blog/templates/default/backup-staging.cron.erb +++ b/cookbooks/blog/templates/default/backup-staging.cron.erb @@ -13,10 +13,10 @@ mkdir "$T/osm-blog-staging-$D" echo '[mysqldump]' > "$T/mysqldump.opts" echo 'user=osm-blog-staging-user' >> "$T/mysqldump.opts" echo 'password=<%= @passwords["osm-blog-staging-user"] %>' >> "$T/mysqldump.opts" -mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces osm-blog-staging > "$T/osm-blog-staging-$D/osm-blog-staging.sql" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces --max-allowed-packet=1G osm-blog-staging > "$T/osm-blog-staging-$D/osm-blog-staging.sql" ln -s /srv/staging.blog.openstreetmap.org "$T/osm-blog-staging-$D/www" -nice tar --create --dereference --directory="$T" --warning=no-file-changed "osm-blog-staging-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name --warning=no-file-changed "osm-blog-staging-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/blog/templates/default/backup.cron.erb b/cookbooks/blog/templates/default/backup.cron.erb index 59ff2f8cc..2b9f31fd8 100644 --- a/cookbooks/blog/templates/default/backup.cron.erb +++ b/cookbooks/blog/templates/default/backup.cron.erb @@ -13,10 +13,10 @@ mkdir "$T/osm-blog-$D" echo '[mysqldump]' > "$T/mysqldump.opts" echo 'user=osm-blog-user' >> "$T/mysqldump.opts" echo 'password=<%= @passwords["osm-blog-user"] %>' >> "$T/mysqldump.opts" -mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces osm-blog > "$T/osm-blog-$D/osm-blog.sql" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces --max-allowed-packet=1G osm-blog > "$T/osm-blog-$D/osm-blog.sql" ln -s /srv/blog.openstreetmap.org "$T/osm-blog-$D/www" -nice tar --create --dereference --directory="$T" --warning=no-file-changed "osm-blog-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name --warning=no-file-changed "osm-blog-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/blogs/templates/default/backup.cron.erb b/cookbooks/blogs/templates/default/backup.cron.erb index d60e95080..d0a8c13d3 100644 --- a/cookbooks/blogs/templates/default/backup.cron.erb +++ b/cookbooks/blogs/templates/default/backup.cron.erb @@ -12,7 +12,7 @@ B="blogs-$D.tar.zst" mkdir "$T/blogs-$D" sqlite3 /srv/blogs.openstreetmap.org/planet.db ".backup $T/blogs-$D/planet.db" -nice tar --create --dereference --directory="$T" "blogs-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name "blogs-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/chef/templates/default/logrotate.erb b/cookbooks/chef/templates/default/logrotate.erb index 451a1c18b..f02dbffbe 100644 --- a/cookbooks/chef/templates/default/logrotate.erb +++ b/cookbooks/chef/templates/default/logrotate.erb @@ -6,7 +6,7 @@ compress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd postrotate systemctl try-restart chef-client.service diff --git a/cookbooks/chef/templates/default/server-backup.cron.erb b/cookbooks/chef/templates/default/server-backup.cron.erb index 82d15d517..afb6db5b1 100644 --- a/cookbooks/chef/templates/default/server-backup.cron.erb +++ b/cookbooks/chef/templates/default/server-backup.cron.erb @@ -15,7 +15,7 @@ chmod g+rwx "$T" "$T/chef-server-$D" sudo -u opscode-pgsql /opt/opscode/embedded/bin/pg_dumpall --file="$T/chef-server-$D/chef.dmp" --clean ln -s /var/opt/opscode/bookshelf/data "$T/chef-server-$D/bookshelf" -nice tar --create --dereference --directory="$T" "chef-server-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name "chef-server-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/civicrm/templates/default/backup.cron.erb b/cookbooks/civicrm/templates/default/backup.cron.erb index bba562483..5c667ec6d 100644 --- a/cookbooks/civicrm/templates/default/backup.cron.erb +++ b/cookbooks/civicrm/templates/default/backup.cron.erb @@ -13,10 +13,10 @@ mkdir "$T/osmf-crm-$D" echo '[mysqldump]' > "$T/mysqldump.opts" echo 'user=civicrm' >> "$T/mysqldump.opts" echo 'password=<%= @passwords["database"] %>' >> "$T/mysqldump.opts" -mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --no-tablespaces civicrm > "$T/osmf-crm-$D/civicrm.sql" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --no-tablespaces --max-allowed-packet=1G civicrm > "$T/osmf-crm-$D/civicrm.sql" ln -s /srv/supporting.openstreetmap.org "$T/osmf-crm-$D/www" -nice tar --create --dereference --directory="$T" "osmf-crm-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name "osmf-crm-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/community/templates/default/backup.cron.erb b/cookbooks/community/templates/default/backup.cron.erb index 72989a3d0..b64abac00 100644 --- a/cookbooks/community/templates/default/backup.cron.erb +++ b/cookbooks/community/templates/default/backup.cron.erb @@ -15,7 +15,7 @@ ln -s /srv/community.openstreetmap.org/shared/web-only "$T/community-$D/shared-w ln -s /srv/community.openstreetmap.org/shared/data/redis_data "$T/community-$D/shared-data-redis_data" ln -s /srv/community.openstreetmap.org/shared/data/postgres_backup "$T/community-$D/shared-data-postgres_backup" -nice tar --create --numeric-owner --sort=name --dereference --directory="$T" --warning=no-file-changed --exclude="community-$D/shared-web-only/log/**/*.gz" "community-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --numeric-owner --dereference --directory="$T" --sort=name --warning=no-file-changed --exclude="community-$D/shared-web-only/log/**/*.gz" "community-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/dev/templates/default/logrotate.apis.erb b/cookbooks/dev/templates/default/logrotate.apis.erb index d2d1dc1e0..d00d5a06f 100644 --- a/cookbooks/dev/templates/default/logrotate.apis.erb +++ b/cookbooks/dev/templates/default/logrotate.apis.erb @@ -8,7 +8,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty create 0660 apis apis diff --git a/cookbooks/git/templates/default/backup.cron.erb b/cookbooks/git/templates/default/backup.cron.erb index fcfe6829f..2582b886e 100644 --- a/cookbooks/git/templates/default/backup.cron.erb +++ b/cookbooks/git/templates/default/backup.cron.erb @@ -11,7 +11,7 @@ B="git-$D.tar.zst" ln -s /var/lib/git "$T/git-$D" -nice tar --create --dereference --directory="$T" "git-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name "git-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/letsencrypt/templates/default/logrotate.erb b/cookbooks/letsencrypt/templates/default/logrotate.erb index 3a80f1b48..3d59e9d4a 100644 --- a/cookbooks/letsencrypt/templates/default/logrotate.erb +++ b/cookbooks/letsencrypt/templates/default/logrotate.erb @@ -5,6 +5,6 @@ compress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd } diff --git a/cookbooks/mailman/templates/default/backup.cron.erb b/cookbooks/mailman/templates/default/backup.cron.erb index a3da0832b..6d79de4b9 100644 --- a/cookbooks/mailman/templates/default/backup.cron.erb +++ b/cookbooks/mailman/templates/default/backup.cron.erb @@ -17,7 +17,8 @@ nice tar --create --dereference \ --warning=no-file-removed \ --exclude='*/archives/*.txt.gz' \ --directory="$T" \ - "lists-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" + --sort=name \ + "lists-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup diff --git a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb index 0cd5f2976..6cd775d9a 100755 --- a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb +++ b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb @@ -13,7 +13,7 @@ mkdir "$T/wiki-<%= @name %>-$D" echo '[mysqldump]' > "$T/mysqldump.opts" echo 'user=<%= @database_params[:username] %>' >> "$T/mysqldump.opts" echo 'password=<%= @database_params[:password] %>' >> "$T/mysqldump.opts" -mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --single-transaction --no-tablespaces --max-allowed-packet=1G "<%= @database_params[:name] %>" | zstd --quiet --adapt --rsyncable -o "$T/wiki-<%= @name %>-$D/wiki.sql.zst" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --single-transaction --no-tablespaces --max-allowed-packet=1G "<%= @database_params[:name] %>" | zstd --quiet --long --rsyncable -o "$T/wiki-<%= @name %>-$D/wiki.sql.zst" ln -s "<%= @directory %>" "$T/wiki-<%= @name %>-$D/www" nice tar --create --dereference --directory="$T" --warning=no-file-changed --warning=no-file-removed \ @@ -21,7 +21,8 @@ nice tar --create --dereference --directory="$T" --warning=no-file-changed --war --exclude="wiki-<%= @name %>-$D/www/w/.git" \ --exclude="wiki-<%= @name %>-$D/www/w/extensions/*/.git" \ --exclude="wiki-<%= @name %>-$D/www/dump" \ - "wiki-<%= @name %>-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" + --sort=name \ + "wiki-<%= @name %>-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/nominatim/templates/default/logrotate.nginx.erb b/cookbooks/nominatim/templates/default/logrotate.nginx.erb index 3cda39725..4b2ec18b4 100644 --- a/cookbooks/nominatim/templates/default/logrotate.nginx.erb +++ b/cookbooks/nominatim/templates/default/logrotate.nginx.erb @@ -8,7 +8,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty create 640 www-data adm @@ -26,7 +26,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty create 640 www-data adm diff --git a/cookbooks/nominatim/templates/default/logrotate.nominatim.erb b/cookbooks/nominatim/templates/default/logrotate.nominatim.erb index 838e3b2e9..0acea0b06 100644 --- a/cookbooks/nominatim/templates/default/logrotate.nominatim.erb +++ b/cookbooks/nominatim/templates/default/logrotate.nominatim.erb @@ -8,7 +8,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty postrotate diff --git a/cookbooks/otrs/templates/default/backup.cron.erb b/cookbooks/otrs/templates/default/backup.cron.erb index 2fa11f6e8..2bea81908 100644 --- a/cookbooks/otrs/templates/default/backup.cron.erb +++ b/cookbooks/otrs/templates/default/backup.cron.erb @@ -16,7 +16,7 @@ sudo -u otrs pg_dump --file="$T/otrs-$D/otrs.dmp" otrs ln -s /var/lib/otrs "$T/otrs-$D/otrs-var" ln -s /etc/apache2/sites-available/otrs.openstreetmap.org.conf "$T/otrs-$D/apache2-otrs.openstreetmap.org.conf" -nice tar --create --dereference --directory="$T" "otrs-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name "otrs-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/overpass/templates/default/logrotate.erb b/cookbooks/overpass/templates/default/logrotate.erb index 21421684e..a4e6359de 100644 --- a/cookbooks/overpass/templates/default/logrotate.erb +++ b/cookbooks/overpass/templates/default/logrotate.erb @@ -5,6 +5,6 @@ compress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd } diff --git a/cookbooks/oxidized/templates/default/logrotate.erb b/cookbooks/oxidized/templates/default/logrotate.erb index 66f2158eb..676f8761f 100644 --- a/cookbooks/oxidized/templates/default/logrotate.erb +++ b/cookbooks/oxidized/templates/default/logrotate.erb @@ -8,7 +8,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd missingok } diff --git a/cookbooks/planet/templates/default/logrotate.apache.erb b/cookbooks/planet/templates/default/logrotate.apache.erb index 87761ba56..1d6123611 100644 --- a/cookbooks/planet/templates/default/logrotate.apache.erb +++ b/cookbooks/planet/templates/default/logrotate.apache.erb @@ -8,7 +8,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty create 640 root adm diff --git a/cookbooks/prometheus/templates/default/backup.cron.erb b/cookbooks/prometheus/templates/default/backup.cron.erb index ed581d09e..5c5a8dc8d 100644 --- a/cookbooks/prometheus/templates/default/backup.cron.erb +++ b/cookbooks/prometheus/templates/default/backup.cron.erb @@ -14,7 +14,7 @@ mkdir "$T/prometheus-$D" ln -s /var/lib/prometheus/alertmanager "$T/prometheus-$D/alertmanager" ln -s /var/lib/grafana "$T/prometheus-$D/grafana" -nice tar --create --dereference --directory="$T" "prometheus-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name "prometheus-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/stateofthemap/templates/default/backup.cron.erb b/cookbooks/stateofthemap/templates/default/backup.cron.erb index e221f9694..92295fd1a 100644 --- a/cookbooks/stateofthemap/templates/default/backup.cron.erb +++ b/cookbooks/stateofthemap/templates/default/backup.cron.erb @@ -15,14 +15,14 @@ mkdir "$T/sotm-$D" echo '[mysqldump]' > "$T/mysqldump.opts" echo 'user=sotm<%= year %>' >> "$T/mysqldump.opts" echo 'password=<%= @passwords["sotm#{year}"] %>' >> "$T/mysqldump.opts" -mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces "sotm<%= year %>" > "$T/sotm-$D/sotm<%= year %>.sql" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces --max-allowed-packet=1G "sotm<%= year %>" > "$T/sotm-$D/sotm<%= year %>.sql" <% end -%> ln -s /srv/2010.stateofthemap.org "$T/sotm-$D/www2010" ln -s /srv/2011.stateofthemap.org "$T/sotm-$D/www2011" ln -s /srv/2012.stateofthemap.org "$T/sotm-$D/www2012" -nice tar --create --dereference --directory="$T" "sotm-$D" | nice zstd --quiet --adapt --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" --sort=name "sotm-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/tile/templates/default/logrotate.apache.erb b/cookbooks/tile/templates/default/logrotate.apache.erb index 1f164db1e..5a57681b2 100644 --- a/cookbooks/tile/templates/default/logrotate.apache.erb +++ b/cookbooks/tile/templates/default/logrotate.apache.erb @@ -9,7 +9,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty create 640 root adm diff --git a/cookbooks/web/templates/default/logrotate.apache.erb b/cookbooks/web/templates/default/logrotate.apache.erb index aad47ce01..db47df2db 100644 --- a/cookbooks/web/templates/default/logrotate.apache.erb +++ b/cookbooks/web/templates/default/logrotate.apache.erb @@ -8,7 +8,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty create 640 root adm diff --git a/cookbooks/web/templates/default/logrotate.web.erb b/cookbooks/web/templates/default/logrotate.web.erb index 0f4004eee..7f30ed458 100644 --- a/cookbooks/web/templates/default/logrotate.web.erb +++ b/cookbooks/web/templates/default/logrotate.web.erb @@ -8,7 +8,7 @@ delaycompress compresscmd /usr/bin/zstd compressext .zst - compressoptions -T0 --adapt + compressoptions -T0 -11 --quiet --long --rsyncable uncompresscmd /usr/bin/unzstd notifempty su rails rails -- 2.39.5