From 5215b9046e2800b60f4bb25ea2da90f4502126e3 Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Fri, 8 May 2026 10:41:05 +0100 Subject: [PATCH] zstd backup - workaround https://github.com/facebook/zstd/issues/4332 --- cookbooks/blog/templates/default/backup-staging.cron.erb | 2 +- cookbooks/blog/templates/default/backup.cron.erb | 3 +-- cookbooks/blogs/templates/default/backup.cron.erb | 3 +-- cookbooks/chef/templates/default/server-backup.cron.erb | 3 +-- cookbooks/civicrm/templates/default/backup.cron.erb | 3 +-- cookbooks/community/templates/default/backup.cron.erb | 3 +-- cookbooks/git/templates/default/backup.cron.erb | 3 +-- cookbooks/mailman/templates/default/backup.cron.erb | 3 +-- .../mediawiki/templates/default/mediawiki-backup.cron.erb | 5 ++--- cookbooks/otrs/templates/default/backup.cron.erb | 3 +-- cookbooks/prometheus/templates/default/backup.cron.erb | 3 +-- cookbooks/stateofthemap/templates/default/backup.cron.erb | 3 +-- 12 files changed, 13 insertions(+), 24 deletions(-) diff --git a/cookbooks/blog/templates/default/backup-staging.cron.erb b/cookbooks/blog/templates/default/backup-staging.cron.erb index 7746b5ba2..478569960 100644 --- a/cookbooks/blog/templates/default/backup-staging.cron.erb +++ b/cookbooks/blog/templates/default/backup-staging.cron.erb @@ -22,7 +22,7 @@ nice tar --create --dereference --directory="$T" \ --sort=name \ --warning=no-file-changed \ --warning=no-file-removed \ - "osm-blog-staging-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + "osm-blog-staging-$D" | nice zstd -T0 --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup diff --git a/cookbooks/blog/templates/default/backup.cron.erb b/cookbooks/blog/templates/default/backup.cron.erb index 7561966e7..1a26da3bc 100644 --- a/cookbooks/blog/templates/default/backup.cron.erb +++ b/cookbooks/blog/templates/default/backup.cron.erb @@ -4,7 +4,6 @@ set -euo pipefail export ZSTD_CLEVEL=16 -export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp osm-blog.XXXXXXXXXX) D=$(date +%Y-%m-%d) @@ -22,7 +21,7 @@ nice tar --create --dereference --directory="$T" \ --sort=name \ --warning=no-file-changed \ --warning=no-file-removed \ - "osm-blog-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + "osm-blog-$D" | nice zstd -T0 --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup diff --git a/cookbooks/blogs/templates/default/backup.cron.erb b/cookbooks/blogs/templates/default/backup.cron.erb index 3c92c2fab..071d07a7c 100644 --- a/cookbooks/blogs/templates/default/backup.cron.erb +++ b/cookbooks/blogs/templates/default/backup.cron.erb @@ -4,7 +4,6 @@ set -euo pipefail export ZSTD_CLEVEL=16 -export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp blogs.XXXXXXXXXX) D=$(date +%Y-%m-%d) @@ -15,7 +14,7 @@ sqlite3 /srv/blogs.openstreetmap.org/planet.db ".backup $T/blogs-$D/planet.db" nice tar --create --dereference --directory="$T" \ --sort=name \ - "blogs-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + "blogs-$D" | nice zstd -T0 --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup diff --git a/cookbooks/chef/templates/default/server-backup.cron.erb b/cookbooks/chef/templates/default/server-backup.cron.erb index aabfc6c70..44b41ae1a 100644 --- a/cookbooks/chef/templates/default/server-backup.cron.erb +++ b/cookbooks/chef/templates/default/server-backup.cron.erb @@ -4,7 +4,6 @@ set -euo pipefail export ZSTD_CLEVEL=16 -export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp chef-server.XXXXXXXXXX) D=$(date +%Y-%m-%d) @@ -18,7 +17,7 @@ ln -s /var/opt/opscode/bookshelf/data "$T/chef-server-$D/bookshelf" nice tar --create --dereference --directory="$T" \ --sort=name \ - "chef-server-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + "chef-server-$D" | nice zstd -T0 --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup diff --git a/cookbooks/civicrm/templates/default/backup.cron.erb b/cookbooks/civicrm/templates/default/backup.cron.erb index 52678c1f7..c3e17d114 100644 --- a/cookbooks/civicrm/templates/default/backup.cron.erb +++ b/cookbooks/civicrm/templates/default/backup.cron.erb @@ -4,7 +4,6 @@ set -euo pipefail export ZSTD_CLEVEL=16 -export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp osmf-crm.XXXXXXXXXX) D=$(date +%Y-%m-%d) @@ -19,7 +18,7 @@ ln -s /srv/supporting.openstreetmap.org "$T/osmf-crm-$D/www" nice tar --create --dereference --directory="$T" \ --sort=name \ - "osmf-crm-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + "osmf-crm-$D" | nice zstd -T0 --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup diff --git a/cookbooks/community/templates/default/backup.cron.erb b/cookbooks/community/templates/default/backup.cron.erb index c6e22550b..e3a6872a0 100644 --- a/cookbooks/community/templates/default/backup.cron.erb +++ b/cookbooks/community/templates/default/backup.cron.erb @@ -4,7 +4,6 @@ set -euo pipefail export ZSTD_CLEVEL=16 -export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp community.XXXXXXXXXX) D=$(date +%Y-%m-%d) @@ -22,7 +21,7 @@ nice tar --create --numeric-owner --dereference --directory="$T" \ --warning=no-file-changed \ --warning=no-file-removed \ --exclude="community-$D/shared-web-only/log" \ - "community-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + "community-$D" | nice zstd -T0 --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup diff --git a/cookbooks/git/templates/default/backup.cron.erb b/cookbooks/git/templates/default/backup.cron.erb index 7de75fd93..c8cc1e151 100644 --- a/cookbooks/git/templates/default/backup.cron.erb +++ b/cookbooks/git/templates/default/backup.cron.erb @@ -4,7 +4,6 @@ set -euo pipefail export ZSTD_CLEVEL=16 -export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp git.XXXXXXXXXX) D=$(date +%Y-%m-%d) @@ -14,7 +13,7 @@ ln -s /var/lib/git "$T/git-$D" nice tar --create --dereference --directory="$T" \ --sort=name \ - "git-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + "git-$D" | nice zstd -T0 --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup diff --git a/cookbooks/mailman/templates/default/backup.cron.erb b/cookbooks/mailman/templates/default/backup.cron.erb index 45473fc31..11b1c23cd 100644 --- a/cookbooks/mailman/templates/default/backup.cron.erb +++ b/cookbooks/mailman/templates/default/backup.cron.erb @@ -4,7 +4,6 @@ set -euo pipefail export ZSTD_CLEVEL=12 -export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp lists.XXXXXXXXXX) D=$(date +%Y-%m-%d) @@ -20,7 +19,7 @@ nice tar --create --dereference \ --exclude='*/archives/*.txt.gz' \ --directory="$T" \ --sort=name \ - "lists-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + "lists-$D" | nice zstd -T0 --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup diff --git a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb index 850b7ec48..1cd910004 100755 --- a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb +++ b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb @@ -4,7 +4,6 @@ set -euo pipefail export ZSTD_CLEVEL=16 -export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp "mediawiki-<%= @name %>.XXXXXXXXXX") D=$(date +%Y-%m-%d) @@ -14,7 +13,7 @@ mkdir "$T/wiki-<%= @name %>-$D" echo '[mysqldump]' > "$T/mysqldump.opts" echo 'user=<%= @database_params[:username] %>' >> "$T/mysqldump.opts" echo 'password=<%= @database_params[:password] %>' >> "$T/mysqldump.opts" -mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --single-transaction --no-tablespaces --max-allowed-packet=1G "<%= @database_params[:name] %>" | zstd --quiet --long --rsyncable -o "$T/wiki-<%= @name %>-$D/wiki.sql.zst" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --single-transaction --no-tablespaces --max-allowed-packet=1G "<%= @database_params[:name] %>" | zstd -T0 --quiet --long --rsyncable -o "$T/wiki-<%= @name %>-$D/wiki.sql.zst" ln -s "<%= @directory %>" "$T/wiki-<%= @name %>-$D/www" set +e @@ -28,7 +27,7 @@ nice tar --create --dereference --directory="$T" \ --exclude="wiki-<%= @name %>-$D/www/dump" \ --exclude="wiki-<%= @name %>-$D/www/sitemap-*" \ --sort=name \ - "wiki-<%= @name %>-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + "wiki-<%= @name %>-$D" | nice zstd -T0 --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup diff --git a/cookbooks/otrs/templates/default/backup.cron.erb b/cookbooks/otrs/templates/default/backup.cron.erb index 1fa0057a4..8a1d947e2 100644 --- a/cookbooks/otrs/templates/default/backup.cron.erb +++ b/cookbooks/otrs/templates/default/backup.cron.erb @@ -4,7 +4,6 @@ set -euo pipefail export ZSTD_CLEVEL=16 -export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp otrs.XXXXXXXXXX) D=$(date +%Y-%m-%d) @@ -20,7 +19,7 @@ ln -s /etc/apache2/sites-available/otrs.openstreetmap.org.conf "$T/otrs-$D/apach nice tar --create --dereference --directory="$T" \ --sort=name \ --exclude="otrs-$D/otrs-var/tmp/CacheFileStorable" \ - "otrs-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + "otrs-$D" | nice zstd -T0 --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup diff --git a/cookbooks/prometheus/templates/default/backup.cron.erb b/cookbooks/prometheus/templates/default/backup.cron.erb index 7447b1934..08c1c8a84 100644 --- a/cookbooks/prometheus/templates/default/backup.cron.erb +++ b/cookbooks/prometheus/templates/default/backup.cron.erb @@ -4,7 +4,6 @@ set -euo pipefail export ZSTD_CLEVEL=16 -export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp prometheus.XXXXXXXXXX) D=$(date +%Y-%m-%d) @@ -17,7 +16,7 @@ ln -s /var/lib/grafana "$T/prometheus-$D/grafana" nice tar --create --dereference --directory="$T" \ --sort=name \ - "prometheus-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + "prometheus-$D" | nice zstd -T0 --quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup diff --git a/cookbooks/stateofthemap/templates/default/backup.cron.erb b/cookbooks/stateofthemap/templates/default/backup.cron.erb index 9fb0d5935..449835093 100644 --- a/cookbooks/stateofthemap/templates/default/backup.cron.erb +++ b/cookbooks/stateofthemap/templates/default/backup.cron.erb @@ -4,7 +4,6 @@ set -euo pipefail export ZSTD_CLEVEL=16 -export ZSTD_NBTHREADS=0 T=$(mktemp -d -t -p /var/tmp sotm.XXXXXXXXXX) D=$(date +%Y-%m-%d) @@ -31,7 +30,7 @@ nice tar --create --dereference --directory="$T" \ --exclude="sotm-<%= @name %>-$D/www2011/wp/.svn/tmp" \ --exclude="sotm-<%= @name %>-$D/www2012/wp/.svn/tmp" \ --sort=name \ - "sotm-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + "sotm-$D" | nice zstd -T0 -quiet --long --rsyncable -o "$T/$B" nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -- 2.39.5