From 1002dcc4dc5079709323a4d40cb4d1d93ba2ba20 Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Thu, 5 Jun 2025 14:34:20 +0100 Subject: [PATCH] Switch backups to using zstd --- .../templates/default/backup-staging.cron.erb | 25 +++++------ .../blog/templates/default/backup.cron.erb | 25 +++++------ .../blogs/templates/default/backup.cron.erb | 15 +++---- .../templates/default/server-backup.cron.erb | 23 ++++++---- .../civicrm/templates/default/backup.cron.erb | 25 +++++------ .../templates/default/backup.cron.erb | 23 +++++----- .../git/templates/default/backup.cron.erb | 15 +++---- .../mailman/templates/default/backup.cron.erb | 17 ++++---- .../default/mediawiki-backup.cron.erb | 42 ++++++++++++------- .../osqa/templates/default/backup.cron.erb | 23 +++++----- .../otrs/templates/default/backup.cron.erb | 25 +++++------ .../templates/default/backup.cron.erb | 19 +++++---- .../templates/default/backup.cron.erb | 29 ++++++------- cookbooks/tools/recipes/default.rb | 2 + 14 files changed, 169 insertions(+), 139 deletions(-) diff --git a/cookbooks/blog/templates/default/backup-staging.cron.erb b/cookbooks/blog/templates/default/backup-staging.cron.erb index 35b0a77ff..6e6f99dab 100644 --- a/cookbooks/blog/templates/default/backup-staging.cron.erb +++ b/cookbooks/blog/templates/default/backup-staging.cron.erb @@ -2,20 +2,21 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp osm-blog-staging.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=osm-blog-staging-$D.tar.gz - -mkdir $T/osm-blog-staging-$D -echo '[mysqldump]' > $T/mysqldump.opts -echo 'user=osm-blog-staging-user' >> $T/mysqldump.opts -echo 'password=<%= @passwords["osm-blog-staging-user"] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt --no-tablespaces osm-blog-staging > $T/osm-blog-staging-$D/osm-blog-staging.sql -ln -s /srv/staging.blog.openstreetmap.org $T/osm-blog-staging-$D/www +B="osm-blog-staging-$D.tar.zst" -export RSYNC_RSH="ssh -ax" +mkdir "$T/osm-blog-staging-$D" +echo '[mysqldump]' > "$T/mysqldump.opts" +echo 'user=osm-blog-staging-user' >> "$T/mysqldump.opts" +echo 'password=<%= @passwords["osm-blog-staging-user"] %>' >> "$T/mysqldump.opts" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces osm-blog-staging > "$T/osm-blog-staging-$D/osm-blog-staging.sql" +ln -s /srv/staging.blog.openstreetmap.org "$T/osm-blog-staging-$D/www" -nice tar --create --dereference --directory=$T --warning=no-file-changed osm-blog-staging-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" --warning=no-file-changed "osm-blog-staging-$D" | nice zstd --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/blog/templates/default/backup.cron.erb b/cookbooks/blog/templates/default/backup.cron.erb index 9e3bea7d8..a36bbe48b 100644 --- a/cookbooks/blog/templates/default/backup.cron.erb +++ b/cookbooks/blog/templates/default/backup.cron.erb @@ -2,20 +2,21 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp osm-blog.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=osm-blog-$D.tar.gz - -mkdir $T/osm-blog-$D -echo '[mysqldump]' > $T/mysqldump.opts -echo 'user=osm-blog-user' >> $T/mysqldump.opts -echo 'password=<%= @passwords["osm-blog-user"] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt --no-tablespaces osm-blog > $T/osm-blog-$D/osm-blog.sql -ln -s /srv/blog.openstreetmap.org $T/osm-blog-$D/www +B="osm-blog-$D.tar.zst" -export RSYNC_RSH="ssh -ax" +mkdir "$T/osm-blog-$D" +echo '[mysqldump]' > "$T/mysqldump.opts" +echo 'user=osm-blog-user' >> "$T/mysqldump.opts" +echo 'password=<%= @passwords["osm-blog-user"] %>' >> "$T/mysqldump.opts" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces osm-blog > "$T/osm-blog-$D/osm-blog.sql" +ln -s /srv/blog.openstreetmap.org "$T/osm-blog-$D/www" -nice tar --create --dereference --directory=$T --warning=no-file-changed osm-blog-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" --warning=no-file-changed "osm-blog-$D" | nice zstd --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/blogs/templates/default/backup.cron.erb b/cookbooks/blogs/templates/default/backup.cron.erb index c020a5e35..36725ef6b 100644 --- a/cookbooks/blogs/templates/default/backup.cron.erb +++ b/cookbooks/blogs/templates/default/backup.cron.erb @@ -2,16 +2,17 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp blogs.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=blogs-$D.tar.gz +B="blogs-$D.tar.zst" -mkdir $T/blogs-$D +mkdir "$T/blogs-$D" sqlite3 /srv/blogs.openstreetmap.org/planet.db ".backup $T/blogs-$D/planet.db" -export RSYNC_RSH="ssh -ax" - -nice tar --create --dereference --directory=$T blogs-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" "blogs-$D" | nice zstd --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/chef/templates/default/server-backup.cron.erb b/cookbooks/chef/templates/default/server-backup.cron.erb index 9b864768e..601432069 100644 --- a/cookbooks/chef/templates/default/server-backup.cron.erb +++ b/cookbooks/chef/templates/default/server-backup.cron.erb @@ -1,16 +1,21 @@ #!/bin/sh +# DO NOT EDIT - This file is being maintained by Chef + +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp chef-server.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=chef-server-$D.tar.gz +B="chef-server-$D.tar.zst" -mkdir $T/chef-server-$D -chgrp opscode-pgsql $T $T/chef-server-$D -chmod g+rwx $T $T/chef-server-$D -sudo -u opscode-pgsql /opt/opscode/embedded/bin/pg_dumpall --file=$T/chef-server-$D/chef.dmp --clean -ln -s /var/opt/opscode/bookshelf/data $T/chef-server-$D/bookshelf +mkdir "$T/chef-server-$D" +chgrp opscode-pgsql "$T" "$T/chef-server-$D" +chmod g+rwx "$T" "$T/chef-server-$D" +sudo -u opscode-pgsql /opt/opscode/embedded/bin/pg_dumpall --file="$T/chef-server-$D/chef.dmp" --clean +ln -s /var/opt/opscode/bookshelf/data "$T/chef-server-$D/bookshelf" -nice tar --create --dereference --directory=$T chef-server-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" "chef-server-$D" | nice zstd --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/civicrm/templates/default/backup.cron.erb b/cookbooks/civicrm/templates/default/backup.cron.erb index 7738f75ca..373090103 100644 --- a/cookbooks/civicrm/templates/default/backup.cron.erb +++ b/cookbooks/civicrm/templates/default/backup.cron.erb @@ -2,20 +2,21 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp osmf-crm.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=osmf-crm-$D.tar.gz - -mkdir $T/osmf-crm-$D -echo '[mysqldump]' > $T/mysqldump.opts -echo 'user=civicrm' >> $T/mysqldump.opts -echo 'password=<%= @passwords["database"] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt --skip-lock-tables --no-tablespaces civicrm > $T/osmf-crm-$D/civicrm.sql -ln -s /srv/supporting.openstreetmap.org $T/osmf-crm-$D/www +B="osmf-crm-$D.tar.zst" -export RSYNC_RSH="ssh -ax" +mkdir "$T/osmf-crm-$D" +echo '[mysqldump]' > "$T/mysqldump.opts" +echo 'user=civicrm' >> "$T/mysqldump.opts" +echo 'password=<%= @passwords["database"] %>' >> "$T/mysqldump.opts" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --no-tablespaces civicrm > "$T/osmf-crm-$D/civicrm.sql" +ln -s /srv/supporting.openstreetmap.org "$T/osmf-crm-$D/www" -nice tar --create --dereference --directory=$T osmf-crm-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" "osmf-crm-$D" | nice zstd --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/community/templates/default/backup.cron.erb b/cookbooks/community/templates/default/backup.cron.erb index d23cd47e9..e3af3df63 100644 --- a/cookbooks/community/templates/default/backup.cron.erb +++ b/cookbooks/community/templates/default/backup.cron.erb @@ -2,19 +2,20 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp community.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=community-$D.tar.gz - -mkdir $T/community-$D -ln -s /srv/community.openstreetmap.org/docker/containers $T/community-$D/containers -ln -s /srv/community.openstreetmap.org/shared/web-only $T/community-$D/shared-web-only -ln -s /srv/community.openstreetmap.org/shared/data/redis_data $T/community-$D/shared-data-redis_data -ln -s /srv/community.openstreetmap.org/shared/data/postgres_backup $T/community-$D/shared-data-postgres_backup +B="community-$D.tar.zst" -export RSYNC_RSH="ssh -ax" +mkdir "$T/community-$D" +ln -s /srv/community.openstreetmap.org/docker/containers "$T/community-$D/containers" +ln -s /srv/community.openstreetmap.org/shared/web-only "$T/community-$D/shared-web-only" +ln -s /srv/community.openstreetmap.org/shared/data/redis_data "$T/community-$D/shared-data-redis_data" +ln -s /srv/community.openstreetmap.org/shared/data/postgres_backup "$T/community-$D/shared-data-postgres_backup" -nice tar --create --numeric-owner --dereference --directory=$T --warning=no-file-changed community-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --numeric-owner --dereference --directory="$T" --warning=no-file-changed "community-$D" | nice zstd --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/git/templates/default/backup.cron.erb b/cookbooks/git/templates/default/backup.cron.erb index b3404c7d5..02fe522e3 100644 --- a/cookbooks/git/templates/default/backup.cron.erb +++ b/cookbooks/git/templates/default/backup.cron.erb @@ -2,15 +2,16 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp git.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=git-$D.tar.gz - -ln -s /var/lib/git $T/git-$D +B="git-$D.tar.zst" -export RSYNC_RSH="ssh -ax" +ln -s /var/lib/git "$T/git-$D" -nice tar --create --dereference --directory=$T git-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" "git-$D" | nice zstd --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/mailman/templates/default/backup.cron.erb b/cookbooks/mailman/templates/default/backup.cron.erb index cb0eda69d..13423f1f8 100644 --- a/cookbooks/mailman/templates/default/backup.cron.erb +++ b/cookbooks/mailman/templates/default/backup.cron.erb @@ -2,16 +2,17 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp lists.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=lists-$D.tar.gz - -mkdir $T/lists-$D -ln -s /var/lib/mailman $T/lists-$D/mailman +B="lists-$D.tar.zst" -export RSYNC_RSH="ssh -ax" +mkdir "$T/lists-$D" +ln -s /var/lib/mailman "$T/lists-$D/mailman" -nice tar --create --dereference --warning=no-file-changed --warning=no-file-removed --directory=$T lists-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --warning=no-file-changed --warning=no-file-removed --directory="$T" "lists-$D" | nice zstd --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb index 533cc76b0..6c1913d26 100755 --- a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb +++ b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb @@ -1,15 +1,27 @@ -#!/bin/bash -T=`mktemp -d -t -p /var/tmp mediawiki-<%= @name %>.XXXXXXXXXX` -D=`date +%Y-%m-%d` -B=wiki-<%= @name %>-$D.tar.gz - -mkdir $T/wiki-<%= @name %>-$D -echo '[mysqldump]' > $T/mysqldump.opts -echo 'user=<%= @database_params[:username] %>' >> $T/mysqldump.opts -echo 'password=<%= @database_params[:password] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt --skip-lock-tables --single-transaction --no-tablespaces "<%= @database_params[:name] %>" | lz4 -9 > $T/wiki-<%= @name %>-$D/wiki.sql.lz4 -ln -s <%= @directory %> $T/wiki-<%= @name %>-$D/www -nice tar --create --dereference --directory=$T --warning=no-file-changed --warning=no-file-removed --exclude=wiki-<%= @name %>-$D/www/w/images/thumb --exclude=wiki-<%= @name %>-$D/www/w/.git --exclude=wiki-<%= @name %>-$D/www/w/extensions/*/.git --exclude=wiki-<%= @name %>-$D/www/dump wiki-<%= @name %>-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup - -rm -rf $T +#!/bin/sh + +# DO NOT EDIT - This file is being maintained by Chef + +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + +T=$(mktemp -d -t -p /var/tmp "mediawiki-<%= @name %>.XXXXXXXXXX") +D=$(date +%Y-%m-%d) +B="wiki-<%= @name %>-$D.tar.zst" + +mkdir "$T/wiki-<%= @name %>-$D" +echo '[mysqldump]' > "$T/mysqldump.opts" +echo 'user=<%= @database_params[:username] %>' >> "$T/mysqldump.opts" +echo 'password=<%= @database_params[:password] %>' >> "$T/mysqldump.opts" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --single-transaction --no-tablespaces "<%= @database_params[:name] %>" | zstd --rsyncable -o "$T/wiki-<%= @name %>-$D/wiki.sql.zst" +ln -s "<%= @directory %>" "$T/wiki-<%= @name %>-$D/www" + +nice tar --create --dereference --directory="$T" --warning=no-file-changed --warning=no-file-removed \ + --exclude="wiki-<%= @name %>-$D/www/w/images/thumb" \ + --exclude="wiki-<%= @name %>-$D/www/w/.git" \ + --exclude="wiki-<%= @name %>-$D/www/w/extensions/*/.git" \ + --exclude="wiki-<%= @name %>-$D/www/dump" \ + "wiki-<%= @name %>-$D" | nice zstd --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup + +rm -rf "$T" diff --git a/cookbooks/osqa/templates/default/backup.cron.erb b/cookbooks/osqa/templates/default/backup.cron.erb index 5734bb1d4..0c89a5e43 100644 --- a/cookbooks/osqa/templates/default/backup.cron.erb +++ b/cookbooks/osqa/templates/default/backup.cron.erb @@ -2,19 +2,20 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp osqa.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=<%= @name %>-$D.tar.gz - -mkdir $T/<%= @name %>-$D -chown <%= @user %> $T -chown <%= @user %> $T/osqa-$D -sudo -u <%= @user %> pg_dump --format=custom --file=$T/<%= @name %>-$D/osqa.dmp <%= @database %> -ln -s <%= @directory %>/upfiles $T/<%= @name %>-$D/upfiles +B="<%= @name %>-$D.tar.zst" -export RSYNC_RSH="ssh -ax" +mkdir "$T/<%= @name %>-$D" +chown "<%= @user %>" "$T" +chown "<%= @user %>" "$T/osqa-$D" +sudo -u "<%= @user %>" pg_dump --format=custom --file="$T/<%= @name %>-$D/osqa.dmp" "<%= @database %>" +ln -s "<%= @directory %>/upfiles" "$T/<%= @name %>-$D/upfiles" -nice tar --create --dereference --directory=$T <%= @name %>-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" "<%= @name %>-$D" | nice zstd --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/otrs/templates/default/backup.cron.erb b/cookbooks/otrs/templates/default/backup.cron.erb index e7af774dc..6fdb74b40 100644 --- a/cookbooks/otrs/templates/default/backup.cron.erb +++ b/cookbooks/otrs/templates/default/backup.cron.erb @@ -2,20 +2,21 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp otrs.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=otrs-$D.tar.gz - -mkdir $T/otrs-$D -chown otrs $T -chown otrs $T/otrs-$D -sudo -u otrs pg_dump --file=$T/otrs-$D/otrs.dmp otrs -ln -s /var/lib/otrs $T/otrs-$D/otrs-var -ln -s /etc/apache2/sites-available/otrs.openstreetmap.org.conf $T/otrs-$D/apache2-otrs.openstreetmap.org.conf +B="otrs-$D.tar.zst" -export RSYNC_RSH="ssh -ax" +mkdir "$T/otrs-$D" +chown otrs "$T" +chown otrs "$T/otrs-$D" +sudo -u otrs pg_dump --file="$T/otrs-$D/otrs.dmp" otrs +ln -s /var/lib/otrs "$T/otrs-$D/otrs-var" +ln -s /etc/apache2/sites-available/otrs.openstreetmap.org.conf "$T/otrs-$D/apache2-otrs.openstreetmap.org.conf" -nice tar --create --dereference --directory=$T otrs-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" "otrs-$D" | nice zstd --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/prometheus/templates/default/backup.cron.erb b/cookbooks/prometheus/templates/default/backup.cron.erb index d23977599..3760820b9 100644 --- a/cookbooks/prometheus/templates/default/backup.cron.erb +++ b/cookbooks/prometheus/templates/default/backup.cron.erb @@ -2,18 +2,19 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp prometheus.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=prometheus-$D.tar.gz - -mkdir $T/prometheus-$D +B="prometheus-$D.tar.zst" -ln -s /var/lib/prometheus/alertmanager $T/prometheus-$D/alertmanager -ln -s /var/lib/grafana $T/prometheus-$D/grafana +mkdir "$T/prometheus-$D" -export RSYNC_RSH="ssh -ax" +ln -s /var/lib/prometheus/alertmanager "$T/prometheus-$D/alertmanager" +ln -s /var/lib/grafana "$T/prometheus-$D/grafana" -nice tar --create --dereference --directory=$T prometheus-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" "prometheus-$D" | nice zstd --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/stateofthemap/templates/default/backup.cron.erb b/cookbooks/stateofthemap/templates/default/backup.cron.erb index a9259ec5c..e35e4017c 100644 --- a/cookbooks/stateofthemap/templates/default/backup.cron.erb +++ b/cookbooks/stateofthemap/templates/default/backup.cron.erb @@ -2,26 +2,27 @@ # DO NOT EDIT - This file is being maintained by Chef +export ZSTD_CLEVEL=11 +export ZSTD_NBTHREADS=0 + T=$(mktemp -d -t -p /var/tmp sotm.XXXXXXXXXX) D=$(date +%Y-%m-%d) -B=sotm-$D.tar.gz +B="sotm-$D.tar.zst" -mkdir $T/sotm-$D +mkdir "$T/sotm-$D" <% %w(2010 2011 2012).each do |year| -%> -echo '[mysqldump]' > $T/mysqldump.opts -echo 'user=sotm<%= year %>' >> $T/mysqldump.opts -echo 'password=<%= @passwords["sotm#{year}"] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt --no-tablespaces sotm<%= year %> > $T/sotm-$D/sotm<%= year %>.sql +echo '[mysqldump]' > "$T/mysqldump.opts" +echo 'user=sotm<%= year %>' >> "$T/mysqldump.opts" +echo 'password=<%= @passwords["sotm#{year}"] %>' >> "$T/mysqldump.opts" +mysqldump --defaults-file="$T/mysqldump.opts" --opt --no-tablespaces "sotm<%= year %>" > "$T/sotm-$D/sotm<%= year %>.sql" <% end -%> -ln -s /srv/2010.stateofthemap.org $T/sotm-$D/www2010 -ln -s /srv/2011.stateofthemap.org $T/sotm-$D/www2011 -ln -s /srv/2012.stateofthemap.org $T/sotm-$D/www2012 - -export RSYNC_RSH="ssh -ax" +ln -s /srv/2010.stateofthemap.org "$T/sotm-$D/www2010" +ln -s /srv/2011.stateofthemap.org "$T/sotm-$D/www2011" +ln -s /srv/2012.stateofthemap.org "$T/sotm-$D/www2012" -nice tar --create --dereference --directory=$T sotm-$D | nice gzip --rsyncable -9 > $T/$B -nice rsync --preallocate --fuzzy $T/$B backup.openstreetmap.org::backup +nice tar --create --dereference --directory="$T" "sotm-$D" | nice zstd --rsyncable -o "$T/$B" +nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup -rm -rf $T +rm -rf "$T" diff --git a/cookbooks/tools/recipes/default.rb b/cookbooks/tools/recipes/default.rb index d07ddcfe5..a51879e67 100644 --- a/cookbooks/tools/recipes/default.rb +++ b/cookbooks/tools/recipes/default.rb @@ -32,6 +32,7 @@ package %w[ mtr-tiny numactl pciutils + rsync rsyslog screen smartmontools @@ -42,6 +43,7 @@ package %w[ usbutils vim xfsprogs + zstd ] service "rsyslog" do -- 2.39.5