From 1b8b3202803ef9e4e430094ea79f84ad06a18266 Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Tue, 28 Oct 2025 14:10:33 +0000 Subject: [PATCH] Minor cleanup backup scripts. Add cache excludes --- cookbooks/blog/templates/default/backup-staging.cron.erb | 2 +- cookbooks/blog/templates/default/backup.cron.erb | 2 +- cookbooks/blogs/templates/default/backup.cron.erb | 5 ++++- cookbooks/chef/templates/default/server-backup.cron.erb | 5 ++++- cookbooks/civicrm/templates/default/backup.cron.erb | 5 ++++- cookbooks/community/templates/default/backup.cron.erb | 4 ++-- cookbooks/git/templates/default/backup.cron.erb | 5 ++++- cookbooks/mailman/templates/default/backup.cron.erb | 1 - .../mediawiki/templates/default/mediawiki-backup.cron.erb | 4 +++- cookbooks/otrs/templates/default/backup.cron.erb | 6 +++++- cookbooks/prometheus/templates/default/backup.cron.erb | 5 ++++- cookbooks/stateofthemap/templates/default/backup.cron.erb | 5 ++++- 12 files changed, 36 insertions(+), 13 deletions(-) diff --git a/cookbooks/blog/templates/default/backup-staging.cron.erb b/cookbooks/blog/templates/default/backup-staging.cron.erb index ff464e9c1..b712e4018 100644 --- a/cookbooks/blog/templates/default/backup-staging.cron.erb +++ b/cookbooks/blog/templates/default/backup-staging.cron.erb @@ -22,8 +22,8 @@ nice tar --create --dereference --directory="$T" \ --sort=name \ --warning=no-file-changed \ --warning=no-file-removed \ - --ignore-failed-read \ "osm-blog-staging-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/blog/templates/default/backup.cron.erb b/cookbooks/blog/templates/default/backup.cron.erb index c14286617..303fa6a70 100644 --- a/cookbooks/blog/templates/default/backup.cron.erb +++ b/cookbooks/blog/templates/default/backup.cron.erb @@ -22,8 +22,8 @@ nice tar --create --dereference --directory="$T" \ --sort=name \ --warning=no-file-changed \ --warning=no-file-removed \ - --ignore-failed-read \ "osm-blog-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/blogs/templates/default/backup.cron.erb b/cookbooks/blogs/templates/default/backup.cron.erb index 38dbc0c6a..fbff448bc 100644 --- a/cookbooks/blogs/templates/default/backup.cron.erb +++ b/cookbooks/blogs/templates/default/backup.cron.erb @@ -13,7 +13,10 @@ B="blogs-$D.tar.zst" mkdir "$T/blogs-$D" sqlite3 /srv/blogs.openstreetmap.org/planet.db ".backup $T/blogs-$D/planet.db" -nice tar --create --dereference --directory="$T" --sort=name "blogs-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" \ + --sort=name \ + "blogs-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/chef/templates/default/server-backup.cron.erb b/cookbooks/chef/templates/default/server-backup.cron.erb index d5f612665..bc1204002 100644 --- a/cookbooks/chef/templates/default/server-backup.cron.erb +++ b/cookbooks/chef/templates/default/server-backup.cron.erb @@ -16,7 +16,10 @@ chmod g+rwx "$T" "$T/chef-server-$D" sudo -u opscode-pgsql /opt/opscode/embedded/bin/pg_dumpall --file="$T/chef-server-$D/chef.dmp" --clean ln -s /var/opt/opscode/bookshelf/data "$T/chef-server-$D/bookshelf" -nice tar --create --dereference --directory="$T" --sort=name "chef-server-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" \ + --sort=name \ + "chef-server-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/civicrm/templates/default/backup.cron.erb b/cookbooks/civicrm/templates/default/backup.cron.erb index b504e2ad3..ed0ea9f10 100644 --- a/cookbooks/civicrm/templates/default/backup.cron.erb +++ b/cookbooks/civicrm/templates/default/backup.cron.erb @@ -17,7 +17,10 @@ echo 'password=<%= @passwords["database"] %>' >> "$T/mysqldump.opts" mysqldump --defaults-file="$T/mysqldump.opts" --opt --skip-lock-tables --no-tablespaces --max-allowed-packet=1G civicrm > "$T/osmf-crm-$D/civicrm.sql" ln -s /srv/supporting.openstreetmap.org "$T/osmf-crm-$D/www" -nice tar --create --dereference --directory="$T" --sort=name "osmf-crm-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" \ + --sort=name \ + "osmf-crm-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/community/templates/default/backup.cron.erb b/cookbooks/community/templates/default/backup.cron.erb index e5932b744..aad8c0f9e 100644 --- a/cookbooks/community/templates/default/backup.cron.erb +++ b/cookbooks/community/templates/default/backup.cron.erb @@ -21,9 +21,9 @@ nice tar --create --numeric-owner --dereference --directory="$T" \ --sort=name \ --warning=no-file-changed \ --warning=no-file-removed \ - --ignore-failed-read \ - --exclude="community-$D/shared-web-only/log/**/*.gz" \ + --exclude="community-$D/shared-web-only/log" \ "community-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/git/templates/default/backup.cron.erb b/cookbooks/git/templates/default/backup.cron.erb index 2c9469028..ae6b1f136 100644 --- a/cookbooks/git/templates/default/backup.cron.erb +++ b/cookbooks/git/templates/default/backup.cron.erb @@ -12,7 +12,10 @@ B="git-$D.tar.zst" ln -s /var/lib/git "$T/git-$D" -nice tar --create --dereference --directory="$T" --sort=name "git-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" \ + --sort=name \ + "git-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/mailman/templates/default/backup.cron.erb b/cookbooks/mailman/templates/default/backup.cron.erb index 33cd61132..92b3f4677 100644 --- a/cookbooks/mailman/templates/default/backup.cron.erb +++ b/cookbooks/mailman/templates/default/backup.cron.erb @@ -17,7 +17,6 @@ set +e nice tar --create --dereference \ --warning=no-file-changed \ --warning=no-file-removed \ - --ignore-failed-read \ --exclude='*/archives/*.txt.gz' \ --directory="$T" \ --sort=name \ diff --git a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb index 1287fbaed..2bfa49c15 100755 --- a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb +++ b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb @@ -21,13 +21,15 @@ set +e nice tar --create --dereference --directory="$T" \ --warning=no-file-changed \ --warning=no-file-removed \ - --ignore-failed-read \ --exclude="wiki-<%= @name %>-$D/www/w/images/thumb" \ --exclude="wiki-<%= @name %>-$D/www/w/.git" \ + --exclude="wiki-<%= @name %>-$D/www/w/cache" \ --exclude="wiki-<%= @name %>-$D/www/w/extensions/*/.git" \ --exclude="wiki-<%= @name %>-$D/www/dump" \ + --exclude="wiki-<%= @name %>-$D/www/sitemap-*" \ --sort=name \ "wiki-<%= @name %>-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/otrs/templates/default/backup.cron.erb b/cookbooks/otrs/templates/default/backup.cron.erb index d0bfd3b35..2bf2f8798 100644 --- a/cookbooks/otrs/templates/default/backup.cron.erb +++ b/cookbooks/otrs/templates/default/backup.cron.erb @@ -17,7 +17,11 @@ sudo -u otrs pg_dump --file="$T/otrs-$D/otrs.dmp" otrs ln -s /var/lib/otrs "$T/otrs-$D/otrs-var" ln -s /etc/apache2/sites-available/otrs.openstreetmap.org.conf "$T/otrs-$D/apache2-otrs.openstreetmap.org.conf" -nice tar --create --dereference --directory="$T" --sort=name "otrs-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" \ + --sort=name \ + --exclude="otrs-$D//otrs-var/tmp/CacheFileStorable" \ + "otrs-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/prometheus/templates/default/backup.cron.erb b/cookbooks/prometheus/templates/default/backup.cron.erb index d65b06554..f07ab4c27 100644 --- a/cookbooks/prometheus/templates/default/backup.cron.erb +++ b/cookbooks/prometheus/templates/default/backup.cron.erb @@ -15,7 +15,10 @@ mkdir "$T/prometheus-$D" ln -s /var/lib/prometheus/alertmanager "$T/prometheus-$D/alertmanager" ln -s /var/lib/grafana "$T/prometheus-$D/grafana" -nice tar --create --dereference --directory="$T" --sort=name "prometheus-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" \ + --sort=name \ + "prometheus-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" diff --git a/cookbooks/stateofthemap/templates/default/backup.cron.erb b/cookbooks/stateofthemap/templates/default/backup.cron.erb index 4476c3819..5898e52aa 100644 --- a/cookbooks/stateofthemap/templates/default/backup.cron.erb +++ b/cookbooks/stateofthemap/templates/default/backup.cron.erb @@ -23,7 +23,10 @@ ln -s /srv/2010.stateofthemap.org "$T/sotm-$D/www2010" ln -s /srv/2011.stateofthemap.org "$T/sotm-$D/www2011" ln -s /srv/2012.stateofthemap.org "$T/sotm-$D/www2012" -nice tar --create --dereference --directory="$T" --sort=name "sotm-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" +nice tar --create --dereference --directory="$T" \ + --sort=name \ + "sotm-$D" | nice zstd --quiet --long --rsyncable -o "$T/$B" + nice rsync --preallocate --fuzzy "$T/$B" backup.openstreetmap.org::backup rm -rf "$T" -- 2.39.5