From 5524a5f2a12c63acc122269b928cf28a4b0ce211 Mon Sep 17 00:00:00 2001
From: Grant Slater <git@firefishy.com>
Date: Sat, 12 Nov 2016 16:30:51 +0000
Subject: [PATCH] Use rsync preallocate files to avoid fragmentation

Enable rsync preallocate to limit filesystem fragmentation of file creation.
Enable rsync fuzzy where suitable to leverage diffing from existing files.
---
 cookbooks/blog/templates/default/backup.cron.erb              | 2 +-
 cookbooks/chef/templates/default/repository-backup.cron.erb   | 2 +-
 cookbooks/chef/templates/default/server-backup.cron.erb       | 2 +-
 cookbooks/civicrm/templates/default/backup.cron.erb           | 2 +-
 cookbooks/db/templates/default/backup-db.erb                  | 2 +-
 cookbooks/donate/templates/default/backup.cron.erb            | 2 +-
 cookbooks/forum/templates/default/backup.cron.erb             | 2 +-
 cookbooks/git/templates/default/backup.cron.erb               | 2 +-
 cookbooks/mailman/templates/default/backup.cron.erb           | 2 +-
 .../mediawiki/templates/default/mediawiki-backup.cron.erb     | 2 +-
 cookbooks/munin/templates/default/backup.cron.erb             | 2 +-
 cookbooks/nominatim/templates/default/backup-nominatim.erb    | 2 +-
 cookbooks/osqa/templates/default/backup.cron.erb              | 2 +-
 cookbooks/otrs/templates/default/backup.cron.erb              | 2 +-
 cookbooks/planet/templates/default/logrotate.apache.erb       | 2 +-
 cookbooks/stateofthemap/templates/default/backup.cron.erb     | 2 +-
 cookbooks/subversion/templates/default/backup.cron.erb        | 2 +-
 cookbooks/switch2osm/templates/default/backup.cron.erb        | 2 +-
 cookbooks/tilecache/templates/default/logrotate.squid.erb     | 2 +-
 cookbooks/trac/templates/default/backup.cron.erb              | 2 +-
 cookbooks/web/templates/default/logrotate.apache.erb          | 2 +-
 cookbooks/web/templates/default/logrotate.web.erb             | 4 ++--
 22 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/cookbooks/blog/templates/default/backup.cron.erb b/cookbooks/blog/templates/default/backup.cron.erb
index 9e791abd9..0903962a7 100644
--- a/cookbooks/blog/templates/default/backup.cron.erb
+++ b/cookbooks/blog/templates/default/backup.cron.erb
@@ -17,6 +17,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B osm-blog-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/chef/templates/default/repository-backup.cron.erb b/cookbooks/chef/templates/default/repository-backup.cron.erb
index 2e55f0575..f82e7b227 100644
--- a/cookbooks/chef/templates/default/repository-backup.cron.erb
+++ b/cookbooks/chef/templates/default/repository-backup.cron.erb
@@ -9,6 +9,6 @@ ln -s /var/lib/git/chef.git $T/chef-repository-$D
 export GZIP="--rsyncable -9"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B chef-repository-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/chef/templates/default/server-backup.cron.erb b/cookbooks/chef/templates/default/server-backup.cron.erb
index c1e2100b3..048bc99e9 100644
--- a/cookbooks/chef/templates/default/server-backup.cron.erb
+++ b/cookbooks/chef/templates/default/server-backup.cron.erb
@@ -13,6 +13,6 @@ ln -s /var/opt/opscode/bookshelf/data $T/chef-server-$D/bookshelf
 export GZIP="--rsyncable -9"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B chef-server-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/civicrm/templates/default/backup.cron.erb b/cookbooks/civicrm/templates/default/backup.cron.erb
index 8ee0c5257..61bb4cd45 100644
--- a/cookbooks/civicrm/templates/default/backup.cron.erb
+++ b/cookbooks/civicrm/templates/default/backup.cron.erb
@@ -17,6 +17,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B osmf-crm-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/db/templates/default/backup-db.erb b/cookbooks/db/templates/default/backup-db.erb
index 492dded40..a625a2ef5 100644
--- a/cookbooks/db/templates/default/backup-db.erb
+++ b/cookbooks/db/templates/default/backup-db.erb
@@ -8,6 +8,6 @@ F=/store/backup/osm-${D}.dmp
 pg_dump --user=backup --format=custom --file=$F openstreetmap
 
 export RSYNC_RSH="ssh -ax"
-rsync $F backup.openstreetmap.org::backup
+rsync --preallocate $F backup.openstreetmap.org::backup
 
 rm -f $F
diff --git a/cookbooks/donate/templates/default/backup.cron.erb b/cookbooks/donate/templates/default/backup.cron.erb
index 7ff3ee08a..92b66cf00 100644
--- a/cookbooks/donate/templates/default/backup.cron.erb
+++ b/cookbooks/donate/templates/default/backup.cron.erb
@@ -17,6 +17,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B osm-donate-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/forum/templates/default/backup.cron.erb b/cookbooks/forum/templates/default/backup.cron.erb
index ec30196d3..c3475b876 100644
--- a/cookbooks/forum/templates/default/backup.cron.erb
+++ b/cookbooks/forum/templates/default/backup.cron.erb
@@ -17,6 +17,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B forum-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/git/templates/default/backup.cron.erb b/cookbooks/git/templates/default/backup.cron.erb
index 29e059b45..ac2167b8c 100644
--- a/cookbooks/git/templates/default/backup.cron.erb
+++ b/cookbooks/git/templates/default/backup.cron.erb
@@ -12,6 +12,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B git-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/mailman/templates/default/backup.cron.erb b/cookbooks/mailman/templates/default/backup.cron.erb
index 80a5c8ba2..12a550c03 100644
--- a/cookbooks/mailman/templates/default/backup.cron.erb
+++ b/cookbooks/mailman/templates/default/backup.cron.erb
@@ -13,6 +13,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B lists-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb
index a99503a23..e42cb8f46 100755
--- a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb
+++ b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb
@@ -11,6 +11,6 @@ mysqldump --defaults-file=$T/mysqldump.opts --opt --skip-lock-tables --single-tr
 ln -s <%= @directory %>  $T/wiki-<%= @name %>-$D/www
 export GZIP="--rsyncable -9" #make backup rsyncable
 nice tar --create --gzip --dereference --directory=$T --exclude=wiki-<%= @name %>-$D/www/w/images/thumb --exclude=wiki-<%= @name %>-$D/www/w/.git --exclude=wiki-<%= @name %>-$D/www/w/extensions/*/.git --file=$T/$B wiki-<%= @name %>-$D
-nice rsync $T/$B backup::backup --fuzzy
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/munin/templates/default/backup.cron.erb b/cookbooks/munin/templates/default/backup.cron.erb
index 9ff94f2e0..6b8460577 100644
--- a/cookbooks/munin/templates/default/backup.cron.erb
+++ b/cookbooks/munin/templates/default/backup.cron.erb
@@ -14,6 +14,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B munin-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/nominatim/templates/default/backup-nominatim.erb b/cookbooks/nominatim/templates/default/backup-nominatim.erb
index b3ef0646a..2b73ab806 100644
--- a/cookbooks/nominatim/templates/default/backup-nominatim.erb
+++ b/cookbooks/nominatim/templates/default/backup-nominatim.erb
@@ -15,6 +15,6 @@ F=/tmp/nominatim-${D}.dmp
 pg_dump --file=$F -F c -Z 9 -t file -t '*columns' -t 'import_polygon_*' -t import_status -t place_addressline -t location_property_osmline -t placex -t search_name -t 'seq_*' -t word <%= @db %>
 
 export RSYNC_RSH="ssh -ax"
-rsync $F backup.openstreetmap.org::backup
+rsync --preallocate --fuzzy $F backup.openstreetmap.org::backup
 
 rm -f $F
diff --git a/cookbooks/osqa/templates/default/backup.cron.erb b/cookbooks/osqa/templates/default/backup.cron.erb
index e9d0e7352..3c2abf6a7 100644
--- a/cookbooks/osqa/templates/default/backup.cron.erb
+++ b/cookbooks/osqa/templates/default/backup.cron.erb
@@ -16,6 +16,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B <%= @name %>-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/otrs/templates/default/backup.cron.erb b/cookbooks/otrs/templates/default/backup.cron.erb
index 6724e3ca4..bfe5242af 100644
--- a/cookbooks/otrs/templates/default/backup.cron.erb
+++ b/cookbooks/otrs/templates/default/backup.cron.erb
@@ -17,6 +17,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B otrs-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/planet/templates/default/logrotate.apache.erb b/cookbooks/planet/templates/default/logrotate.apache.erb
index c25e4853f..f1c5dde56 100644
--- a/cookbooks/planet/templates/default/logrotate.apache.erb
+++ b/cookbooks/planet/templates/default/logrotate.apache.erb
@@ -11,7 +11,7 @@
 	sharedscripts
 	postrotate
 		/etc/init.d/apache2 reload > /dev/null
-		rsync /var/log/apache2/planet.openstreetmap.org-access.log.2.gz ironbelly::logs/planet.openstreetmap.org/`date -d "-7 days" +%Y-%m-%d`.gz
+		rsync --preallocate /var/log/apache2/planet.openstreetmap.org-access.log.2.gz ironbelly::logs/planet.openstreetmap.org/`date -d "-7 days" +%Y-%m-%d`.gz
 	endscript
 	prerotate
 		if [ -d /etc/logrotate.d/httpd-prerotate ]; then \
diff --git a/cookbooks/stateofthemap/templates/default/backup.cron.erb b/cookbooks/stateofthemap/templates/default/backup.cron.erb
index 3f2fe9776..cb7171b6f 100644
--- a/cookbooks/stateofthemap/templates/default/backup.cron.erb
+++ b/cookbooks/stateofthemap/templates/default/backup.cron.erb
@@ -27,6 +27,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B sotm-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/subversion/templates/default/backup.cron.erb b/cookbooks/subversion/templates/default/backup.cron.erb
index 750cf47ce..df7b4fe28 100644
--- a/cookbooks/subversion/templates/default/backup.cron.erb
+++ b/cookbooks/subversion/templates/default/backup.cron.erb
@@ -12,6 +12,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --directory=$T --file=$T/$B svn-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/switch2osm/templates/default/backup.cron.erb b/cookbooks/switch2osm/templates/default/backup.cron.erb
index 5f913f3d5..54717f49c 100644
--- a/cookbooks/switch2osm/templates/default/backup.cron.erb
+++ b/cookbooks/switch2osm/templates/default/backup.cron.erb
@@ -16,6 +16,6 @@ ln -s /srv/switch2osm.org $T/switch2osm-$D/www
 export GZIP="--rsyncable -9"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B switch2osm-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/tilecache/templates/default/logrotate.squid.erb b/cookbooks/tilecache/templates/default/logrotate.squid.erb
index fdc426cfe..2112b85cd 100644
--- a/cookbooks/tilecache/templates/default/logrotate.squid.erb
+++ b/cookbooks/tilecache/templates/default/logrotate.squid.erb
@@ -14,6 +14,6 @@
     test ! -e /var/run/squid.pid || /usr/sbin/squid -k rotate
   endscript
   lastaction
-    /usr/bin/rsync /var/log/squid/zere.log.1.xz ironbelly::logs/tile.openstreetmap.org/<%= node[:hostname] %>-`date -d "-1 days" +%Y-%m-%d`.xz || true
+    /usr/bin/rsync --preallocate /var/log/squid/zere.log.1.xz ironbelly::logs/tile.openstreetmap.org/<%= node[:hostname] %>-`date -d "-1 days" +%Y-%m-%d`.xz || true
   endscript
 }
diff --git a/cookbooks/trac/templates/default/backup.cron.erb b/cookbooks/trac/templates/default/backup.cron.erb
index 6d92f3a7e..e4e4bd8d6 100644
--- a/cookbooks/trac/templates/default/backup.cron.erb
+++ b/cookbooks/trac/templates/default/backup.cron.erb
@@ -12,6 +12,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --directory=$T --file=$T/$B trac-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T
diff --git a/cookbooks/web/templates/default/logrotate.apache.erb b/cookbooks/web/templates/default/logrotate.apache.erb
index 8d64879f5..b4d171be8 100644
--- a/cookbooks/web/templates/default/logrotate.apache.erb
+++ b/cookbooks/web/templates/default/logrotate.apache.erb
@@ -11,6 +11,6 @@
   sharedscripts
   postrotate
     /etc/init.d/apache2 reload > /dev/null
-    /usr/bin/rsync /var/log/apache2/access.log.2.gz ironbelly::logs/www.openstreetmap.org/<%= node[:hostname] %>-`date -d "-2 days" +%Y-%m-%d`.gz
+    /usr/bin/rsync --preallocate /var/log/apache2/access.log.2.gz ironbelly::logs/www.openstreetmap.org/<%= node[:hostname] %>-`date -d "-2 days" +%Y-%m-%d`.gz
   endscript
 }
diff --git a/cookbooks/web/templates/default/logrotate.web.erb b/cookbooks/web/templates/default/logrotate.web.erb
index 8cb1da526..bc38d4515 100644
--- a/cookbooks/web/templates/default/logrotate.web.erb
+++ b/cookbooks/web/templates/default/logrotate.web.erb
@@ -3,7 +3,7 @@
 <%= node[:web][:log_directory] %>/*.log {
   daily
   missingok
-  rotate 7 
+  rotate 7
   compress
   delaycompress
   notifempty
@@ -21,7 +21,7 @@
 <% end -%>
 <% if File.directory?("#{node[:web][:base_directory]}/cgimap") -%>
     /bin/systemctl reload cgimap
-    /usr/bin/rsync <%= node[:web][:log_directory] %>/cgimap.log.2.gz ironbelly::logs/www.openstreetmap.org/cgimap-<%= node[:hostname] %>-`date -d "-2 days" +%Y-%m-%d`.gz
+    /usr/bin/rsync --preallocate <%= node[:web][:log_directory] %>/cgimap.log.2.gz ironbelly::logs/www.openstreetmap.org/cgimap-<%= node[:hostname] %>-`date -d "-2 days" +%Y-%m-%d`.gz
 <% end -%>
   endscript
 }
-- 
2.43.2