Use rsync preallocate files to avoid fragmentation

Enable rsync preallocate to limit filesystem fragmentation of file creation.
Enable rsync fuzzy where suitable to leverage diffing from existing files.

diff --git a/cookbooks/blog/templates/default/backup.cron.erb b/cookbooks/blog/templates/default/backup.cron.erb
index 9e791abd9f681e0ca63db4d859de0974534e8279..0903962a794a84c4096cf0ff1a67d347cf24c734 100644 (file)
--- a/cookbooks/blog/templates/default/backup.cron.erb
+++ b/cookbooks/blog/templates/default/backup.cron.erb
@@ -17,6 +17,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B osm-blog-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/chef/templates/default/repository-backup.cron.erb b/cookbooks/chef/templates/default/repository-backup.cron.erb
index 2e55f057561e5d5a0cb5f77c0b93b5f569677856..f82e7b2273a58d1a097b2a7085c25516df75f9d2 100644 (file)
--- a/cookbooks/chef/templates/default/repository-backup.cron.erb
+++ b/cookbooks/chef/templates/default/repository-backup.cron.erb
@@ -9,6 +9,6 @@ ln -s /var/lib/git/chef.git $T/chef-repository-$D
 export GZIP="--rsyncable -9"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B chef-repository-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/chef/templates/default/server-backup.cron.erb b/cookbooks/chef/templates/default/server-backup.cron.erb
index c1e2100b3387e8b89de88960e914198f2df72ca4..048bc99e9679e2b41ad5598f1d730f18056e8ec4 100644 (file)
--- a/cookbooks/chef/templates/default/server-backup.cron.erb
+++ b/cookbooks/chef/templates/default/server-backup.cron.erb
@@ -13,6 +13,6 @@ ln -s /var/opt/opscode/bookshelf/data $T/chef-server-$D/bookshelf
 export GZIP="--rsyncable -9"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B chef-server-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/civicrm/templates/default/backup.cron.erb b/cookbooks/civicrm/templates/default/backup.cron.erb
index 8ee0c5257b3033c9c6a7a487b44940e783c89078..61bb4cd453898600d7414bddc15268004a6ed591 100644 (file)
--- a/cookbooks/civicrm/templates/default/backup.cron.erb
+++ b/cookbooks/civicrm/templates/default/backup.cron.erb
@@ -17,6 +17,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B osmf-crm-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/db/templates/default/backup-db.erb b/cookbooks/db/templates/default/backup-db.erb
index 492dded40be76318cc17b0612f4275237965834b..a625a2ef5433289662ca80f6a2803b2e800c9206 100644 (file)
--- a/cookbooks/db/templates/default/backup-db.erb
+++ b/cookbooks/db/templates/default/backup-db.erb
@@ -8,6 +8,6 @@ F=/store/backup/osm-${D}.dmp
 pg_dump --user=backup --format=custom --file=$F openstreetmap
 
 export RSYNC_RSH="ssh -ax"
-rsync $F backup.openstreetmap.org::backup
+rsync --preallocate $F backup.openstreetmap.org::backup
 
 rm -f $F

diff --git a/cookbooks/donate/templates/default/backup.cron.erb b/cookbooks/donate/templates/default/backup.cron.erb
index 7ff3ee08aa6413d7e453eb45f63c3c7e0a0fd6ee..92b66cf005bcf36f9dfab5a09937cc2ec47c402f 100644 (file)
--- a/cookbooks/donate/templates/default/backup.cron.erb
+++ b/cookbooks/donate/templates/default/backup.cron.erb
@@ -17,6 +17,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B osm-donate-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/forum/templates/default/backup.cron.erb b/cookbooks/forum/templates/default/backup.cron.erb
index ec30196d3097cbe8a836004f981ed1be3ac602ea..c3475b87663bd2dfd47b39b7161424382c76699f 100644 (file)
--- a/cookbooks/forum/templates/default/backup.cron.erb
+++ b/cookbooks/forum/templates/default/backup.cron.erb
@@ -17,6 +17,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B forum-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/git/templates/default/backup.cron.erb b/cookbooks/git/templates/default/backup.cron.erb
index 29e059b45e614592d612c5632887b9c45893bcb7..ac2167b8c20459b7408f62f68024fdad73a2a0f5 100644 (file)
--- a/cookbooks/git/templates/default/backup.cron.erb
+++ b/cookbooks/git/templates/default/backup.cron.erb
@@ -12,6 +12,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B git-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/mailman/templates/default/backup.cron.erb b/cookbooks/mailman/templates/default/backup.cron.erb
index 80a5c8ba2d7c89d745f60d8b55cc2f965913cbf1..12a550c03a36695ef18482fb13f9a3fbd334d464 100644 (file)
--- a/cookbooks/mailman/templates/default/backup.cron.erb
+++ b/cookbooks/mailman/templates/default/backup.cron.erb
@@ -13,6 +13,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B lists-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb
index a99503a235a35c294dd5bbd056627c3367dcf8c8..e42cb8f469e0943f2c23ea4ece1b9f0a5e7968c3 100755 (executable)
--- a/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb
+++ b/cookbooks/mediawiki/templates/default/mediawiki-backup.cron.erb
@@ -11,6 +11,6 @@ mysqldump --defaults-file=$T/mysqldump.opts --opt --skip-lock-tables --single-tr
 ln -s <%= @directory %>  $T/wiki-<%= @name %>-$D/www
 export GZIP="--rsyncable -9" #make backup rsyncable
 nice tar --create --gzip --dereference --directory=$T --exclude=wiki-<%= @name %>-$D/www/w/images/thumb --exclude=wiki-<%= @name %>-$D/www/w/.git --exclude=wiki-<%= @name %>-$D/www/w/extensions/*/.git --file=$T/$B wiki-<%= @name %>-$D
-nice rsync $T/$B backup::backup --fuzzy
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/munin/templates/default/backup.cron.erb b/cookbooks/munin/templates/default/backup.cron.erb
index 9ff94f2e03bfff0699403059b17644fa5abe3d49..6b846057769f8d03d443bfeba94a516a817d341b 100644 (file)
--- a/cookbooks/munin/templates/default/backup.cron.erb
+++ b/cookbooks/munin/templates/default/backup.cron.erb
@@ -14,6 +14,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B munin-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/nominatim/templates/default/backup-nominatim.erb b/cookbooks/nominatim/templates/default/backup-nominatim.erb
index b3ef0646a1b1c57ca1aaac0f2dd1c51b3c22eec5..2b73ab80605ab84fb7db3d4a6c0a872577ea818b 100644 (file)
--- a/cookbooks/nominatim/templates/default/backup-nominatim.erb
+++ b/cookbooks/nominatim/templates/default/backup-nominatim.erb
@@ -15,6 +15,6 @@ F=/tmp/nominatim-${D}.dmp
 pg_dump --file=$F -F c -Z 9 -t file -t '*columns' -t 'import_polygon_*' -t import_status -t place_addressline -t location_property_osmline -t placex -t search_name -t 'seq_*' -t word <%= @db %>
 
 export RSYNC_RSH="ssh -ax"
-rsync $F backup.openstreetmap.org::backup
+rsync --preallocate --fuzzy $F backup.openstreetmap.org::backup
 
 rm -f $F

diff --git a/cookbooks/osqa/templates/default/backup.cron.erb b/cookbooks/osqa/templates/default/backup.cron.erb
index e9d0e7352a8ad1ca48bbe5cf6299ccb98449de0f..3c2abf6a79c259a493b56127a96bc75b0291937f 100644 (file)
--- a/cookbooks/osqa/templates/default/backup.cron.erb
+++ b/cookbooks/osqa/templates/default/backup.cron.erb
@@ -16,6 +16,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B <%= @name %>-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/otrs/templates/default/backup.cron.erb b/cookbooks/otrs/templates/default/backup.cron.erb
index 6724e3ca4c50d576f6a82a621b6f7b120d1da125..bfe5242afb3a34f0eaaaa1e149ffa749b8524270 100644 (file)
--- a/cookbooks/otrs/templates/default/backup.cron.erb
+++ b/cookbooks/otrs/templates/default/backup.cron.erb
@@ -17,6 +17,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B otrs-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/planet/templates/default/logrotate.apache.erb b/cookbooks/planet/templates/default/logrotate.apache.erb
index c25e4853fd7f43f8f6cef5891c5adc79ba415784..f1c5dde56843eb164e2562d52813c3ee6eff7f47 100644 (file)
--- a/cookbooks/planet/templates/default/logrotate.apache.erb
+++ b/cookbooks/planet/templates/default/logrotate.apache.erb
@@ -11,7 +11,7 @@
        sharedscripts
        postrotate
                /etc/init.d/apache2 reload > /dev/null
-               rsync /var/log/apache2/planet.openstreetmap.org-access.log.2.gz ironbelly::logs/planet.openstreetmap.org/`date -d "-7 days" +%Y-%m-%d`.gz
+               rsync --preallocate /var/log/apache2/planet.openstreetmap.org-access.log.2.gz ironbelly::logs/planet.openstreetmap.org/`date -d "-7 days" +%Y-%m-%d`.gz
        endscript
        prerotate
                if [ -d /etc/logrotate.d/httpd-prerotate ]; then \

diff --git a/cookbooks/stateofthemap/templates/default/backup.cron.erb b/cookbooks/stateofthemap/templates/default/backup.cron.erb
index 3f2fe977653886c24d33022c5782a84bee77fe02..cb7171b6f5dd82464c922b70849c0f69b93c15bd 100644 (file)
--- a/cookbooks/stateofthemap/templates/default/backup.cron.erb
+++ b/cookbooks/stateofthemap/templates/default/backup.cron.erb
@@ -27,6 +27,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B sotm-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/subversion/templates/default/backup.cron.erb b/cookbooks/subversion/templates/default/backup.cron.erb
index 750cf47ce33ccf72f745139817cd8ea17a7aacc0..df7b4fe28f880d38b60508b698b080daa78efa59 100644 (file)
--- a/cookbooks/subversion/templates/default/backup.cron.erb
+++ b/cookbooks/subversion/templates/default/backup.cron.erb
@@ -12,6 +12,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --directory=$T --file=$T/$B svn-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/switch2osm/templates/default/backup.cron.erb b/cookbooks/switch2osm/templates/default/backup.cron.erb
index 5f913f3d535720e9b63fd46b085a1416134f2b8f..54717f49cc17062f22918baf58e965dd6bd88717 100644 (file)
--- a/cookbooks/switch2osm/templates/default/backup.cron.erb
+++ b/cookbooks/switch2osm/templates/default/backup.cron.erb
@@ -16,6 +16,6 @@ ln -s /srv/switch2osm.org $T/switch2osm-$D/www
 export GZIP="--rsyncable -9"
 
 nice tar --create --gzip --dereference --directory=$T --file=$T/$B switch2osm-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/tilecache/templates/default/logrotate.squid.erb b/cookbooks/tilecache/templates/default/logrotate.squid.erb
index fdc426cfe6f7d0fc30fd037487482a996bbc0e4d..2112b85cde2f631e38679ad42cd1eba2c7852b53 100644 (file)
--- a/cookbooks/tilecache/templates/default/logrotate.squid.erb
+++ b/cookbooks/tilecache/templates/default/logrotate.squid.erb
@@ -14,6 +14,6 @@
     test ! -e /var/run/squid.pid || /usr/sbin/squid -k rotate
   endscript
   lastaction
-    /usr/bin/rsync /var/log/squid/zere.log.1.xz ironbelly::logs/tile.openstreetmap.org/<%= node[:hostname] %>-`date -d "-1 days" +%Y-%m-%d`.xz || true
+    /usr/bin/rsync --preallocate /var/log/squid/zere.log.1.xz ironbelly::logs/tile.openstreetmap.org/<%= node[:hostname] %>-`date -d "-1 days" +%Y-%m-%d`.xz || true
   endscript
 }

diff --git a/cookbooks/trac/templates/default/backup.cron.erb b/cookbooks/trac/templates/default/backup.cron.erb
index 6d92f3a7e4f2063e4d0a54b5a52e80fd2a7ac034..e4e4bd8d6673fc738daccb7b95dd2802b5f6c53c 100644 (file)
--- a/cookbooks/trac/templates/default/backup.cron.erb
+++ b/cookbooks/trac/templates/default/backup.cron.erb
@@ -12,6 +12,6 @@ export GZIP="--rsyncable -9"
 export RSYNC_RSH="ssh -ax"
 
 nice tar --create --gzip --directory=$T --file=$T/$B trac-$D
-nice rsync $T/$B backup::backup
+nice rsync --preallocate --fuzzy $T/$B backup::backup
 
 rm -rf $T

diff --git a/cookbooks/web/templates/default/logrotate.apache.erb b/cookbooks/web/templates/default/logrotate.apache.erb
index 8d64879f573dc9fc826867d708a7660232078e00..b4d171be83a210b32d2f12af2e86349efb6fe5ae 100644 (file)
--- a/cookbooks/web/templates/default/logrotate.apache.erb
+++ b/cookbooks/web/templates/default/logrotate.apache.erb
@@ -11,6 +11,6 @@
   sharedscripts
   postrotate
     /etc/init.d/apache2 reload > /dev/null
-    /usr/bin/rsync /var/log/apache2/access.log.2.gz ironbelly::logs/www.openstreetmap.org/<%= node[:hostname] %>-`date -d "-2 days" +%Y-%m-%d`.gz
+    /usr/bin/rsync --preallocate /var/log/apache2/access.log.2.gz ironbelly::logs/www.openstreetmap.org/<%= node[:hostname] %>-`date -d "-2 days" +%Y-%m-%d`.gz
   endscript
 }

diff --git a/cookbooks/web/templates/default/logrotate.web.erb b/cookbooks/web/templates/default/logrotate.web.erb
index 8cb1da52641d4877060be5de76ee31e244eeb024..bc38d45154b42c83d3c3b23baa0e2fab2864e19e 100644 (file)
--- a/cookbooks/web/templates/default/logrotate.web.erb
+++ b/cookbooks/web/templates/default/logrotate.web.erb
@@ -3,7 +3,7 @@
 <%= node[:web][:log_directory] %>/*.log {
   daily
   missingok
-  rotate 7 
+  rotate 7
   compress
   delaycompress
   notifempty
@@ -21,7 +21,7 @@
 <% end -%>
 <% if File.directory?("#{node[:web][:base_directory]}/cgimap") -%>
     /bin/systemctl reload cgimap
-    /usr/bin/rsync <%= node[:web][:log_directory] %>/cgimap.log.2.gz ironbelly::logs/www.openstreetmap.org/cgimap-<%= node[:hostname] %>-`date -d "-2 days" +%Y-%m-%d`.gz
+    /usr/bin/rsync --preallocate <%= node[:web][:log_directory] %>/cgimap.log.2.gz ironbelly::logs/www.openstreetmap.org/cgimap-<%= node[:hostname] %>-`date -d "-2 days" +%Y-%m-%d`.gz
 <% end -%>
   endscript
 }