From: Grant Slater <github@firefishy.com>
Date: Thu, 28 Sep 2023 14:43:15 +0000 (+0100)
Subject: Do not report progress on scripted s3 uploads
X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/b426085a43bc5dfc0ffa8deba338160c960db43c

Do not report progress on scripted s3 uploads
---

diff --git a/cookbooks/planet/files/default/replication-bin/replicate-changesets b/cookbooks/planet/files/default/replication-bin/replicate-changesets
index 87334f21a..5d47266e6 100755
--- a/cookbooks/planet/files/default/replication-bin/replicate-changesets
+++ b/cookbooks/planet/files/default/replication-bin/replicate-changesets
@@ -264,8 +264,8 @@ class Replicator
       s3_file = s3_stem + ".osm.gz"
       s3_state_file = s3_stem + ".state.txt"
 
-      system("/opt/awscli/v2/current/bin/aws", "--profile=osm-pds-upload", "s3", "cp", "--storage-class=INTELLIGENT_TIERING", data_file, s3_file)
-      system("/opt/awscli/v2/current/bin/aws", "--profile=osm-pds-upload", "s3", "cp", "--storage-class=INTELLIGENT_TIERING", data_state_file, s3_state_file)
+      system("/opt/awscli/v2/current/bin/aws", "--profile=osm-pds-upload", "s3", "cp", "--storage-class=INTELLIGENT_TIERING", "--no-progress", data_file, s3_file)
+      system("/opt/awscli/v2/current/bin/aws", "--profile=osm-pds-upload", "s3", "cp", "--storage-class=INTELLIGENT_TIERING", "--no-progress", data_state_file, s3_state_file)
     end
   end
 
diff --git a/cookbooks/planet/files/default/replication-bin/replicate-day b/cookbooks/planet/files/default/replication-bin/replicate-day
index 2db770c16..4e8fd34e8 100644
--- a/cookbooks/planet/files/default/replication-bin/replicate-day
+++ b/cookbooks/planet/files/default/replication-bin/replicate-day
@@ -11,6 +11,6 @@ sequencePart2=$(($sequenceNumber / 1000 % 1000))
 sequencePart3=$(($sequenceNumber % 1000))
 diffPath=$(printf "%03d/%03d/%03d" $sequencePart1 $sequencePart2 $sequencePart3)
 
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/day/${diffPath}.osc.gz" "s3://osm-planet-eu-central-1/planet/replication/day/${diffPath}.osc.gz"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/day/${diffPath}.state.txt" "s3://osm-planet-eu-central-1/planet/replication/day/${diffPath}.state.txt"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/day/state.txt" "s3://osm-planet-eu-central-1/planet/replication/day/state.txt"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/day/${diffPath}.osc.gz" "s3://osm-planet-eu-central-1/planet/replication/day/${diffPath}.osc.gz"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/day/${diffPath}.state.txt" "s3://osm-planet-eu-central-1/planet/replication/day/${diffPath}.state.txt"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/day/state.txt" "s3://osm-planet-eu-central-1/planet/replication/day/state.txt"
diff --git a/cookbooks/planet/files/default/replication-bin/replicate-hour b/cookbooks/planet/files/default/replication-bin/replicate-hour
index 849cbb417..184c4a3b2 100644
--- a/cookbooks/planet/files/default/replication-bin/replicate-hour
+++ b/cookbooks/planet/files/default/replication-bin/replicate-hour
@@ -11,6 +11,6 @@ sequencePart2=$(($sequenceNumber / 1000 % 1000))
 sequencePart3=$(($sequenceNumber % 1000))
 diffPath=$(printf "%03d/%03d/%03d" $sequencePart1 $sequencePart2 $sequencePart3)
 
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/hour/${diffPath}.osc.gz" "s3://osm-planet-eu-central-1/planet/replication/hour/${diffPath}.osc.gz"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/hour/${diffPath}.state.txt" "s3://osm-planet-eu-central-1/planet/replication/hour/${diffPath}.state.txt"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/hour/state.txt" "s3://osm-planet-eu-central-1/planet/replication/hour/state.txt"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/hour/${diffPath}.osc.gz" "s3://osm-planet-eu-central-1/planet/replication/hour/${diffPath}.osc.gz"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/hour/${diffPath}.state.txt" "s3://osm-planet-eu-central-1/planet/replication/hour/${diffPath}.state.txt"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/hour/state.txt" "s3://osm-planet-eu-central-1/planet/replication/hour/state.txt"
diff --git a/cookbooks/planet/files/default/replication-bin/replicate-minute b/cookbooks/planet/files/default/replication-bin/replicate-minute
index 9c54581d8..943cfa6a3 100755
--- a/cookbooks/planet/files/default/replication-bin/replicate-minute
+++ b/cookbooks/planet/files/default/replication-bin/replicate-minute
@@ -16,6 +16,6 @@ sequencePart2=$(($sequenceNumber / 1000 % 1000))
 sequencePart3=$(($sequenceNumber % 1000))
 diffPath=$(printf "%03d/%03d/%03d" $sequencePart1 $sequencePart2 $sequencePart3)
 
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/minute/${diffPath}.osc.gz" "s3://osm-planet-eu-central-1/planet/replication/minute/${diffPath}.osc.gz"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/minute/${diffPath}.state.txt" "s3://osm-planet-eu-central-1/planet/replication/minute/${diffPath}.state.txt"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/minute/state.txt" "s3://osm-planet-eu-central-1/planet/replication/minute/state.txt"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/minute/${diffPath}.osc.gz" "s3://osm-planet-eu-central-1/planet/replication/minute/${diffPath}.osc.gz"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/minute/${diffPath}.state.txt" "s3://osm-planet-eu-central-1/planet/replication/minute/${diffPath}.state.txt"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/minute/state.txt" "s3://osm-planet-eu-central-1/planet/replication/minute/state.txt"
diff --git a/cookbooks/planet/templates/default/planet-notes-dump.erb b/cookbooks/planet/templates/default/planet-notes-dump.erb
index 9f266b40a..ebfb9db13 100644
--- a/cookbooks/planet/templates/default/planet-notes-dump.erb
+++ b/cookbooks/planet/templates/default/planet-notes-dump.erb
@@ -55,7 +55,7 @@ rm -f planet-notes-latest.osn.bz2.md5
 
 sed -e "s/${cur_planet_notes}.bz2/planet-notes-latest.osn.bz2/" "${cur_year}/${cur_planet_notes}.bz2.md5" > planet-notes-latest.osn.bz2.md5
 
-/opt/awscli/v2/current/bin/aws --profile osm-pds-upload s3 cp --storage-class INTELLIGENT_TIERING "${cur_year}/${cur_planet_notes}.bz2.md5" "s3://osm-planet-eu-central-1/notes/osn/${cur_year}/${cur_planet_notes}.bz2.md5"
-/opt/awscli/v2/current/bin/aws --profile osm-pds-upload s3 cp --storage-class INTELLIGENT_TIERING "${cur_year}/${cur_planet_notes}.bz2" "s3://osm-planet-eu-central-1/notes/osn/${cur_year}/${cur_planet_notes}.bz2"
+/opt/awscli/v2/current/bin/aws --profile osm-pds-upload s3 cp --storage-class INTELLIGENT_TIERING  --no-progress "${cur_year}/${cur_planet_notes}.bz2.md5" "s3://osm-planet-eu-central-1/notes/osn/${cur_year}/${cur_planet_notes}.bz2.md5"
+/opt/awscli/v2/current/bin/aws --profile osm-pds-upload s3 cp --storage-class INTELLIGENT_TIERING  --no-progress "${cur_year}/${cur_planet_notes}.bz2" "s3://osm-planet-eu-central-1/notes/osn/${cur_year}/${cur_planet_notes}.bz2"
 
 rm /tmp/planet-notes-dump.lock
diff --git a/cookbooks/planet/templates/default/planetdump.erb b/cookbooks/planet/templates/default/planetdump.erb
index 8ffd8f9cc..dadc24adc 100644
--- a/cookbooks/planet/templates/default/planetdump.erb
+++ b/cookbooks/planet/templates/default/planetdump.erb
@@ -157,9 +157,9 @@ function install_dump {
   rm -f "${dir}/${latest}.md5"
   sed -e "s/${name}/${latest}/" "${dir}/${year}/${name}.md5" > "${dir}/${latest}.md5"
 
-  /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "${dir}/${year}/${name}.md5" "${s3dir}/${name}.md5"
-  /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "${dir}/${year}/${name}" "${s3dir}/${name}"
-  /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "${dir}/${year}/${name}.torrent" "${s3dir}/${name}.torrent"
+  /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING  --no-progress "${dir}/${year}/${name}.md5" "${s3dir}/${name}.md5"
+  /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING  --no-progress "${dir}/${year}/${name}" "${s3dir}/${name}"
+  /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING  --no-progress "${dir}/${year}/${name}.torrent" "${s3dir}/${name}.torrent"
 }
 
 # Create *.torrent files
diff --git a/cookbooks/tilelog/templates/default/tilelog.erb b/cookbooks/tilelog/templates/default/tilelog.erb
index f00f2cb5d..978974d89 100644
--- a/cookbooks/tilelog/templates/default/tilelog.erb
+++ b/cookbooks/tilelog/templates/default/tilelog.erb
@@ -27,9 +27,9 @@ nice -n 19 /opt/tilelog/bin/tilelog --date "${DATE}" \
 
 mv "${TILEFILE}" "${HOSTFILE}" "${APPFILE}" "${COUNTRYFILE}" "${OUTDIR}"
 
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "${OUTDIR}/${TILEFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/tiles/${YEAR}/${TILEFILE}"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "${OUTDIR}/${HOSTFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/hosts/${YEAR}/${HOSTFILE}"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "${OUTDIR}/${APPFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/apps/${YEAR}/${APPFILE}"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "${OUTDIR}/${COUNTRYFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/countries/${YEAR}/${COUNTRYFILE}"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "${OUTDIR}/${TILEFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/tiles/${YEAR}/${TILEFILE}"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "${OUTDIR}/${HOSTFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/hosts/${YEAR}/${HOSTFILE}"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "${OUTDIR}/${APPFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/apps/${YEAR}/${APPFILE}"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "${OUTDIR}/${COUNTRYFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/countries/${YEAR}/${COUNTRYFILE}"
 
 rm -rf "$TMPDIR"