Do not report progress on scripted s3 uploads

diff --git a/cookbooks/planet/files/default/replication-bin/replicate-changesets b/cookbooks/planet/files/default/replication-bin/replicate-changesets
index 87334f21a06b51f840b01e708a6a03c4fce48a71..5d47266e62ba3eac96d9e99ac0a7a8d257d8c7d1 100755 (executable)
--- a/cookbooks/planet/files/default/replication-bin/replicate-changesets
+++ b/cookbooks/planet/files/default/replication-bin/replicate-changesets
@@ -264,8 +264,8 @@ class Replicator
       s3_file = s3_stem + ".osm.gz"
       s3_state_file = s3_stem + ".state.txt"
 
-      system("/opt/awscli/v2/current/bin/aws", "--profile=osm-pds-upload", "s3", "cp", "--storage-class=INTELLIGENT_TIERING", data_file, s3_file)
-      system("/opt/awscli/v2/current/bin/aws", "--profile=osm-pds-upload", "s3", "cp", "--storage-class=INTELLIGENT_TIERING", data_state_file, s3_state_file)
+      system("/opt/awscli/v2/current/bin/aws", "--profile=osm-pds-upload", "s3", "cp", "--storage-class=INTELLIGENT_TIERING", "--no-progress", data_file, s3_file)
+      system("/opt/awscli/v2/current/bin/aws", "--profile=osm-pds-upload", "s3", "cp", "--storage-class=INTELLIGENT_TIERING", "--no-progress", data_state_file, s3_state_file)
     end
   end
 

diff --git a/cookbooks/planet/files/default/replication-bin/replicate-day b/cookbooks/planet/files/default/replication-bin/replicate-day
index 2db770c16680aeda77d9ae65266668ae6427bf46..4e8fd34e8c8eca01c665136ec85c2803784b9678 100644 (file)
--- a/cookbooks/planet/files/default/replication-bin/replicate-day
+++ b/cookbooks/planet/files/default/replication-bin/replicate-day
@@ -11,6 +11,6 @@ sequencePart2=$(($sequenceNumber / 1000 % 1000))
 sequencePart3=$(($sequenceNumber % 1000))
 diffPath=$(printf "%03d/%03d/%03d" $sequencePart1 $sequencePart2 $sequencePart3)
 
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/day/${diffPath}.osc.gz" "s3://osm-planet-eu-central-1/planet/replication/day/${diffPath}.osc.gz"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/day/${diffPath}.state.txt" "s3://osm-planet-eu-central-1/planet/replication/day/${diffPath}.state.txt"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/day/state.txt" "s3://osm-planet-eu-central-1/planet/replication/day/state.txt"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/day/${diffPath}.osc.gz" "s3://osm-planet-eu-central-1/planet/replication/day/${diffPath}.osc.gz"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/day/${diffPath}.state.txt" "s3://osm-planet-eu-central-1/planet/replication/day/${diffPath}.state.txt"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/day/state.txt" "s3://osm-planet-eu-central-1/planet/replication/day/state.txt"

diff --git a/cookbooks/planet/files/default/replication-bin/replicate-hour b/cookbooks/planet/files/default/replication-bin/replicate-hour
index 849cbb41774a760bf61d14c14f04824c9313a32c..184c4a3b2a75c3036bc08cafb54f8bfd143f7601 100644 (file)
--- a/cookbooks/planet/files/default/replication-bin/replicate-hour
+++ b/cookbooks/planet/files/default/replication-bin/replicate-hour
@@ -11,6 +11,6 @@ sequencePart2=$(($sequenceNumber / 1000 % 1000))
 sequencePart3=$(($sequenceNumber % 1000))
 diffPath=$(printf "%03d/%03d/%03d" $sequencePart1 $sequencePart2 $sequencePart3)
 
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/hour/${diffPath}.osc.gz" "s3://osm-planet-eu-central-1/planet/replication/hour/${diffPath}.osc.gz"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/hour/${diffPath}.state.txt" "s3://osm-planet-eu-central-1/planet/replication/hour/${diffPath}.state.txt"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/hour/state.txt" "s3://osm-planet-eu-central-1/planet/replication/hour/state.txt"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/hour/${diffPath}.osc.gz" "s3://osm-planet-eu-central-1/planet/replication/hour/${diffPath}.osc.gz"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/hour/${diffPath}.state.txt" "s3://osm-planet-eu-central-1/planet/replication/hour/${diffPath}.state.txt"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/hour/state.txt" "s3://osm-planet-eu-central-1/planet/replication/hour/state.txt"

diff --git a/cookbooks/planet/files/default/replication-bin/replicate-minute b/cookbooks/planet/files/default/replication-bin/replicate-minute
index 9c54581d8ac1d093913e5048ef5cd223ded4f5bd..943cfa6a3b6a850845e4b3cf6a8bcc38c259b745 100755 (executable)
--- a/cookbooks/planet/files/default/replication-bin/replicate-minute
+++ b/cookbooks/planet/files/default/replication-bin/replicate-minute
@@ -16,6 +16,6 @@ sequencePart2=$(($sequenceNumber / 1000 % 1000))
 sequencePart3=$(($sequenceNumber % 1000))
 diffPath=$(printf "%03d/%03d/%03d" $sequencePart1 $sequencePart2 $sequencePart3)
 
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/minute/${diffPath}.osc.gz" "s3://osm-planet-eu-central-1/planet/replication/minute/${diffPath}.osc.gz"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/minute/${diffPath}.state.txt" "s3://osm-planet-eu-central-1/planet/replication/minute/${diffPath}.state.txt"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "/store/planet/replication/minute/state.txt" "s3://osm-planet-eu-central-1/planet/replication/minute/state.txt"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/minute/${diffPath}.osc.gz" "s3://osm-planet-eu-central-1/planet/replication/minute/${diffPath}.osc.gz"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/minute/${diffPath}.state.txt" "s3://osm-planet-eu-central-1/planet/replication/minute/${diffPath}.state.txt"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "/store/planet/replication/minute/state.txt" "s3://osm-planet-eu-central-1/planet/replication/minute/state.txt"

diff --git a/cookbooks/planet/templates/default/planet-notes-dump.erb b/cookbooks/planet/templates/default/planet-notes-dump.erb
index 9f266b40a6f2feb2f842ff8264f936ebedc5daf4..ebfb9db13bfa3256a54088cb24edb26499a0f595 100644 (file)
--- a/cookbooks/planet/templates/default/planet-notes-dump.erb
+++ b/cookbooks/planet/templates/default/planet-notes-dump.erb
@@ -55,7 +55,7 @@ rm -f planet-notes-latest.osn.bz2.md5
 
 sed -e "s/${cur_planet_notes}.bz2/planet-notes-latest.osn.bz2/" "${cur_year}/${cur_planet_notes}.bz2.md5" > planet-notes-latest.osn.bz2.md5
 
-/opt/awscli/v2/current/bin/aws --profile osm-pds-upload s3 cp --storage-class INTELLIGENT_TIERING "${cur_year}/${cur_planet_notes}.bz2.md5" "s3://osm-planet-eu-central-1/notes/osn/${cur_year}/${cur_planet_notes}.bz2.md5"
-/opt/awscli/v2/current/bin/aws --profile osm-pds-upload s3 cp --storage-class INTELLIGENT_TIERING "${cur_year}/${cur_planet_notes}.bz2" "s3://osm-planet-eu-central-1/notes/osn/${cur_year}/${cur_planet_notes}.bz2"
+/opt/awscli/v2/current/bin/aws --profile osm-pds-upload s3 cp --storage-class INTELLIGENT_TIERING  --no-progress "${cur_year}/${cur_planet_notes}.bz2.md5" "s3://osm-planet-eu-central-1/notes/osn/${cur_year}/${cur_planet_notes}.bz2.md5"
+/opt/awscli/v2/current/bin/aws --profile osm-pds-upload s3 cp --storage-class INTELLIGENT_TIERING  --no-progress "${cur_year}/${cur_planet_notes}.bz2" "s3://osm-planet-eu-central-1/notes/osn/${cur_year}/${cur_planet_notes}.bz2"
 
 rm /tmp/planet-notes-dump.lock

diff --git a/cookbooks/planet/templates/default/planetdump.erb b/cookbooks/planet/templates/default/planetdump.erb
index 8ffd8f9cc0994aaa5c26c44f22dc2f2e074d2d7c..dadc24adcf9a9a883107c15b35fd9c4cd808773a 100644 (file)
--- a/cookbooks/planet/templates/default/planetdump.erb
+++ b/cookbooks/planet/templates/default/planetdump.erb
@@ -157,9 +157,9 @@ function install_dump {
   rm -f "${dir}/${latest}.md5"
   sed -e "s/${name}/${latest}/" "${dir}/${year}/${name}.md5" > "${dir}/${latest}.md5"
 
-  /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "${dir}/${year}/${name}.md5" "${s3dir}/${name}.md5"
-  /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "${dir}/${year}/${name}" "${s3dir}/${name}"
-  /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "${dir}/${year}/${name}.torrent" "${s3dir}/${name}.torrent"
+  /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING  --no-progress "${dir}/${year}/${name}.md5" "${s3dir}/${name}.md5"
+  /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING  --no-progress "${dir}/${year}/${name}" "${s3dir}/${name}"
+  /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING  --no-progress "${dir}/${year}/${name}.torrent" "${s3dir}/${name}.torrent"
 }
 
 # Create *.torrent files

diff --git a/cookbooks/tilelog/templates/default/tilelog.erb b/cookbooks/tilelog/templates/default/tilelog.erb
index f00f2cb5d2d8eda9681fcce09bf8278b9d00e74d..978974d8964e8c2f55c51d3998e90374c9c9c169 100644 (file)
--- a/cookbooks/tilelog/templates/default/tilelog.erb
+++ b/cookbooks/tilelog/templates/default/tilelog.erb
@@ -27,9 +27,9 @@ nice -n 19 /opt/tilelog/bin/tilelog --date "${DATE}" \
 
 mv "${TILEFILE}" "${HOSTFILE}" "${APPFILE}" "${COUNTRYFILE}" "${OUTDIR}"
 
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "${OUTDIR}/${TILEFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/tiles/${YEAR}/${TILEFILE}"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "${OUTDIR}/${HOSTFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/hosts/${YEAR}/${HOSTFILE}"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "${OUTDIR}/${APPFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/apps/${YEAR}/${APPFILE}"
-/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING "${OUTDIR}/${COUNTRYFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/countries/${YEAR}/${COUNTRYFILE}"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "${OUTDIR}/${TILEFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/tiles/${YEAR}/${TILEFILE}"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "${OUTDIR}/${HOSTFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/hosts/${YEAR}/${HOSTFILE}"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "${OUTDIR}/${APPFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/apps/${YEAR}/${APPFILE}"
+/opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "${OUTDIR}/${COUNTRYFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/countries/${YEAR}/${COUNTRYFILE}"
 
 rm -rf "$TMPDIR"