]> git.openstreetmap.org Git - chef.git/blob - cookbooks/tilelog/templates/default/tilelog.erb
Do not report progress on scripted s3 uploads
[chef.git] / cookbooks / tilelog / templates / default / tilelog.erb
1 #!/bin/sh
2 set -e
3
4 if [ -z "$DATE" ]
5 then
6   DATE=$(date -u -d "1 day ago" "+%Y-%m-%d")
7   YEAR=$(date -u -d "1 day ago" "+%Y")
8 fi
9
10 OUTDIR="<%= @output_dir %>"
11 TMPDIR=$(mktemp -d -t tilelog.XXXXXXXXX)
12
13 cd "$TMPDIR"
14
15 export AWS_ACCESS_KEY_ID="AKIASQUXHPE7JFCFMOUP"
16 export AWS_SECRET_ACCESS_KEY="<%= @aws_key %>"
17 export AWS_REGION="eu-west-1"
18
19 TILEFILE="tiles-${DATE}.txt.xz"
20 HOSTFILE="hosts-${DATE}.csv"
21 APPFILE="apps-${DATE}.csv"
22 COUNTRYFILE="countries-${DATE}.csv"
23
24 nice -n 19 /opt/tilelog/bin/tilelog --date "${DATE}" \
25   --generate-success --generate-minimise --generate-location \
26   --tile "${TILEFILE}" --host "${HOSTFILE}" --app "${APPFILE}" --country "${COUNTRYFILE}"
27
28 mv "${TILEFILE}" "${HOSTFILE}" "${APPFILE}" "${COUNTRYFILE}" "${OUTDIR}"
29
30 /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "${OUTDIR}/${TILEFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/tiles/${YEAR}/${TILEFILE}"
31 /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "${OUTDIR}/${HOSTFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/hosts/${YEAR}/${HOSTFILE}"
32 /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "${OUTDIR}/${APPFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/apps/${YEAR}/${APPFILE}"
33 /opt/awscli/v2/current/bin/aws --profile=osm-pds-upload s3 cp --storage-class=INTELLIGENT_TIERING --no-progress "${OUTDIR}/${COUNTRYFILE}" "s3://osm-planet-eu-central-1/tile_logs/standard_layer/countries/${YEAR}/${COUNTRYFILE}"
34
35 rm -rf "$TMPDIR"