From 08e273c0c7ebcbb7f8e4c512c58b22da1bac95b7 Mon Sep 17 00:00:00 2001
From: K Rahul Reddy <k_rahul_reddy@outlook.com>
Date: Wed, 15 Apr 2020 17:18:02 +0530
Subject: [PATCH] Added scripts for multiple country setup and updates

---
 utils/import_multiple_regions.sh | 91 ++++++++++++++++++++++++++++++++
 utils/update_database.sh         | 80 ++++++++++++++++++++++++++++
 2 files changed, 171 insertions(+)
 create mode 100644 utils/import_multiple_regions.sh
 create mode 100644 utils/update_database.sh

diff --git a/utils/import_multiple_regions.sh b/utils/import_multiple_regions.sh
new file mode 100644
index 00000000..83323c2e
--- /dev/null
+++ b/utils/import_multiple_regions.sh
@@ -0,0 +1,91 @@
+#!/bin/bash -xv
+
+# Script to set up Nominatim database for multiple countries
+
+# Steps to follow:
+
+#     *) Get the pbf files from server
+
+#     *) Set up sequence.state for updates
+
+#     *) Merge the pbf files into a single file.
+
+#     *) Setup nominatim db using 'setup.php --osm-file'
+
+# Hint:
+#
+# Use "bashdb ./update_database.sh" and bashdb's "next" command for step-by-step
+# execution.
+
+# ******************************************************************************
+
+touch2() { mkdir -p "$(dirname "$1")" && touch "$1" ; }
+
+# ******************************************************************************
+# Configuration section: Variables in this section should be set according to your requirements
+
+# REPLACE WITH LIST OF YOUR "COUNTRIES":
+
+COUNTRIES="europe/monaco europe/andorra"
+
+# SET TO YOUR NOMINATIM build FOLDER PATH:
+
+NOMINATIMBUILD="/srv/nominatim/build"
+SETUPFILE="$NOMINATIMBUILD/utils/setup.php"
+UPDATEFILE="$NOMINATIMBUILD/utils/update.php"
+
+# SET TO YOUR update FOLDER PATH:
+
+UPDATEDIR="/srv/nominatim/update"
+
+# SET TO YOUR replication server URL:
+
+BASEURL="https://download.geofabrik.de"
+DOWNCOUNTRYPOSTFIX="-latest.osm.pbf"
+
+# End of configuration section
+# ******************************************************************************
+
+COMBINEFILES="osmium merge"
+
+mkdir -p ${UPDATEDIR}
+cd ${UPDATEDIR}
+rm -rf tmp
+mkdir -p tmp
+cd tmp
+
+for COUNTRY in $COUNTRIES;
+do
+    
+    echo "===================================================================="
+    echo "$COUNTRY"
+    echo "===================================================================="
+    DIR="$UPDATEDIR/$COUNTRY"
+    FILE="$DIR/configuration.txt"
+    DOWNURL="$BASEURL/$COUNTRY$DOWNCOUNTRYPOSTFIX"
+    IMPORTFILE=$COUNTRY$DOWNCOUNTRYPOSTFIX
+    IMPORTFILEPATH=${UPDATEDIR}/tmp/${IMPORTFILE}
+    FILENAME=${COUNTRY//[\/]/_}
+    
+
+    touch2 $IMPORTFILEPATH
+    wget ${DOWNURL} -O $IMPORTFILEPATH
+
+    touch2 ${DIR}/sequence.state
+    pyosmium-get-changes -O $IMPORTFILEPATH -f ${DIR}/sequence.state -v
+
+    COMBINEFILES="${COMBINEFILES} ${IMPORTFILEPATH}"
+    echo $IMPORTFILE
+    echo "===================================================================="
+done
+
+
+echo "${COMBINEFILES} -o combined.osm.pbf"
+${COMBINEFILES} -o combined.osm.pbf
+
+echo "===================================================================="
+echo "Setting up nominatim db"
+${SETUPFILE} --osm-file ${UPDATEDIR}/tmp/combined.osm.pbf --all 2>&1
+
+# ${UPDATEFILE} --import-file ${UPDATEDIR}/tmp/combined.osm.pbf 2>&1
+echo "===================================================================="
\ No newline at end of file
diff --git a/utils/update_database.sh b/utils/update_database.sh
new file mode 100644
index 00000000..75d0de5d
--- /dev/null
+++ b/utils/update_database.sh
@@ -0,0 +1,80 @@
+#!/bin/bash -xv
+
+# Derived from https://gist.github.com/RhinoDevel/8a35ebd2a08166f328eca01ab005c6de and edited to work with Pyosmium
+# Related to https://github.com/osm-search/Nominatim/issues/1683
+
+# Steps being followed:
+
+#     *) Get the diff file from server
+#         1) pyosmium-get-changes (with -f sequence.state for getting sequenceNumber)
+
+#     *) Import diff
+#         1) utils/update.php --import-diff
+
+#     *) Index for all the countries at the end
+
+# Hint:
+#
+# Use "bashdb ./update_database.sh" and bashdb's "next" command for step-by-step
+# execution.
+
+# ******************************************************************************
+
+# REPLACE WITH LIST OF YOUR "COUNTRIES":
+#
+
+
+COUNTRIES="europe/monaco europe/andorra"
+
+# SET TO YOUR NOMINATIM build FOLDER PATH:
+#
+NOMINATIMBUILD="/srv/nominatim/build"
+UPDATEFILE="$NOMINATIMBUILD/utils/update.php"
+
+# SET TO YOUR update data FOLDER PATH:
+#
+UPDATEDIR="/srv/nominatim/update"
+
+UPDATEBASEURL="https://download.geofabrik.de"
+UPDATECOUNTRYPOSTFIX="-updates"
+
+# If you do not use Photon, let Nominatim handle (re-)indexing:
+#
+FOLLOWUP="$UPDATEFILE --index"
+#
+# If you use Photon, update Photon and let it handle the index
+# (Photon server must be running and must have been started with "-database",
+# "-user" and "-password" parameters):
+#
+#FOLLOWUP="curl http://localhost:2322/nominatim-update"
+
+# ******************************************************************************
+
+
+for COUNTRY in $COUNTRIES;
+do
+    
+    echo "===================================================================="
+    echo "$COUNTRY"
+    echo "===================================================================="
+    DIR="$UPDATEDIR/$COUNTRY"
+    FILE="$DIR/sequence.state"
+    BASEURL="$UPDATEBASEURL/$COUNTRY$UPDATECOUNTRYPOSTFIX"
+    FILENAME=${COUNTRY//[\/]/_}
+    
+    # mkdir -p ${DIR}
+    cd ${DIR}
+
+    echo "Attempting to get changes"
+    pyosmium-get-changes -o ${DIR}/${FILENAME}.osc.gz -f ${FILE} --server $BASEURL -v
+
+    echo "Attempting to import diffs"
+    ${NOMINATIMBUILD}/utils/update.php --import-diff ${DIR}/${FILENAME}.osc.gz
+    rm ${DIR}/${FILENAME}.osc.gz
+
+done
+
+echo "===================================================================="
+echo "Reindexing" 
+${FOLLOWUP}
+echo "===================================================================="
\ No newline at end of file
-- 
2.45.2