From: Sarah Hoffmann <lonvia@denofr.de>
Date: Fri, 9 Jun 2017 19:49:31 +0000 (+0200)
Subject: Guard against network failures in getDatabaseDate()
X-Git-Tag: v3.0.0~5
X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/4e6c75040e0deaabba44e86bb87c80e176a06896?hp=7448d3f171510a5fa98d96ab9f273687bb6af8e2

Guard against network failures in getDatabaseDate()

When updating use the date from the diff file instead
as we are guaranteed to get this if the file has been
successfully loaded.
---

diff --git a/lib/lib.php b/lib/lib.php
index 9691f88a..cb599ae8 100644
--- a/lib/lib.php
+++ b/lib/lib.php
@@ -38,8 +38,13 @@ function getDatabaseDate(&$oDB)
     // Find the newest node in the DB
     $iLastOSMID = $oDB->getOne("select max(osm_id) from place where osm_type = 'N'");
     // Lookup the timestamp that node was created
-    $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID."/1";
+    $sLastNodeURL = 'https://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID."/1";
     $sLastNodeXML = file_get_contents($sLastNodeURL);
+
+    if ($sLastNodeXML === false) {
+        return false;
+    }
+
     preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 
     return $aLastNodeDate[1];
diff --git a/utils/osm_file_date.py b/utils/osm_file_date.py
new file mode 100755
index 00000000..6418efb0
--- /dev/null
+++ b/utils/osm_file_date.py
@@ -0,0 +1,33 @@
+#!/usr/bin/python
+
+import osmium
+import sys
+import datetime
+
+
+class Datecounter(osmium.SimpleHandler):
+
+    filedate = None
+
+    def date(self, o):
+        ts = o.timestamp
+        if self.filedate is None or ts > self.filedate:
+            self.filedate = ts
+
+    node = date
+    way = date
+    relation = date
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print("Usage: python osm_file_date.py <osmfile>")
+        sys.exit(-1)
+
+    h = Datecounter()
+
+    h.apply_file(sys.argv[1])
+
+    print(h.filedate)
+
+
diff --git a/utils/setup.php b/utils/setup.php
index eeba0a28..7490d820 100755
--- a/utils/setup.php
+++ b/utils/setup.php
@@ -407,9 +407,13 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) {
 
     $sDatabaseDate = getDatabaseDate($oDB);
     pg_query($oDB->connection, 'TRUNCATE import_status');
-    $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
-    pg_query($oDB->connection, $sSQL);
-    echo "Latest data imported from $sDatabaseDate.\n";
+    if ($sDatabaseDate === false) {
+        echo "WARNING: could not determine database date.\n";
+    } else {
+        $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
+        pg_query($oDB->connection, $sSQL);
+        echo "Latest data imported from $sDatabaseDate.\n";
+    }
 }
 
 if ($aCMDResult['import-tiger-data']) {
diff --git a/utils/update.php b/utils/update.php
index 2b72fe82..f4716768 100755
--- a/utils/update.php
+++ b/utils/update.php
@@ -67,6 +67,9 @@ if ($aResult['init-updates']) {
     }
 
     $sDatabaseDate = getDatabaseDate($oDB);
+    if ($sDatabaseDate === false) {
+        fail("Cannot determine date of database.");
+    }
     $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ',
                           strtotime($sDatabaseDate) - (3*60*60));
 
@@ -293,7 +296,14 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
             // write the update logs
             $iFileSize = filesize($sImportFile);
-            $sBatchEnd = getDatabaseDate($oDB);
+            // get the newest object from the diff file
+            $sBatchEnd = 0;
+            $iRet = 0;
+            exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
+            if ($iRet != 0) {
+                fail('Error getting date from diff file.');
+            }
+            $sBatchEnd = $sBatchEnd[0];
             $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
             var_Dump($sSQL);
             chksql($oDB->query($sSQL));