]> git.openstreetmap.org Git - nominatim.git/blob - utils/update.php
fix nominatim paths
[nominatim.git] / utils / update.php
1 #!/usr/bin/php -Cq
2 <?php
3
4         require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
5         require_once(CONST_BasePath.'/lib/init-cmd.php');
6         ini_set('memory_limit', '800M');
7
8         $aCMDOptions = array(
9                 "Import / update / index osm data",
10                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
11                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
12                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
13
14                 array('max-load', '', 0, 1, 1, 1, 'float', 'Maximum load average - indexing is paused if this is exceeded'),
15                 array('max-blocking', '', 0, 1, 1, 1, 'int', 'Maximum blocking processes - indexing is aborted / paused if this is exceeded'),
16
17                 array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import using osmosis'),
18                 array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import using osmosis forever'),
19                 array('no-npi', '', 0, 1, 0, 0, 'bool', 'Do not write npi index files'),
20                 array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
21
22                 array('import-npi-all', '', 0, 1, 0, 0, 'bool', 'Import npi pre-indexed files'),
23
24                 array('import-hourly', '', 0, 1, 0, 0, 'bool', 'Import hourly diffs'),
25                 array('import-daily', '', 0, 1, 0, 0, 'bool', 'Import daily diffs'),
26                 array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
27
28                 array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
29                 array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
30                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
31
32                 array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
33                 array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
34                 array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
35                 array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
36
37                 array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
38                 array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
39                 array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
40                 array('index-estrate', '', 0, 1, 1, 1, 'int', 'Estimated indexed items per second (def:30)'),
41
42                 array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
43         );
44         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
45
46         if ($aResult['import-hourly'] + $aResult['import-daily'] + isset($aResult['import-diff']) > 1)
47         {
48                 showUsage($aCMDOptions, true, 'Select either import of hourly or daily');
49         }
50
51         if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
52         if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
53
54 /*
55         // Lock to prevent multiple copies running
56         if (exec('/bin/ps uww | grep '.basename(__FILE__).' | grep -v /dev/null | grep -v grep -c', $aOutput2, $iResult) > 1)
57         {
58                 fail("Copy already running\n");
59         }
60         if (!isset($aResult['max-load'])) $aResult['max-load'] = 1.9;
61         if (!isset($aResult['max-blocking'])) $aResult['max-blocking'] = 3;
62         if (getBlockingProcesses() > $aResult['max-blocking'])
63         {
64                 fail("Too many blocking processes for import\n");
65         }
66 */
67
68         date_default_timezone_set('Etc/UTC');
69
70         $oDB =& getDB();
71
72         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
73         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
74
75         // cache memory to be used by osm2pgsql, should not be more than the available memory
76         $iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
77         if ($iCacheMemory + 500 > getTotalMemoryMB())
78         {
79                 $iCacheMemory = getCacheMemoryMB();
80                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
81         }
82         $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
83         if (!is_null(CONST_Osm2pgsql_Flatnode_File))
84         {
85                 $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
86         }
87
88
89         $bFirst = true;
90         $bContinue = $aResult['import-all'];
91         while ($bContinue || $bFirst)
92         {
93                 $bFirst = false;
94
95                 if ($aResult['import-hourly'])
96                 {
97                         // Mirror the hourly diffs
98                         exec('wget --quiet --mirror -l 1 -P '.$sMirrorDir.' http://planet.openstreetmap.org/hourly');
99                         $sNextFile = $oDB->getOne('select TO_CHAR(lastimportdate,\'YYYYMMDDHH24\')||\'-\'||TO_CHAR(lastimportdate+\'1 hour\'::interval,\'YYYYMMDDHH24\')||\'.osc.gz\' from import_status');
100                         $sNextFile = $sMirrorDir.'planet.openstreetmap.org/hourly/'.$sNextFile;
101                         $sUpdateSQL = 'update import_status set lastimportdate = lastimportdate+\'1 hour\'::interval';
102                 }
103
104                 if ($aResult['import-daily'])
105                 {
106                         // Mirror the daily diffs
107                         exec('wget --quiet --mirror -l 1 -P '.$sMirrorDir.' http://planet.openstreetmap.org/daily');
108                         $sNextFile = $oDB->getOne('select TO_CHAR(lastimportdate,\'YYYYMMDD\')||\'-\'||TO_CHAR(lastimportdate+\'1 day\'::interval,\'YYYYMMDD\')||\'.osc.gz\' from import_status');
109                         $sNextFile = $sMirrorDir.'planet.openstreetmap.org/daily/'.$sNextFile;
110                         $sUpdateSQL = 'update import_status set lastimportdate = lastimportdate::date + 1';
111                 }
112                 
113                 if (isset($aResult['import-diff']))
114                 {
115                         // import diff directly (e.g. from osmosis --rri)
116                         $sNextFile = $aResult['import-diff'];
117                         if (!file_exists($sNextFile))
118                         {
119                                 fail("Cannot open $sNextFile\n");
120                         }
121                         // Don't update the import status - we don't know what this file contains
122                         $sUpdateSQL = 'update import_status set lastimportdate = now() where false';
123                 }
124
125                 // Missing file is not an error - it might not be created yet
126                 if (($aResult['import-hourly'] || $aResult['import-daily'] || isset($aResult['import-diff'])) && file_exists($sNextFile))
127                 {
128                         // Import the file
129                         $sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
130                         echo $sCMD."\n";
131                         exec($sCMD, $sJunk, $iErrorLevel);
132
133                         if ($iErrorLevel)
134                         {
135                                 fail("Error from osm2pgsql, $iErrorLevel\n");
136                         }
137         
138                         // Move the date onwards
139                         $oDB->query($sUpdateSQL);
140                 }
141                 else
142                 {
143                         $bContinue = false;
144                 }
145         }
146
147         $bModifyXML = false;
148         $sModifyXMLstr = '';
149         $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
150         if (isset($aResult['import-file']) && $aResult['import-file'])
151         {
152                 $bModifyXML = true;
153         }
154         if (isset($aResult['import-node']) && $aResult['import-node'])
155         {
156                 $bModifyXML = true;
157                 if ($bUseOSMApi)
158                 {
159                         $sModifyXMLstr = file_get_contents('http://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node']);
160                 }
161                 else
162                 {
163                         $sModifyXMLstr = file_get_contents('http://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;');
164                 }
165         }
166         if (isset($aResult['import-way']) && $aResult['import-way'])
167         {
168                 $bModifyXML = true;
169                 if ($bUseOSMApi)
170                 {
171                         $sCmd = 'http://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
172                 }
173                 else
174                 {
175                         $sCmd = 'http://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
176                 }
177                 $sModifyXMLstr = file_get_contents($sCmd);
178         }
179         if (isset($aResult['import-relation']) && $aResult['import-relation'])
180         {
181                 $bModifyXML = true;
182                 if ($bUseOSMApi)
183                 {
184                         $sModifyXMLstr = file_get_contents('http://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full');
185                 }
186                 else
187                 {
188                         $sModifyXMLstr = file_get_contents('http://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;');
189                 }
190         }
191         if ($bModifyXML)
192         {
193                 // derive change from normal osm file with osmosis
194                 $sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
195                 if (isset($aResult['import-file']) && $aResult['import-file'])
196                 {
197                         $sCMD = CONST_Osmosis_Binary.' --read-xml \''.$aResult['import-file'].'\' --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
198                         echo $sCMD."\n";
199                         exec($sCMD, $sJunk, $iErrorLevel);
200                         if ($iErrorLevel)
201                         {
202                                 fail("Error converting osm to osc, osmosis returned: $iErrorLevel\n");
203                         }
204                 }
205                 else
206                 {
207                         $aSpec = array(
208                                 0 => array("pipe", "r"),  // stdin
209                                 1 => array("pipe", "w"),  // stdout
210                                 2 => array("pipe", "w") // stderr
211                         );
212                         $sCMD = CONST_Osmosis_Binary.' --read-xml - --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
213                         echo $sCMD."\n";
214                         $hProc = proc_open($sCMD, $aSpec, $aPipes);
215                         if (!is_resource($hProc))
216                         {
217                                 fail("Error converting osm to osc, osmosis failed\n");
218                         }
219                         fwrite($aPipes[0], $sModifyXMLstr);
220                         fclose($aPipes[0]);
221                         $sOut = stream_get_contents($aPipes[1]);
222                         if ($aResult['verbose']) echo $sOut;
223                         fclose($aPipes[1]);
224                         $sErrors = stream_get_contents($aPipes[2]);
225                         if ($aResult['verbose']) echo $sErrors;
226                         fclose($aPipes[2]);
227                         if ($iError = proc_close($hProc))
228                         {
229                                 echo "Error converting osm to osc, osmosis returned: $iError\n";
230                                 echo $sOut;
231                                 echo $sErrors;
232                                 exit(-1);
233                         }
234                 }
235
236                 // import generated change file
237                 $sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
238                 echo $sCMD."\n";
239                 exec($sCMD, $sJunk, $iErrorLevel);
240                 if ($iErrorLevel)
241                 {
242                         fail("osm2pgsql exited with error level $iErrorLevel\n");
243                 }
244         }
245
246         if ($aResult['deduplicate'])
247         {
248
249                 $pgver = (float) CONST_Postgresql_Version;
250                 if ($pgver < 9.3) {
251                         fail("ERROR: deduplicate is only currently supported in postgresql 9.3");
252                 }
253
254                 $oDB =& getDB();
255                 $sSQL = 'select partition from country_name order by country_code';
256                 $aPartitions = $oDB->getCol($sSQL);
257                 if (PEAR::isError($aPartitions))
258                 {
259                         fail($aPartitions->getMessage());
260                 }
261                 $aPartitions[] = 0;
262
263                 $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' ' and class is null and type is null and country_code is null group by word_token having count(*) > 1 order by word_token";
264                 $aDuplicateTokens = $oDB->getAll($sSQL);
265                 foreach($aDuplicateTokens as $aToken)
266                 {
267                         if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
268                         echo "Deduping ".$aToken['word_token']."\n";
269                         $sSQL = "select word_id,(select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num from word where word_token = '".$aToken['word_token']."' and class is null and type is null and country_code is null order by num desc";
270                         $aTokenSet = $oDB->getAll($sSQL);
271                         if (PEAR::isError($aTokenSet))
272                         {
273                                 var_dump($aTokenSet, $sSQL);
274                                 exit(1);
275                         }
276
277                         $aKeep = array_shift($aTokenSet);
278                         $iKeepID = $aKeep['word_id'];
279
280                         foreach($aTokenSet as $aRemove)
281                         {
282                                 $sSQL = "update search_name set";
283                                 $sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID."),";
284                                 $sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
285                                 $sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
286                                 $x = $oDB->query($sSQL);
287                                 if (PEAR::isError($x))
288                                 {
289                                         var_dump($x);
290                                         exit(1);
291                                 }
292
293                                 $sSQL = "update search_name set";
294                                 $sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
295                                 $sSQL .= " where nameaddress_vector @> ARRAY[".$aRemove['word_id']."]";
296                                 $x = $oDB->query($sSQL);
297                                 if (PEAR::isError($x))
298                                 {
299                                         var_dump($x);
300                                         exit(1);
301                                 }
302
303                                 $sSQL = "update location_area_country set";
304                                 $sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
305                                 $sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
306                                 $x = $oDB->query($sSQL);
307                                 if (PEAR::isError($x))
308                                 {
309                                         var_dump($x);
310                                         exit(1);
311                                 }
312
313                                 foreach ($aPartitions as $sPartition)
314                                 {
315                                         $sSQL = "update search_name_".$sPartition." set";
316                                         $sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID.")";
317                                         $sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
318                                         $x = $oDB->query($sSQL);
319                                         if (PEAR::isError($x))
320                                         {
321                                                 var_dump($x);
322                                                 exit(1);
323                                         }
324
325                                         $sSQL = "update location_area_country set";
326                                         $sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
327                                         $sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
328                                         $x = $oDB->query($sSQL);
329                                         if (PEAR::isError($x))
330                                         {
331                                                 var_dump($x);
332                                                 exit(1);
333                                         }
334                                 }
335
336                                 $sSQL = "delete from word where word_id = ".$aRemove['word_id'];
337                                 $x = $oDB->query($sSQL);
338                                 if (PEAR::isError($x))
339                                 {
340                                         var_dump($x);
341                                         exit(1);
342                                 }
343                         }
344
345                 }
346         }
347
348         if ($aResult['index'])
349         {
350                 passthru(CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
351         }
352
353         if ($aResult['import-osmosis'] || $aResult['import-osmosis-all'])
354         {
355
356                 if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
357                         fail("Error: Update interval too low for download.geofabrik.de.  Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n");
358                 }
359
360                 $sImportFile = CONST_BasePath.'/data/osmosischange.osc';
361                 $sOsmosisCMD = CONST_Osmosis_Binary;
362                 $sOsmosisConfigDirectory = CONST_InstallPath.'/settings';
363                 $sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
364                 $sCMDCheckReplicationLag = $sOsmosisCMD.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
365                 $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
366                 $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
367                 if (!$aResult['no-npi']) {
368                         $sCMDIndex .= '-F ';
369                 }
370                 while(true)
371                 {
372                         $fStartTime = time();
373                         $iFileSize = 1001;
374
375                         // Logic behind this is that osm2pgsql locks the database quite a bit
376                         // So it is better to import lots of small files
377                         // But indexing works most efficiently on large amounts of data
378                         // So do lots of small imports and a BIG index
379
380 //                      while($aResult['import-osmosis-all'] && $iFileSize > 1000)
381 //                      {
382                                 if (!file_exists($sImportFile))
383                                 {
384                                         // First check if there are new updates published (except for minutelies - there's always new diffs to process)
385                                         if ( CONST_Replication_Update_Interval > 60 )
386                                         {
387
388                                                 unset($aReplicationLag);
389                                                 exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
390                                                 while ($iErrorLevel > 0 || $aReplicationLag[0] < 1)
391                                                 {
392                                                         if ($iErrorLevel)
393                                                         {
394                                                                 echo "Error: $iErrorLevel. ";
395                                                                 echo "Re-trying: ".$sCMDCheckReplicationLag." in ".CONST_Replication_Recheck_Interval." secs\n";
396                                                         }
397                                                         else
398                                                         {
399                                                                 echo ".";
400                                                         }
401                                                         sleep(CONST_Replication_Recheck_Interval);
402                                                         unset($aReplicationLag);
403                                                         exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
404                                                 }
405                                                 // There are new replication files - use osmosis to download the file
406                                                 echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n";
407                                         }
408                                         $fStartTime = time();
409                                         $fCMDStartTime = time();
410                                         echo $sCMDDownload."\n";
411                                         exec($sCMDDownload, $sJunk, $iErrorLevel);
412                                         while ($iErrorLevel > 0)
413                                         {
414                                                 echo "Error: $iErrorLevel\n";
415                                                 sleep(60);
416                                                 echo 'Re-trying: '.$sCMDDownload."\n";
417                                                 exec($sCMDDownload, $sJunk, $iErrorLevel);
418                                         }
419                                         $iFileSize = filesize($sImportFile);
420                                         $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
421                                         $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
422                                         var_Dump($sSQL);
423                                         $oDB->query($sSQL);
424                                         echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
425                                 }
426
427                                 $iFileSize = filesize($sImportFile);
428                                 $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
429                 
430                                 // Import the file
431                                 $fCMDStartTime = time();
432                                 echo $sCMDImport."\n";
433                                 exec($sCMDImport, $sJunk, $iErrorLevel);
434                                 if ($iErrorLevel)
435                                 {
436                                         echo "Error: $iErrorLevel\n";
437                                         exit($iErrorLevel);
438                                 }
439                                 $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
440                                 var_Dump($sSQL);
441                                 $oDB->query($sSQL);
442                                 echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
443
444                                 // Archive for debug?
445                                 unlink($sImportFile);
446 //                      }
447
448                         $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
449
450                         // Index file
451                         $sThisIndexCmd = $sCMDIndex;
452                         $fCMDStartTime = time();
453
454                         if (!$aResult['no-npi'])
455                         {
456                                 $iFileID = $oDB->getOne('select nextval(\'file\')');
457                                 if (PEAR::isError($iFileID))
458                                 {
459                                         echo $iFileID->getMessage()."\n";
460                                         exit(-1);
461                                 } 
462                                 $sFileDir = CONST_BasePath.'/export/diff/';
463                                 $sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT);
464                                 $sFileDir .= '/'.str_pad(floor($iFileID/1000) % 1000, 3, '0', STR_PAD_LEFT);
465
466                                 if (!is_dir($sFileDir)) mkdir($sFileDir, 0777, true);
467                                 $sThisIndexCmd .= $sFileDir;
468                                 $sThisIndexCmd .= '/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT);
469                                 $sThisIndexCmd .= ".npi.out";
470
471                                 preg_match('#^([0-9]{4})-([0-9]{2})-([0-9]{2})#', $sBatchEnd, $aBatchMatch);
472                                 $sFileDir = CONST_BasePath.'/export/index/';
473                                 $sFileDir .= $aBatchMatch[1].'/'.$aBatchMatch[2];
474
475                                 if (!is_dir($sFileDir)) mkdir($sFileDir, 0777, true);
476                                 file_put_contents($sFileDir.'/'.$aBatchMatch[3].'.idx', "$sBatchEnd\t$iFileID\n", FILE_APPEND);
477                         }
478
479                         if (!$aResult['no-index'])
480                         {
481                                 echo "$sThisIndexCmd\n";
482                                 exec($sThisIndexCmd, $sJunk, $iErrorLevel);
483                                 if ($iErrorLevel)
484                                 {
485                                         echo "Error: $iErrorLevel\n";
486                                         exit($iErrorLevel);
487                                 }
488
489                                 if (!$aResult['no-npi'])
490                                 {
491                                         $sFileDir = CONST_BasePath.'/export/diff/';
492                                         $sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT);
493                                         $sFileDir .= '/'.str_pad(floor($iFileID/1000) % 1000, 3, '0', STR_PAD_LEFT);
494
495                                         $sThisIndexCmd = 'bzip2 -z9 '.$sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out";
496                                         echo "$sThisIndexCmd\n";
497                                         exec($sThisIndexCmd, $sJunk, $iErrorLevel);
498                                         if ($iErrorLevel)
499                                         {
500                                                 echo "Error: $iErrorLevel\n";
501                                                 exit($iErrorLevel);
502                                         }
503
504                                         rename($sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out.bz2",
505                                                 $sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.bz2");
506                                 }
507                         }
508
509                         $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
510                         var_Dump($sSQL);
511                         $oDB->query($sSQL);
512                         echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
513
514                         $sSQL = "update import_status set lastimportdate = '$sBatchEnd'";
515                         $oDB->query($sSQL);
516
517                         $fDuration = time() - $fStartTime;
518                         echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n";
519                         if (!$aResult['import-osmosis-all']) exit(0);
520
521                         if ( CONST_Replication_Update_Interval > 60 )
522                         {
523                                 $iSleep = max(0,(strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time()));
524                         }
525                         else
526                         {
527                                 $iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
528                         }
529                         echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n";
530                         sleep($iSleep);
531                 }
532
533         }
534
535         if ($aResult['import-npi-all'])
536         {
537                 $iNPIID = $oDB->getOne('select max(npiid) from import_npi_log');
538                 if (PEAR::isError($iNPIID))
539                 {
540                         var_dump($iNPIID);
541                         exit(1);
542                 }
543                 $sConfigDirectory = CONST_InstallPath.'/settings';
544                 $sCMDImportTemplate = CONST_InstallPath.'/nominatim/nominatim -d gazetteer -P 5433 -I -T '.CONST_BasePath.'/settings/partitionedtags.def -F ';
545                 while(true)
546                 {
547                         $fStartTime = time();
548
549                         $iNPIID++;
550
551                         $sImportFile = CONST_BasePath.'/export/diff/';
552                         $sImportFile .= str_pad(floor($iNPIID/1000000), 3, '0', STR_PAD_LEFT);
553                         $sImportFile .= '/'.str_pad(floor($iNPIID/1000) % 1000, 3, '0', STR_PAD_LEFT);
554                         $sImportFile .= '/'.str_pad($iNPIID % 1000, 3, '0', STR_PAD_LEFT);
555                         $sImportFile .= ".npi";
556                         while(!file_exists($sImportFile) && !file_exists($sImportFile.'.bz2'))
557                         {
558                                 echo "sleep (waiting for $sImportFile)\n";
559                                 sleep(10);
560                         }
561                         if (file_exists($sImportFile.'.bz2')) $sImportFile .= '.bz2';
562
563                         $iFileSize = filesize($sImportFile);
564                 
565                         // Import the file
566                         $fCMDStartTime = time();
567                         $sCMDImport = $sCMDImportTemplate . $sImportFile;
568                         echo $sCMDImport."\n";
569                         exec($sCMDImport, $sJunk, $iErrorLevel);
570                         if ($iErrorLevel)
571                         {
572                                 fail("Error: $iErrorLevel\n");
573                         }
574                         $sBatchEnd = $iNPIID;
575                         echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
576                         $sSQL = "INSERT INTO import_npi_log values ($iNPIID, null, $iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
577                         var_Dump($sSQL);
578                         $oDB->query($sSQL);
579                 }
580                 
581         }
582
583         function getosmosistimestamp($sOsmosisConfigDirectory)
584         {
585                 $sStateFile = file_get_contents($sOsmosisConfigDirectory.'/state.txt');
586                 preg_match('#timestamp=(.+)#', $sStateFile, $aResult);
587                 return str_replace('\:',':',$aResult[1]);
588         }