]> git.openstreetmap.org Git - nominatim.git/blob - utils/update.php
introduce chksql function for phpscripts
[nominatim.git] / utils / update.php
1 #!/usr/bin/php -Cq
2 <?php
3
4         require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
5         require_once(CONST_BasePath.'/lib/init-cmd.php');
6         ini_set('memory_limit', '800M');
7
8         $aCMDOptions = array(
9                 "Import / update / index osm data",
10                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
11                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
12                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
13
14                 array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import using osmosis'),
15                 array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import using osmosis forever'),
16                 array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolate)'),
17                 array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
18
19                 array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
20
21                 array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
22                 array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
23                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
24
25                 array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
26                 array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
27                 array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
28                 array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
29
30                 array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
31                 array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
32                 array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
33
34                 array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
35         );
36         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
37
38         if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
39         if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
40
41         date_default_timezone_set('Etc/UTC');
42
43         $oDB =& getDB();
44
45         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
46         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
47
48         // cache memory to be used by osm2pgsql, should not be more than the available memory
49         $iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
50         if ($iCacheMemory + 500 > getTotalMemoryMB())
51         {
52                 $iCacheMemory = getCacheMemoryMB();
53                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
54         }
55         $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
56         if (!is_null(CONST_Osm2pgsql_Flatnode_File))
57         {
58                 $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
59         }
60
61
62         if (isset($aResult['import-diff']))
63         {
64                 // import diff directly (e.g. from osmosis --rri)
65                 $sNextFile = $aResult['import-diff'];
66                 if (!file_exists($sNextFile))
67                 {
68                         fail("Cannot open $sNextFile\n");
69                 }
70
71                 // Import the file
72                 $sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
73                 echo $sCMD."\n";
74                 exec($sCMD, $sJunk, $iErrorLevel);
75
76                 if ($iErrorLevel)
77                 {
78                         fail("Error from osm2pgsql, $iErrorLevel\n");
79                 }
80
81                 // Don't update the import status - we don't know what this file contains
82         }
83
84         $sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
85         $bHaveDiff = false;
86         if (isset($aResult['import-file']) && $aResult['import-file'])
87         {
88                 $bHaveDiff = true;
89                 $sCMD = CONST_Osmosis_Binary.' --read-xml \''.$aResult['import-file'].'\' --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
90                 echo $sCMD."\n";
91                 exec($sCMD, $sJunk, $iErrorLevel);
92                 if ($iErrorLevel)
93                 {
94                         fail("Error converting osm to osc, osmosis returned: $iErrorLevel\n");
95                 }
96         }
97
98         $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
99         $sContentURL = '';
100         if (isset($aResult['import-node']) && $aResult['import-node'])
101         {
102                 if ($bUseOSMApi)
103                 {
104                         $sContentURL = 'http://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
105                 }
106                 else
107                 {
108                         $sContentURL = 'http://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
109                 }
110         }
111         if (isset($aResult['import-way']) && $aResult['import-way'])
112         {
113                 if ($bUseOSMApi)
114                 {
115                         $sContentURL = 'http://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
116                 }
117                 else
118                 {
119                         $sContentURL = 'http://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
120                 }
121         }
122         if (isset($aResult['import-relation']) && $aResult['import-relation'])
123         {
124                 if ($bUseOSMApi)
125                 {
126                         $sContentURLsModifyXMLstr = 'http://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
127                 }
128                 else
129                 {
130                         $sContentURL = 'http://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
131                 }
132         }
133         if ($sContentURL)
134         {
135                 $sModifyXMLstr = file_get_contents($sContentURL);
136                 $bHaveDiff = true;
137
138                 $aSpec = array(
139                         0 => array("pipe", "r"),  // stdin
140                         1 => array("pipe", "w"),  // stdout
141                         2 => array("pipe", "w") // stderr
142                 );
143                 $sCMD = CONST_Osmosis_Binary.' --read-xml - --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
144                 echo $sCMD."\n";
145                 $hProc = proc_open($sCMD, $aSpec, $aPipes);
146                 if (!is_resource($hProc))
147                 {
148                         fail("Error converting osm to osc, osmosis failed\n");
149                 }
150                 fwrite($aPipes[0], $sModifyXMLstr);
151                 fclose($aPipes[0]);
152                 $sOut = stream_get_contents($aPipes[1]);
153                 if ($aResult['verbose']) echo $sOut;
154                 fclose($aPipes[1]);
155                 $sErrors = stream_get_contents($aPipes[2]);
156                 if ($aResult['verbose']) echo $sErrors;
157                 fclose($aPipes[2]);
158                 if ($iError = proc_close($hProc))
159                 {
160                         echo $sOut;
161                         echo $sErrors;
162                         fail("Error converting osm to osc, osmosis returned: $iError\n");
163                 }
164         }
165
166         if ($bHaveDiff)
167         {
168                 // import generated change file
169                 $sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
170                 echo $sCMD."\n";
171                 exec($sCMD, $sJunk, $iErrorLevel);
172                 if ($iErrorLevel)
173                 {
174                         fail("osm2pgsql exited with error level $iErrorLevel\n");
175                 }
176         }
177
178         if ($aResult['deduplicate'])
179         {
180
181                 if (getPostgresVersion() < 9.3)
182                 {
183                         fail("ERROR: deduplicate is only currently supported in postgresql 9.3");
184                 }
185
186                 $oDB =& getDB();
187                 $sSQL = 'select partition from country_name order by country_code';
188                 $aPartitions = chksql($oDB->getCol($sSQL));
189                 $aPartitions[] = 0;
190
191                 $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' ' and class is null and type is null and country_code is null group by word_token having count(*) > 1 order by word_token";
192                 $aDuplicateTokens = chksql($oDB->getAll($sSQL));
193                 foreach($aDuplicateTokens as $aToken)
194                 {
195                         if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
196                         echo "Deduping ".$aToken['word_token']."\n";
197                         $sSQL = "select word_id,(select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num from word where word_token = '".$aToken['word_token']."' and class is null and type is null and country_code is null order by num desc";
198                         $aTokenSet = chksql($oDB->getAll($sSQL));
199
200                         $aKeep = array_shift($aTokenSet);
201                         $iKeepID = $aKeep['word_id'];
202
203                         foreach($aTokenSet as $aRemove)
204                         {
205                                 $sSQL = "update search_name set";
206                                 $sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID."),";
207                                 $sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
208                                 $sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
209                                 chksql($oDB->query($sSQL));
210
211                                 $sSQL = "update search_name set";
212                                 $sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
213                                 $sSQL .= " where nameaddress_vector @> ARRAY[".$aRemove['word_id']."]";
214                                 chksql($oDB->query($sSQL));
215
216                                 $sSQL = "update location_area_country set";
217                                 $sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
218                                 $sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
219                                 chksql($oDB->query($sSQL));
220
221                                 foreach ($aPartitions as $sPartition)
222                                 {
223                                         $sSQL = "update search_name_".$sPartition." set";
224                                         $sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID.")";
225                                         $sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
226                                         chksql($oDB->query($sSQL));
227
228                                         $sSQL = "update location_area_country set";
229                                         $sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
230                                         $sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
231                                         chksql($oDB->query($sSQL));
232                                 }
233
234                                 $sSQL = "delete from word where word_id = ".$aRemove['word_id'];
235                                 chksql($oDB->query($sSQL));
236                         }
237                 }
238         }
239
240         if ($aResult['index'])
241         {
242                 passthru(CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
243         }
244
245         if ($aResult['import-osmosis'] || $aResult['import-osmosis-all'])
246         {
247
248                 if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
249                         fail("Error: Update interval too low for download.geofabrik.de.  Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n");
250                 }
251
252                 $sImportFile = CONST_BasePath.'/data/osmosischange.osc';
253                 $sOsmosisConfigDirectory = CONST_InstallPath.'/settings';
254                 $sCMDDownload = CONST_Osmosis_Binary.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
255                 $sCMDCheckReplicationLag = CONST_Osmosis_Binary.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
256                 $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
257                 $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
258
259                 while(true)
260                 {
261                         $fStartTime = time();
262                         $iFileSize = 1001;
263
264                         if (!file_exists($sImportFile))
265                         {
266                                 // First check if there are new updates published (except for minutelies - there's always new diffs to process)
267                                 if ( CONST_Replication_Update_Interval > 60 )
268                                 {
269
270                                         unset($aReplicationLag);
271                                         exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
272                                         while ($iErrorLevel > 0 || $aReplicationLag[0] < 1)
273                                         {
274                                                 if ($iErrorLevel)
275                                                 {
276                                                         echo "Error: $iErrorLevel. ";
277                                                         echo "Re-trying: ".$sCMDCheckReplicationLag." in ".CONST_Replication_Recheck_Interval." secs\n";
278                                                 }
279                                                 else
280                                                 {
281                                                         echo ".";
282                                                 }
283                                                 sleep(CONST_Replication_Recheck_Interval);
284                                                 unset($aReplicationLag);
285                                                 exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
286                                         }
287                                         // There are new replication files - use osmosis to download the file
288                                         echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n";
289                                 }
290                                 $fStartTime = time();
291                                 $fCMDStartTime = time();
292                                 echo $sCMDDownload."\n";
293                                 exec($sCMDDownload, $sJunk, $iErrorLevel);
294                                 while ($iErrorLevel > 0)
295                                 {
296                                         echo "Error: $iErrorLevel\n";
297                                         sleep(60);
298                                         echo 'Re-trying: '.$sCMDDownload."\n";
299                                         exec($sCMDDownload, $sJunk, $iErrorLevel);
300                                 }
301                                 $iFileSize = filesize($sImportFile);
302                                 $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
303                                 $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
304                                 var_Dump($sSQL);
305                                 $oDB->query($sSQL);
306                                 echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
307                         }
308
309                         $iFileSize = filesize($sImportFile);
310                         $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
311         
312                         // Import the file
313                         $fCMDStartTime = time();
314                         echo $sCMDImport."\n";
315                         exec($sCMDImport, $sJunk, $iErrorLevel);
316                         if ($iErrorLevel)
317                         {
318                                 echo "Error: $iErrorLevel\n";
319                                 exit($iErrorLevel);
320                         }
321                         $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
322                         var_Dump($sSQL);
323                         $oDB->query($sSQL);
324                         echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
325
326                         // Archive for debug?
327                         unlink($sImportFile);
328
329                         $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
330
331                         // Index file
332                         $sThisIndexCmd = $sCMDIndex;
333                         $fCMDStartTime = time();
334
335                         if (!$aResult['no-index'])
336                         {
337                                 echo "$sThisIndexCmd\n";
338                                 exec($sThisIndexCmd, $sJunk, $iErrorLevel);
339                                 if ($iErrorLevel)
340                                 {
341                                         echo "Error: $iErrorLevel\n";
342                                         exit($iErrorLevel);
343                                 }
344                         }
345
346                         $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
347                         var_Dump($sSQL);
348                         $oDB->query($sSQL);
349                         echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
350
351                         $sSQL = "update import_status set lastimportdate = '$sBatchEnd'";
352                         $oDB->query($sSQL);
353
354                         $fDuration = time() - $fStartTime;
355                         echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n";
356                         if (!$aResult['import-osmosis-all']) exit(0);
357
358                         if ( CONST_Replication_Update_Interval > 60 )
359                         {
360                                 $iSleep = max(0,(strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time()));
361                         }
362                         else
363                         {
364                                 $iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
365                         }
366                         echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n";
367                         sleep($iSleep);
368                 }
369         }
370
371         function getosmosistimestamp($sOsmosisConfigDirectory)
372         {
373                 $sStateFile = file_get_contents($sOsmosisConfigDirectory.'/state.txt');
374                 preg_match('#timestamp=(.+)#', $sStateFile, $aResult);
375                 return str_replace('\:',':',$aResult[1]);
376         }