]> git.openstreetmap.org Git - nominatim.git/blob - utils/setup.php
Merge branch 'master' of github.com:twain47/Nominatim
[nominatim.git] / utils / setup.php
1 #!/usr/bin/php -Cq
2 <?php
3
4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
5         ini_set('memory_limit', '800M');
6
7         $aCMDOptions = array(
8                 "Create and setup nominatim search system",
9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
12
13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
15
16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
17
18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
35                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
36                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
37                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
38                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
39                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
40         );
41         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
42
43         $bDidSomething = false;
44
45         // Check if osm-file is set and points to a valid file if --all or --import-data is given
46         if ($aCMDResult['import-data'] || $aCMDResult['all'])
47         {
48                 if (!isset($aCMDResult['osm-file']))
49                 {
50                         fail('missing --osm-file for data import');
51                 }
52
53                 if (!file_exists($aCMDResult['osm-file']))
54                 {
55                         fail('the path supplied to --osm-file does not exist');
56                 }
57
58                 if (!is_readable($aCMDResult['osm-file']))
59                 {
60                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
61                 }
62         }
63
64
65         // This is a pretty hard core default - the number of processors in the box - 1
66         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
67         if ($iInstances < 1)
68         {
69                 $iInstances = 1;
70                 echo "WARNING: resetting threads to $iInstances\n";
71         }
72         if ($iInstances > getProcessorCount())
73         {
74                 $iInstances = getProcessorCount();
75                 echo "WARNING: resetting threads to $iInstances\n";
76         }
77
78         // Assume we can steal all the cache memory in the box (unless told otherwise)
79         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
80         if ($iCacheMemory > getTotalMemoryMB())
81         {
82                 $iCacheMemory = getCacheMemoryMB();
83                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
84         }
85
86         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
87         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
88
89         if ($aCMDResult['create-db'] || $aCMDResult['all'])
90         {
91                 echo "Create DB\n";
92                 $bDidSomething = true;
93                 $oDB =& DB::connect(CONST_Database_DSN, false);
94                 if (!PEAR::isError($oDB))
95                 {
96                         fail('database already exists ('.CONST_Database_DSN.')');
97                 }
98                 passthruCheckReturn('createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
99         }
100
101         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
102         {
103                 echo "Setup DB\n";
104                 $bDidSomething = true;
105                 // TODO: path detection, detection memory, etc.
106
107                 $oDB =& getDB();
108
109                 $sVersionString = $oDB->getOne('select version()');
110                 preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[.]([0-9]+) #', $sVersionString, $aMatches);
111                 if (CONST_Postgresql_Version != $aMatches[1].'.'.$aMatches[2])
112                 {
113                         echo "ERROR: PostgreSQL version is not correct.  Expected ".CONST_Postgresql_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
114                         exit;
115                 }
116
117                 passthru('createlang plpgsql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
118                 $pgver = (float) CONST_Postgresql_Version;
119                 if ($pgver < 9.1) {
120                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
121                         pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
122                 } else {
123                         pgsqlRunScript('CREATE EXTENSION hstore');
124                 }
125
126                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
127                 $sVersionString = $oDB->getOne('select postgis_full_version()');
128                 preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
129                 if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
130                 {
131                         echo "ERROR: PostGIS version is not correct.  Expected ".CONST_Postgis_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
132                         exit;
133                 }
134
135                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
136                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
137                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
138                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
139                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
140                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
141                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
142                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
143                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
144         }
145
146         if ($aCMDResult['import-data'] || $aCMDResult['all'])
147         {
148                 echo "Import\n";
149                 $bDidSomething = true;
150
151                 $osm2pgsql = CONST_Osm2pgsql_Binary;
152                 if (!file_exists($osm2pgsql))
153                 {
154                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
155                         fail("osm2pgsql not found in '$osm2pgsql'");
156                 }
157                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
158                 $osm2pgsql .= ' -C '.$iCacheMemory;
159                 $osm2pgsql .= ' -P '.$aDSNInfo['port'];
160                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
161                 passthruCheckReturn($osm2pgsql);
162
163                 $oDB =& getDB();
164                 $x = $oDB->getRow('select * from place limit 1');
165                 if (PEAR::isError($x)) {
166                         fail($x->getMessage());
167                 }
168                 if (!$x) fail('No Data');
169         }
170
171         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
172         {
173                 echo "Functions\n";
174                 $bDidSomething = true;
175                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
176                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
177                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
178                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
179                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
180                 pgsqlRunScript($sTemplate);
181         }
182
183         if ($aCMDResult['create-minimal-tables'])
184         {
185                 echo "Minimal Tables\n";
186                 $bDidSomething = true;
187                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
188
189                 $sScript = '';
190
191                 // Backstop the import process - easliest possible import id
192                 $sScript .= "insert into import_npi_log values (18022);\n";
193
194                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
195                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
196
197                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
198                 {
199                         list($sClass, $sType) = explode(' ', trim($sLine));
200                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
201                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
202
203                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
204                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
205
206                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
207                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
208                 }
209                 fclose($hFile);
210                 pgsqlRunScript($sScript);
211         }
212
213         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
214         {
215                 echo "Tables\n";
216                 $bDidSomething = true;
217                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
218
219                 // re-run the functions
220                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
221                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
222                 pgsqlRunScript($sTemplate);
223         }
224
225         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
226         {
227                 echo "Partitions\n";
228                 $bDidSomething = true;
229                 $oDB =& getDB();
230                 $sSQL = 'select partition from country_name order by country_code';
231                 $aPartitions = $oDB->getCol($sSQL);
232                 if (PEAR::isError($aPartitions))
233                 {
234                         fail($aPartitions->getMessage());
235                 }
236                 $aPartitions[] = 0;
237
238                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
239                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
240                 foreach($aMatches as $aMatch)
241                 {
242                         $sResult = '';
243                         foreach($aPartitions as $sPartitionName)
244                         {
245                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
246                         }
247                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
248                 }
249
250                 pgsqlRunScript($sTemplate);
251         }
252
253         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
254         {
255                 $bDidSomething = true;
256                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
257                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
258                 if (file_exists($sWikiArticlesFile))
259                 {
260                         echo "Importing wikipedia articles...";
261                         pgsqlRunDropAndRestore($sWikiArticlesFile);
262                         echo "...done\n";
263                 }
264                 else
265                 {
266                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
267                 }
268                 if (file_exists($sWikiRedirectsFile))
269                 {
270                         echo "Importing wikipedia redirects...";
271                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
272                         echo "...done\n";
273                 }
274                 else
275                 {
276                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
277                 }
278         }
279
280
281         if ($aCMDResult['load-data'] || $aCMDResult['all'])
282         {
283                 echo "Drop old Data\n";
284                 $bDidSomething = true;
285
286                 $oDB =& getDB();
287                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
288                 echo '.';
289                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
290                 echo '.';
291                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
292                 echo '.';
293                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
294                 echo '.';
295                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
296                 echo '.';
297                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
298                 echo '.';
299                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
300                 echo '.';
301                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
302                 echo '.';
303                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
304                 echo '.';
305
306                 $sSQL = 'select partition from country_name order by country_code';
307                 $aPartitions = $oDB->getCol($sSQL);
308                 if (PEAR::isError($aPartitions))
309                 {
310                         fail($aPartitions->getMessage());
311                 }
312                 $aPartitions[] = 0;
313                 foreach($aPartitions as $sPartition)
314                 {
315                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
316                         echo '.';
317                 }
318
319                 // used by getorcreate_word_id to ignore frequent partial words
320                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
321                 echo ".\n";
322
323                 // pre-create the word list
324                 if (!$aCMDResult['disable-token-precalc'])
325                 {
326                         echo "Loading word list\n";
327                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
328                 }
329
330                 echo "Load Data\n";
331                 $aDBInstances = array();
332                 for($i = 0; $i < $iInstances; $i++)
333                 {
334                         $aDBInstances[$i] =& getDB(true);
335                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
336                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
337                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
338                         if ($aCMDResult['verbose']) echo "$sSQL\n";
339                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
340                 }
341                 $bAnyBusy = true;
342                 while($bAnyBusy)
343                 {
344                         $bAnyBusy = false;
345                         for($i = 0; $i < $iInstances; $i++)
346                         {
347                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
348                         }
349                         sleep(1);
350                         echo '.';
351                 }
352                 echo "\n";
353                 echo "Reanalysing database...\n";
354                 pgsqlRunScript('ANALYSE');
355         }
356
357         if ($aCMDResult['create-roads'])
358         {
359                 $bDidSomething = true;
360
361                 $oDB =& getDB();
362                 $aDBInstances = array();
363                 for($i = 0; $i < $iInstances; $i++)
364                 {
365                         $aDBInstances[$i] =& getDB(true);
366                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
367                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
368                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
369                         if ($aCMDResult['verbose']) echo "$sSQL\n";
370                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
371                 }
372                 $bAnyBusy = true;
373                 while($bAnyBusy)
374                 {
375                         $bAnyBusy = false;
376                         for($i = 0; $i < $iInstances; $i++)
377                         {
378                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
379                         }
380                         sleep(1);
381                         echo '.';
382                 }
383                 echo "\n";
384         }
385
386         if ($aCMDResult['import-tiger-data'])
387         {
388                 $bDidSomething = true;
389
390                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
391
392                 $aDBInstances = array();
393                 for($i = 0; $i < $iInstances; $i++)
394                 {
395                         $aDBInstances[$i] =& getDB(true);
396                 }
397
398                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
399                 {
400                         echo $sFile.': ';
401                         $hFile = fopen($sFile, "r");
402                         $sSQL = fgets($hFile, 100000);
403                         $iLines = 0;
404
405                         while(true)
406                         {
407                                 for($i = 0; $i < $iInstances; $i++)
408                                 {
409                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
410                                         {
411                                                 while(pg_get_result($aDBInstances[$i]->connection));
412                                                 $sSQL = fgets($hFile, 100000);
413                                                 if (!$sSQL) break 2;
414                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
415                                                 $iLines++;
416                                                 if ($iLines == 1000)
417                                                 {
418                                                         echo ".";
419                                                         $iLines = 0;
420                                                 }
421                                         }
422                                 }
423                                 usleep(10);
424                         }
425
426                         fclose($hFile);
427
428                         $bAnyBusy = true;
429                         while($bAnyBusy)
430                         {
431                                 $bAnyBusy = false;
432                                 for($i = 0; $i < $iInstances; $i++)
433                                 {
434                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
435                                 }
436                                 usleep(10);
437                         }
438                         echo "\n";
439                 }
440
441                 echo "Creating indexes\n";
442                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
443         }
444
445         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
446         {
447                 $bDidSomething = true;
448                 $oDB =& getDB();
449                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
450                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
451                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
452                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
453                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
454                 $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
455                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
456
457                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
458                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
459                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
460                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
461         }
462
463         if ($aCMDResult['osmosis-init'] || $aCMDResult['all'])
464         {
465                 $bDidSomething = true;
466                 $oDB =& getDB();
467
468                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
469                 if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
470                 {
471                         echo "settings/configuration.txt already exists\n";
472                 }
473                 else
474                 {
475                         passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
476                         // server layout changed afer license change, fix path to minutely diffs
477                         passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
478                 }
479
480                 // Find the last node in the DB
481                 $iLastOSMID = $oDB->getOne("select max(osm_id) as osm_id from place where osm_type = 'N'");
482
483                 // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
484                 $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID;
485                 $sLastNodeXML = file_get_contents($sLastNodeURL);
486                 preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
487                 $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
488
489
490                 // Search for the correct state file - uses file timestamps
491                 $sRepURL = 'http://planet.openstreetmap.org/replication/minute/';
492                 $sRep = file_get_contents($sRepURL);
493                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
494                 $aPrevRepMatch = false;
495                 foreach($aRepMatches as $aRepMatch)
496                 {
497                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
498                         $aPrevRepMatch = $aRepMatch;
499                 }
500                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
501
502                 $sRepURL .= $aRepMatch[1];
503                 $sRep = file_get_contents($sRepURL);
504                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
505                 $aPrevRepMatch = false;
506                 foreach($aRepMatches as $aRepMatch)
507                 {
508                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
509                         $aPrevRepMatch = $aRepMatch;
510                 }
511                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
512
513                 $sRepURL .= $aRepMatch[1];
514                 $sRep = file_get_contents($sRepURL);
515                 preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
516                 $aPrevRepMatch = false;
517                 foreach($aRepMatches as $aRepMatch)
518                 {
519                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
520                         $aPrevRepMatch = $aRepMatch;
521                 }
522                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
523
524                 $sRepURL .= $aRepMatch[1].'.state.txt';
525                 echo "Getting state file: $sRepURL\n";
526                 $sStateFile = file_get_contents($sRepURL);
527                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
528                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
529                 echo "Updating DB status\n";
530                 pg_query($oDB->connection, 'TRUNCATE import_status');
531                 $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
532                 pg_query($oDB->connection, $sSQL);
533         }
534
535         if ($aCMDResult['index'] || $aCMDResult['all'])
536         {
537                 $bDidSomething = true;
538                 $sOutputFile = '';
539                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
540                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
541                 passthruCheckReturn($sBaseCmd.' -R 4');
542                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
543                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
544                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
545                 passthruCheckReturn($sBaseCmd.' -r 26');
546         }
547
548         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
549         {
550                 echo "Search indices\n";
551                 $bDidSomething = true;
552                 $oDB =& getDB();
553                 $sSQL = 'select partition from country_name order by country_code';
554                 $aPartitions = $oDB->getCol($sSQL);
555                 if (PEAR::isError($aPartitions))
556                 {
557                         fail($aPartitions->getMessage());
558                 }
559                 $aPartitions[] = 0;
560
561                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
562                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
563                 foreach($aMatches as $aMatch)
564                 {
565                         $sResult = '';
566                         foreach($aPartitions as $sPartitionName)
567                         {
568                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
569                         }
570                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
571                 }
572
573                 pgsqlRunScript($sTemplate);
574         }
575
576         if (isset($aCMDResult['create-website']))
577         {
578                 $bDidSomething = true;
579                 $sTargetDir = $aCMDResult['create-website'];
580                 if (!is_dir($sTargetDir))
581                 {
582                         echo "You must create the website directory before calling this function.\n";
583                         fail("Target directory does not exist.");
584                 }
585
586                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
587                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
588                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
589                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
590                 @symlink(CONST_BasePath.'/website/deletable.php', $sTargetDir.'/deletable.php');
591                 @symlink(CONST_BasePath.'/website/polygons.php', $sTargetDir.'/polygons.php');
592                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
593                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
594                 @symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css');
595                 echo "Symlinks created\n";
596         }
597
598         if (!$bDidSomething)
599         {
600                 showUsage($aCMDOptions, true);
601         }
602
603         function pgsqlRunScriptFile($sFilename)
604         {
605                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
606
607                 // Convert database DSN to psql parameters
608                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
609                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
610                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
611
612                 $aDescriptors = array(
613                         0 => array('pipe', 'r'),
614                         1 => array('pipe', 'w'),
615                         2 => array('file', '/dev/null', 'a')
616                 );
617                 $ahPipes = null;
618                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
619                 if (!is_resource($hProcess)) fail('unable to start pgsql');
620
621                 fclose($ahPipes[0]);
622
623                 // TODO: error checking
624                 while(!feof($ahPipes[1]))
625                 {
626                         echo fread($ahPipes[1], 4096);
627                 }
628                 fclose($ahPipes[1]);
629
630                 proc_close($hProcess);
631         }
632
633         function pgsqlRunScript($sScript)
634         {
635                 // Convert database DSN to psql parameters
636                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
637                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
638                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
639                 $aDescriptors = array(
640                         0 => array('pipe', 'r'),
641                         1 => STDOUT, 
642                         2 => STDERR
643                 );
644                 $ahPipes = null;
645                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
646                 if (!is_resource($hProcess)) fail('unable to start pgsql');
647
648                 while(strlen($sScript))
649                 {
650                         $written = fwrite($ahPipes[0], $sScript);
651                         $sScript = substr($sScript, $written);
652                 }
653                 fclose($ahPipes[0]);
654                 proc_close($hProcess);
655         }
656
657         function pgsqlRunRestoreData($sDumpFile)
658         {
659                 // Convert database DSN to psql parameters
660                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
661                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
662                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
663
664                 $aDescriptors = array(
665                         0 => array('pipe', 'r'),
666                         1 => array('pipe', 'w'),
667                         2 => array('file', '/dev/null', 'a')
668                 );
669                 $ahPipes = null;
670                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
671                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
672
673                 fclose($ahPipes[0]);
674
675                 // TODO: error checking
676                 while(!feof($ahPipes[1]))
677                 {
678                         echo fread($ahPipes[1], 4096);
679                 }
680                 fclose($ahPipes[1]);
681
682                 proc_close($hProcess);
683         }
684
685         function pgsqlRunDropAndRestore($sDumpFile)
686         {
687                 // Convert database DSN to psql parameters
688                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
689                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
690                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
691
692                 $aDescriptors = array(
693                         0 => array('pipe', 'r'),
694                         1 => array('pipe', 'w'),
695                         2 => array('file', '/dev/null', 'a')
696                 );
697                 $ahPipes = null;
698                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
699                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
700
701                 fclose($ahPipes[0]);
702
703                 // TODO: error checking
704                 while(!feof($ahPipes[1]))
705                 {
706                         echo fread($ahPipes[1], 4096);
707                 }
708                 fclose($ahPipes[1]);
709
710                 proc_close($hProcess);
711         }
712
713         function passthruCheckReturn($cmd)
714         {
715                 $result = -1;
716                 passthru($cmd, $result);
717                 if ($result != 0) fail('Error executing external command: '.$cmd);
718         }