]> git.openstreetmap.org Git - nominatim.git/blob - utils/setup.php
hstore in 9.2 replaces k=>v with hstore(k,v). Update sql to reflect change and intro...
[nominatim.git] / utils / setup.php
1 #!/usr/bin/php -Cq
2 <?php
3
4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
5         ini_set('memory_limit', '800M');
6
7         $aCMDOptions = array(
8                 "Create and setup nominatim search system",
9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
12
13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
15
16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
17
18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
35                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
36                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
37                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
38                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
39                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
40         );
41         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
42
43         $bDidSomething = false;
44
45         // Check if osm-file is set and points to a valid file if --all or --import-data is given
46         if ($aCMDResult['import-data'] || $aCMDResult['all'])
47         {
48                 if (!isset($aCMDResult['osm-file']))
49                 {
50                         fail('missing --osm-file for data import');
51                 }
52
53                 if (!file_exists($aCMDResult['osm-file']))
54                 {
55                         fail('the path supplied to --osm-file does not exist');
56                 }
57
58                 if (!is_readable($aCMDResult['osm-file']))
59                 {
60                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
61                 }
62         }
63
64
65         // This is a pretty hard core default - the number of processors in the box - 1
66         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
67         if ($iInstances < 1)
68         {
69                 $iInstances = 1;
70                 echo "WARNING: resetting threads to $iInstances\n";
71         }
72         if ($iInstances > getProcessorCount())
73         {
74                 $iInstances = getProcessorCount();
75                 echo "WARNING: resetting threads to $iInstances\n";
76         }
77
78         // Assume we can steal all the cache memory in the box (unless told otherwise)
79         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
80         if ($iCacheMemory > getTotalMemoryMB())
81         {
82                 $iCacheMemory = getCacheMemoryMB();
83                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
84         }
85
86         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
87         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
88
89         if ($aCMDResult['create-db'] || $aCMDResult['all'])
90         {
91                 echo "Create DB\n";
92                 $bDidSomething = true;
93                 $oDB =& DB::connect(CONST_Database_DSN, false);
94                 if (!PEAR::isError($oDB))
95                 {
96                         fail('database already exists ('.CONST_Database_DSN.')');
97                 }
98                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
99         }
100
101         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
102         {
103                 echo "Setup DB\n";
104                 $bDidSomething = true;
105                 // TODO: path detection, detection memory, etc.
106
107                 $oDB =& getDB();
108                 passthru('createlang plpgsql '.$aDSNInfo['database']);
109                 $pgver = (float) CONST_Postgresql_Version;
110                 if ($pgver < 9.1) {
111                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
112                         pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
113                 } else {
114                         pgsqlRunScript('CREATE EXTENSION hstore');
115                 }
116                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
117                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
118                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
119                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
120                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
121                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
122                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
123                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
124                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
125                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
126         }
127
128         if ($aCMDResult['import-data'] || $aCMDResult['all'])
129         {
130                 echo "Import\n";
131                 $bDidSomething = true;
132
133                 $osm2pgsql = CONST_Osm2pgsql_Binary;
134                 if (!file_exists($osm2pgsql))
135                 {
136                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
137                         fail("osm2pgsql not found in '$osm2pgsql'");
138                 }
139                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
140                 $osm2pgsql .= ' -C '.$iCacheMemory;
141                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
142                 passthruCheckReturn($osm2pgsql);
143
144                 $oDB =& getDB();
145                 $x = $oDB->getRow('select * from place limit 1');
146                 if (PEAR::isError($x)) {
147                         fail($x->getMessage());
148                 }
149                 if (!$x) fail('No Data');
150         }
151
152         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
153         {
154                 echo "Functions\n";
155                 $bDidSomething = true;
156                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
157                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
158                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
159                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
160                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
161                 pgsqlRunScript($sTemplate);
162         }
163
164         if ($aCMDResult['create-minimal-tables'])
165         {
166                 echo "Minimal Tables\n";
167                 $bDidSomething = true;
168                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
169
170                 $sScript = '';
171
172                 // Backstop the import process - easliest possible import id
173                 $sScript .= "insert into import_npi_log values (18022);\n";
174
175                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
176                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
177
178                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
179                 {
180                         list($sClass, $sType) = explode(' ', trim($sLine));
181                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
182                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
183
184                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
185                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
186
187                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
188                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
189                 }
190                 fclose($hFile);
191                 pgsqlRunScript($sScript);
192         }
193
194         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
195         {
196                 echo "Tables\n";
197                 $bDidSomething = true;
198                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
199
200                 // re-run the functions
201                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
202                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
203                 pgsqlRunScript($sTemplate);
204         }
205
206         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
207         {
208                 echo "Partitions\n";
209                 $bDidSomething = true;
210                 $oDB =& getDB();
211                 $sSQL = 'select partition from country_name order by country_code';
212                 $aPartitions = $oDB->getCol($sSQL);
213                 if (PEAR::isError($aPartitions))
214                 {
215                         fail($aPartitions->getMessage());
216                 }
217                 $aPartitions[] = 0;
218
219                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
220                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
221                 foreach($aMatches as $aMatch)
222                 {
223                         $sResult = '';
224                         foreach($aPartitions as $sPartitionName)
225                         {
226                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
227                         }
228                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
229                 }
230
231                 pgsqlRunScript($sTemplate);
232         }
233
234         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
235         {
236                 $bDidSomething = true;
237                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
238                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
239                 if (file_exists($sWikiArticlesFile))
240                 {
241                         echo "Importing wikipedia articles...";
242                         pgsqlRunDropAndRestore($sWikiArticlesFile);
243                         echo "...done\n";
244                 }
245                 else
246                 {
247                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
248                 }
249                 if (file_exists($sWikiRedirectsFile))
250                 {
251                         echo "Importing wikipedia redirects...";
252                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
253                         echo "...done\n";
254                 }
255                 else
256                 {
257                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
258                 }
259         }
260
261
262         if ($aCMDResult['load-data'] || $aCMDResult['all'])
263         {
264                 echo "Drop old Data\n";
265                 $bDidSomething = true;
266
267                 $oDB =& getDB();
268                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
269                 echo '.';
270                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
271                 echo '.';
272                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
273                 echo '.';
274                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
275                 echo '.';
276                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
277                 echo '.';
278                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
279                 echo '.';
280                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
281                 echo '.';
282                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
283                 echo '.';
284                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
285                 echo '.';
286
287                 $sSQL = 'select partition from country_name order by country_code';
288                 $aPartitions = $oDB->getCol($sSQL);
289                 if (PEAR::isError($aPartitions))
290                 {
291                         fail($aPartitions->getMessage());
292                 }
293                 $aPartitions[] = 0;
294                 foreach($aPartitions as $sPartition)
295                 {
296                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
297                         echo '.';
298                 }
299
300                 // used by getorcreate_word_id to ignore frequent partial words
301                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
302                 echo ".\n";
303
304                 // pre-create the word list
305                 if (!$aCMDResult['disable-token-precalc'])
306                 {
307                         echo "Loading word list\n";
308                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
309                 }
310
311                 echo "Load Data\n";
312                 $aDBInstances = array();
313                 for($i = 0; $i < $iInstances; $i++)
314                 {
315                         $aDBInstances[$i] =& getDB(true);
316                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
317                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
318                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
319                         if ($aCMDResult['verbose']) echo "$sSQL\n";
320                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
321                 }
322                 $bAnyBusy = true;
323                 while($bAnyBusy)
324                 {
325                         $bAnyBusy = false;
326                         for($i = 0; $i < $iInstances; $i++)
327                         {
328                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
329                         }
330                         sleep(1);
331                         echo '.';
332                 }
333                 echo "\n";
334                 echo "Reanalysing database...\n";
335                 pgsqlRunScript('ANALYSE');
336         }
337
338         if ($aCMDResult['create-roads'])
339         {
340                 $bDidSomething = true;
341
342                 $oDB =& getDB();
343                 $aDBInstances = array();
344                 for($i = 0; $i < $iInstances; $i++)
345                 {
346                         $aDBInstances[$i] =& getDB(true);
347                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
348                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
349                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
350                         if ($aCMDResult['verbose']) echo "$sSQL\n";
351                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
352                 }
353                 $bAnyBusy = true;
354                 while($bAnyBusy)
355                 {
356                         $bAnyBusy = false;
357                         for($i = 0; $i < $iInstances; $i++)
358                         {
359                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
360                         }
361                         sleep(1);
362                         echo '.';
363                 }
364                 echo "\n";
365         }
366
367         if ($aCMDResult['import-tiger-data'])
368         {
369                 $bDidSomething = true;
370
371                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
372
373                 $aDBInstances = array();
374                 for($i = 0; $i < $iInstances; $i++)
375                 {
376                         $aDBInstances[$i] =& getDB(true);
377                 }
378
379                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
380                 {
381                         echo $sFile.': ';
382                         $hFile = fopen($sFile, "r");
383                         $sSQL = fgets($hFile, 100000);
384                         $iLines = 0;
385
386                         while(true)
387                         {
388                                 for($i = 0; $i < $iInstances; $i++)
389                                 {
390                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
391                                         {
392                                                 while(pg_get_result($aDBInstances[$i]->connection));
393                                                 $sSQL = fgets($hFile, 100000);
394                                                 if (!$sSQL) break 2;
395                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
396                                                 $iLines++;
397                                                 if ($iLines == 1000)
398                                                 {
399                                                         echo ".";
400                                                         $iLines = 0;
401                                                 }
402                                         }
403                                 }
404                                 usleep(10);
405                         }
406
407                         fclose($hFile);
408
409                         $bAnyBusy = true;
410                         while($bAnyBusy)
411                         {
412                                 $bAnyBusy = false;
413                                 for($i = 0; $i < $iInstances; $i++)
414                                 {
415                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
416                                 }
417                                 usleep(10);
418                         }
419                         echo "\n";
420                 }
421
422                 echo "Creating indexes\n";
423                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
424         }
425
426         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
427         {
428                 $bDidSomething = true;
429                 $oDB =& getDB();
430                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
431                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
432                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
433                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
434                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
435                 $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
436                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
437
438                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
439                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
440                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
441                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
442         }
443
444         if ($aCMDResult['osmosis-init'] || $aCMDResult['all'])
445         {
446                 $bDidSomething = true;
447                 $oDB =& getDB();
448
449                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
450                 if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
451                 {
452                         echo "settings/configuration.txt already exists\n";
453                 }
454                 else
455                 {
456                         passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
457                         // server layout changed afer license change, fix path to minutely diffs
458                         passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
459                 }
460
461                 // Find the last node in the DB
462                 $iLastOSMID = $oDB->getOne("select max(osm_id) as osm_id from place where osm_type = 'N'");
463
464                 // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
465                 $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID;
466                 $sLastNodeXML = file_get_contents($sLastNodeURL);
467                 preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
468                 $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
469
470
471                 // Search for the correct state file - uses file timestamps
472                 $sRepURL = 'http://planet.openstreetmap.org/replication/minute/';
473                 $sRep = file_get_contents($sRepURL);
474                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
475                 $aPrevRepMatch = false;
476                 foreach($aRepMatches as $aRepMatch)
477                 {
478                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
479                         $aPrevRepMatch = $aRepMatch;
480                 }
481                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
482
483                 $sRepURL .= $aRepMatch[1];
484                 $sRep = file_get_contents($sRepURL);
485                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
486                 $aPrevRepMatch = false;
487                 foreach($aRepMatches as $aRepMatch)
488                 {
489                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
490                         $aPrevRepMatch = $aRepMatch;
491                 }
492                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
493
494                 $sRepURL .= $aRepMatch[1];
495                 $sRep = file_get_contents($sRepURL);
496                 preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
497                 $aPrevRepMatch = false;
498                 foreach($aRepMatches as $aRepMatch)
499                 {
500                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
501                         $aPrevRepMatch = $aRepMatch;
502                 }
503                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
504
505                 $sRepURL .= $aRepMatch[1].'.state.txt';
506                 echo "Getting state file: $sRepURL\n";
507                 $sStateFile = file_get_contents($sRepURL);
508                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
509                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
510                 echo "Updating DB status\n";
511                 pg_query($oDB->connection, 'TRUNCATE import_status');
512                 $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
513                 pg_query($oDB->connection, $sSQL);
514         }
515
516         if ($aCMDResult['index'] || $aCMDResult['all'])
517         {
518                 $bDidSomething = true;
519                 $sOutputFile = '';
520                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
521                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
522                 passthruCheckReturn($sBaseCmd.' -R 4');
523                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
524                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
525                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
526                 passthruCheckReturn($sBaseCmd.' -r 26');
527         }
528
529         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
530         {
531                 echo "Search indices\n";
532                 $bDidSomething = true;
533                 $oDB =& getDB();
534                 $sSQL = 'select partition from country_name order by country_code';
535                 $aPartitions = $oDB->getCol($sSQL);
536                 if (PEAR::isError($aPartitions))
537                 {
538                         fail($aPartitions->getMessage());
539                 }
540                 $aPartitions[] = 0;
541
542                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
543                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
544                 foreach($aMatches as $aMatch)
545                 {
546                         $sResult = '';
547                         foreach($aPartitions as $sPartitionName)
548                         {
549                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
550                         }
551                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
552                 }
553
554                 pgsqlRunScript($sTemplate);
555         }
556
557         if (isset($aCMDResult['create-website']))
558         {
559                 $bDidSomething = true;
560                 $sTargetDir = $aCMDResult['create-website'];
561                 if (!is_dir($sTargetDir))
562                 {
563                         echo "You must create the website directory before calling this function.\n";
564                         fail("Target directory does not exist.");
565                 }
566
567                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
568                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
569                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
570                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
571                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
572                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
573                 echo "Symlinks created\n";
574         }
575
576         if (!$bDidSomething)
577         {
578                 showUsage($aCMDOptions, true);
579         }
580
581         function pgsqlRunScriptFile($sFilename)
582         {
583                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
584
585                 // Convert database DSN to psql parameters
586                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
587                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
588                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
589
590                 $aDescriptors = array(
591                         0 => array('pipe', 'r'),
592                         1 => array('pipe', 'w'),
593                         2 => array('file', '/dev/null', 'a')
594                 );
595                 $ahPipes = null;
596                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
597                 if (!is_resource($hProcess)) fail('unable to start pgsql');
598
599                 fclose($ahPipes[0]);
600
601                 // TODO: error checking
602                 while(!feof($ahPipes[1]))
603                 {
604                         echo fread($ahPipes[1], 4096);
605                 }
606                 fclose($ahPipes[1]);
607
608                 proc_close($hProcess);
609         }
610
611         function pgsqlRunScript($sScript)
612         {
613                 // Convert database DSN to psql parameters
614                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
615                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
616                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
617                 $aDescriptors = array(
618                         0 => array('pipe', 'r'),
619                         1 => STDOUT, 
620                         2 => STDERR
621                 );
622                 $ahPipes = null;
623                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
624                 if (!is_resource($hProcess)) fail('unable to start pgsql');
625
626                 while(strlen($sScript))
627                 {
628                         $written = fwrite($ahPipes[0], $sScript);
629                         $sScript = substr($sScript, $written);
630                 }
631                 fclose($ahPipes[0]);
632                 proc_close($hProcess);
633         }
634
635         function pgsqlRunRestoreData($sDumpFile)
636         {
637                 // Convert database DSN to psql parameters
638                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
639                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
640                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
641
642                 $aDescriptors = array(
643                         0 => array('pipe', 'r'),
644                         1 => array('pipe', 'w'),
645                         2 => array('file', '/dev/null', 'a')
646                 );
647                 $ahPipes = null;
648                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
649                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
650
651                 fclose($ahPipes[0]);
652
653                 // TODO: error checking
654                 while(!feof($ahPipes[1]))
655                 {
656                         echo fread($ahPipes[1], 4096);
657                 }
658                 fclose($ahPipes[1]);
659
660                 proc_close($hProcess);
661         }
662
663         function pgsqlRunDropAndRestore($sDumpFile)
664         {
665                 // Convert database DSN to psql parameters
666                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
667                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
668                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
669
670                 $aDescriptors = array(
671                         0 => array('pipe', 'r'),
672                         1 => array('pipe', 'w'),
673                         2 => array('file', '/dev/null', 'a')
674                 );
675                 $ahPipes = null;
676                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
677                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
678
679                 fclose($ahPipes[0]);
680
681                 // TODO: error checking
682                 while(!feof($ahPipes[1]))
683                 {
684                         echo fread($ahPipes[1], 4096);
685                 }
686                 fclose($ahPipes[1]);
687
688                 proc_close($hProcess);
689         }
690
691         function passthruCheckReturn($cmd)
692         {
693                 $result = -1;
694                 passthru($cmd, $result);
695                 if ($result != 0) fail('Error executing external command: '.$cmd);
696         }