]> git.openstreetmap.org Git - nominatim.git/blob - utils/setup.php
switch to distribution-supplied osmosis
[nominatim.git] / utils / setup.php
1 #!/usr/bin/php -Cq
2 <?php
3
4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
5         ini_set('memory_limit', '800M');
6
7         $aCMDOptions = array(
8                 "Create and setup nominatim search system",
9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
12
13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
15
16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
17
18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
35                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
36                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
37                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
38                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
39                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
40                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
41         );
42         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
43
44         $bDidSomething = false;
45
46         // Check if osm-file is set and points to a valid file if --all or --import-data is given
47         if ($aCMDResult['import-data'] || $aCMDResult['all'])
48         {
49                 if (!isset($aCMDResult['osm-file']))
50                 {
51                         fail('missing --osm-file for data import');
52                 }
53
54                 if (!file_exists($aCMDResult['osm-file']))
55                 {
56                         fail('the path supplied to --osm-file does not exist');
57                 }
58
59                 if (!is_readable($aCMDResult['osm-file']))
60                 {
61                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
62                 }
63         }
64
65
66         // This is a pretty hard core default - the number of processors in the box - 1
67         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
68         if ($iInstances < 1)
69         {
70                 $iInstances = 1;
71                 echo "WARNING: resetting threads to $iInstances\n";
72         }
73         if ($iInstances > getProcessorCount())
74         {
75                 $iInstances = getProcessorCount();
76                 echo "WARNING: resetting threads to $iInstances\n";
77         }
78
79         // Assume we can steal all the cache memory in the box (unless told otherwise)
80         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
81         if ($iCacheMemory > getTotalMemoryMB())
82         {
83                 $iCacheMemory = getCacheMemoryMB();
84                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
85         }
86
87         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
88         {
89                 $sBaseFile = basename($aCMDResult['osm-file']);
90                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
91                 {
92                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
93                         $iTime -= (60*60*24);
94                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
95                 }
96         }
97         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
98         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
99
100         if ($aCMDResult['create-db'] || $aCMDResult['all'])
101         {
102                 echo "Create DB\n";
103                 $bDidSomething = true;
104                 $oDB =& DB::connect(CONST_Database_DSN, false);
105                 if (!PEAR::isError($oDB))
106                 {
107                         fail('database already exists ('.CONST_Database_DSN.')');
108                 }
109                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
110         }
111
112         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
113         {
114                 echo "Setup DB\n";
115                 $bDidSomething = true;
116                 // TODO: path detection, detection memory, etc.
117
118                 $oDB =& getDB();
119                 passthru('createlang plpgsql '.$aDSNInfo['database']);
120                 $pgver = (float) CONST_Postgresql_Version;
121                 if ($pgver < 9.1) {
122                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
123                 } else {
124                         pgsqlRunScript('CREATE EXTENSION hstore');
125                 }
126                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
127                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
128                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
129                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
130                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
131                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
132                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
133                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
134                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
135                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
136         }
137
138         if ($aCMDResult['import-data'] || $aCMDResult['all'])
139         {
140                 echo "Import\n";
141                 $bDidSomething = true;
142
143                 $osm2pgsql = CONST_Osm2pgsql_Binary;
144                 if (!file_exists($osm2pgsql))
145                 {
146                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
147                         fail("osm2pgsql not found in '$osm2pgsql'");
148                 }
149                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
150                 $osm2pgsql .= ' -C '.$iCacheMemory;
151                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
152                 passthruCheckReturn($osm2pgsql);
153
154                 $oDB =& getDB();
155                 $x = $oDB->getRow('select * from place limit 1');
156                 if (PEAR::isError($x)) {
157                         fail($x->getMessage());
158                 }
159                 if (!$x) fail('No Data');
160         }
161
162         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
163         {
164                 echo "Functions\n";
165                 $bDidSomething = true;
166                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
167                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
168                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
169                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
170                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
171                 pgsqlRunScript($sTemplate);
172         }
173
174         if ($aCMDResult['create-minimal-tables'])
175         {
176                 echo "Minimal Tables\n";
177                 $bDidSomething = true;
178                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
179
180                 $sScript = '';
181
182                 // Backstop the import process - easliest possible import id
183                 $sScript .= "insert into import_npi_log values (18022);\n";
184
185                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
186                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
187
188                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
189                 {
190                         list($sClass, $sType) = explode(' ', trim($sLine));
191                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
192                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
193
194                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
195                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
196
197                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
198                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
199                 }
200                 fclose($hFile);
201                 pgsqlRunScript($sScript);
202         }
203
204         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
205         {
206                 echo "Tables\n";
207                 $bDidSomething = true;
208                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
209
210                 // re-run the functions
211                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
212                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
213                 pgsqlRunScript($sTemplate);
214         }
215
216         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
217         {
218                 echo "Partitions\n";
219                 $bDidSomething = true;
220                 $oDB =& getDB();
221                 $sSQL = 'select partition from country_name order by country_code';
222                 $aPartitions = $oDB->getCol($sSQL);
223                 if (PEAR::isError($aPartitions))
224                 {
225                         fail($aPartitions->getMessage());
226                 }
227                 $aPartitions[] = 0;
228
229                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
230                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
231                 foreach($aMatches as $aMatch)
232                 {
233                         $sResult = '';
234                         foreach($aPartitions as $sPartitionName)
235                         {
236                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
237                         }
238                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
239                 }
240
241                 pgsqlRunScript($sTemplate);
242         }
243
244         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
245         {
246                 $bDidSomething = true;
247                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
248                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
249                 if (file_exists($sWikiArticlesFile))
250                 {
251                         echo "Importing wikipedia articles...";
252                         pgsqlRunDropAndRestore($sWikiArticlesFile);
253                         echo "...done\n";
254                 }
255                 else
256                 {
257                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
258                 }
259                 if (file_exists($sWikiRedirectsFile))
260                 {
261                         echo "Importing wikipedia redirects...";
262                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
263                         echo "...done\n";
264                 }
265                 else
266                 {
267                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
268                 }
269         }
270
271
272         if ($aCMDResult['load-data'] || $aCMDResult['all'])
273         {
274                 echo "Load Data\n";
275                 $bDidSomething = true;
276
277                 $oDB =& getDB();
278                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
279                 echo '.';
280                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
281                 echo '.';
282                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
283                 echo '.';
284                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
285                 echo '.';
286                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
287                 echo '.';
288                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
289                 echo '.';
290                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
291                 echo '.';
292                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
293                 echo '.';
294                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
295                 echo '.';
296
297                 $sSQL = 'select partition from country_name order by country_code';
298                 $aPartitions = $oDB->getCol($sSQL);
299                 if (PEAR::isError($aPartitions))
300                 {
301                         fail($aPartitions->getMessage());
302                 }
303                 $aPartitions[] = 0;
304                 foreach($aPartitions as $sPartition)
305                 {
306                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
307                         echo '.';
308                 }
309
310                 // pre-create the word list
311                 if (!$aCMDResult['disable-token-precalc'])
312                 {
313                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
314                         echo '.';
315                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct postcode as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
316                         echo '.';
317                         if (!pg_query($oDB->connection, 'select count(getorcreate_housenumber_id(v)) from (select distinct housenumber as v from place where housenumber is not null) as w;')) fail(pg_last_error($oDB->connection));
318                         echo '.';
319                 }
320
321                 $aDBInstances = array();
322                 for($i = 0; $i < $iInstances; $i++)
323                 {
324                         $aDBInstances[$i] =& getDB(true);
325                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
326                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
327                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
328                         if ($aCMDResult['verbose']) echo "$sSQL\n";
329                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
330                 }
331                 $bAnyBusy = true;
332                 while($bAnyBusy)
333                 {
334                         $bAnyBusy = false;
335                         for($i = 0; $i < $iInstances; $i++)
336                         {
337                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
338                         }
339                         sleep(1);
340                         echo '.';
341                 }
342                 echo "\n";
343                 echo "Reanalysing database...\n";
344                 pgsqlRunScript('ANALYSE');
345         }
346
347         if ($aCMDResult['create-roads'])
348         {
349                 $bDidSomething = true;
350
351                 $oDB =& getDB();
352                 $aDBInstances = array();
353                 for($i = 0; $i < $iInstances; $i++)
354                 {
355                         $aDBInstances[$i] =& getDB(true);
356                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
357                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, country_code, geometry) from ';
358                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
359                         if ($aCMDResult['verbose']) echo "$sSQL\n";
360                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
361                 }
362                 $bAnyBusy = true;
363                 while($bAnyBusy)
364                 {
365                         $bAnyBusy = false;
366                         for($i = 0; $i < $iInstances; $i++)
367                         {
368                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
369                         }
370                         sleep(1);
371                         echo '.';
372                 }
373                 echo "\n";
374         }
375
376         if ($aCMDResult['import-tiger-data'])
377         {
378                 $bDidSomething = true;
379
380                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
381
382                 $aDBInstances = array();
383                 for($i = 0; $i < $iInstances; $i++)
384                 {
385                         $aDBInstances[$i] =& getDB(true);
386                 }
387
388                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
389                 {
390                         echo $sFile.': ';
391                         $hFile = fopen($sFile, "r");
392                         $sSQL = fgets($hFile, 100000);
393                         $iLines = 0;
394
395                         while(true)
396                         {
397                                 for($i = 0; $i < $iInstances; $i++)
398                                 {
399                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
400                                         {
401                                                 while(pg_get_result($aDBInstances[$i]->connection));
402                                                 $sSQL = fgets($hFile, 100000);
403                                                 if (!$sSQL) break 2;
404                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
405                                                 $iLines++;
406                                                 if ($iLines == 1000)
407                                                 {
408                                                         echo ".";
409                                                         $iLines = 0;
410                                                 }
411                                         }
412                                 }
413                                 usleep(10);
414                         }
415
416                         fclose($hFile);
417
418                         $bAnyBusy = true;
419                         while($bAnyBusy)
420                         {
421                                 $bAnyBusy = false;
422                                 for($i = 0; $i < $iInstances; $i++)
423                                 {
424                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
425                                 }
426                                 usleep(10);
427                         }
428                         echo "\n";
429                 }
430
431                 echo "Creating indexes\n";
432                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
433         }
434
435         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
436         {
437                 $bDidSomething = true;
438                 $oDB =& getDB();
439                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
440                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
441                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,country_code,";
442                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select country_code,postcode,";
443                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
444                 $sSQL .= "from placex where postcode is not null group by country_code,postcode) as x";
445                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
446
447                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
448                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
449                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
450                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
451         }
452
453         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
454         {
455                 $bDidSomething = true;
456                 $oDB =& getDB();
457
458                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
459                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
460                 else passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
461
462                 $sDate = $aCMDResult['osmosis-init-date'];
463                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
464                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
465                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
466                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
467                 $sURL .= '&stream=minute';
468                 echo "Getting state file: $sURL\n";
469                 $sStateFile = file_get_contents($sURL);
470                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
471                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
472                 echo "Updating DB status\n";
473                 pg_query($oDB->connection, 'TRUNCATE import_status');
474                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
475                 pg_query($oDB->connection, $sSQL);
476
477         }
478
479         if ($aCMDResult['index'] || $aCMDResult['all'])
480         {
481                 $bDidSomething = true;
482                 $sOutputFile = '';
483                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
484                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
485                 passthruCheckReturn($sBaseCmd.' -R 4');
486                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
487                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
488                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
489                 passthruCheckReturn($sBaseCmd.' -r 26');
490         }
491
492         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
493         {
494                 echo "Search indices\n";
495                 $bDidSomething = true;
496                 $oDB =& getDB();
497                 $sSQL = 'select partition from country_name order by country_code';
498                 $aPartitions = $oDB->getCol($sSQL);
499                 if (PEAR::isError($aPartitions))
500                 {
501                         fail($aPartitions->getMessage());
502                 }
503                 $aPartitions[] = 0;
504
505                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
506                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
507                 foreach($aMatches as $aMatch)
508                 {
509                         $sResult = '';
510                         foreach($aPartitions as $sPartitionName)
511                         {
512                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
513                         }
514                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
515                 }
516
517                 pgsqlRunScript($sTemplate);
518         }
519
520         if (isset($aCMDResult['create-website']))
521         {
522                 $bDidSomething = true;
523                 $sTargetDir = $aCMDResult['create-website'];
524                 if (!is_dir($sTargetDir))
525                 {
526                         echo "You must create the website directory before calling this function.\n";
527                         fail("Target directory does not exist.");
528                 }
529
530                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
531                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
532                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
533                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
534                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
535                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
536                 echo "Symlinks created\n";
537         }
538
539         if (!$bDidSomething)
540         {
541                 showUsage($aCMDOptions, true);
542         }
543
544         function pgsqlRunScriptFile($sFilename)
545         {
546                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
547
548                 // Convert database DSN to psql parameters
549                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
550                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
551                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
552
553                 $aDescriptors = array(
554                         0 => array('pipe', 'r'),
555                         1 => array('pipe', 'w'),
556                         2 => array('file', '/dev/null', 'a')
557                 );
558                 $ahPipes = null;
559                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
560                 if (!is_resource($hProcess)) fail('unable to start pgsql');
561
562                 fclose($ahPipes[0]);
563
564                 // TODO: error checking
565                 while(!feof($ahPipes[1]))
566                 {
567                         echo fread($ahPipes[1], 4096);
568                 }
569                 fclose($ahPipes[1]);
570
571                 proc_close($hProcess);
572         }
573
574         function pgsqlRunScript($sScript)
575         {
576                 // Convert database DSN to psql parameters
577                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
578                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
579                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
580                 $aDescriptors = array(
581                         0 => array('pipe', 'r'),
582                         1 => STDOUT, 
583                         2 => STDERR
584                 );
585                 $ahPipes = null;
586                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
587                 if (!is_resource($hProcess)) fail('unable to start pgsql');
588
589                 while(strlen($sScript))
590                 {
591                         $written = fwrite($ahPipes[0], $sScript);
592                         $sScript = substr($sScript, $written);
593                 }
594                 fclose($ahPipes[0]);
595                 proc_close($hProcess);
596         }
597
598         function pgsqlRunRestoreData($sDumpFile)
599         {
600                 // Convert database DSN to psql parameters
601                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
602                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
603                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
604
605                 $aDescriptors = array(
606                         0 => array('pipe', 'r'),
607                         1 => array('pipe', 'w'),
608                         2 => array('file', '/dev/null', 'a')
609                 );
610                 $ahPipes = null;
611                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
612                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
613
614                 fclose($ahPipes[0]);
615
616                 // TODO: error checking
617                 while(!feof($ahPipes[1]))
618                 {
619                         echo fread($ahPipes[1], 4096);
620                 }
621                 fclose($ahPipes[1]);
622
623                 proc_close($hProcess);
624         }
625
626         function pgsqlRunDropAndRestore($sDumpFile)
627         {
628                 // Convert database DSN to psql parameters
629                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
630                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
631                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
632
633                 $aDescriptors = array(
634                         0 => array('pipe', 'r'),
635                         1 => array('pipe', 'w'),
636                         2 => array('file', '/dev/null', 'a')
637                 );
638                 $ahPipes = null;
639                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
640                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
641
642                 fclose($ahPipes[0]);
643
644                 // TODO: error checking
645                 while(!feof($ahPipes[1]))
646                 {
647                         echo fread($ahPipes[1], 4096);
648                 }
649                 fclose($ahPipes[1]);
650
651                 proc_close($hProcess);
652         }
653
654         function passthruCheckReturn($cmd)
655         {
656                 $result = -1;
657                 passthru($cmd, $result);
658                 if ($result != 0) fail('Error executing external command: '.$cmd);
659         }