]> git.openstreetmap.org Git - nominatim.git/blob - utils/setup.php
leave osm2pgsql tables on ssd
[nominatim.git] / utils / setup.php
1 #!/usr/bin/php -Cq
2 <?php
3
4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
5         ini_set('memory_limit', '800M');
6
7         $aCMDOptions = array(
8                 "Create and setup nominatim search system",
9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
12
13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
15
16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
17
18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
35                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
36                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
37                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
38                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
39                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
40                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
41         );
42         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
43
44         $bDidSomething = false;
45
46         // Check if osm-file is set and points to a valid file if --all or --import-data is given
47         if ($aCMDResult['import-data'] || $aCMDResult['all'])
48         {
49                 if (!isset($aCMDResult['osm-file']))
50                 {
51                         fail('missing --osm-file for data import');
52                 }
53
54                 if (!file_exists($aCMDResult['osm-file']))
55                 {
56                         fail('the path supplied to --osm-file does not exist');
57                 }
58
59                 if (!is_readable($aCMDResult['osm-file']))
60                 {
61                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
62                 }
63         }
64
65
66         // This is a pretty hard core default - the number of processors in the box - 1
67         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
68         if ($iInstances < 1)
69         {
70                 $iInstances = 1;
71                 echo "WARNING: resetting threads to $iInstances\n";
72         }
73         if ($iInstances > getProcessorCount())
74         {
75                 $iInstances = getProcessorCount();
76                 echo "WARNING: resetting threads to $iInstances\n";
77         }
78
79         // Assume we can steal all the cache memory in the box (unless told otherwise)
80         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
81         if ($iCacheMemory > getTotalMemoryMB())
82         {
83                 $iCacheMemory = getCacheMemoryMB();
84                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
85         }
86
87         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
88         {
89                 $sBaseFile = basename($aCMDResult['osm-file']);
90                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
91                 {
92                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
93                         $iTime -= (60*60*24);
94                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
95                 }
96         }
97         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
98         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
99
100         if ($aCMDResult['create-db'] || $aCMDResult['all'])
101         {
102                 echo "Create DB\n";
103                 $bDidSomething = true;
104                 $oDB =& DB::connect(CONST_Database_DSN, false);
105                 if (!PEAR::isError($oDB))
106                 {
107                         fail('database already exists ('.CONST_Database_DSN.')');
108                 }
109                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
110         }
111
112         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
113         {
114                 echo "Setup DB\n";
115                 $bDidSomething = true;
116                 // TODO: path detection, detection memory, etc.
117
118                 $oDB =& getDB();
119                 passthru('createlang plpgsql '.$aDSNInfo['database']);
120                 $pgver = (float) CONST_Postgresql_Version;
121                 if ($pgver < 9.1) {
122                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
123                 } else {
124                         pgsqlRunScript('CREATE EXTENSION hstore');
125                 }
126                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
127                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
128                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
129                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
130                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
131                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
132                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
133                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
134                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
135                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
136         }
137
138         if ($aCMDResult['import-data'] || $aCMDResult['all'])
139         {
140                 echo "Import\n";
141                 $bDidSomething = true;
142
143                 $osm2pgsql = CONST_Osm2pgsql_Binary;
144                 if (!file_exists($osm2pgsql))
145                 {
146                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
147                         fail("osm2pgsql not found in '$osm2pgsql'");
148                 }
149                 $osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data';
150                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
151                 $osm2pgsql .= ' -C 16000';
152                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
153                 passthruCheckReturn($osm2pgsql);
154
155                 $oDB =& getDB();
156                 $x = $oDB->getRow('select * from place limit 1');
157                 if (PEAR::isError($x)) {
158                         fail($x->getMessage());
159                 }
160                 if (!$x) fail('No Data');
161         }
162
163         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
164         {
165                 echo "Functions\n";
166                 $bDidSomething = true;
167                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
168                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
169                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
170                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
171                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
172                 pgsqlRunScript($sTemplate);
173         }
174
175         if ($aCMDResult['create-minimal-tables'])
176         {
177                 echo "Minimal Tables\n";
178                 $bDidSomething = true;
179                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
180
181                 $sScript = '';
182
183                 // Backstop the import process - easliest possible import id
184                 $sScript .= "insert into import_npi_log values (18022);\n";
185
186                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
187                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
188
189                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
190                 {
191                         list($sClass, $sType) = explode(' ', trim($sLine));
192                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
193                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
194
195                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
196                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
197
198                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
199                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
200                 }
201                 fclose($hFile);
202                 pgsqlRunScript($sScript);
203         }
204
205         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
206         {
207                 echo "Tables\n";
208                 $bDidSomething = true;
209                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
210
211                 // re-run the functions
212                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
213                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
214                 pgsqlRunScript($sTemplate);
215         }
216
217         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
218         {
219                 echo "Partitions\n";
220                 $bDidSomething = true;
221                 $oDB =& getDB();
222                 $sSQL = 'select partition from country_name order by country_code';
223                 $aPartitions = $oDB->getCol($sSQL);
224                 if (PEAR::isError($aPartitions))
225                 {
226                         fail($aPartitions->getMessage());
227                 }
228                 $aPartitions[] = 0;
229
230                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
231                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
232                 foreach($aMatches as $aMatch)
233                 {
234                         $sResult = '';
235                         foreach($aPartitions as $sPartitionName)
236                         {
237                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
238                         }
239                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
240                 }
241
242                 pgsqlRunScript($sTemplate);
243         }
244
245         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
246         {
247                 $bDidSomething = true;
248                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
249                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
250                 if (file_exists($sWikiArticlesFile))
251                 {
252                         echo "Importing wikipedia articles...";
253                         pgsqlRunDropAndRestore($sWikiArticlesFile);
254                         echo "...done\n";
255                 }
256                 else
257                 {
258                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
259                 }
260                 if (file_exists($sWikiRedirectsFile))
261                 {
262                         echo "Importing wikipedia redirects...";
263                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
264                         echo "...done\n";
265                 }
266                 else
267                 {
268                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
269                 }
270         }
271
272
273         if ($aCMDResult['load-data'] || $aCMDResult['all'])
274         {
275                 echo "Drop old Data\n";
276                 $bDidSomething = true;
277
278                 $oDB =& getDB();
279                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
280                 echo '.';
281                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
282                 echo '.';
283                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
284                 echo '.';
285                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
286                 echo '.';
287                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
288                 echo '.';
289                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
290                 echo '.';
291                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
292                 echo '.';
293                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
294                 echo '.';
295                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
296                 echo '.';
297
298                 $sSQL = 'select partition from country_name order by country_code';
299                 $aPartitions = $oDB->getCol($sSQL);
300                 if (PEAR::isError($aPartitions))
301                 {
302                         fail($aPartitions->getMessage());
303                 }
304                 $aPartitions[] = 0;
305                 foreach($aPartitions as $sPartition)
306                 {
307                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
308                         echo '.';
309                 }
310
311                 // used by getorcreate_word_id to ignore frequent partial words
312                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
313                 echo ".\n";
314
315                 // pre-create the word list
316                 if (!$aCMDResult['disable-token-precalc'])
317                 {
318                         echo "Loading word list\n";
319                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
320                 }
321
322                 echo "Load Data\n";
323                 $aDBInstances = array();
324                 for($i = 0; $i < $iInstances; $i++)
325                 {
326                         $aDBInstances[$i] =& getDB(true);
327                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
328                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
329                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
330                         if ($aCMDResult['verbose']) echo "$sSQL\n";
331                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
332                 }
333                 $bAnyBusy = true;
334                 while($bAnyBusy)
335                 {
336                         $bAnyBusy = false;
337                         for($i = 0; $i < $iInstances; $i++)
338                         {
339                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
340                         }
341                         sleep(1);
342                         echo '.';
343                 }
344                 echo "\n";
345                 echo "Reanalysing database...\n";
346                 pgsqlRunScript('ANALYSE');
347         }
348
349         if ($aCMDResult['create-roads'])
350         {
351                 $bDidSomething = true;
352
353                 $oDB =& getDB();
354                 $aDBInstances = array();
355                 for($i = 0; $i < $iInstances; $i++)
356                 {
357                         $aDBInstances[$i] =& getDB(true);
358                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
359                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
360                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
361                         if ($aCMDResult['verbose']) echo "$sSQL\n";
362                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
363                 }
364                 $bAnyBusy = true;
365                 while($bAnyBusy)
366                 {
367                         $bAnyBusy = false;
368                         for($i = 0; $i < $iInstances; $i++)
369                         {
370                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
371                         }
372                         sleep(1);
373                         echo '.';
374                 }
375                 echo "\n";
376         }
377
378         if ($aCMDResult['import-tiger-data'])
379         {
380                 $bDidSomething = true;
381
382                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
383
384                 $aDBInstances = array();
385                 for($i = 0; $i < $iInstances; $i++)
386                 {
387                         $aDBInstances[$i] =& getDB(true);
388                 }
389
390                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
391                 {
392                         echo $sFile.': ';
393                         $hFile = fopen($sFile, "r");
394                         $sSQL = fgets($hFile, 100000);
395                         $iLines = 0;
396
397                         while(true)
398                         {
399                                 for($i = 0; $i < $iInstances; $i++)
400                                 {
401                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
402                                         {
403                                                 while(pg_get_result($aDBInstances[$i]->connection));
404                                                 $sSQL = fgets($hFile, 100000);
405                                                 if (!$sSQL) break 2;
406                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
407                                                 $iLines++;
408                                                 if ($iLines == 1000)
409                                                 {
410                                                         echo ".";
411                                                         $iLines = 0;
412                                                 }
413                                         }
414                                 }
415                                 usleep(10);
416                         }
417
418                         fclose($hFile);
419
420                         $bAnyBusy = true;
421                         while($bAnyBusy)
422                         {
423                                 $bAnyBusy = false;
424                                 for($i = 0; $i < $iInstances; $i++)
425                                 {
426                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
427                                 }
428                                 usleep(10);
429                         }
430                         echo "\n";
431                 }
432
433                 echo "Creating indexes\n";
434                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
435         }
436
437         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
438         {
439                 $bDidSomething = true;
440                 $oDB =& getDB();
441                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
442                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
443                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
444                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
445                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
446                 $sSQL .= "from placex where postcode is not null and calculated_country_code not in ('ie') group by calculated_country_code,postcode) as x";
447                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
448
449                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
450                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
451                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
452                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
453         }
454
455         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
456         {
457                 $bDidSomething = true;
458                 $oDB =& getDB();
459
460                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
461                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
462                 else
463                 {
464                         passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
465                         // server layout changed afer license change, fix path to minutely diffs
466                         passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
467                 }
468
469                 $sDate = $aCMDResult['osmosis-init-date'];
470                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
471                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
472                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
473                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
474                 $sURL .= '&stream=minute';
475                 echo "Getting state file: $sURL\n";
476                 $sStateFile = file_get_contents($sURL);
477                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
478                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
479                 echo "Updating DB status\n";
480                 pg_query($oDB->connection, 'TRUNCATE import_status');
481                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
482                 pg_query($oDB->connection, $sSQL);
483
484         }
485
486         if ($aCMDResult['index'] || $aCMDResult['all'])
487         {
488                 $bDidSomething = true;
489                 $sOutputFile = '';
490                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
491                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
492                 passthruCheckReturn($sBaseCmd.' -R 4');
493                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
494                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
495                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
496                 passthruCheckReturn($sBaseCmd.' -r 26');
497         }
498
499         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
500         {
501                 echo "Search indices\n";
502                 $bDidSomething = true;
503                 $oDB =& getDB();
504                 $sSQL = 'select partition from country_name order by country_code';
505                 $aPartitions = $oDB->getCol($sSQL);
506                 if (PEAR::isError($aPartitions))
507                 {
508                         fail($aPartitions->getMessage());
509                 }
510                 $aPartitions[] = 0;
511
512                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
513                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
514                 foreach($aMatches as $aMatch)
515                 {
516                         $sResult = '';
517                         foreach($aPartitions as $sPartitionName)
518                         {
519                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
520                         }
521                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
522                 }
523
524                 pgsqlRunScript($sTemplate);
525         }
526
527         if (isset($aCMDResult['create-website']))
528         {
529                 $bDidSomething = true;
530                 $sTargetDir = $aCMDResult['create-website'];
531                 if (!is_dir($sTargetDir))
532                 {
533                         echo "You must create the website directory before calling this function.\n";
534                         fail("Target directory does not exist.");
535                 }
536
537                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
538                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
539                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
540                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
541                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
542                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
543                 echo "Symlinks created\n";
544         }
545
546         if (!$bDidSomething)
547         {
548                 showUsage($aCMDOptions, true);
549         }
550
551         function pgsqlRunScriptFile($sFilename)
552         {
553                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
554
555                 // Convert database DSN to psql parameters
556                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
557                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
558                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
559
560                 $aDescriptors = array(
561                         0 => array('pipe', 'r'),
562                         1 => array('pipe', 'w'),
563                         2 => array('file', '/dev/null', 'a')
564                 );
565                 $ahPipes = null;
566                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
567                 if (!is_resource($hProcess)) fail('unable to start pgsql');
568
569                 fclose($ahPipes[0]);
570
571                 // TODO: error checking
572                 while(!feof($ahPipes[1]))
573                 {
574                         echo fread($ahPipes[1], 4096);
575                 }
576                 fclose($ahPipes[1]);
577
578                 proc_close($hProcess);
579         }
580
581         function pgsqlRunScript($sScript)
582         {
583                 // Convert database DSN to psql parameters
584                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
585                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
586                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
587                 $aDescriptors = array(
588                         0 => array('pipe', 'r'),
589                         1 => STDOUT, 
590                         2 => STDERR
591                 );
592                 $ahPipes = null;
593                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
594                 if (!is_resource($hProcess)) fail('unable to start pgsql');
595
596                 while(strlen($sScript))
597                 {
598                         $written = fwrite($ahPipes[0], $sScript);
599                         $sScript = substr($sScript, $written);
600                 }
601                 fclose($ahPipes[0]);
602                 proc_close($hProcess);
603         }
604
605         function pgsqlRunRestoreData($sDumpFile)
606         {
607                 // Convert database DSN to psql parameters
608                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
609                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
610                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
611
612                 $aDescriptors = array(
613                         0 => array('pipe', 'r'),
614                         1 => array('pipe', 'w'),
615                         2 => array('file', '/dev/null', 'a')
616                 );
617                 $ahPipes = null;
618                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
619                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
620
621                 fclose($ahPipes[0]);
622
623                 // TODO: error checking
624                 while(!feof($ahPipes[1]))
625                 {
626                         echo fread($ahPipes[1], 4096);
627                 }
628                 fclose($ahPipes[1]);
629
630                 proc_close($hProcess);
631         }
632
633         function pgsqlRunDropAndRestore($sDumpFile)
634         {
635                 // Convert database DSN to psql parameters
636                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
637                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
638                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
639
640                 $aDescriptors = array(
641                         0 => array('pipe', 'r'),
642                         1 => array('pipe', 'w'),
643                         2 => array('file', '/dev/null', 'a')
644                 );
645                 $ahPipes = null;
646                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
647                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
648
649                 fclose($ahPipes[0]);
650
651                 // TODO: error checking
652                 while(!feof($ahPipes[1]))
653                 {
654                         echo fread($ahPipes[1], 4096);
655                 }
656                 fclose($ahPipes[1]);
657
658                 proc_close($hProcess);
659         }
660
661         function passthruCheckReturn($cmd)
662         {
663                 $result = -1;
664                 passthru($cmd, $result);
665                 if ($result != 0) fail('Error executing external command: '.$cmd);
666         }