utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
  36                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  37                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse opertions during index (EXPERT)'),
  38                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  39                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  40                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  41         );
  42         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  43
  44         $bDidSomething = false;
  45
  46         // This is a pretty hard core defult - the number of processors in the box - 1
  47         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  48         if ($iInstances < 1)
  49         {
  50                 $iInstances = 1;
  51                 echo "WARNING: resetting threads to $iInstances\n";
  52         }
  53         if ($iInstances > getProcessorCount())
  54         {
  55                 $iInstances = getProcessorCount();
  56                 echo "WARNING: resetting threads to $iInstances\n";
  57         }
  58
  59         // Assume we can steal all the cache memory in the box (unless told otherwise)
  60         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  61         if ($iCacheMemory > getTotalMemoryMB())
  62         {
  63                 $iCacheMemory = getCacheMemoryMB();
  64                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  65         }
  66
  67         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
  68         {
  69                 $sBaseFile = basename($aCMDResult['osm-file']);
  70                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
  71                 {
  72                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
  73                         $iTime -= (60*60*24);
  74                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
  75                 }
  76         }
  77         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  78         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  79
  80         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  81         {
  82                 echo "Create DB\n";
  83                 $bDidSomething = true;
  84                 $oDB =& DB::connect(CONST_Database_DSN, false);
  85                 if (!PEAR::isError($oDB))
  86                 {
  87                         fail('database already exists ('.CONST_Database_DSN.')');
  88                 }
  89                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
  90         }
  91
  92         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  93         {
  94                 echo "Create DB (2)\n";
  95                 $bDidSomething = true;
  96                 // TODO: path detection, detection memory, etc.
  97
  98                 $oDB =& getDB();
  99                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 100                 $pgver = (float) CONST_Postgresql_Version;
 101                 if ($pgver < 9.1) {
 102                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 103                 } else {
 104                         pgsqlRunScript('CREATE EXTENSION hstore');
 105                 }
 106                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 107                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 108                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 109                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 110                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 111                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 112                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 113                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 114                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 115                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 116         }
 117
 118         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 119         {
 120                 echo "Import\n";
 121                 $bDidSomething = true;
 122
 123                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 124                 if (!file_exists($osm2pgsql)) fail("please download and build osm2pgsql");
 125                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 126                 $osm2pgsql .= ' -C '.$iCacheMemory;
 127                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 128                 passthru($osm2pgsql);
 129
 130                 $oDB =& getDB();
 131                 $x = $oDB->getRow('select * from place limit 1');
 132                 if (PEAR::isError($x)) {
 133                         fail($x->getMessage());
 134                 }
 135                 if (!$x) fail('No Data');
 136         }
 137
 138         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 139         {
 140                 echo "Functions\n";
 141                 $bDidSomething = true;
 142                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 143                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 144                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 145                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 146                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 147                 pgsqlRunScript($sTemplate);
 148         }
 149
 150         if ($aCMDResult['create-minimal-tables'])
 151         {
 152                 echo "Minimal Tables\n";
 153                 $bDidSomething = true;
 154                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 155
 156                 $sScript = '';
 157
 158                 // Backstop the import process - easliest possible import id
 159                 $sScript .= "insert into import_npi_log values (18022);\n";
 160
 161                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 162                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 163
 164                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 165                 {
 166                         list($sClass, $sType) = explode(' ', trim($sLine));
 167                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 168                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 169
 170                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 171                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 172
 173                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 174                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 175                 }
 176                 fclose($hFile);
 177                 pgsqlRunScript($sScript);
 178         }
 179
 180         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 181         {
 182                 echo "Tables\n";
 183                 $bDidSomething = true;
 184                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 185
 186                 // re-run the functions
 187                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 188                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 189                 pgsqlRunScript($sTemplate);
 190         }
 191
 192         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 193         {
 194                 echo "Partitions\n";
 195                 $bDidSomething = true;
 196                 $oDB =& getDB();
 197                 $sSQL = 'select partition from country_name order by country_code';
 198                 $aPartitions = $oDB->getCol($sSQL);
 199                 if (PEAR::isError($aPartitions))
 200                 {
 201                         fail($aPartitions->getMessage());
 202                 }
 203                 $aPartitions[] = 0;
 204
 205                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 206                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 207                 foreach($aMatches as $aMatch)
 208                 {
 209                         $sResult = '';
 210                         foreach($aPartitions as $sPartitionName)
 211                         {
 212                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 213                         }
 214                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 215                 }
 216
 217                 pgsqlRunScript($sTemplate);
 218         }
 219
 220         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 221         {
 222                 $bDidSomething = true;
 223                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 224                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 225                 if (file_exists($sWikiArticlesFile))
 226                 {
 227                         echo "Importing wikipedia articles...";
 228                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 229                         echo "...done\n";
 230                 }
 231                 else
 232                 {
 233                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 234                 }
 235                 if (file_exists($sWikiRedirectsFile))
 236                 {
 237                         echo "Importing wikipedia redirects...";
 238                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 239                         echo "...done\n";
 240                 }
 241                 else
 242                 {
 243                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 244                 }
 245         }
 246
 247
 248         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 249         {
 250                 echo "Load Data\n";
 251                 $bDidSomething = true;
 252
 253                 $oDB =& getDB();
 254                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 255                 echo '.';
 256                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 257                 echo '.';
 258                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 259                 echo '.';
 260                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 261                 echo '.';
 262                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 263                 echo '.';
 264                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 265                 echo '.';
 266                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 267                 echo '.';
 268                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 269                 echo '.';
 270                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 271                 echo '.';
 272
 273                 $sSQL = 'select partition from country_name order by country_code';
 274                 $aPartitions = $oDB->getCol($sSQL);
 275                 if (PEAR::isError($aPartitions))
 276                 {
 277                         fail($aPartitions->getMessage());
 278                 }
 279                 $aPartitions[] = 0;
 280                 foreach($aPartitions as $sPartition)
 281                 {
 282                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 283                         echo '.';
 284                 }
 285
 286                 // pre-create the word list
 287                 if (!$aCMDResult['disable-token-precalc'])
 288                 {
 289                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 290                         echo '.';
 291                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct postcode as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 292                         echo '.';
 293                         if (!pg_query($oDB->connection, 'select count(getorcreate_housenumber_id(v)) from (select distinct housenumber as v from place where housenumber is not null) as w;')) fail(pg_last_error($oDB->connection));
 294                         echo '.';
 295                 }
 296
 297                 $aDBInstances = array();
 298                 for($i = 0; $i < $iInstances; $i++)
 299                 {
 300                         $aDBInstances[$i] =& getDB(true);
 301                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 302                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 303                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 304                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 305                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 306                 }
 307                 $bAnyBusy = true;
 308                 while($bAnyBusy)
 309                 {
 310                         $bAnyBusy = false;
 311                         for($i = 0; $i < $iInstances; $i++)
 312                         {
 313                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 314                         }
 315                         sleep(1);
 316                         echo '.';
 317                 }
 318                 echo "\n";
 319                 echo "Reanalysing database...\n";
 320                 pgsqlRunScript('ANALYSE');
 321         }
 322
 323         if ($aCMDResult['create-roads'])
 324         {
 325                 $bDidSomething = true;
 326
 327                 $oDB =& getDB();
 328                 $aDBInstances = array();
 329                 for($i = 0; $i < $iInstances; $i++)
 330                 {
 331                         $aDBInstances[$i] =& getDB(true);
 332                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 333                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, country_code, geometry) from ';
 334                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 335                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 336                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 337                 }
 338                 $bAnyBusy = true;
 339                 while($bAnyBusy)
 340                 {
 341                         $bAnyBusy = false;
 342                         for($i = 0; $i < $iInstances; $i++)
 343                         {
 344                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 345                         }
 346                         sleep(1);
 347                         echo '.';
 348                 }
 349                 echo "\n";
 350         }
 351
 352         if ($aCMDResult['import-tiger-data'])
 353         {
 354                 $bDidSomething = true;
 355
 356                 $aDBInstances = array();
 357                 for($i = 0; $i < $iInstances; $i++)
 358                 {
 359                         $aDBInstances[$i] =& getDB(true);
 360                 }
 361
 362                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 363                 {
 364                         echo $sFile.': ';
 365                         $hFile = fopen($sFile, "r");
 366                         $sSQL = fgets($hFile, 100000);
 367                         $iLines = 0;
 368
 369                         while(true)
 370                         {
 371                                 for($i = 0; $i < $iInstances; $i++)
 372                                 {
 373                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 374                                         {
 375                                                 while(pg_get_result($aDBInstances[$i]->connection));
 376                                                 $sSQL = fgets($hFile, 100000);
 377                                                 if (!$sSQL) break 2;
 378                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 379                                                 $iLines++;
 380                                                 if ($iLines == 1000)
 381                                                 {
 382                                                         echo ".";
 383                                                         $iLines = 0;
 384                                                 }
 385                                         }
 386                                 }
 387                                 usleep(10);
 388                         }
 389
 390                         fclose($hFile);
 391
 392                         $bAnyBusy = true;
 393                         while($bAnyBusy)
 394                         {
 395                                 $bAnyBusy = false;
 396                                 for($i = 0; $i < $iInstances; $i++)
 397                                 {
 398                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 399                                 }
 400                                 usleep(10);
 401                         }
 402                         echo "\n";
 403                 }
 404         }
 405
 406         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 407         {
 408                 $bDidSomething = true;
 409                 $oDB =& getDB();
 410                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 411                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 412                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,country_code,";
 413                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select country_code,postcode,";
 414                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 415                 $sSQL .= "from placex where postcode is not null group by country_code,postcode) as x";
 416                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 417
 418                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 419                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 420                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 421                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 422         }
 423
 424         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
 425         {
 426                 $bDidSomething = true;
 427                 $oDB =& getDB();
 428
 429                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 430                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
 431                 else passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 432
 433                 $sDate = $aCMDResult['osmosis-init-date'];
 434                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
 435                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
 436                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
 437                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
 438                 $sURL .= '&stream=minute';
 439                 echo "Getting state file: $sURL\n";
 440                 $sStateFile = file_get_contents($sURL);
 441                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 442                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 443                 echo "Updating DB status\n";
 444                 pg_query($oDB->connection, 'TRUNCATE import_status');
 445                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
 446                 pg_query($oDB->connection, $sSQL);
 447
 448         }
 449
 450         if ($aCMDResult['index'] || $aCMDResult['all'])
 451         {
 452                 $bDidSomething = true;
 453                 $sOutputFile = '';
 454                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 455                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 456                 passthru($sBaseCmd.' -R 4');
 457                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 458                 passthru($sBaseCmd.' -r 5 -R 25');
 459                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 460                 passthru($sBaseCmd.' -r 26');
 461         }
 462
 463         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 464         {
 465                 echo "Search indices\n";
 466                 $bDidSomething = true;
 467                 $oDB =& getDB();
 468                 $sSQL = 'select partition from country_name order by country_code';
 469                 $aPartitions = $oDB->getCol($sSQL);
 470                 if (PEAR::isError($aPartitions))
 471                 {
 472                         fail($aPartitions->getMessage());
 473                 }
 474                 $aPartitions[] = 0;
 475
 476                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 477                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 478                 foreach($aMatches as $aMatch)
 479                 {
 480                         $sResult = '';
 481                         foreach($aPartitions as $sPartitionName)
 482                         {
 483                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 484                         }
 485                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 486                 }
 487
 488                 pgsqlRunScript($sTemplate);
 489         }
 490
 491         if (isset($aCMDResult['create-website']))
 492         {
 493                 $bDidSomething = true;
 494                 $sTargetDir = $aCMDResult['create-website'];
 495                 if (!is_dir($sTargetDir)) fail('please specify a directory to setup');
 496                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 497                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 498                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 499                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 500                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 501                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 502                 echo "Symlinks created\n";
 503         }
 504
 505         if (!$bDidSomething)
 506         {
 507                 showUsage($aCMDOptions, true);
 508         }
 509
 510         function pgsqlRunScriptFile($sFilename)
 511         {
 512                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 513
 514                 // Convert database DSN to psql paramaters
 515                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 516                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 517                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 518
 519                 $aDescriptors = array(
 520                         0 => array('pipe', 'r'),
 521                         1 => array('pipe', 'w'),
 522                         2 => array('file', '/dev/null', 'a')
 523                 );
 524                 $ahPipes = null;
 525                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 526                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 527
 528                 fclose($ahPipes[0]);
 529
 530                 // TODO: error checking
 531                 while(!feof($ahPipes[1]))
 532                 {
 533                         echo fread($ahPipes[1], 4096);
 534                 }
 535                 fclose($ahPipes[1]);
 536
 537                 proc_close($hProcess);
 538         }
 539
 540         function pgsqlRunScript($sScript)
 541         {
 542                 // Convert database DSN to psql paramaters
 543                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 544                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 545                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 546                 $aDescriptors = array(
 547                         0 => array('pipe', 'r'),
 548                         1 => STDOUT,
 549                         2 => STDERR
 550                 );
 551                 $ahPipes = null;
 552                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 553                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 554
 555                 while(strlen($sScript))
 556                 {
 557                         $written = fwrite($ahPipes[0], $sScript);
 558                         $sScript = substr($sScript, $written);
 559                 }
 560                 fclose($ahPipes[0]);
 561                 proc_close($hProcess);
 562         }
 563
 564         function pgsqlRunRestoreData($sDumpFile)
 565         {
 566                 // Convert database DSN to psql paramaters
 567                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 568                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 569                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 570
 571                 $aDescriptors = array(
 572                         0 => array('pipe', 'r'),
 573                         1 => array('pipe', 'w'),
 574                         2 => array('file', '/dev/null', 'a')
 575                 );
 576                 $ahPipes = null;
 577                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 578                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 579
 580                 fclose($ahPipes[0]);
 581
 582                 // TODO: error checking
 583                 while(!feof($ahPipes[1]))
 584                 {
 585                         echo fread($ahPipes[1], 4096);
 586                 }
 587                 fclose($ahPipes[1]);
 588
 589                 proc_close($hProcess);
 590         }
 591
 592         function pgsqlRunDropAndRestore($sDumpFile)
 593         {
 594                 // Convert database DSN to psql paramaters
 595                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 596                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 597                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 598
 599                 $aDescriptors = array(
 600                         0 => array('pipe', 'r'),
 601                         1 => array('pipe', 'w'),
 602                         2 => array('file', '/dev/null', 'a')
 603                 );
 604                 $ahPipes = null;
 605                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 606                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 607
 608                 fclose($ahPipes[0]);
 609
 610                 // TODO: error checking
 611                 while(!feof($ahPipes[1]))
 612                 {
 613                         echo fread($ahPipes[1], 4096);
 614                 }
 615                 fclose($ahPipes[1]);
 616
 617                 proc_close($hProcess);
 618         }