utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
  36                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  37                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  38                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  39                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  40                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  41         );
  42         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  43
  44         $bDidSomething = false;
  45
  46         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  47         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  48         {
  49                 if (!isset($aCMDResult['osm-file']))
  50                 {
  51                         fail('missing --osm-file for data import');
  52                 }
  53
  54                 if (!file_exists($aCMDResult['osm-file']))
  55                 {
  56                         fail('the path supplied to --osm-file does not exist');
  57                 }
  58
  59                 if (!is_readable($aCMDResult['osm-file']))
  60                 {
  61                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  62                 }
  63         }
  64
  65
  66         // This is a pretty hard core default - the number of processors in the box - 1
  67         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  68         if ($iInstances < 1)
  69         {
  70                 $iInstances = 1;
  71                 echo "WARNING: resetting threads to $iInstances\n";
  72         }
  73         if ($iInstances > getProcessorCount())
  74         {
  75                 $iInstances = getProcessorCount();
  76                 echo "WARNING: resetting threads to $iInstances\n";
  77         }
  78
  79         // Assume we can steal all the cache memory in the box (unless told otherwise)
  80         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  81         if ($iCacheMemory > getTotalMemoryMB())
  82         {
  83                 $iCacheMemory = getCacheMemoryMB();
  84                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  85         }
  86
  87         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
  88         {
  89                 $sBaseFile = basename($aCMDResult['osm-file']);
  90                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
  91                 {
  92                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
  93                         $iTime -= (60*60*24);
  94                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
  95                 }
  96         }
  97         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  98         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  99
 100         if ($aCMDResult['create-db'] || $aCMDResult['all'])
 101         {
 102                 echo "Create DB\n";
 103                 $bDidSomething = true;
 104                 $oDB =& DB::connect(CONST_Database_DSN, false);
 105                 if (!PEAR::isError($oDB))
 106                 {
 107                         fail('database already exists ('.CONST_Database_DSN.')');
 108                 }
 109                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
 110         }
 111
 112         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 113         {
 114                 echo "Setup DB\n";
 115                 $bDidSomething = true;
 116                 // TODO: path detection, detection memory, etc.
 117
 118                 $oDB =& getDB();
 119                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 120                 $pgver = (float) CONST_Postgresql_Version;
 121                 if ($pgver < 9.1) {
 122                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 123                 } else {
 124                         pgsqlRunScript('CREATE EXTENSION hstore');
 125                 }
 126                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 127                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 128                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 129                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 130                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 131                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 132                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 133                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 134                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 135                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 136         }
 137
 138         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 139         {
 140                 echo "Import\n";
 141                 $bDidSomething = true;
 142
 143                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 144                 if (!file_exists($osm2pgsql))
 145                 {
 146                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 147                         fail("osm2pgsql not found in '$osm2pgsql'");
 148                 }
 149                 $osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data';
 150                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 151                 $osm2pgsql .= ' -C 16000';
 152                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 153                 passthruCheckReturn($osm2pgsql);
 154
 155                 $oDB =& getDB();
 156                 $x = $oDB->getRow('select * from place limit 1');
 157                 if (PEAR::isError($x)) {
 158                         fail($x->getMessage());
 159                 }
 160                 if (!$x) fail('No Data');
 161         }
 162
 163         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 164         {
 165                 echo "Functions\n";
 166                 $bDidSomething = true;
 167                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 168                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 169                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 170                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 171                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 172                 pgsqlRunScript($sTemplate);
 173         }
 174
 175         if ($aCMDResult['create-minimal-tables'])
 176         {
 177                 echo "Minimal Tables\n";
 178                 $bDidSomething = true;
 179                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 180
 181                 $sScript = '';
 182
 183                 // Backstop the import process - easliest possible import id
 184                 $sScript .= "insert into import_npi_log values (18022);\n";
 185
 186                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 187                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 188
 189                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 190                 {
 191                         list($sClass, $sType) = explode(' ', trim($sLine));
 192                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 193                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 194
 195                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 196                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 197
 198                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 199                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 200                 }
 201                 fclose($hFile);
 202                 pgsqlRunScript($sScript);
 203         }
 204
 205         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 206         {
 207                 echo "Tables\n";
 208                 $bDidSomething = true;
 209                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 210
 211                 // re-run the functions
 212                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 213                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 214                 pgsqlRunScript($sTemplate);
 215         }
 216
 217         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 218         {
 219                 echo "Partitions\n";
 220                 $bDidSomething = true;
 221                 $oDB =& getDB();
 222                 $sSQL = 'select partition from country_name order by country_code';
 223                 $aPartitions = $oDB->getCol($sSQL);
 224                 if (PEAR::isError($aPartitions))
 225                 {
 226                         fail($aPartitions->getMessage());
 227                 }
 228                 $aPartitions[] = 0;
 229
 230                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 231                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 232                 foreach($aMatches as $aMatch)
 233                 {
 234                         $sResult = '';
 235                         foreach($aPartitions as $sPartitionName)
 236                         {
 237                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 238                         }
 239                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 240                 }
 241
 242                 pgsqlRunScript($sTemplate);
 243         }
 244
 245         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 246         {
 247                 $bDidSomething = true;
 248                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 249                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 250                 if (file_exists($sWikiArticlesFile))
 251                 {
 252                         echo "Importing wikipedia articles...";
 253                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 254                         echo "...done\n";
 255                 }
 256                 else
 257                 {
 258                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 259                 }
 260                 if (file_exists($sWikiRedirectsFile))
 261                 {
 262                         echo "Importing wikipedia redirects...";
 263                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 264                         echo "...done\n";
 265                 }
 266                 else
 267                 {
 268                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 269                 }
 270         }
 271
 272
 273         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 274         {
 275                 echo "Load Data\n";
 276                 $bDidSomething = true;
 277
 278                 $oDB =& getDB();
 279                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 280                 echo '.';
 281                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 282                 echo '.';
 283                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 284                 echo '.';
 285                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 286                 echo '.';
 287                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 288                 echo '.';
 289                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 290                 echo '.';
 291                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 292                 echo '.';
 293                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 294                 echo '.';
 295                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 296                 echo '.';
 297
 298                 $sSQL = 'select partition from country_name order by country_code';
 299                 $aPartitions = $oDB->getCol($sSQL);
 300                 if (PEAR::isError($aPartitions))
 301                 {
 302                         fail($aPartitions->getMessage());
 303                 }
 304                 $aPartitions[] = 0;
 305                 foreach($aPartitions as $sPartition)
 306                 {
 307                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 308                         echo '.';
 309                 }
 310
 311                 // pre-create the word list
 312                 if (!$aCMDResult['disable-token-precalc'])
 313                 {
 314                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 315                         echo '.';
 316                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct postcode as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 317                         echo '.';
 318                         if (!pg_query($oDB->connection, 'select count(getorcreate_housenumber_id(v)) from (select distinct housenumber as v from place where housenumber is not null) as w;')) fail(pg_last_error($oDB->connection));
 319                         echo '.';
 320                 }
 321
 322                 $aDBInstances = array();
 323                 for($i = 0; $i < $iInstances; $i++)
 324                 {
 325                         $aDBInstances[$i] =& getDB(true);
 326                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 327                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 328                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 329                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 330                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 331                 }
 332                 $bAnyBusy = true;
 333                 while($bAnyBusy)
 334                 {
 335                         $bAnyBusy = false;
 336                         for($i = 0; $i < $iInstances; $i++)
 337                         {
 338                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 339                         }
 340                         sleep(1);
 341                         echo '.';
 342                 }
 343                 echo "\n";
 344                 pgsqlRunScript('ALTER TABLE place SET TABLESPACE "data"');
 345                 echo "Reanalysing database...\n";
 346                 pgsqlRunScript('ANALYSE');
 347         }
 348
 349         if ($aCMDResult['create-roads'])
 350         {
 351                 $bDidSomething = true;
 352
 353                 $oDB =& getDB();
 354                 $aDBInstances = array();
 355                 for($i = 0; $i < $iInstances; $i++)
 356                 {
 357                         $aDBInstances[$i] =& getDB(true);
 358                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 359                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, country_code, geometry) from ';
 360                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 361                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 362                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 363                 }
 364                 $bAnyBusy = true;
 365                 while($bAnyBusy)
 366                 {
 367                         $bAnyBusy = false;
 368                         for($i = 0; $i < $iInstances; $i++)
 369                         {
 370                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 371                         }
 372                         sleep(1);
 373                         echo '.';
 374                 }
 375                 echo "\n";
 376         }
 377
 378         if ($aCMDResult['import-tiger-data'])
 379         {
 380                 $bDidSomething = true;
 381
 382                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 383
 384                 $aDBInstances = array();
 385                 for($i = 0; $i < $iInstances; $i++)
 386                 {
 387                         $aDBInstances[$i] =& getDB(true);
 388                 }
 389
 390                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 391                 {
 392                         echo $sFile.': ';
 393                         $hFile = fopen($sFile, "r");
 394                         $sSQL = fgets($hFile, 100000);
 395                         $iLines = 0;
 396
 397                         while(true)
 398                         {
 399                                 for($i = 0; $i < $iInstances; $i++)
 400                                 {
 401                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 402                                         {
 403                                                 while(pg_get_result($aDBInstances[$i]->connection));
 404                                                 $sSQL = fgets($hFile, 100000);
 405                                                 if (!$sSQL) break 2;
 406                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 407                                                 $iLines++;
 408                                                 if ($iLines == 1000)
 409                                                 {
 410                                                         echo ".";
 411                                                         $iLines = 0;
 412                                                 }
 413                                         }
 414                                 }
 415                                 usleep(10);
 416                         }
 417
 418                         fclose($hFile);
 419
 420                         $bAnyBusy = true;
 421                         while($bAnyBusy)
 422                         {
 423                                 $bAnyBusy = false;
 424                                 for($i = 0; $i < $iInstances; $i++)
 425                                 {
 426                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 427                                 }
 428                                 usleep(10);
 429                         }
 430                         echo "\n";
 431                 }
 432
 433                 echo "Creating indexes\n";
 434                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 435         }
 436
 437         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 438         {
 439                 $bDidSomething = true;
 440                 $oDB =& getDB();
 441                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 442                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 443                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,country_code,";
 444                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select country_code,postcode,";
 445                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 446                 $sSQL .= "from placex where postcode is not null group by country_code,postcode) as x";
 447                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 448
 449                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 450                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 451                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 452                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 453         }
 454
 455         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
 456         {
 457                 $bDidSomething = true;
 458                 $oDB =& getDB();
 459
 460                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 461                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
 462                 else passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 463
 464                 $sDate = $aCMDResult['osmosis-init-date'];
 465                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
 466                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
 467                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
 468                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
 469                 $sURL .= '&stream=minute';
 470                 echo "Getting state file: $sURL\n";
 471                 $sStateFile = file_get_contents($sURL);
 472                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 473                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 474                 echo "Updating DB status\n";
 475                 pg_query($oDB->connection, 'TRUNCATE import_status');
 476                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
 477                 pg_query($oDB->connection, $sSQL);
 478
 479         }
 480
 481         if ($aCMDResult['index'] || $aCMDResult['all'])
 482         {
 483                 $bDidSomething = true;
 484                 $sOutputFile = '';
 485                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 486                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 487                 passthruCheckReturn($sBaseCmd.' -R 4');
 488                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 489                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 490                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 491                 passthruCheckReturn($sBaseCmd.' -r 26');
 492         }
 493
 494         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 495         {
 496                 echo "Search indices\n";
 497                 $bDidSomething = true;
 498                 $oDB =& getDB();
 499                 $sSQL = 'select partition from country_name order by country_code';
 500                 $aPartitions = $oDB->getCol($sSQL);
 501                 if (PEAR::isError($aPartitions))
 502                 {
 503                         fail($aPartitions->getMessage());
 504                 }
 505                 $aPartitions[] = 0;
 506
 507                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 508                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 509                 foreach($aMatches as $aMatch)
 510                 {
 511                         $sResult = '';
 512                         foreach($aPartitions as $sPartitionName)
 513                         {
 514                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 515                         }
 516                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 517                 }
 518
 519                 pgsqlRunScript($sTemplate);
 520         }
 521
 522         if (isset($aCMDResult['create-website']))
 523         {
 524                 $bDidSomething = true;
 525                 $sTargetDir = $aCMDResult['create-website'];
 526                 if (!is_dir($sTargetDir))
 527                 {
 528                         echo "You must create the website directory before calling this function.\n";
 529                         fail("Target directory does not exist.");
 530                 }
 531
 532                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 533                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 534                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 535                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 536                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 537                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 538                 echo "Symlinks created\n";
 539         }
 540
 541         if (!$bDidSomething)
 542         {
 543                 showUsage($aCMDOptions, true);
 544         }
 545
 546         function pgsqlRunScriptFile($sFilename)
 547         {
 548                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 549
 550                 // Convert database DSN to psql parameters
 551                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 552                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 553                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 554
 555                 $aDescriptors = array(
 556                         0 => array('pipe', 'r'),
 557                         1 => array('pipe', 'w'),
 558                         2 => array('file', '/dev/null', 'a')
 559                 );
 560                 $ahPipes = null;
 561                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 562                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 563
 564                 fclose($ahPipes[0]);
 565
 566                 // TODO: error checking
 567                 while(!feof($ahPipes[1]))
 568                 {
 569                         echo fread($ahPipes[1], 4096);
 570                 }
 571                 fclose($ahPipes[1]);
 572
 573                 proc_close($hProcess);
 574         }
 575
 576         function pgsqlRunScript($sScript)
 577         {
 578                 // Convert database DSN to psql parameters
 579                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 580                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 581                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 582                 $aDescriptors = array(
 583                         0 => array('pipe', 'r'),
 584                         1 => STDOUT,
 585                         2 => STDERR
 586                 );
 587                 $ahPipes = null;
 588                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 589                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 590
 591                 while(strlen($sScript))
 592                 {
 593                         $written = fwrite($ahPipes[0], $sScript);
 594                         $sScript = substr($sScript, $written);
 595                 }
 596                 fclose($ahPipes[0]);
 597                 proc_close($hProcess);
 598         }
 599
 600         function pgsqlRunRestoreData($sDumpFile)
 601         {
 602                 // Convert database DSN to psql parameters
 603                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 604                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 605                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 606
 607                 $aDescriptors = array(
 608                         0 => array('pipe', 'r'),
 609                         1 => array('pipe', 'w'),
 610                         2 => array('file', '/dev/null', 'a')
 611                 );
 612                 $ahPipes = null;
 613                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 614                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 615
 616                 fclose($ahPipes[0]);
 617
 618                 // TODO: error checking
 619                 while(!feof($ahPipes[1]))
 620                 {
 621                         echo fread($ahPipes[1], 4096);
 622                 }
 623                 fclose($ahPipes[1]);
 624
 625                 proc_close($hProcess);
 626         }
 627
 628         function pgsqlRunDropAndRestore($sDumpFile)
 629         {
 630                 // Convert database DSN to psql parameters
 631                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 632                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 633                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 634
 635                 $aDescriptors = array(
 636                         0 => array('pipe', 'r'),
 637                         1 => array('pipe', 'w'),
 638                         2 => array('file', '/dev/null', 'a')
 639                 );
 640                 $ahPipes = null;
 641                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 642                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 643
 644                 fclose($ahPipes[0]);
 645
 646                 // TODO: error checking
 647                 while(!feof($ahPipes[1]))
 648                 {
 649                         echo fread($ahPipes[1], 4096);
 650                 }
 651                 fclose($ahPipes[1]);
 652
 653                 proc_close($hProcess);
 654         }
 655
 656         function passthruCheckReturn($cmd)
 657         {
 658                 $result = -1;
 659                 passthru($cmd, $result);
 660                 if ($result != 0) fail('Error executing external command: '.$cmd);
 661         }