utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
  36                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  37                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  38                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  39                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  40                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  41         );
  42         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  43
  44         $bDidSomething = false;
  45
  46         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  47         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  48         {
  49                 if (!isset($aCMDResult['osm-file']))
  50                 {
  51                         fail('missing --osm-file for data import');
  52                 }
  53
  54                 if (!file_exists($aCMDResult['osm-file']))
  55                 {
  56                         fail('the path supplied to --osm-file does not exist');
  57                 }
  58
  59                 if (!is_readable($aCMDResult['osm-file']))
  60                 {
  61                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  62                 }
  63         }
  64
  65
  66         // This is a pretty hard core default - the number of processors in the box - 1
  67         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  68         if ($iInstances < 1)
  69         {
  70                 $iInstances = 1;
  71                 echo "WARNING: resetting threads to $iInstances\n";
  72         }
  73         if ($iInstances > getProcessorCount())
  74         {
  75                 $iInstances = getProcessorCount();
  76                 echo "WARNING: resetting threads to $iInstances\n";
  77         }
  78
  79         // Assume we can steal all the cache memory in the box (unless told otherwise)
  80         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  81         if ($iCacheMemory > getTotalMemoryMB())
  82         {
  83                 $iCacheMemory = getCacheMemoryMB();
  84                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  85         }
  86
  87         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
  88         {
  89                 $sBaseFile = basename($aCMDResult['osm-file']);
  90                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
  91                 {
  92                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
  93                         $iTime -= (60*60*24);
  94                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
  95                 }
  96         }
  97         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  98         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  99
 100         if ($aCMDResult['create-db'] || $aCMDResult['all'])
 101         {
 102                 echo "Create DB\n";
 103                 $bDidSomething = true;
 104                 $oDB =& DB::connect(CONST_Database_DSN, false);
 105                 if (!PEAR::isError($oDB))
 106                 {
 107                         fail('database already exists ('.CONST_Database_DSN.')');
 108                 }
 109                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
 110         }
 111
 112         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 113         {
 114                 echo "Setup DB\n";
 115                 $bDidSomething = true;
 116                 // TODO: path detection, detection memory, etc.
 117
 118                 $oDB =& getDB();
 119                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 120                 $pgver = (float) CONST_Postgresql_Version;
 121                 if ($pgver < 9.1) {
 122                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 123                 } else {
 124                         pgsqlRunScript('CREATE EXTENSION hstore');
 125                 }
 126                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 127                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 128                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 129                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 130                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 131                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 132                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 133                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 134                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 135                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 136         }
 137
 138         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 139         {
 140                 echo "Import\n";
 141                 $bDidSomething = true;
 142
 143                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 144                 if (!file_exists($osm2pgsql))
 145                 {
 146                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 147                         fail("osm2pgsql not found in '$osm2pgsql'");
 148                 }
 149                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 150                 $osm2pgsql .= ' -C '.$iCacheMemory;
 151                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 152                 passthruCheckReturn($osm2pgsql);
 153
 154                 $oDB =& getDB();
 155                 $x = $oDB->getRow('select * from place limit 1');
 156                 if (PEAR::isError($x)) {
 157                         fail($x->getMessage());
 158                 }
 159                 if (!$x) fail('No Data');
 160         }
 161
 162         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 163         {
 164                 echo "Functions\n";
 165                 $bDidSomething = true;
 166                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 167                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 168                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 169                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 170                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 171                 pgsqlRunScript($sTemplate);
 172         }
 173
 174         if ($aCMDResult['create-minimal-tables'])
 175         {
 176                 echo "Minimal Tables\n";
 177                 $bDidSomething = true;
 178                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 179
 180                 $sScript = '';
 181
 182                 // Backstop the import process - easliest possible import id
 183                 $sScript .= "insert into import_npi_log values (18022);\n";
 184
 185                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 186                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 187
 188                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 189                 {
 190                         list($sClass, $sType) = explode(' ', trim($sLine));
 191                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 192                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 193
 194                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 195                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 196
 197                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 198                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 199                 }
 200                 fclose($hFile);
 201                 pgsqlRunScript($sScript);
 202         }
 203
 204         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 205         {
 206                 echo "Tables\n";
 207                 $bDidSomething = true;
 208                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 209
 210                 // re-run the functions
 211                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 212                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 213                 pgsqlRunScript($sTemplate);
 214         }
 215
 216         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 217         {
 218                 echo "Partitions\n";
 219                 $bDidSomething = true;
 220                 $oDB =& getDB();
 221                 $sSQL = 'select partition from country_name order by country_code';
 222                 $aPartitions = $oDB->getCol($sSQL);
 223                 if (PEAR::isError($aPartitions))
 224                 {
 225                         fail($aPartitions->getMessage());
 226                 }
 227                 $aPartitions[] = 0;
 228
 229                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 230                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 231                 foreach($aMatches as $aMatch)
 232                 {
 233                         $sResult = '';
 234                         foreach($aPartitions as $sPartitionName)
 235                         {
 236                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 237                         }
 238                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 239                 }
 240
 241                 pgsqlRunScript($sTemplate);
 242         }
 243
 244         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 245         {
 246                 $bDidSomething = true;
 247                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 248                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 249                 if (file_exists($sWikiArticlesFile))
 250                 {
 251                         echo "Importing wikipedia articles...";
 252                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 253                         echo "...done\n";
 254                 }
 255                 else
 256                 {
 257                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 258                 }
 259                 if (file_exists($sWikiRedirectsFile))
 260                 {
 261                         echo "Importing wikipedia redirects...";
 262                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 263                         echo "...done\n";
 264                 }
 265                 else
 266                 {
 267                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 268                 }
 269         }
 270
 271
 272         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 273         {
 274                 echo "Drop old Data\n";
 275                 $bDidSomething = true;
 276
 277                 $oDB =& getDB();
 278                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 279                 echo '.';
 280                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 281                 echo '.';
 282                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 283                 echo '.';
 284                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 285                 echo '.';
 286                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 287                 echo '.';
 288                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 289                 echo '.';
 290                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 291                 echo '.';
 292                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 293                 echo '.';
 294                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 295                 echo '.';
 296
 297                 $sSQL = 'select partition from country_name order by country_code';
 298                 $aPartitions = $oDB->getCol($sSQL);
 299                 if (PEAR::isError($aPartitions))
 300                 {
 301                         fail($aPartitions->getMessage());
 302                 }
 303                 $aPartitions[] = 0;
 304                 foreach($aPartitions as $sPartition)
 305                 {
 306                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 307                         echo '.';
 308                 }
 309
 310                 // used by getorcreate_word_id to ignore frequent partial words
 311                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 312                 echo ".\n";
 313
 314                 // pre-create the word list
 315                 if (!$aCMDResult['disable-token-precalc'])
 316                 {
 317                         echo "Loading word list\n";
 318                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 319                 }
 320
 321                 echo "Load Data\n";
 322                 $aDBInstances = array();
 323                 for($i = 0; $i < $iInstances; $i++)
 324                 {
 325                         $aDBInstances[$i] =& getDB(true);
 326                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 327                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 328                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 329                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 330                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 331                 }
 332                 $bAnyBusy = true;
 333                 while($bAnyBusy)
 334                 {
 335                         $bAnyBusy = false;
 336                         for($i = 0; $i < $iInstances; $i++)
 337                         {
 338                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 339                         }
 340                         sleep(1);
 341                         echo '.';
 342                 }
 343                 echo "\n";
 344                 echo "Reanalysing database...\n";
 345                 pgsqlRunScript('ANALYSE');
 346         }
 347
 348         if ($aCMDResult['create-roads'])
 349         {
 350                 $bDidSomething = true;
 351
 352                 $oDB =& getDB();
 353                 $aDBInstances = array();
 354                 for($i = 0; $i < $iInstances; $i++)
 355                 {
 356                         $aDBInstances[$i] =& getDB(true);
 357                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 358                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, country_code, geometry) from ';
 359                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 360                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 361                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 362                 }
 363                 $bAnyBusy = true;
 364                 while($bAnyBusy)
 365                 {
 366                         $bAnyBusy = false;
 367                         for($i = 0; $i < $iInstances; $i++)
 368                         {
 369                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 370                         }
 371                         sleep(1);
 372                         echo '.';
 373                 }
 374                 echo "\n";
 375         }
 376
 377         if ($aCMDResult['import-tiger-data'])
 378         {
 379                 $bDidSomething = true;
 380
 381                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 382
 383                 $aDBInstances = array();
 384                 for($i = 0; $i < $iInstances; $i++)
 385                 {
 386                         $aDBInstances[$i] =& getDB(true);
 387                 }
 388
 389                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 390                 {
 391                         echo $sFile.': ';
 392                         $hFile = fopen($sFile, "r");
 393                         $sSQL = fgets($hFile, 100000);
 394                         $iLines = 0;
 395
 396                         while(true)
 397                         {
 398                                 for($i = 0; $i < $iInstances; $i++)
 399                                 {
 400                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 401                                         {
 402                                                 while(pg_get_result($aDBInstances[$i]->connection));
 403                                                 $sSQL = fgets($hFile, 100000);
 404                                                 if (!$sSQL) break 2;
 405                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 406                                                 $iLines++;
 407                                                 if ($iLines == 1000)
 408                                                 {
 409                                                         echo ".";
 410                                                         $iLines = 0;
 411                                                 }
 412                                         }
 413                                 }
 414                                 usleep(10);
 415                         }
 416
 417                         fclose($hFile);
 418
 419                         $bAnyBusy = true;
 420                         while($bAnyBusy)
 421                         {
 422                                 $bAnyBusy = false;
 423                                 for($i = 0; $i < $iInstances; $i++)
 424                                 {
 425                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 426                                 }
 427                                 usleep(10);
 428                         }
 429                         echo "\n";
 430                 }
 431
 432                 echo "Creating indexes\n";
 433                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 434         }
 435
 436         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 437         {
 438                 $bDidSomething = true;
 439                 $oDB =& getDB();
 440                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 441                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 442                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,country_code,";
 443                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select country_code,postcode,";
 444                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 445                 $sSQL .= "from placex where postcode is not null group by country_code,postcode) as x";
 446                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 447
 448                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 449                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 450                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 451                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 452         }
 453
 454         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
 455         {
 456                 $bDidSomething = true;
 457                 $oDB =& getDB();
 458
 459                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 460                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
 461                 else
 462                 {
 463                         passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 464                         // server layout changed afer license change, fix path to minutely diffs
 465                         passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
 466                 }
 467
 468                 $sDate = $aCMDResult['osmosis-init-date'];
 469                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
 470                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
 471                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
 472                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
 473                 $sURL .= '&stream=minute';
 474                 echo "Getting state file: $sURL\n";
 475                 $sStateFile = file_get_contents($sURL);
 476                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 477                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 478                 echo "Updating DB status\n";
 479                 pg_query($oDB->connection, 'TRUNCATE import_status');
 480                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
 481                 pg_query($oDB->connection, $sSQL);
 482
 483         }
 484
 485         if ($aCMDResult['index'] || $aCMDResult['all'])
 486         {
 487                 $bDidSomething = true;
 488                 $sOutputFile = '';
 489                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 490                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 491                 passthruCheckReturn($sBaseCmd.' -R 4');
 492                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 493                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 494                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 495                 passthruCheckReturn($sBaseCmd.' -r 26');
 496         }
 497
 498         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 499         {
 500                 echo "Search indices\n";
 501                 $bDidSomething = true;
 502                 $oDB =& getDB();
 503                 $sSQL = 'select partition from country_name order by country_code';
 504                 $aPartitions = $oDB->getCol($sSQL);
 505                 if (PEAR::isError($aPartitions))
 506                 {
 507                         fail($aPartitions->getMessage());
 508                 }
 509                 $aPartitions[] = 0;
 510
 511                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 512                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 513                 foreach($aMatches as $aMatch)
 514                 {
 515                         $sResult = '';
 516                         foreach($aPartitions as $sPartitionName)
 517                         {
 518                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 519                         }
 520                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 521                 }
 522
 523                 pgsqlRunScript($sTemplate);
 524         }
 525
 526         if (isset($aCMDResult['create-website']))
 527         {
 528                 $bDidSomething = true;
 529                 $sTargetDir = $aCMDResult['create-website'];
 530                 if (!is_dir($sTargetDir))
 531                 {
 532                         echo "You must create the website directory before calling this function.\n";
 533                         fail("Target directory does not exist.");
 534                 }
 535
 536                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 537                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 538                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 539                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 540                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 541                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 542                 echo "Symlinks created\n";
 543         }
 544
 545         if (!$bDidSomething)
 546         {
 547                 showUsage($aCMDOptions, true);
 548         }
 549
 550         function pgsqlRunScriptFile($sFilename)
 551         {
 552                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 553
 554                 // Convert database DSN to psql parameters
 555                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 556                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 557                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 558
 559                 $aDescriptors = array(
 560                         0 => array('pipe', 'r'),
 561                         1 => array('pipe', 'w'),
 562                         2 => array('file', '/dev/null', 'a')
 563                 );
 564                 $ahPipes = null;
 565                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 566                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 567
 568                 fclose($ahPipes[0]);
 569
 570                 // TODO: error checking
 571                 while(!feof($ahPipes[1]))
 572                 {
 573                         echo fread($ahPipes[1], 4096);
 574                 }
 575                 fclose($ahPipes[1]);
 576
 577                 proc_close($hProcess);
 578         }
 579
 580         function pgsqlRunScript($sScript)
 581         {
 582                 // Convert database DSN to psql parameters
 583                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 584                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 585                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 586                 $aDescriptors = array(
 587                         0 => array('pipe', 'r'),
 588                         1 => STDOUT,
 589                         2 => STDERR
 590                 );
 591                 $ahPipes = null;
 592                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 593                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 594
 595                 while(strlen($sScript))
 596                 {
 597                         $written = fwrite($ahPipes[0], $sScript);
 598                         $sScript = substr($sScript, $written);
 599                 }
 600                 fclose($ahPipes[0]);
 601                 proc_close($hProcess);
 602         }
 603
 604         function pgsqlRunRestoreData($sDumpFile)
 605         {
 606                 // Convert database DSN to psql parameters
 607                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 608                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 609                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 610
 611                 $aDescriptors = array(
 612                         0 => array('pipe', 'r'),
 613                         1 => array('pipe', 'w'),
 614                         2 => array('file', '/dev/null', 'a')
 615                 );
 616                 $ahPipes = null;
 617                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 618                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 619
 620                 fclose($ahPipes[0]);
 621
 622                 // TODO: error checking
 623                 while(!feof($ahPipes[1]))
 624                 {
 625                         echo fread($ahPipes[1], 4096);
 626                 }
 627                 fclose($ahPipes[1]);
 628
 629                 proc_close($hProcess);
 630         }
 631
 632         function pgsqlRunDropAndRestore($sDumpFile)
 633         {
 634                 // Convert database DSN to psql parameters
 635                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 636                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 637                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 638
 639                 $aDescriptors = array(
 640                         0 => array('pipe', 'r'),
 641                         1 => array('pipe', 'w'),
 642                         2 => array('file', '/dev/null', 'a')
 643                 );
 644                 $ahPipes = null;
 645                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 646                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 647
 648                 fclose($ahPipes[0]);
 649
 650                 // TODO: error checking
 651                 while(!feof($ahPipes[1]))
 652                 {
 653                         echo fread($ahPipes[1], 4096);
 654                 }
 655                 fclose($ahPipes[1]);
 656
 657                 proc_close($hProcess);
 658         }
 659
 660         function passthruCheckReturn($cmd)
 661         {
 662                 $result = -1;
 663                 passthru($cmd, $result);
 664                 if ($result != 0) fail('Error executing external command: '.$cmd);
 665         }