utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
  28                 array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
  29                 array('no-partitions', '', 0, 1, 0, 0, 'bool', "Do not partition search indices (speeds up import of single country extracts)"),
  30                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  31                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  32                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  33                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  34                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  35                 array('create-roads', '', 0, 1, 0, 0, 'bool', ''),
  36                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  37                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  38                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  39                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  40                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  41                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  42         );
  43         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  44
  45         $bDidSomething = false;
  46
  47         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  48         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  49         {
  50                 if (!isset($aCMDResult['osm-file']))
  51                 {
  52                         fail('missing --osm-file for data import');
  53                 }
  54
  55                 if (!file_exists($aCMDResult['osm-file']))
  56                 {
  57                         fail('the path supplied to --osm-file does not exist');
  58                 }
  59
  60                 if (!is_readable($aCMDResult['osm-file']))
  61                 {
  62                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  63                 }
  64         }
  65
  66
  67         // This is a pretty hard core default - the number of processors in the box - 1
  68         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  69         if ($iInstances < 1)
  70         {
  71                 $iInstances = 1;
  72                 echo "WARNING: resetting threads to $iInstances\n";
  73         }
  74         if ($iInstances > getProcessorCount())
  75         {
  76                 $iInstances = getProcessorCount();
  77                 echo "WARNING: resetting threads to $iInstances\n";
  78         }
  79
  80         // Assume we can steal all the cache memory in the box (unless told otherwise)
  81         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  82         if ($iCacheMemory > getTotalMemoryMB())
  83         {
  84                 $iCacheMemory = getCacheMemoryMB();
  85                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  86         }
  87
  88         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  89         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  90
  91         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  92         {
  93                 echo "Create DB\n";
  94                 $bDidSomething = true;
  95                 $oDB =& DB::connect(CONST_Database_DSN, false);
  96                 if (!PEAR::isError($oDB))
  97                 {
  98                         fail('database already exists ('.CONST_Database_DSN.')');
  99                 }
 100                 passthruCheckReturn('createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 101         }
 102
 103         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 104         {
 105                 echo "Setup DB\n";
 106                 $bDidSomething = true;
 107                 // TODO: path detection, detection memory, etc.
 108
 109                 $oDB =& getDB();
 110
 111                 $sVersionString = $oDB->getOne('select version()');
 112                 preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[.]([0-9]+) #', $sVersionString, $aMatches);
 113                 if (CONST_Postgresql_Version != $aMatches[1].'.'.$aMatches[2])
 114                 {
 115                         echo "ERROR: PostgreSQL version is not correct.  Expected ".CONST_Postgresql_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 116                         exit;
 117                 }
 118
 119                 passthru('createlang plpgsql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 120                 $pgver = (float) CONST_Postgresql_Version;
 121                 if ($pgver < 9.1) {
 122                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 123                         pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
 124                 } else {
 125                         pgsqlRunScript('CREATE EXTENSION hstore');
 126                 }
 127
 128                 $fPostgisVersion = (float) CONST_Postgis_Version;
 129                 if ($fPostgisVersion < 2.0) {
 130                         pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 131                         pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 132                 } else {
 133                         pgsqlRunScript('CREATE EXTENSION postgis');
 134                 }
 135                 $sVersionString = $oDB->getOne('select postgis_full_version()');
 136                 preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
 137                 if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
 138                 {
 139                         echo "ERROR: PostGIS version is not correct.  Expected ".CONST_Postgis_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 140                         exit;
 141                 }
 142
 143                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 144                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 145                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 146                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 147                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 148                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 149                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 150
 151                 if ($aCMDResult['no-partitions'])
 152                 {
 153                         pgsqlRunScript('update country_name set partition = 0');
 154                 }
 155         }
 156
 157         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 158         {
 159                 echo "Import\n";
 160                 $bDidSomething = true;
 161
 162                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 163                 if (!file_exists($osm2pgsql))
 164                 {
 165                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 166                         fail("osm2pgsql not found in '$osm2pgsql'");
 167                 }
 168
 169                 if (!is_null(CONST_Osm2pgsql_Flatnode_File))
 170                 {
 171                         $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
 172                 }
 173                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 174                 $osm2pgsql .= ' -C '.$iCacheMemory;
 175                 $osm2pgsql .= ' -P '.$aDSNInfo['port'];
 176                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 177                 passthruCheckReturn($osm2pgsql);
 178
 179                 $oDB =& getDB();
 180                 $x = $oDB->getRow('select * from place limit 1');
 181                 if (PEAR::isError($x)) {
 182                         fail($x->getMessage());
 183                 }
 184                 if (!$x) fail('No Data');
 185         }
 186
 187         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 188         {
 189                 echo "Functions\n";
 190                 $bDidSomething = true;
 191                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 192                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 193                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 194                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 195                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 196                 pgsqlRunScript($sTemplate);
 197         }
 198
 199         if ($aCMDResult['create-minimal-tables'])
 200         {
 201                 echo "Minimal Tables\n";
 202                 $bDidSomething = true;
 203                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 204
 205                 $sScript = '';
 206
 207                 // Backstop the import process - easliest possible import id
 208                 $sScript .= "insert into import_npi_log values (18022);\n";
 209
 210                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 211                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 212
 213                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 214                 {
 215                         list($sClass, $sType) = explode(' ', trim($sLine));
 216                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 217                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 218
 219                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 220                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 221
 222                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 223                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 224                 }
 225                 fclose($hFile);
 226                 pgsqlRunScript($sScript);
 227         }
 228
 229         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 230         {
 231                 echo "Tables\n";
 232                 $bDidSomething = true;
 233                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 234
 235                 // re-run the functions
 236                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 237                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 238                 pgsqlRunScript($sTemplate);
 239         }
 240
 241         if ($aCMDResult['create-partition-tables'] || $aCMDResult['all'])
 242         {
 243                 echo "Partition Tables\n";
 244                 $bDidSomething = true;
 245                 $oDB =& getDB();
 246                 $sSQL = 'select distinct partition from country_name';
 247                 $aPartitions = $oDB->getCol($sSQL);
 248                 if (PEAR::isError($aPartitions))
 249                 {
 250                         fail($aPartitions->getMessage());
 251                 }
 252                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 253
 254                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
 255                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 256                 foreach($aMatches as $aMatch)
 257                 {
 258                         $sResult = '';
 259                         foreach($aPartitions as $sPartitionName)
 260                         {
 261                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 262                         }
 263                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 264                 }
 265
 266                 pgsqlRunScript($sTemplate);
 267         }
 268
 269
 270         if ($aCMDResult['create-partition-functions'] || $aCMDResult['all'])
 271         {
 272                 echo "Partition Functions\n";
 273                 $bDidSomething = true;
 274                 $oDB =& getDB();
 275                 $sSQL = 'select distinct partition from country_name';
 276                 $aPartitions = $oDB->getCol($sSQL);
 277                 if (PEAR::isError($aPartitions))
 278                 {
 279                         fail($aPartitions->getMessage());
 280                 }
 281                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 282
 283                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
 284                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 285                 foreach($aMatches as $aMatch)
 286                 {
 287                         $sResult = '';
 288                         foreach($aPartitions as $sPartitionName)
 289                         {
 290                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 291                         }
 292                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 293                 }
 294
 295                 pgsqlRunScript($sTemplate);
 296         }
 297
 298         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 299         {
 300                 $bDidSomething = true;
 301                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 302                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 303                 if (file_exists($sWikiArticlesFile))
 304                 {
 305                         echo "Importing wikipedia articles...";
 306                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 307                         echo "...done\n";
 308                 }
 309                 else
 310                 {
 311                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 312                 }
 313                 if (file_exists($sWikiRedirectsFile))
 314                 {
 315                         echo "Importing wikipedia redirects...";
 316                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 317                         echo "...done\n";
 318                 }
 319                 else
 320                 {
 321                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 322                 }
 323         }
 324
 325
 326         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 327         {
 328                 echo "Drop old Data\n";
 329                 $bDidSomething = true;
 330
 331                 $oDB =& getDB();
 332                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 333                 echo '.';
 334                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 335                 echo '.';
 336                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 337                 echo '.';
 338                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 339                 echo '.';
 340                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 341                 echo '.';
 342                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 343                 echo '.';
 344                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 345                 echo '.';
 346                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 347                 echo '.';
 348                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 349                 echo '.';
 350
 351                 $sSQL = 'select distinct partition from country_name';
 352                 $aPartitions = $oDB->getCol($sSQL);
 353                 if (PEAR::isError($aPartitions))
 354                 {
 355                         fail($aPartitions->getMessage());
 356                 }
 357                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 358                 foreach($aPartitions as $sPartition)
 359                 {
 360                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 361                         echo '.';
 362                 }
 363
 364                 // used by getorcreate_word_id to ignore frequent partial words
 365                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 366                 echo ".\n";
 367
 368                 // pre-create the word list
 369                 if (!$aCMDResult['disable-token-precalc'])
 370                 {
 371                         echo "Loading word list\n";
 372                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 373                 }
 374
 375                 echo "Load Data\n";
 376                 $aDBInstances = array();
 377                 for($i = 0; $i < $iInstances; $i++)
 378                 {
 379                         $aDBInstances[$i] =& getDB(true);
 380                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 381                         $sSQL .= 'housenumber, street, addr_place, isin, postcode, country_code, extratags, ';
 382                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 383                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 384                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 385                 }
 386                 $bAnyBusy = true;
 387                 while($bAnyBusy)
 388                 {
 389                         $bAnyBusy = false;
 390                         for($i = 0; $i < $iInstances; $i++)
 391                         {
 392                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 393                         }
 394                         sleep(1);
 395                         echo '.';
 396                 }
 397                 echo "\n";
 398                 echo "Reanalysing database...\n";
 399                 pgsqlRunScript('ANALYSE');
 400         }
 401
 402         if ($aCMDResult['create-roads'])
 403         {
 404                 $bDidSomething = true;
 405
 406                 $oDB =& getDB();
 407                 $aDBInstances = array();
 408                 for($i = 0; $i < $iInstances; $i++)
 409                 {
 410                         $aDBInstances[$i] =& getDB(true);
 411                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 412                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
 413                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 414                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 415                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 416                 }
 417                 $bAnyBusy = true;
 418                 while($bAnyBusy)
 419                 {
 420                         $bAnyBusy = false;
 421                         for($i = 0; $i < $iInstances; $i++)
 422                         {
 423                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 424                         }
 425                         sleep(1);
 426                         echo '.';
 427                 }
 428                 echo "\n";
 429         }
 430
 431         if ($aCMDResult['import-tiger-data'])
 432         {
 433                 $bDidSomething = true;
 434
 435                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 436
 437                 $aDBInstances = array();
 438                 for($i = 0; $i < $iInstances; $i++)
 439                 {
 440                         $aDBInstances[$i] =& getDB(true);
 441                 }
 442
 443                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 444                 {
 445                         echo $sFile.': ';
 446                         $hFile = fopen($sFile, "r");
 447                         $sSQL = fgets($hFile, 100000);
 448                         $iLines = 0;
 449
 450                         while(true)
 451                         {
 452                                 for($i = 0; $i < $iInstances; $i++)
 453                                 {
 454                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 455                                         {
 456                                                 while(pg_get_result($aDBInstances[$i]->connection));
 457                                                 $sSQL = fgets($hFile, 100000);
 458                                                 if (!$sSQL) break 2;
 459                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 460                                                 $iLines++;
 461                                                 if ($iLines == 1000)
 462                                                 {
 463                                                         echo ".";
 464                                                         $iLines = 0;
 465                                                 }
 466                                         }
 467                                 }
 468                                 usleep(10);
 469                         }
 470
 471                         fclose($hFile);
 472
 473                         $bAnyBusy = true;
 474                         while($bAnyBusy)
 475                         {
 476                                 $bAnyBusy = false;
 477                                 for($i = 0; $i < $iInstances; $i++)
 478                                 {
 479                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 480                                 }
 481                                 usleep(10);
 482                         }
 483                         echo "\n";
 484                 }
 485
 486                 echo "Creating indexes\n";
 487                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 488         }
 489
 490         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 491         {
 492                 $bDidSomething = true;
 493                 $oDB =& getDB();
 494                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 495                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 496                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 497                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 498                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 499                 $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
 500                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 501
 502                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 503                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 504                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 505                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 506         }
 507
 508         if ($aCMDResult['osmosis-init'] || $aCMDResult['all'])
 509         {
 510                 $bDidSomething = true;
 511                 $oDB =& getDB();
 512
 513                 if (!file_exists(CONST_Osmosis_Binary))
 514                 {
 515                         echo "Please download osmosis.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 516                         if (!$aCMDResult['all'])
 517                         {
 518                                 fail("osmosis not found in '".CONST_Osmosis_Binary."'");
 519                         }
 520                 }
 521                 else
 522                 {
 523                         if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
 524                         {
 525                                 echo "settings/configuration.txt already exists\n";
 526                         }
 527                         else
 528                         {
 529                                 passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 530                                 // update osmosis configuration.txt with our settings
 531                                 passthru("sed -i 's!baseUrl=.*!baseUrl=".CONST_Replication_Url."!' ".CONST_BasePath.'/settings/configuration.txt');
 532                                 passthru("sed -i 's:maxInterval = .*:maxInterval = ".CONST_Replication_MaxInterval.":' ".CONST_BasePath.'/settings/configuration.txt');
 533                         }
 534
 535                         // Find the last node in the DB
 536                         $iLastOSMID = $oDB->getOne("select max(id) from planet_osm_nodes");
 537
 538                         // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
 539                         $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID."/1";
 540                         $sLastNodeXML = file_get_contents($sLastNodeURL);
 541                         preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 542                         $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
 543
 544                         // Search for the correct state file - uses file timestamps so need to sort by date descending
 545                         $sRepURL = CONST_Replication_Url."/";
 546                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 547                         // download.geofabrik.de:    <a href="000/">000/</a></td><td align="right">26-Feb-2013 11:53  </td>
 548                         // planet.openstreetmap.org: <a href="273/">273/</a>                    22-Mar-2013 07:41    -
 549                         preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 550                         $aPrevRepMatch = false;
 551                         foreach($aRepMatches as $aRepMatch)
 552                         {
 553                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 554                                 $aPrevRepMatch = $aRepMatch;
 555                         }
 556                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 557
 558                         $sRepURL .= $aRepMatch[1];
 559                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 560                         preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 561                         $aPrevRepMatch = false;
 562                         foreach($aRepMatches as $aRepMatch)
 563                         {
 564                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 565                                 $aPrevRepMatch = $aRepMatch;
 566                         }
 567                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 568
 569                         $sRepURL .= $aRepMatch[1];
 570                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 571                         preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 572                         $aPrevRepMatch = false;
 573                         foreach($aRepMatches as $aRepMatch)
 574                         {
 575                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 576                                 $aPrevRepMatch = $aRepMatch;
 577                         }
 578                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 579
 580                         $sRepURL .= $aRepMatch[1].'.state.txt';
 581                         echo "Getting state file: $sRepURL\n";
 582                         $sStateFile = file_get_contents($sRepURL);
 583                         if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 584                         file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 585                         echo "Updating DB status\n";
 586                         pg_query($oDB->connection, 'TRUNCATE import_status');
 587                         $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
 588                         pg_query($oDB->connection, $sSQL);
 589                 }
 590         }
 591
 592         if ($aCMDResult['index'] || $aCMDResult['all'])
 593         {
 594                 $bDidSomething = true;
 595                 $sOutputFile = '';
 596                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 597                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
 598                 passthruCheckReturn($sBaseCmd.' -R 4');
 599                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 600                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 601                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 602                 passthruCheckReturn($sBaseCmd.' -r 26');
 603         }
 604
 605         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 606         {
 607                 echo "Search indices\n";
 608                 $bDidSomething = true;
 609                 $oDB =& getDB();
 610                 $sSQL = 'select distinct partition from country_name';
 611                 $aPartitions = $oDB->getCol($sSQL);
 612                 if (PEAR::isError($aPartitions))
 613                 {
 614                         fail($aPartitions->getMessage());
 615                 }
 616                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 617
 618                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 619                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 620                 foreach($aMatches as $aMatch)
 621                 {
 622                         $sResult = '';
 623                         foreach($aPartitions as $sPartitionName)
 624                         {
 625                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 626                         }
 627                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 628                 }
 629
 630                 pgsqlRunScript($sTemplate);
 631         }
 632
 633         if (isset($aCMDResult['create-website']))
 634         {
 635                 $bDidSomething = true;
 636                 $sTargetDir = $aCMDResult['create-website'];
 637                 if (!is_dir($sTargetDir))
 638                 {
 639                         echo "You must create the website directory before calling this function.\n";
 640                         fail("Target directory does not exist.");
 641                 }
 642
 643                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 644                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 645                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 646                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 647                 @symlink(CONST_BasePath.'/website/deletable.php', $sTargetDir.'/deletable.php');
 648                 @symlink(CONST_BasePath.'/website/polygons.php', $sTargetDir.'/polygons.php');
 649                 @symlink(CONST_BasePath.'/website/status.php', $sTargetDir.'/status.php');
 650                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 651                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 652                 @symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css');
 653                 echo "Symlinks created\n";
 654
 655                 $sTestFile = @file_get_contents(CONST_Website_BaseURL.'js/tiles.js');
 656                 if (!$sTestFile)
 657                 {
 658                         echo "\nWARNING: Unable to access the website at ".CONST_Website_BaseURL."\n";
 659                         echo "You may want to update settings/local.php with @define('CONST_Website_BaseURL', 'http://[HOST]/[PATH]/');\n";
 660                 }
 661         }
 662
 663         if (!$bDidSomething)
 664         {
 665                 showUsage($aCMDOptions, true);
 666         }
 667
 668         function pgsqlRunScriptFile($sFilename)
 669         {
 670                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 671
 672                 // Convert database DSN to psql parameters
 673                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 674                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 675                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 676
 677                 $aDescriptors = array(
 678                         0 => array('pipe', 'r'),
 679                         1 => array('pipe', 'w'),
 680                         2 => array('file', '/dev/null', 'a')
 681                 );
 682                 $ahPipes = null;
 683                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 684                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 685
 686                 fclose($ahPipes[0]);
 687
 688                 // TODO: error checking
 689                 while(!feof($ahPipes[1]))
 690                 {
 691                         echo fread($ahPipes[1], 4096);
 692                 }
 693                 fclose($ahPipes[1]);
 694
 695                 proc_close($hProcess);
 696         }
 697
 698         function pgsqlRunScript($sScript)
 699         {
 700                 // Convert database DSN to psql parameters
 701                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 702                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 703                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 704                 $aDescriptors = array(
 705                         0 => array('pipe', 'r'),
 706                         1 => STDOUT,
 707                         2 => STDERR
 708                 );
 709                 $ahPipes = null;
 710                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 711                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 712
 713                 while(strlen($sScript))
 714                 {
 715                         $written = fwrite($ahPipes[0], $sScript);
 716                         $sScript = substr($sScript, $written);
 717                 }
 718                 fclose($ahPipes[0]);
 719                 proc_close($hProcess);
 720         }
 721
 722         function pgsqlRunRestoreData($sDumpFile)
 723         {
 724                 // Convert database DSN to psql parameters
 725                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 726                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 727                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 728
 729                 $aDescriptors = array(
 730                         0 => array('pipe', 'r'),
 731                         1 => array('pipe', 'w'),
 732                         2 => array('file', '/dev/null', 'a')
 733                 );
 734                 $ahPipes = null;
 735                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 736                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 737
 738                 fclose($ahPipes[0]);
 739
 740                 // TODO: error checking
 741                 while(!feof($ahPipes[1]))
 742                 {
 743                         echo fread($ahPipes[1], 4096);
 744                 }
 745                 fclose($ahPipes[1]);
 746
 747                 proc_close($hProcess);
 748         }
 749
 750         function pgsqlRunDropAndRestore($sDumpFile)
 751         {
 752                 // Convert database DSN to psql parameters
 753                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 754                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 755                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 756
 757                 $aDescriptors = array(
 758                         0 => array('pipe', 'r'),
 759                         1 => array('pipe', 'w'),
 760                         2 => array('file', '/dev/null', 'a')
 761                 );
 762                 $ahPipes = null;
 763                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 764                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 765
 766                 fclose($ahPipes[0]);
 767
 768                 // TODO: error checking
 769                 while(!feof($ahPipes[1]))
 770                 {
 771                         echo fread($ahPipes[1], 4096);
 772                 }
 773                 fclose($ahPipes[1]);
 774
 775                 proc_close($hProcess);
 776         }
 777
 778         function passthruCheckReturn($cmd)
 779         {
 780                 $result = -1;
 781                 passthru($cmd, $result);
 782                 if ($result != 0) fail('Error executing external command: '.$cmd);
 783         }