utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
  28                 array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
  29                 array('no-partitions', '', 0, 1, 0, 0, 'bool', "Do not partition search indices (speeds up import of single country extracts)"),
  30                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  31                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  32                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  33                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  34                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  35                 array('create-roads', '', 0, 1, 0, 0, 'bool', ''),
  36                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  37                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  38                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  39                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  40                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  41                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  42         );
  43         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  44
  45         $bDidSomething = false;
  46
  47         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  48         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  49         {
  50                 if (!isset($aCMDResult['osm-file']))
  51                 {
  52                         fail('missing --osm-file for data import');
  53                 }
  54
  55                 if (!file_exists($aCMDResult['osm-file']))
  56                 {
  57                         fail('the path supplied to --osm-file does not exist');
  58                 }
  59
  60                 if (!is_readable($aCMDResult['osm-file']))
  61                 {
  62                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  63                 }
  64         }
  65
  66
  67         // This is a pretty hard core default - the number of processors in the box - 1
  68         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  69         if ($iInstances < 1)
  70         {
  71                 $iInstances = 1;
  72                 echo "WARNING: resetting threads to $iInstances\n";
  73         }
  74         if ($iInstances > getProcessorCount())
  75         {
  76                 $iInstances = getProcessorCount();
  77                 echo "WARNING: resetting threads to $iInstances\n";
  78         }
  79
  80         // Assume we can steal all the cache memory in the box (unless told otherwise)
  81         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  82         if ($iCacheMemory > getTotalMemoryMB())
  83         {
  84                 $iCacheMemory = getCacheMemoryMB();
  85                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  86         }
  87
  88         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  89         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  90
  91         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  92         {
  93                 echo "Create DB\n";
  94                 $bDidSomething = true;
  95                 $oDB =& DB::connect(CONST_Database_DSN, false);
  96                 if (!PEAR::isError($oDB))
  97                 {
  98                         fail('database already exists ('.CONST_Database_DSN.')');
  99                 }
 100                 passthruCheckReturn('createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 101         }
 102
 103         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 104         {
 105                 echo "Setup DB\n";
 106                 $bDidSomething = true;
 107                 // TODO: path detection, detection memory, etc.
 108
 109                 $oDB =& getDB();
 110
 111                 $sVersionString = $oDB->getOne('select version()');
 112                 preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[.]([0-9]+) #', $sVersionString, $aMatches);
 113                 if (CONST_Postgresql_Version != $aMatches[1].'.'.$aMatches[2])
 114                 {
 115                         echo "ERROR: PostgreSQL version is not correct.  Expected ".CONST_Postgresql_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 116                         exit;
 117                 }
 118
 119                 passthru('createlang plpgsql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 120                 $pgver = (float) CONST_Postgresql_Version;
 121                 if ($pgver < 9.1) {
 122                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 123                         pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
 124                 } else {
 125                         pgsqlRunScript('CREATE EXTENSION hstore');
 126                 }
 127
 128                 $fPostgisVersion = (float) CONST_Postgis_Version;
 129                 if ($fPostgisVersion < 2.0) {
 130                         pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 131                         pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 132                 } else {
 133                         pgsqlRunScript('CREATE EXTENSION postgis');
 134                 }
 135                 if ($fPostgisVersion < 2.1) {
 136                         // Function was renamed in 2.1 and throws an annoying deprecation warning
 137                         pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint');
 138                 }
 139                 $sVersionString = $oDB->getOne('select postgis_full_version()');
 140                 preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
 141                 if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
 142                 {
 143                         echo "ERROR: PostGIS version is not correct.  Expected ".CONST_Postgis_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 144                         exit;
 145                 }
 146
 147                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 148                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 149                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 150                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 151                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 152                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 153                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 154
 155                 if ($aCMDResult['no-partitions'])
 156                 {
 157                         pgsqlRunScript('update country_name set partition = 0');
 158                 }
 159         }
 160
 161         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 162         {
 163                 echo "Import\n";
 164                 $bDidSomething = true;
 165
 166                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 167                 if (!file_exists($osm2pgsql))
 168                 {
 169                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 170                         fail("osm2pgsql not found in '$osm2pgsql'");
 171                 }
 172
 173                 if (!is_null(CONST_Osm2pgsql_Flatnode_File))
 174                 {
 175                         $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
 176                 }
 177                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 178                 $osm2pgsql .= ' -C '.$iCacheMemory;
 179                 $osm2pgsql .= ' -P '.$aDSNInfo['port'];
 180                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 181                 passthruCheckReturn($osm2pgsql);
 182
 183                 $oDB =& getDB();
 184                 $x = $oDB->getRow('select * from place limit 1');
 185                 if (PEAR::isError($x)) {
 186                         fail($x->getMessage());
 187                 }
 188                 if (!$x) fail('No Data');
 189         }
 190
 191         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 192         {
 193                 echo "Functions\n";
 194                 $bDidSomething = true;
 195                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 196                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 197                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 198                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 199                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 200                 pgsqlRunScript($sTemplate);
 201         }
 202
 203         if ($aCMDResult['create-minimal-tables'])
 204         {
 205                 echo "Minimal Tables\n";
 206                 $bDidSomething = true;
 207                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 208
 209                 $sScript = '';
 210
 211                 // Backstop the import process - easliest possible import id
 212                 $sScript .= "insert into import_npi_log values (18022);\n";
 213
 214                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 215                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 216
 217                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 218                 {
 219                         list($sClass, $sType) = explode(' ', trim($sLine));
 220                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 221                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 222
 223                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 224                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 225
 226                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 227                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 228                 }
 229                 fclose($hFile);
 230                 pgsqlRunScript($sScript);
 231         }
 232
 233         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 234         {
 235                 echo "Tables\n";
 236                 $bDidSomething = true;
 237                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 238
 239                 // re-run the functions
 240                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 241                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 242                 pgsqlRunScript($sTemplate);
 243         }
 244
 245         if ($aCMDResult['create-partition-tables'] || $aCMDResult['all'])
 246         {
 247                 echo "Partition Tables\n";
 248                 $bDidSomething = true;
 249                 $oDB =& getDB();
 250                 $sSQL = 'select distinct partition from country_name';
 251                 $aPartitions = $oDB->getCol($sSQL);
 252                 if (PEAR::isError($aPartitions))
 253                 {
 254                         fail($aPartitions->getMessage());
 255                 }
 256                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 257
 258                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
 259                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 260                 foreach($aMatches as $aMatch)
 261                 {
 262                         $sResult = '';
 263                         foreach($aPartitions as $sPartitionName)
 264                         {
 265                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 266                         }
 267                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 268                 }
 269
 270                 pgsqlRunScript($sTemplate);
 271         }
 272
 273
 274         if ($aCMDResult['create-partition-functions'] || $aCMDResult['all'])
 275         {
 276                 echo "Partition Functions\n";
 277                 $bDidSomething = true;
 278                 $oDB =& getDB();
 279                 $sSQL = 'select distinct partition from country_name';
 280                 $aPartitions = $oDB->getCol($sSQL);
 281                 if (PEAR::isError($aPartitions))
 282                 {
 283                         fail($aPartitions->getMessage());
 284                 }
 285                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 286
 287                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
 288                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 289                 foreach($aMatches as $aMatch)
 290                 {
 291                         $sResult = '';
 292                         foreach($aPartitions as $sPartitionName)
 293                         {
 294                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 295                         }
 296                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 297                 }
 298
 299                 pgsqlRunScript($sTemplate);
 300         }
 301
 302         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 303         {
 304                 $bDidSomething = true;
 305                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 306                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 307                 if (file_exists($sWikiArticlesFile))
 308                 {
 309                         echo "Importing wikipedia articles...";
 310                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 311                         echo "...done\n";
 312                 }
 313                 else
 314                 {
 315                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 316                 }
 317                 if (file_exists($sWikiRedirectsFile))
 318                 {
 319                         echo "Importing wikipedia redirects...";
 320                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 321                         echo "...done\n";
 322                 }
 323                 else
 324                 {
 325                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 326                 }
 327         }
 328
 329
 330         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 331         {
 332                 echo "Drop old Data\n";
 333                 $bDidSomething = true;
 334
 335                 $oDB =& getDB();
 336                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 337                 echo '.';
 338                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 339                 echo '.';
 340                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 341                 echo '.';
 342                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 343                 echo '.';
 344                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 345                 echo '.';
 346                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 347                 echo '.';
 348                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 349                 echo '.';
 350                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 351                 echo '.';
 352                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 353                 echo '.';
 354
 355                 $sSQL = 'select distinct partition from country_name';
 356                 $aPartitions = $oDB->getCol($sSQL);
 357                 if (PEAR::isError($aPartitions))
 358                 {
 359                         fail($aPartitions->getMessage());
 360                 }
 361                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 362                 foreach($aPartitions as $sPartition)
 363                 {
 364                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 365                         echo '.';
 366                 }
 367
 368                 // used by getorcreate_word_id to ignore frequent partial words
 369                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 370                 echo ".\n";
 371
 372                 // pre-create the word list
 373                 if (!$aCMDResult['disable-token-precalc'])
 374                 {
 375                         echo "Loading word list\n";
 376                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 377                 }
 378
 379                 echo "Load Data\n";
 380                 $aDBInstances = array();
 381                 for($i = 0; $i < $iInstances; $i++)
 382                 {
 383                         $aDBInstances[$i] =& getDB(true);
 384                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 385                         $sSQL .= 'housenumber, street, addr_place, isin, postcode, country_code, extratags, ';
 386                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 387                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 388                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 389                 }
 390                 $bAnyBusy = true;
 391                 while($bAnyBusy)
 392                 {
 393                         $bAnyBusy = false;
 394                         for($i = 0; $i < $iInstances; $i++)
 395                         {
 396                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 397                         }
 398                         sleep(1);
 399                         echo '.';
 400                 }
 401                 echo "\n";
 402                 echo "Reanalysing database...\n";
 403                 pgsqlRunScript('ANALYSE');
 404         }
 405
 406         if ($aCMDResult['create-roads'])
 407         {
 408                 $bDidSomething = true;
 409
 410                 $oDB =& getDB();
 411                 $aDBInstances = array();
 412                 for($i = 0; $i < $iInstances; $i++)
 413                 {
 414                         $aDBInstances[$i] =& getDB(true);
 415                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 416                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
 417                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 418                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 419                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 420                 }
 421                 $bAnyBusy = true;
 422                 while($bAnyBusy)
 423                 {
 424                         $bAnyBusy = false;
 425                         for($i = 0; $i < $iInstances; $i++)
 426                         {
 427                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 428                         }
 429                         sleep(1);
 430                         echo '.';
 431                 }
 432                 echo "\n";
 433         }
 434
 435         if ($aCMDResult['import-tiger-data'])
 436         {
 437                 $bDidSomething = true;
 438
 439                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 440
 441                 $aDBInstances = array();
 442                 for($i = 0; $i < $iInstances; $i++)
 443                 {
 444                         $aDBInstances[$i] =& getDB(true);
 445                 }
 446
 447                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 448                 {
 449                         echo $sFile.': ';
 450                         $hFile = fopen($sFile, "r");
 451                         $sSQL = fgets($hFile, 100000);
 452                         $iLines = 0;
 453
 454                         while(true)
 455                         {
 456                                 for($i = 0; $i < $iInstances; $i++)
 457                                 {
 458                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 459                                         {
 460                                                 while(pg_get_result($aDBInstances[$i]->connection));
 461                                                 $sSQL = fgets($hFile, 100000);
 462                                                 if (!$sSQL) break 2;
 463                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 464                                                 $iLines++;
 465                                                 if ($iLines == 1000)
 466                                                 {
 467                                                         echo ".";
 468                                                         $iLines = 0;
 469                                                 }
 470                                         }
 471                                 }
 472                                 usleep(10);
 473                         }
 474
 475                         fclose($hFile);
 476
 477                         $bAnyBusy = true;
 478                         while($bAnyBusy)
 479                         {
 480                                 $bAnyBusy = false;
 481                                 for($i = 0; $i < $iInstances; $i++)
 482                                 {
 483                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 484                                 }
 485                                 usleep(10);
 486                         }
 487                         echo "\n";
 488                 }
 489
 490                 echo "Creating indexes\n";
 491                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 492         }
 493
 494         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 495         {
 496                 $bDidSomething = true;
 497                 $oDB =& getDB();
 498                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 499                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 500                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 501                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 502                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 503                 $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
 504                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 505
 506                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 507                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 508                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 509                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 510         }
 511
 512         if ($aCMDResult['osmosis-init'] || $aCMDResult['all'])
 513         {
 514                 $bDidSomething = true;
 515                 $oDB =& getDB();
 516
 517                 if (!file_exists(CONST_Osmosis_Binary))
 518                 {
 519                         echo "Please download osmosis.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 520                         if (!$aCMDResult['all'])
 521                         {
 522                                 fail("osmosis not found in '".CONST_Osmosis_Binary."'");
 523                         }
 524                 }
 525                 else
 526                 {
 527                         if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
 528                         {
 529                                 echo "settings/configuration.txt already exists\n";
 530                         }
 531                         else
 532                         {
 533                                 passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 534                                 // update osmosis configuration.txt with our settings
 535                                 passthru("sed -i 's!baseUrl=.*!baseUrl=".CONST_Replication_Url."!' ".CONST_BasePath.'/settings/configuration.txt');
 536                                 passthru("sed -i 's:maxInterval = .*:maxInterval = ".CONST_Replication_MaxInterval.":' ".CONST_BasePath.'/settings/configuration.txt');
 537                         }
 538
 539                         // Find the last node in the DB
 540                         $iLastOSMID = $oDB->getOne("select max(id) from planet_osm_nodes");
 541
 542                         // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
 543                         $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID."/1";
 544                         $sLastNodeXML = file_get_contents($sLastNodeURL);
 545                         preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 546                         $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
 547
 548                         // Search for the correct state file - uses file timestamps so need to sort by date descending
 549                         $sRepURL = CONST_Replication_Url."/";
 550                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 551                         // download.geofabrik.de:    <a href="000/">000/</a></td><td align="right">26-Feb-2013 11:53  </td>
 552                         // planet.openstreetmap.org: <a href="273/">273/</a>                    22-Mar-2013 07:41    -
 553                         preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 554                         $aPrevRepMatch = false;
 555                         foreach($aRepMatches as $aRepMatch)
 556                         {
 557                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 558                                 $aPrevRepMatch = $aRepMatch;
 559                         }
 560                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 561
 562                         $sRepURL .= $aRepMatch[1];
 563                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 564                         preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 565                         $aPrevRepMatch = false;
 566                         foreach($aRepMatches as $aRepMatch)
 567                         {
 568                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 569                                 $aPrevRepMatch = $aRepMatch;
 570                         }
 571                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 572
 573                         $sRepURL .= $aRepMatch[1];
 574                         $sRep = file_get_contents($sRepURL."?C=M;O=D");
 575                         preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 576                         $aPrevRepMatch = false;
 577                         foreach($aRepMatches as $aRepMatch)
 578                         {
 579                                 if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 580                                 $aPrevRepMatch = $aRepMatch;
 581                         }
 582                         if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 583
 584                         $sRepURL .= $aRepMatch[1].'.state.txt';
 585                         echo "Getting state file: $sRepURL\n";
 586                         $sStateFile = file_get_contents($sRepURL);
 587                         if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 588                         file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 589                         echo "Updating DB status\n";
 590                         pg_query($oDB->connection, 'TRUNCATE import_status');
 591                         $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
 592                         pg_query($oDB->connection, $sSQL);
 593                 }
 594         }
 595
 596         if ($aCMDResult['index'] || $aCMDResult['all'])
 597         {
 598                 $bDidSomething = true;
 599                 $sOutputFile = '';
 600                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 601                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
 602                 passthruCheckReturn($sBaseCmd.' -R 4');
 603                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 604                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 605                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 606                 passthruCheckReturn($sBaseCmd.' -r 26');
 607         }
 608
 609         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 610         {
 611                 echo "Search indices\n";
 612                 $bDidSomething = true;
 613                 $oDB =& getDB();
 614                 $sSQL = 'select distinct partition from country_name';
 615                 $aPartitions = $oDB->getCol($sSQL);
 616                 if (PEAR::isError($aPartitions))
 617                 {
 618                         fail($aPartitions->getMessage());
 619                 }
 620                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 621
 622                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 623                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 624                 foreach($aMatches as $aMatch)
 625                 {
 626                         $sResult = '';
 627                         foreach($aPartitions as $sPartitionName)
 628                         {
 629                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 630                         }
 631                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 632                 }
 633
 634                 pgsqlRunScript($sTemplate);
 635         }
 636
 637         if (isset($aCMDResult['create-website']))
 638         {
 639                 $bDidSomething = true;
 640                 $sTargetDir = $aCMDResult['create-website'];
 641                 if (!is_dir($sTargetDir))
 642                 {
 643                         echo "You must create the website directory before calling this function.\n";
 644                         fail("Target directory does not exist.");
 645                 }
 646
 647                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 648                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 649                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 650                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 651                 @symlink(CONST_BasePath.'/website/deletable.php', $sTargetDir.'/deletable.php');
 652                 @symlink(CONST_BasePath.'/website/polygons.php', $sTargetDir.'/polygons.php');
 653                 @symlink(CONST_BasePath.'/website/status.php', $sTargetDir.'/status.php');
 654                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 655                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 656                 @symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css');
 657                 echo "Symlinks created\n";
 658
 659                 $sTestFile = @file_get_contents(CONST_Website_BaseURL.'js/tiles.js');
 660                 if (!$sTestFile)
 661                 {
 662                         echo "\nWARNING: Unable to access the website at ".CONST_Website_BaseURL."\n";
 663                         echo "You may want to update settings/local.php with @define('CONST_Website_BaseURL', 'http://[HOST]/[PATH]/');\n";
 664                 }
 665         }
 666
 667         if (!$bDidSomething)
 668         {
 669                 showUsage($aCMDOptions, true);
 670         }
 671
 672         function pgsqlRunScriptFile($sFilename)
 673         {
 674                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 675
 676                 // Convert database DSN to psql parameters
 677                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 678                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 679                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 680
 681                 $aDescriptors = array(
 682                         0 => array('pipe', 'r'),
 683                         1 => array('pipe', 'w'),
 684                         2 => array('file', '/dev/null', 'a')
 685                 );
 686                 $ahPipes = null;
 687                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 688                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 689
 690                 fclose($ahPipes[0]);
 691
 692                 // TODO: error checking
 693                 while(!feof($ahPipes[1]))
 694                 {
 695                         echo fread($ahPipes[1], 4096);
 696                 }
 697                 fclose($ahPipes[1]);
 698
 699                 proc_close($hProcess);
 700         }
 701
 702         function pgsqlRunScript($sScript)
 703         {
 704                 // Convert database DSN to psql parameters
 705                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 706                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 707                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 708                 $aDescriptors = array(
 709                         0 => array('pipe', 'r'),
 710                         1 => STDOUT,
 711                         2 => STDERR
 712                 );
 713                 $ahPipes = null;
 714                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 715                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 716
 717                 while(strlen($sScript))
 718                 {
 719                         $written = fwrite($ahPipes[0], $sScript);
 720                         $sScript = substr($sScript, $written);
 721                 }
 722                 fclose($ahPipes[0]);
 723                 proc_close($hProcess);
 724         }
 725
 726         function pgsqlRunRestoreData($sDumpFile)
 727         {
 728                 // Convert database DSN to psql parameters
 729                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 730                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 731                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 732
 733                 $aDescriptors = array(
 734                         0 => array('pipe', 'r'),
 735                         1 => array('pipe', 'w'),
 736                         2 => array('file', '/dev/null', 'a')
 737                 );
 738                 $ahPipes = null;
 739                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 740                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 741
 742                 fclose($ahPipes[0]);
 743
 744                 // TODO: error checking
 745                 while(!feof($ahPipes[1]))
 746                 {
 747                         echo fread($ahPipes[1], 4096);
 748                 }
 749                 fclose($ahPipes[1]);
 750
 751                 proc_close($hProcess);
 752         }
 753
 754         function pgsqlRunDropAndRestore($sDumpFile)
 755         {
 756                 // Convert database DSN to psql parameters
 757                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 758                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 759                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 760
 761                 $aDescriptors = array(
 762                         0 => array('pipe', 'r'),
 763                         1 => array('pipe', 'w'),
 764                         2 => array('file', '/dev/null', 'a')
 765                 );
 766                 $ahPipes = null;
 767                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 768                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 769
 770                 fclose($ahPipes[0]);
 771
 772                 // TODO: error checking
 773                 while(!feof($ahPipes[1]))
 774                 {
 775                         echo fread($ahPipes[1], 4096);
 776                 }
 777                 fclose($ahPipes[1]);
 778
 779                 proc_close($hProcess);
 780         }
 781
 782         function passthruCheckReturn($cmd)
 783         {
 784                 $result = -1;
 785                 passthru($cmd, $result);
 786                 if ($result != 0) fail('Error executing external command: '.$cmd);
 787         }