utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  36                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  37                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  38                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  39                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  40         );
  41         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  42
  43         $bDidSomething = false;
  44
  45         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  46         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  47         {
  48                 if (!isset($aCMDResult['osm-file']))
  49                 {
  50                         fail('missing --osm-file for data import');
  51                 }
  52
  53                 if (!file_exists($aCMDResult['osm-file']))
  54                 {
  55                         fail('the path supplied to --osm-file does not exist');
  56                 }
  57
  58                 if (!is_readable($aCMDResult['osm-file']))
  59                 {
  60                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  61                 }
  62         }
  63
  64
  65         // This is a pretty hard core default - the number of processors in the box - 1
  66         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  67         if ($iInstances < 1)
  68         {
  69                 $iInstances = 1;
  70                 echo "WARNING: resetting threads to $iInstances\n";
  71         }
  72         if ($iInstances > getProcessorCount())
  73         {
  74                 $iInstances = getProcessorCount();
  75                 echo "WARNING: resetting threads to $iInstances\n";
  76         }
  77
  78         // Assume we can steal all the cache memory in the box (unless told otherwise)
  79         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  80         if ($iCacheMemory > getTotalMemoryMB())
  81         {
  82                 $iCacheMemory = getCacheMemoryMB();
  83                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  84         }
  85
  86         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  87         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  88
  89         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  90         {
  91                 echo "Create DB\n";
  92                 $bDidSomething = true;
  93                 $oDB =& DB::connect(CONST_Database_DSN, false);
  94                 if (!PEAR::isError($oDB))
  95                 {
  96                         fail('database already exists ('.CONST_Database_DSN.')');
  97                 }
  98                 passthruCheckReturn('createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
  99         }
 100
 101         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 102         {
 103                 echo "Setup DB\n";
 104                 $bDidSomething = true;
 105                 // TODO: path detection, detection memory, etc.
 106
 107                 $oDB =& getDB();
 108
 109                 $sVersionString = $oDB->getOne('select version()');
 110                 preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[.]([0-9]+) #', $sVersionString, $aMatches);
 111                 if (CONST_Postgresql_Version != $aMatches[1].'.'.$aMatches[2])
 112                 {
 113                         echo "ERROR: PostgreSQL version is not correct.  Expected ".CONST_Postgresql_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 114                         exit;
 115                 }
 116
 117                 passthru('createlang plpgsql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 118                 $pgver = (float) CONST_Postgresql_Version;
 119                 if ($pgver < 9.1) {
 120                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 121                         pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
 122                 } else {
 123                         pgsqlRunScript('CREATE EXTENSION hstore');
 124                 }
 125
 126                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 127                 $sVersionString = $oDB->getOne('select postgis_full_version()');
 128                 preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
 129                 if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
 130                 {
 131                         echo "ERROR: PostGIS version is not correct.  Expected ".CONST_Postgis_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 132                         exit;
 133                 }
 134
 135                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 136                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 137                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 138                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 139                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 140                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 141                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 142                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 143                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 144         }
 145
 146         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 147         {
 148                 echo "Import\n";
 149                 $bDidSomething = true;
 150
 151                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 152                 if (!file_exists($osm2pgsql))
 153                 {
 154                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 155                         fail("osm2pgsql not found in '$osm2pgsql'");
 156                 }
 157                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 158                 $osm2pgsql .= ' -C '.$iCacheMemory;
 159                 $osm2pgsql .= ' -P '.$aDSNInfo['port'];
 160                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 161                 passthruCheckReturn($osm2pgsql);
 162
 163                 $oDB =& getDB();
 164                 $x = $oDB->getRow('select * from place limit 1');
 165                 if (PEAR::isError($x)) {
 166                         fail($x->getMessage());
 167                 }
 168                 if (!$x) fail('No Data');
 169         }
 170
 171         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 172         {
 173                 echo "Functions\n";
 174                 $bDidSomething = true;
 175                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 176                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 177                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 178                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 179                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 180                 pgsqlRunScript($sTemplate);
 181         }
 182
 183         if ($aCMDResult['create-minimal-tables'])
 184         {
 185                 echo "Minimal Tables\n";
 186                 $bDidSomething = true;
 187                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 188
 189                 $sScript = '';
 190
 191                 // Backstop the import process - easliest possible import id
 192                 $sScript .= "insert into import_npi_log values (18022);\n";
 193
 194                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 195                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 196
 197                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 198                 {
 199                         list($sClass, $sType) = explode(' ', trim($sLine));
 200                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 201                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 202
 203                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 204                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 205
 206                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 207                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 208                 }
 209                 fclose($hFile);
 210                 pgsqlRunScript($sScript);
 211         }
 212
 213         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 214         {
 215                 echo "Tables\n";
 216                 $bDidSomething = true;
 217                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 218
 219                 // re-run the functions
 220                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 221                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 222                 pgsqlRunScript($sTemplate);
 223         }
 224
 225         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 226         {
 227                 echo "Partitions\n";
 228                 $bDidSomething = true;
 229                 $oDB =& getDB();
 230                 $sSQL = 'select partition from country_name order by country_code';
 231                 $aPartitions = $oDB->getCol($sSQL);
 232                 if (PEAR::isError($aPartitions))
 233                 {
 234                         fail($aPartitions->getMessage());
 235                 }
 236                 $aPartitions[] = 0;
 237
 238                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 239                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 240                 foreach($aMatches as $aMatch)
 241                 {
 242                         $sResult = '';
 243                         foreach($aPartitions as $sPartitionName)
 244                         {
 245                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 246                         }
 247                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 248                 }
 249
 250                 pgsqlRunScript($sTemplate);
 251         }
 252
 253         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 254         {
 255                 $bDidSomething = true;
 256                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 257                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 258                 if (file_exists($sWikiArticlesFile))
 259                 {
 260                         echo "Importing wikipedia articles...";
 261                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 262                         echo "...done\n";
 263                 }
 264                 else
 265                 {
 266                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 267                 }
 268                 if (file_exists($sWikiRedirectsFile))
 269                 {
 270                         echo "Importing wikipedia redirects...";
 271                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 272                         echo "...done\n";
 273                 }
 274                 else
 275                 {
 276                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 277                 }
 278         }
 279
 280
 281         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 282         {
 283                 echo "Drop old Data\n";
 284                 $bDidSomething = true;
 285
 286                 $oDB =& getDB();
 287                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 288                 echo '.';
 289                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 290                 echo '.';
 291                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 292                 echo '.';
 293                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 294                 echo '.';
 295                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 296                 echo '.';
 297                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 298                 echo '.';
 299                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 300                 echo '.';
 301                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 302                 echo '.';
 303                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 304                 echo '.';
 305
 306                 $sSQL = 'select partition from country_name order by country_code';
 307                 $aPartitions = $oDB->getCol($sSQL);
 308                 if (PEAR::isError($aPartitions))
 309                 {
 310                         fail($aPartitions->getMessage());
 311                 }
 312                 $aPartitions[] = 0;
 313                 foreach($aPartitions as $sPartition)
 314                 {
 315                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 316                         echo '.';
 317                 }
 318
 319                 // used by getorcreate_word_id to ignore frequent partial words
 320                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 321                 echo ".\n";
 322
 323                 // pre-create the word list
 324                 if (!$aCMDResult['disable-token-precalc'])
 325                 {
 326                         echo "Loading word list\n";
 327                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 328                 }
 329
 330                 echo "Load Data\n";
 331                 $aDBInstances = array();
 332                 for($i = 0; $i < $iInstances; $i++)
 333                 {
 334                         $aDBInstances[$i] =& getDB(true);
 335                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 336                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 337                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 338                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 339                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 340                 }
 341                 $bAnyBusy = true;
 342                 while($bAnyBusy)
 343                 {
 344                         $bAnyBusy = false;
 345                         for($i = 0; $i < $iInstances; $i++)
 346                         {
 347                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 348                         }
 349                         sleep(1);
 350                         echo '.';
 351                 }
 352                 echo "\n";
 353                 echo "Reanalysing database...\n";
 354                 pgsqlRunScript('ANALYSE');
 355         }
 356
 357         if ($aCMDResult['create-roads'])
 358         {
 359                 $bDidSomething = true;
 360
 361                 $oDB =& getDB();
 362                 $aDBInstances = array();
 363                 for($i = 0; $i < $iInstances; $i++)
 364                 {
 365                         $aDBInstances[$i] =& getDB(true);
 366                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 367                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
 368                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 369                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 370                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 371                 }
 372                 $bAnyBusy = true;
 373                 while($bAnyBusy)
 374                 {
 375                         $bAnyBusy = false;
 376                         for($i = 0; $i < $iInstances; $i++)
 377                         {
 378                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 379                         }
 380                         sleep(1);
 381                         echo '.';
 382                 }
 383                 echo "\n";
 384         }
 385
 386         if ($aCMDResult['import-tiger-data'])
 387         {
 388                 $bDidSomething = true;
 389
 390                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 391
 392                 $aDBInstances = array();
 393                 for($i = 0; $i < $iInstances; $i++)
 394                 {
 395                         $aDBInstances[$i] =& getDB(true);
 396                 }
 397
 398                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 399                 {
 400                         echo $sFile.': ';
 401                         $hFile = fopen($sFile, "r");
 402                         $sSQL = fgets($hFile, 100000);
 403                         $iLines = 0;
 404
 405                         while(true)
 406                         {
 407                                 for($i = 0; $i < $iInstances; $i++)
 408                                 {
 409                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 410                                         {
 411                                                 while(pg_get_result($aDBInstances[$i]->connection));
 412                                                 $sSQL = fgets($hFile, 100000);
 413                                                 if (!$sSQL) break 2;
 414                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 415                                                 $iLines++;
 416                                                 if ($iLines == 1000)
 417                                                 {
 418                                                         echo ".";
 419                                                         $iLines = 0;
 420                                                 }
 421                                         }
 422                                 }
 423                                 usleep(10);
 424                         }
 425
 426                         fclose($hFile);
 427
 428                         $bAnyBusy = true;
 429                         while($bAnyBusy)
 430                         {
 431                                 $bAnyBusy = false;
 432                                 for($i = 0; $i < $iInstances; $i++)
 433                                 {
 434                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 435                                 }
 436                                 usleep(10);
 437                         }
 438                         echo "\n";
 439                 }
 440
 441                 echo "Creating indexes\n";
 442                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 443         }
 444
 445         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 446         {
 447                 $bDidSomething = true;
 448                 $oDB =& getDB();
 449                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 450                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 451                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 452                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 453                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 454                 $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
 455                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 456
 457                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 458                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 459                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 460                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 461         }
 462
 463         if ($aCMDResult['osmosis-init'] || $aCMDResult['all'])
 464         {
 465                 $bDidSomething = true;
 466                 $oDB =& getDB();
 467
 468                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 469                 if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
 470                 {
 471                         echo "settings/configuration.txt already exists\n";
 472                 }
 473                 else
 474                 {
 475                         passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 476                         // server layout changed afer license change, fix path to minutely diffs
 477                         passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
 478                 }
 479
 480                 // Find the last node in the DB
 481                 $iLastOSMID = $oDB->getOne("select max(osm_id) as osm_id from place where osm_type = 'N'");
 482
 483                 // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
 484                 $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID;
 485                 $sLastNodeXML = file_get_contents($sLastNodeURL);
 486                 preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 487                 $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
 488
 489
 490                 // Search for the correct state file - uses file timestamps
 491                 $sRepURL = 'http://planet.openstreetmap.org/replication/minute/';
 492                 $sRep = file_get_contents($sRepURL);
 493                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 494                 $aPrevRepMatch = false;
 495                 foreach($aRepMatches as $aRepMatch)
 496                 {
 497                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 498                         $aPrevRepMatch = $aRepMatch;
 499                 }
 500                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 501
 502                 $sRepURL .= $aRepMatch[1];
 503                 $sRep = file_get_contents($sRepURL);
 504                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 505                 $aPrevRepMatch = false;
 506                 foreach($aRepMatches as $aRepMatch)
 507                 {
 508                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 509                         $aPrevRepMatch = $aRepMatch;
 510                 }
 511                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 512
 513                 $sRepURL .= $aRepMatch[1];
 514                 $sRep = file_get_contents($sRepURL);
 515                 preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 516                 $aPrevRepMatch = false;
 517                 foreach($aRepMatches as $aRepMatch)
 518                 {
 519                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 520                         $aPrevRepMatch = $aRepMatch;
 521                 }
 522                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 523
 524                 $sRepURL .= $aRepMatch[1].'.state.txt';
 525                 echo "Getting state file: $sRepURL\n";
 526                 $sStateFile = file_get_contents($sRepURL);
 527                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 528                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 529                 echo "Updating DB status\n";
 530                 pg_query($oDB->connection, 'TRUNCATE import_status');
 531                 $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
 532                 pg_query($oDB->connection, $sSQL);
 533         }
 534
 535         if ($aCMDResult['index'] || $aCMDResult['all'])
 536         {
 537                 $bDidSomething = true;
 538                 $sOutputFile = '';
 539                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 540                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
 541                 passthruCheckReturn($sBaseCmd.' -R 4');
 542                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 543                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 544                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 545                 passthruCheckReturn($sBaseCmd.' -r 26');
 546         }
 547
 548         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 549         {
 550                 echo "Search indices\n";
 551                 $bDidSomething = true;
 552                 $oDB =& getDB();
 553                 $sSQL = 'select partition from country_name order by country_code';
 554                 $aPartitions = $oDB->getCol($sSQL);
 555                 if (PEAR::isError($aPartitions))
 556                 {
 557                         fail($aPartitions->getMessage());
 558                 }
 559                 $aPartitions[] = 0;
 560
 561                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 562                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 563                 foreach($aMatches as $aMatch)
 564                 {
 565                         $sResult = '';
 566                         foreach($aPartitions as $sPartitionName)
 567                         {
 568                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 569                         }
 570                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 571                 }
 572
 573                 pgsqlRunScript($sTemplate);
 574         }
 575
 576         if (isset($aCMDResult['create-website']))
 577         {
 578                 $bDidSomething = true;
 579                 $sTargetDir = $aCMDResult['create-website'];
 580                 if (!is_dir($sTargetDir))
 581                 {
 582                         echo "You must create the website directory before calling this function.\n";
 583                         fail("Target directory does not exist.");
 584                 }
 585
 586                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 587                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 588                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 589                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 590                 @symlink(CONST_BasePath.'/website/deletable.php', $sTargetDir.'/deletable.php');
 591                 @symlink(CONST_BasePath.'/website/polygons.php', $sTargetDir.'/polygons.php');
 592                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 593                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 594                 @symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css');
 595                 echo "Symlinks created\n";
 596         }
 597
 598         if (!$bDidSomething)
 599         {
 600                 showUsage($aCMDOptions, true);
 601         }
 602
 603         function pgsqlRunScriptFile($sFilename)
 604         {
 605                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 606
 607                 // Convert database DSN to psql parameters
 608                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 609                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 610                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 611
 612                 $aDescriptors = array(
 613                         0 => array('pipe', 'r'),
 614                         1 => array('pipe', 'w'),
 615                         2 => array('file', '/dev/null', 'a')
 616                 );
 617                 $ahPipes = null;
 618                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 619                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 620
 621                 fclose($ahPipes[0]);
 622
 623                 // TODO: error checking
 624                 while(!feof($ahPipes[1]))
 625                 {
 626                         echo fread($ahPipes[1], 4096);
 627                 }
 628                 fclose($ahPipes[1]);
 629
 630                 proc_close($hProcess);
 631         }
 632
 633         function pgsqlRunScript($sScript)
 634         {
 635                 // Convert database DSN to psql parameters
 636                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 637                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 638                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 639                 $aDescriptors = array(
 640                         0 => array('pipe', 'r'),
 641                         1 => STDOUT,
 642                         2 => STDERR
 643                 );
 644                 $ahPipes = null;
 645                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 646                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 647
 648                 while(strlen($sScript))
 649                 {
 650                         $written = fwrite($ahPipes[0], $sScript);
 651                         $sScript = substr($sScript, $written);
 652                 }
 653                 fclose($ahPipes[0]);
 654                 proc_close($hProcess);
 655         }
 656
 657         function pgsqlRunRestoreData($sDumpFile)
 658         {
 659                 // Convert database DSN to psql parameters
 660                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 661                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 662                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 663
 664                 $aDescriptors = array(
 665                         0 => array('pipe', 'r'),
 666                         1 => array('pipe', 'w'),
 667                         2 => array('file', '/dev/null', 'a')
 668                 );
 669                 $ahPipes = null;
 670                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 671                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 672
 673                 fclose($ahPipes[0]);
 674
 675                 // TODO: error checking
 676                 while(!feof($ahPipes[1]))
 677                 {
 678                         echo fread($ahPipes[1], 4096);
 679                 }
 680                 fclose($ahPipes[1]);
 681
 682                 proc_close($hProcess);
 683         }
 684
 685         function pgsqlRunDropAndRestore($sDumpFile)
 686         {
 687                 // Convert database DSN to psql parameters
 688                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 689                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 690                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 691
 692                 $aDescriptors = array(
 693                         0 => array('pipe', 'r'),
 694                         1 => array('pipe', 'w'),
 695                         2 => array('file', '/dev/null', 'a')
 696                 );
 697                 $ahPipes = null;
 698                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 699                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 700
 701                 fclose($ahPipes[0]);
 702
 703                 // TODO: error checking
 704                 while(!feof($ahPipes[1]))
 705                 {
 706                         echo fread($ahPipes[1], 4096);
 707                 }
 708                 fclose($ahPipes[1]);
 709
 710                 proc_close($hProcess);
 711         }
 712
 713         function passthruCheckReturn($cmd)
 714         {
 715                 $result = -1;
 716                 passthru($cmd, $result);
 717                 if ($result != 0) fail('Error executing external command: '.$cmd);
 718         }