utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  36                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  37                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  38                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  39                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  40         );
  41         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  42
  43         $bDidSomething = false;
  44
  45         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  46         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  47         {
  48                 if (!isset($aCMDResult['osm-file']))
  49                 {
  50                         fail('missing --osm-file for data import');
  51                 }
  52
  53                 if (!file_exists($aCMDResult['osm-file']))
  54                 {
  55                         fail('the path supplied to --osm-file does not exist');
  56                 }
  57
  58                 if (!is_readable($aCMDResult['osm-file']))
  59                 {
  60                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  61                 }
  62         }
  63
  64
  65         // This is a pretty hard core default - the number of processors in the box - 1
  66         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  67         if ($iInstances < 1)
  68         {
  69                 $iInstances = 1;
  70                 echo "WARNING: resetting threads to $iInstances\n";
  71         }
  72         if ($iInstances > getProcessorCount())
  73         {
  74                 $iInstances = getProcessorCount();
  75                 echo "WARNING: resetting threads to $iInstances\n";
  76         }
  77
  78         // Assume we can steal all the cache memory in the box (unless told otherwise)
  79         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  80         if ($iCacheMemory > getTotalMemoryMB())
  81         {
  82                 $iCacheMemory = getCacheMemoryMB();
  83                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  84         }
  85
  86         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  87         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  88
  89         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  90         {
  91                 echo "Create DB\n";
  92                 $bDidSomething = true;
  93                 $oDB =& DB::connect(CONST_Database_DSN, false);
  94                 if (!PEAR::isError($oDB))
  95                 {
  96                         fail('database already exists ('.CONST_Database_DSN.')');
  97                 }
  98                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
  99         }
 100
 101         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 102         {
 103                 echo "Setup DB\n";
 104                 $bDidSomething = true;
 105                 // TODO: path detection, detection memory, etc.
 106
 107                 $oDB =& getDB();
 108                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 109                 $pgver = (float) CONST_Postgresql_Version;
 110                 if ($pgver < 9.1) {
 111                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 112                 } else {
 113                         pgsqlRunScript('CREATE EXTENSION hstore');
 114                 }
 115                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 116                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 117                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 118                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 119                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 120                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 121                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 122                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 123                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 124                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 125         }
 126
 127         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 128         {
 129                 echo "Import\n";
 130                 $bDidSomething = true;
 131
 132                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 133                 if (!file_exists($osm2pgsql))
 134                 {
 135                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 136                         fail("osm2pgsql not found in '$osm2pgsql'");
 137                 }
 138                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 139                 $osm2pgsql .= ' -C '.$iCacheMemory;
 140                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 141                 passthruCheckReturn($osm2pgsql);
 142
 143                 $oDB =& getDB();
 144                 $x = $oDB->getRow('select * from place limit 1');
 145                 if (PEAR::isError($x)) {
 146                         fail($x->getMessage());
 147                 }
 148                 if (!$x) fail('No Data');
 149         }
 150
 151         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 152         {
 153                 echo "Functions\n";
 154                 $bDidSomething = true;
 155                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 156                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 157                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 158                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 159                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 160                 pgsqlRunScript($sTemplate);
 161         }
 162
 163         if ($aCMDResult['create-minimal-tables'])
 164         {
 165                 echo "Minimal Tables\n";
 166                 $bDidSomething = true;
 167                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 168
 169                 $sScript = '';
 170
 171                 // Backstop the import process - easliest possible import id
 172                 $sScript .= "insert into import_npi_log values (18022);\n";
 173
 174                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 175                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 176
 177                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 178                 {
 179                         list($sClass, $sType) = explode(' ', trim($sLine));
 180                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 181                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 182
 183                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 184                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 185
 186                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 187                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 188                 }
 189                 fclose($hFile);
 190                 pgsqlRunScript($sScript);
 191         }
 192
 193         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 194         {
 195                 echo "Tables\n";
 196                 $bDidSomething = true;
 197                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 198
 199                 // re-run the functions
 200                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 201                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 202                 pgsqlRunScript($sTemplate);
 203         }
 204
 205         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 206         {
 207                 echo "Partitions\n";
 208                 $bDidSomething = true;
 209                 $oDB =& getDB();
 210                 $sSQL = 'select partition from country_name order by country_code';
 211                 $aPartitions = $oDB->getCol($sSQL);
 212                 if (PEAR::isError($aPartitions))
 213                 {
 214                         fail($aPartitions->getMessage());
 215                 }
 216                 $aPartitions[] = 0;
 217
 218                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 219                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 220                 foreach($aMatches as $aMatch)
 221                 {
 222                         $sResult = '';
 223                         foreach($aPartitions as $sPartitionName)
 224                         {
 225                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 226                         }
 227                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 228                 }
 229
 230                 pgsqlRunScript($sTemplate);
 231         }
 232
 233         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 234         {
 235                 $bDidSomething = true;
 236                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 237                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 238                 if (file_exists($sWikiArticlesFile))
 239                 {
 240                         echo "Importing wikipedia articles...";
 241                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 242                         echo "...done\n";
 243                 }
 244                 else
 245                 {
 246                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 247                 }
 248                 if (file_exists($sWikiRedirectsFile))
 249                 {
 250                         echo "Importing wikipedia redirects...";
 251                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 252                         echo "...done\n";
 253                 }
 254                 else
 255                 {
 256                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 257                 }
 258         }
 259
 260
 261         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 262         {
 263                 echo "Drop old Data\n";
 264                 $bDidSomething = true;
 265
 266                 $oDB =& getDB();
 267                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 268                 echo '.';
 269                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 270                 echo '.';
 271                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 272                 echo '.';
 273                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 274                 echo '.';
 275                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 276                 echo '.';
 277                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 278                 echo '.';
 279                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 280                 echo '.';
 281                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 282                 echo '.';
 283                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 284                 echo '.';
 285
 286                 $sSQL = 'select partition from country_name order by country_code';
 287                 $aPartitions = $oDB->getCol($sSQL);
 288                 if (PEAR::isError($aPartitions))
 289                 {
 290                         fail($aPartitions->getMessage());
 291                 }
 292                 $aPartitions[] = 0;
 293                 foreach($aPartitions as $sPartition)
 294                 {
 295                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 296                         echo '.';
 297                 }
 298
 299                 // used by getorcreate_word_id to ignore frequent partial words
 300                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 301                 echo ".\n";
 302
 303                 // pre-create the word list
 304                 if (!$aCMDResult['disable-token-precalc'])
 305                 {
 306                         echo "Loading word list\n";
 307                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 308                 }
 309
 310                 echo "Load Data\n";
 311                 $aDBInstances = array();
 312                 for($i = 0; $i < $iInstances; $i++)
 313                 {
 314                         $aDBInstances[$i] =& getDB(true);
 315                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 316                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 317                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 318                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 319                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 320                 }
 321                 $bAnyBusy = true;
 322                 while($bAnyBusy)
 323                 {
 324                         $bAnyBusy = false;
 325                         for($i = 0; $i < $iInstances; $i++)
 326                         {
 327                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 328                         }
 329                         sleep(1);
 330                         echo '.';
 331                 }
 332                 echo "\n";
 333                 echo "Reanalysing database...\n";
 334                 pgsqlRunScript('ANALYSE');
 335         }
 336
 337         if ($aCMDResult['create-roads'])
 338         {
 339                 $bDidSomething = true;
 340
 341                 $oDB =& getDB();
 342                 $aDBInstances = array();
 343                 for($i = 0; $i < $iInstances; $i++)
 344                 {
 345                         $aDBInstances[$i] =& getDB(true);
 346                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 347                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
 348                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 349                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 350                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 351                 }
 352                 $bAnyBusy = true;
 353                 while($bAnyBusy)
 354                 {
 355                         $bAnyBusy = false;
 356                         for($i = 0; $i < $iInstances; $i++)
 357                         {
 358                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 359                         }
 360                         sleep(1);
 361                         echo '.';
 362                 }
 363                 echo "\n";
 364         }
 365
 366         if ($aCMDResult['import-tiger-data'])
 367         {
 368                 $bDidSomething = true;
 369
 370                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 371
 372                 $aDBInstances = array();
 373                 for($i = 0; $i < $iInstances; $i++)
 374                 {
 375                         $aDBInstances[$i] =& getDB(true);
 376                 }
 377
 378                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 379                 {
 380                         echo $sFile.': ';
 381                         $hFile = fopen($sFile, "r");
 382                         $sSQL = fgets($hFile, 100000);
 383                         $iLines = 0;
 384
 385                         while(true)
 386                         {
 387                                 for($i = 0; $i < $iInstances; $i++)
 388                                 {
 389                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 390                                         {
 391                                                 while(pg_get_result($aDBInstances[$i]->connection));
 392                                                 $sSQL = fgets($hFile, 100000);
 393                                                 if (!$sSQL) break 2;
 394                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 395                                                 $iLines++;
 396                                                 if ($iLines == 1000)
 397                                                 {
 398                                                         echo ".";
 399                                                         $iLines = 0;
 400                                                 }
 401                                         }
 402                                 }
 403                                 usleep(10);
 404                         }
 405
 406                         fclose($hFile);
 407
 408                         $bAnyBusy = true;
 409                         while($bAnyBusy)
 410                         {
 411                                 $bAnyBusy = false;
 412                                 for($i = 0; $i < $iInstances; $i++)
 413                                 {
 414                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 415                                 }
 416                                 usleep(10);
 417                         }
 418                         echo "\n";
 419                 }
 420
 421                 echo "Creating indexes\n";
 422                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 423         }
 424
 425         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 426         {
 427                 $bDidSomething = true;
 428                 $oDB =& getDB();
 429                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 430                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 431                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 432                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 433                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 434                 $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
 435                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 436
 437                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 438                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 439                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 440                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 441         }
 442
 443         if ($aCMDResult['osmosis-init'] || $aCMDResult['all'])
 444         {
 445                 $bDidSomething = true;
 446                 $oDB =& getDB();
 447
 448                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 449                 if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
 450                 {
 451                         echo "settings/configuration.txt already exists\n";
 452                 }
 453                 else
 454                 {
 455                         passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 456                         // server layout changed afer license change, fix path to minutely diffs
 457                         passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
 458                 }
 459
 460                 // Find the last node in the DB
 461                 $iLastOSMID = $oDB->getOne("select max(osm_id) as osm_id from place where osm_type = 'N'");
 462
 463                 // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
 464                 $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID;
 465                 $sLastNodeXML = file_get_contents($sLastNodeURL);
 466                 preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 467                 $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
 468
 469
 470                 // Search for the correct state file - uses file timestamps
 471                 $sRepURL = 'http://planet.openstreetmap.org/replication/minute/';
 472                 $sRep = file_get_contents($sRepURL);
 473                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 474                 $aPrevRepMatch = false;
 475                 foreach($aRepMatches as $aRepMatch)
 476                 {
 477                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 478                         $aPrevRepMatch = $aRepMatch;
 479                 }
 480                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 481
 482                 $sRepURL .= $aRepMatch[1];
 483                 $sRep = file_get_contents($sRepURL);
 484                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 485                 $aPrevRepMatch = false;
 486                 foreach($aRepMatches as $aRepMatch)
 487                 {
 488                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 489                         $aPrevRepMatch = $aRepMatch;
 490                 }
 491                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 492
 493                 $sRepURL .= $aRepMatch[1];
 494                 $sRep = file_get_contents($sRepURL);
 495                 preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
 496                 $aPrevRepMatch = false;
 497                 foreach($aRepMatches as $aRepMatch)
 498                 {
 499                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 500                         $aPrevRepMatch = $aRepMatch;
 501                 }
 502                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 503
 504                 $sRepURL .= $aRepMatch[1].'.state.txt';
 505                 echo "Getting state file: $sRepURL\n";
 506                 $sStateFile = file_get_contents($sRepURL);
 507                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 508                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 509                 echo "Updating DB status\n";
 510                 pg_query($oDB->connection, 'TRUNCATE import_status');
 511                 $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
 512                 pg_query($oDB->connection, $sSQL);
 513                 exit;
 514         }
 515
 516         if ($aCMDResult['index'] || $aCMDResult['all'])
 517         {
 518                 $bDidSomething = true;
 519                 $sOutputFile = '';
 520                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 521                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 522                 passthruCheckReturn($sBaseCmd.' -R 4');
 523                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 524                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 525                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 526                 passthruCheckReturn($sBaseCmd.' -r 26');
 527         }
 528
 529         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 530         {
 531                 echo "Search indices\n";
 532                 $bDidSomething = true;
 533                 $oDB =& getDB();
 534                 $sSQL = 'select partition from country_name order by country_code';
 535                 $aPartitions = $oDB->getCol($sSQL);
 536                 if (PEAR::isError($aPartitions))
 537                 {
 538                         fail($aPartitions->getMessage());
 539                 }
 540                 $aPartitions[] = 0;
 541
 542                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 543                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 544                 foreach($aMatches as $aMatch)
 545                 {
 546                         $sResult = '';
 547                         foreach($aPartitions as $sPartitionName)
 548                         {
 549                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 550                         }
 551                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 552                 }
 553
 554                 pgsqlRunScript($sTemplate);
 555         }
 556
 557         if (isset($aCMDResult['create-website']))
 558         {
 559                 $bDidSomething = true;
 560                 $sTargetDir = $aCMDResult['create-website'];
 561                 if (!is_dir($sTargetDir))
 562                 {
 563                         echo "You must create the website directory before calling this function.\n";
 564                         fail("Target directory does not exist.");
 565                 }
 566
 567                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 568                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 569                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 570                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 571                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 572                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 573                 echo "Symlinks created\n";
 574         }
 575
 576         if (!$bDidSomething)
 577         {
 578                 showUsage($aCMDOptions, true);
 579         }
 580
 581         function pgsqlRunScriptFile($sFilename)
 582         {
 583                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 584
 585                 // Convert database DSN to psql parameters
 586                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 587                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 588                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 589
 590                 $aDescriptors = array(
 591                         0 => array('pipe', 'r'),
 592                         1 => array('pipe', 'w'),
 593                         2 => array('file', '/dev/null', 'a')
 594                 );
 595                 $ahPipes = null;
 596                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 597                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 598
 599                 fclose($ahPipes[0]);
 600
 601                 // TODO: error checking
 602                 while(!feof($ahPipes[1]))
 603                 {
 604                         echo fread($ahPipes[1], 4096);
 605                 }
 606                 fclose($ahPipes[1]);
 607
 608                 proc_close($hProcess);
 609         }
 610
 611         function pgsqlRunScript($sScript)
 612         {
 613                 // Convert database DSN to psql parameters
 614                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 615                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 616                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 617                 $aDescriptors = array(
 618                         0 => array('pipe', 'r'),
 619                         1 => STDOUT,
 620                         2 => STDERR
 621                 );
 622                 $ahPipes = null;
 623                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 624                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 625
 626                 while(strlen($sScript))
 627                 {
 628                         $written = fwrite($ahPipes[0], $sScript);
 629                         $sScript = substr($sScript, $written);
 630                 }
 631                 fclose($ahPipes[0]);
 632                 proc_close($hProcess);
 633         }
 634
 635         function pgsqlRunRestoreData($sDumpFile)
 636         {
 637                 // Convert database DSN to psql parameters
 638                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 639                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 640                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 641
 642                 $aDescriptors = array(
 643                         0 => array('pipe', 'r'),
 644                         1 => array('pipe', 'w'),
 645                         2 => array('file', '/dev/null', 'a')
 646                 );
 647                 $ahPipes = null;
 648                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 649                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 650
 651                 fclose($ahPipes[0]);
 652
 653                 // TODO: error checking
 654                 while(!feof($ahPipes[1]))
 655                 {
 656                         echo fread($ahPipes[1], 4096);
 657                 }
 658                 fclose($ahPipes[1]);
 659
 660                 proc_close($hProcess);
 661         }
 662
 663         function pgsqlRunDropAndRestore($sDumpFile)
 664         {
 665                 // Convert database DSN to psql parameters
 666                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 667                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 668                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 669
 670                 $aDescriptors = array(
 671                         0 => array('pipe', 'r'),
 672                         1 => array('pipe', 'w'),
 673                         2 => array('file', '/dev/null', 'a')
 674                 );
 675                 $ahPipes = null;
 676                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 677                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 678
 679                 fclose($ahPipes[0]);
 680
 681                 // TODO: error checking
 682                 while(!feof($ahPipes[1]))
 683                 {
 684                         echo fread($ahPipes[1], 4096);
 685                 }
 686                 fclose($ahPipes[1]);
 687
 688                 proc_close($hProcess);
 689         }
 690
 691         function passthruCheckReturn($cmd)
 692         {
 693                 $result = -1;
 694                 passthru($cmd, $result);
 695                 if ($result != 0) fail('Error executing external command: '.$cmd);
 696         }