utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
  36                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  37                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  38                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  39                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  40                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  41         );
  42         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  43
  44         $bDidSomething = false;
  45
  46         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  47         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  48         {
  49                 if (!isset($aCMDResult['osm-file']))
  50                 {
  51                         fail('missing --osm-file for data import');
  52                 }
  53
  54                 if (!file_exists($aCMDResult['osm-file']))
  55                 {
  56                         fail('the path supplied to --osm-file does not exist');
  57                 }
  58
  59                 if (!is_readable($aCMDResult['osm-file']))
  60                 {
  61                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  62                 }
  63         }
  64
  65
  66         // This is a pretty hard core default - the number of processors in the box - 1
  67         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  68         if ($iInstances < 1)
  69         {
  70                 $iInstances = 1;
  71                 echo "WARNING: resetting threads to $iInstances\n";
  72         }
  73         if ($iInstances > getProcessorCount())
  74         {
  75                 $iInstances = getProcessorCount();
  76                 echo "WARNING: resetting threads to $iInstances\n";
  77         }
  78
  79         // Assume we can steal all the cache memory in the box (unless told otherwise)
  80         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  81         if ($iCacheMemory > getTotalMemoryMB())
  82         {
  83                 $iCacheMemory = getCacheMemoryMB();
  84                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  85         }
  86
  87         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
  88         {
  89                 $sBaseFile = basename($aCMDResult['osm-file']);
  90                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
  91                 {
  92                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
  93                         $iTime -= (60*60*24);
  94                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
  95                 }
  96         }
  97         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  98         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  99
 100         if ($aCMDResult['create-db'] || $aCMDResult['all'])
 101         {
 102                 echo "Create DB\n";
 103                 $bDidSomething = true;
 104                 $oDB =& DB::connect(CONST_Database_DSN, false);
 105                 if (!PEAR::isError($oDB))
 106                 {
 107                         fail('database already exists ('.CONST_Database_DSN.')');
 108                 }
 109                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
 110         }
 111
 112         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 113         {
 114                 echo "Setup DB\n";
 115                 $bDidSomething = true;
 116                 // TODO: path detection, detection memory, etc.
 117
 118                 $oDB =& getDB();
 119                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 120                 $pgver = (float) CONST_Postgresql_Version;
 121                 if ($pgver < 9.1) {
 122                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 123                 } else {
 124                         pgsqlRunScript('CREATE EXTENSION hstore');
 125                 }
 126                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 127                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 128                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 129                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 130                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 131                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 132                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 133                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 134                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 135                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 136         }
 137
 138         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 139         {
 140                 echo "Import\n";
 141                 $bDidSomething = true;
 142
 143                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 144                 if (!file_exists($osm2pgsql))
 145                 {
 146                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 147                         fail("osm2pgsql not found in '$osm2pgsql'");
 148                 }
 149                 $osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data';
 150                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 151                 $osm2pgsql .= ' -C 16000';
 152                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 153                 passthruCheckReturn($osm2pgsql);
 154
 155                 $oDB =& getDB();
 156                 $x = $oDB->getRow('select * from place limit 1');
 157                 if (PEAR::isError($x)) {
 158                         fail($x->getMessage());
 159                 }
 160                 if (!$x) fail('No Data');
 161         }
 162
 163         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 164         {
 165                 echo "Functions\n";
 166                 $bDidSomething = true;
 167                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 168                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 169                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 170                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 171                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 172                 pgsqlRunScript($sTemplate);
 173         }
 174
 175         if ($aCMDResult['create-minimal-tables'])
 176         {
 177                 echo "Minimal Tables\n";
 178                 $bDidSomething = true;
 179                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 180
 181                 $sScript = '';
 182
 183                 // Backstop the import process - easliest possible import id
 184                 $sScript .= "insert into import_npi_log values (18022);\n";
 185
 186                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 187                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 188
 189                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 190                 {
 191                         list($sClass, $sType) = explode(' ', trim($sLine));
 192                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 193                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 194
 195                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 196                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 197
 198                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 199                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 200                 }
 201                 fclose($hFile);
 202                 pgsqlRunScript($sScript);
 203         }
 204
 205         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 206         {
 207                 echo "Tables\n";
 208                 $bDidSomething = true;
 209                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 210
 211                 // re-run the functions
 212                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 213                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 214                 pgsqlRunScript($sTemplate);
 215         }
 216
 217         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 218         {
 219                 echo "Partitions\n";
 220                 $bDidSomething = true;
 221                 $oDB =& getDB();
 222                 $sSQL = 'select partition from country_name order by country_code';
 223                 $aPartitions = $oDB->getCol($sSQL);
 224                 if (PEAR::isError($aPartitions))
 225                 {
 226                         fail($aPartitions->getMessage());
 227                 }
 228                 $aPartitions[] = 0;
 229
 230                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 231                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 232                 foreach($aMatches as $aMatch)
 233                 {
 234                         $sResult = '';
 235                         foreach($aPartitions as $sPartitionName)
 236                         {
 237                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 238                         }
 239                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 240                 }
 241
 242                 pgsqlRunScript($sTemplate);
 243         }
 244
 245         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 246         {
 247                 $bDidSomething = true;
 248                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 249                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 250                 if (file_exists($sWikiArticlesFile))
 251                 {
 252                         echo "Importing wikipedia articles...";
 253                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 254                         echo "...done\n";
 255                 }
 256                 else
 257                 {
 258                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 259                 }
 260                 if (file_exists($sWikiRedirectsFile))
 261                 {
 262                         echo "Importing wikipedia redirects...";
 263                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 264                         echo "...done\n";
 265                 }
 266                 else
 267                 {
 268                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 269                 }
 270         }
 271
 272
 273         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 274         {
 275                 echo "Drop old Data\n";
 276                 $bDidSomething = true;
 277
 278                 $oDB =& getDB();
 279                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 280                 echo '.';
 281                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 282                 echo '.';
 283                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 284                 echo '.';
 285                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 286                 echo '.';
 287                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 288                 echo '.';
 289                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 290                 echo '.';
 291                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 292                 echo '.';
 293                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 294                 echo '.';
 295                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 296                 echo '.';
 297
 298                 $sSQL = 'select partition from country_name order by country_code';
 299                 $aPartitions = $oDB->getCol($sSQL);
 300                 if (PEAR::isError($aPartitions))
 301                 {
 302                         fail($aPartitions->getMessage());
 303                 }
 304                 $aPartitions[] = 0;
 305                 foreach($aPartitions as $sPartition)
 306                 {
 307                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 308                         echo '.';
 309                 }
 310
 311                 // used by getorcreate_word_id to ignore frequent partial words
 312                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 313                 echo ".\n";
 314
 315                 // pre-create the word list
 316                 if (!$aCMDResult['disable-token-precalc'])
 317                 {
 318                         echo "Loading word list\n";
 319                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 320                 }
 321
 322                 echo "Load Data\n";
 323                 $aDBInstances = array();
 324                 for($i = 0; $i < $iInstances; $i++)
 325                 {
 326                         $aDBInstances[$i] =& getDB(true);
 327                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 328                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 329                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 330                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 331                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 332                 }
 333                 $bAnyBusy = true;
 334                 while($bAnyBusy)
 335                 {
 336                         $bAnyBusy = false;
 337                         for($i = 0; $i < $iInstances; $i++)
 338                         {
 339                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 340                         }
 341                         sleep(1);
 342                         echo '.';
 343                 }
 344                 echo "\n";
 345                 pgsqlRunScript('ALTER TABLE place SET TABLESPACE "data"');
 346                 echo "Reanalysing database...\n";
 347                 pgsqlRunScript('ANALYSE');
 348         }
 349
 350         if ($aCMDResult['create-roads'])
 351         {
 352                 $bDidSomething = true;
 353
 354                 $oDB =& getDB();
 355                 $aDBInstances = array();
 356                 for($i = 0; $i < $iInstances; $i++)
 357                 {
 358                         $aDBInstances[$i] =& getDB(true);
 359                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 360                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, calculated_country_code, geometry) from ';
 361                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 362                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 363                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 364                 }
 365                 $bAnyBusy = true;
 366                 while($bAnyBusy)
 367                 {
 368                         $bAnyBusy = false;
 369                         for($i = 0; $i < $iInstances; $i++)
 370                         {
 371                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 372                         }
 373                         sleep(1);
 374                         echo '.';
 375                 }
 376                 echo "\n";
 377         }
 378
 379         if ($aCMDResult['import-tiger-data'])
 380         {
 381                 $bDidSomething = true;
 382
 383                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 384
 385                 $aDBInstances = array();
 386                 for($i = 0; $i < $iInstances; $i++)
 387                 {
 388                         $aDBInstances[$i] =& getDB(true);
 389                 }
 390
 391                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 392                 {
 393                         echo $sFile.': ';
 394                         $hFile = fopen($sFile, "r");
 395                         $sSQL = fgets($hFile, 100000);
 396                         $iLines = 0;
 397
 398                         while(true)
 399                         {
 400                                 for($i = 0; $i < $iInstances; $i++)
 401                                 {
 402                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 403                                         {
 404                                                 while(pg_get_result($aDBInstances[$i]->connection));
 405                                                 $sSQL = fgets($hFile, 100000);
 406                                                 if (!$sSQL) break 2;
 407                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 408                                                 $iLines++;
 409                                                 if ($iLines == 1000)
 410                                                 {
 411                                                         echo ".";
 412                                                         $iLines = 0;
 413                                                 }
 414                                         }
 415                                 }
 416                                 usleep(10);
 417                         }
 418
 419                         fclose($hFile);
 420
 421                         $bAnyBusy = true;
 422                         while($bAnyBusy)
 423                         {
 424                                 $bAnyBusy = false;
 425                                 for($i = 0; $i < $iInstances; $i++)
 426                                 {
 427                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 428                                 }
 429                                 usleep(10);
 430                         }
 431                         echo "\n";
 432                 }
 433
 434                 echo "Creating indexes\n";
 435                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 436         }
 437
 438         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 439         {
 440                 $bDidSomething = true;
 441                 $oDB =& getDB();
 442                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 443                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 444                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 445                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 446                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 447                 $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
 448                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 449
 450                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calcuclated_country_code,geometry) ";
 451                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 452                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 453                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 454         }
 455
 456         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
 457         {
 458                 $bDidSomething = true;
 459                 $oDB =& getDB();
 460
 461                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 462                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
 463                 else
 464                 {
 465                         passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 466                         // server layout changed afer license change, fix path to minutely diffs
 467                         passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
 468                 }
 469
 470                 $sDate = $aCMDResult['osmosis-init-date'];
 471                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
 472                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
 473                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
 474                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
 475                 $sURL .= '&stream=minute';
 476                 echo "Getting state file: $sURL\n";
 477                 $sStateFile = file_get_contents($sURL);
 478                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 479                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 480                 echo "Updating DB status\n";
 481                 pg_query($oDB->connection, 'TRUNCATE import_status');
 482                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
 483                 pg_query($oDB->connection, $sSQL);
 484
 485         }
 486
 487         if ($aCMDResult['index'] || $aCMDResult['all'])
 488         {
 489                 $bDidSomething = true;
 490                 $sOutputFile = '';
 491                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 492                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 493                 passthruCheckReturn($sBaseCmd.' -R 4');
 494                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 495                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 496                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 497                 passthruCheckReturn($sBaseCmd.' -r 26');
 498         }
 499
 500         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 501         {
 502                 echo "Search indices\n";
 503                 $bDidSomething = true;
 504                 $oDB =& getDB();
 505                 $sSQL = 'select partition from country_name order by country_code';
 506                 $aPartitions = $oDB->getCol($sSQL);
 507                 if (PEAR::isError($aPartitions))
 508                 {
 509                         fail($aPartitions->getMessage());
 510                 }
 511                 $aPartitions[] = 0;
 512
 513                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 514                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 515                 foreach($aMatches as $aMatch)
 516                 {
 517                         $sResult = '';
 518                         foreach($aPartitions as $sPartitionName)
 519                         {
 520                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 521                         }
 522                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 523                 }
 524
 525                 pgsqlRunScript($sTemplate);
 526         }
 527
 528         if (isset($aCMDResult['create-website']))
 529         {
 530                 $bDidSomething = true;
 531                 $sTargetDir = $aCMDResult['create-website'];
 532                 if (!is_dir($sTargetDir))
 533                 {
 534                         echo "You must create the website directory before calling this function.\n";
 535                         fail("Target directory does not exist.");
 536                 }
 537
 538                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 539                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 540                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 541                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 542                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 543                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 544                 echo "Symlinks created\n";
 545         }
 546
 547         if (!$bDidSomething)
 548         {
 549                 showUsage($aCMDOptions, true);
 550         }
 551
 552         function pgsqlRunScriptFile($sFilename)
 553         {
 554                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 555
 556                 // Convert database DSN to psql parameters
 557                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 558                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 559                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 560
 561                 $aDescriptors = array(
 562                         0 => array('pipe', 'r'),
 563                         1 => array('pipe', 'w'),
 564                         2 => array('file', '/dev/null', 'a')
 565                 );
 566                 $ahPipes = null;
 567                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 568                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 569
 570                 fclose($ahPipes[0]);
 571
 572                 // TODO: error checking
 573                 while(!feof($ahPipes[1]))
 574                 {
 575                         echo fread($ahPipes[1], 4096);
 576                 }
 577                 fclose($ahPipes[1]);
 578
 579                 proc_close($hProcess);
 580         }
 581
 582         function pgsqlRunScript($sScript)
 583         {
 584                 // Convert database DSN to psql parameters
 585                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 586                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 587                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 588                 $aDescriptors = array(
 589                         0 => array('pipe', 'r'),
 590                         1 => STDOUT,
 591                         2 => STDERR
 592                 );
 593                 $ahPipes = null;
 594                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 595                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 596
 597                 while(strlen($sScript))
 598                 {
 599                         $written = fwrite($ahPipes[0], $sScript);
 600                         $sScript = substr($sScript, $written);
 601                 }
 602                 fclose($ahPipes[0]);
 603                 proc_close($hProcess);
 604         }
 605
 606         function pgsqlRunRestoreData($sDumpFile)
 607         {
 608                 // Convert database DSN to psql parameters
 609                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 610                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 611                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 612
 613                 $aDescriptors = array(
 614                         0 => array('pipe', 'r'),
 615                         1 => array('pipe', 'w'),
 616                         2 => array('file', '/dev/null', 'a')
 617                 );
 618                 $ahPipes = null;
 619                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 620                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 621
 622                 fclose($ahPipes[0]);
 623
 624                 // TODO: error checking
 625                 while(!feof($ahPipes[1]))
 626                 {
 627                         echo fread($ahPipes[1], 4096);
 628                 }
 629                 fclose($ahPipes[1]);
 630
 631                 proc_close($hProcess);
 632         }
 633
 634         function pgsqlRunDropAndRestore($sDumpFile)
 635         {
 636                 // Convert database DSN to psql parameters
 637                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 638                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 639                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 640
 641                 $aDescriptors = array(
 642                         0 => array('pipe', 'r'),
 643                         1 => array('pipe', 'w'),
 644                         2 => array('file', '/dev/null', 'a')
 645                 );
 646                 $ahPipes = null;
 647                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 648                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 649
 650                 fclose($ahPipes[0]);
 651
 652                 // TODO: error checking
 653                 while(!feof($ahPipes[1]))
 654                 {
 655                         echo fread($ahPipes[1], 4096);
 656                 }
 657                 fclose($ahPipes[1]);
 658
 659                 proc_close($hProcess);
 660         }
 661
 662         function passthruCheckReturn($cmd)
 663         {
 664                 $result = -1;
 665                 passthru($cmd, $result);
 666                 if ($result != 0) fail('Error executing external command: '.$cmd);
 667         }