utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  35                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
  36                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  37                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  38                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  39                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  40                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  41         );
  42         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  43
  44         $bDidSomething = false;
  45
  46         // This is a pretty hard core default - the number of processors in the box - 1
  47         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  48         if ($iInstances < 1)
  49         {
  50                 $iInstances = 1;
  51                 echo "WARNING: resetting threads to $iInstances\n";
  52         }
  53         if ($iInstances > getProcessorCount())
  54         {
  55                 $iInstances = getProcessorCount();
  56                 echo "WARNING: resetting threads to $iInstances\n";
  57         }
  58
  59         // Assume we can steal all the cache memory in the box (unless told otherwise)
  60         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  61         if ($iCacheMemory > getTotalMemoryMB())
  62         {
  63                 $iCacheMemory = getCacheMemoryMB();
  64                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  65         }
  66
  67         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
  68         {
  69                 $sBaseFile = basename($aCMDResult['osm-file']);
  70                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
  71                 {
  72                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
  73                         $iTime -= (60*60*24);
  74                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
  75                 }
  76         }
  77         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  78         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  79
  80         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  81         {
  82                 echo "Create DB\n";
  83                 $bDidSomething = true;
  84                 $oDB =& DB::connect(CONST_Database_DSN, false);
  85                 if (!PEAR::isError($oDB))
  86                 {
  87                         fail('database already exists ('.CONST_Database_DSN.')');
  88                 }
  89                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
  90         }
  91
  92         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  93         {
  94                 echo "Create DB (2)\n";
  95                 $bDidSomething = true;
  96                 // TODO: path detection, detection memory, etc.
  97
  98                 $oDB =& getDB();
  99                 passthru('createlang plpgsql '.$aDSNInfo['database']);
 100                 $pgver = (float) CONST_Postgresql_Version;
 101                 if ($pgver < 9.1) {
 102                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 103                 } else {
 104                         pgsqlRunScript('CREATE EXTENSION hstore');
 105                 }
 106                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 107                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 108                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 109                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 110                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 111                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 112                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 113                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 114                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 115                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 116         }
 117
 118         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 119         {
 120                 echo "Import\n";
 121                 $bDidSomething = true;
 122
 123                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 124                 if (!file_exists($osm2pgsql))
 125                 {
 126                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 127                         fail("osm2pgsql not found in '$osm2pgsql'");
 128                 }
 129                 $osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data';
 130                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 131                 $osm2pgsql .= ' -C 16000';
 132                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 133                 passthruCheckReturn($osm2pgsql);
 134
 135                 $oDB =& getDB();
 136                 $x = $oDB->getRow('select * from place limit 1');
 137                 if (PEAR::isError($x)) {
 138                         fail($x->getMessage());
 139                 }
 140                 if (!$x) fail('No Data');
 141         }
 142
 143         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 144         {
 145                 echo "Functions\n";
 146                 $bDidSomething = true;
 147                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 148                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 149                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 150                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 151                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 152                 pgsqlRunScript($sTemplate);
 153         }
 154
 155         if ($aCMDResult['create-minimal-tables'])
 156         {
 157                 echo "Minimal Tables\n";
 158                 $bDidSomething = true;
 159                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 160
 161                 $sScript = '';
 162
 163                 // Backstop the import process - easliest possible import id
 164                 $sScript .= "insert into import_npi_log values (18022);\n";
 165
 166                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 167                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 168
 169                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 170                 {
 171                         list($sClass, $sType) = explode(' ', trim($sLine));
 172                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 173                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 174
 175                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 176                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 177
 178                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 179                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 180                 }
 181                 fclose($hFile);
 182                 pgsqlRunScript($sScript);
 183         }
 184
 185         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 186         {
 187                 echo "Tables\n";
 188                 $bDidSomething = true;
 189                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 190
 191                 // re-run the functions
 192                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 193                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 194                 pgsqlRunScript($sTemplate);
 195         }
 196
 197         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 198         {
 199                 echo "Partitions\n";
 200                 $bDidSomething = true;
 201                 $oDB =& getDB();
 202                 $sSQL = 'select partition from country_name order by country_code';
 203                 $aPartitions = $oDB->getCol($sSQL);
 204                 if (PEAR::isError($aPartitions))
 205                 {
 206                         fail($aPartitions->getMessage());
 207                 }
 208                 $aPartitions[] = 0;
 209
 210                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 211                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 212                 foreach($aMatches as $aMatch)
 213                 {
 214                         $sResult = '';
 215                         foreach($aPartitions as $sPartitionName)
 216                         {
 217                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 218                         }
 219                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 220                 }
 221
 222                 pgsqlRunScript($sTemplate);
 223         }
 224
 225         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 226         {
 227                 $bDidSomething = true;
 228                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 229                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 230                 if (file_exists($sWikiArticlesFile))
 231                 {
 232                         echo "Importing wikipedia articles...";
 233                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 234                         echo "...done\n";
 235                 }
 236                 else
 237                 {
 238                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 239                 }
 240                 if (file_exists($sWikiRedirectsFile))
 241                 {
 242                         echo "Importing wikipedia redirects...";
 243                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 244                         echo "...done\n";
 245                 }
 246                 else
 247                 {
 248                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 249                 }
 250         }
 251
 252
 253         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 254         {
 255                 echo "Load Data\n";
 256                 $bDidSomething = true;
 257
 258                 $oDB =& getDB();
 259                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 260                 echo '.';
 261                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 262                 echo '.';
 263                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 264                 echo '.';
 265                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 266                 echo '.';
 267                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 268                 echo '.';
 269                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 270                 echo '.';
 271                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 272                 echo '.';
 273                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 274                 echo '.';
 275                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 276                 echo '.';
 277
 278                 $sSQL = 'select partition from country_name order by country_code';
 279                 $aPartitions = $oDB->getCol($sSQL);
 280                 if (PEAR::isError($aPartitions))
 281                 {
 282                         fail($aPartitions->getMessage());
 283                 }
 284                 $aPartitions[] = 0;
 285                 foreach($aPartitions as $sPartition)
 286                 {
 287                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 288                         echo '.';
 289                 }
 290
 291                 // pre-create the word list
 292                 if (!$aCMDResult['disable-token-precalc'])
 293                 {
 294                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 295                         echo '.';
 296                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct postcode as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 297                         echo '.';
 298                         if (!pg_query($oDB->connection, 'select count(getorcreate_housenumber_id(v)) from (select distinct housenumber as v from place where housenumber is not null) as w;')) fail(pg_last_error($oDB->connection));
 299                         echo '.';
 300                 }
 301
 302                 $aDBInstances = array();
 303                 for($i = 0; $i < $iInstances; $i++)
 304                 {
 305                         $aDBInstances[$i] =& getDB(true);
 306                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 307                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 308                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 309                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 310                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 311                 }
 312                 $bAnyBusy = true;
 313                 while($bAnyBusy)
 314                 {
 315                         $bAnyBusy = false;
 316                         for($i = 0; $i < $iInstances; $i++)
 317                         {
 318                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 319                         }
 320                         sleep(1);
 321                         echo '.';
 322                 }
 323                 echo "\n";
 324                 pgsqlRunScript('ALTER TABLE place SET TABLESPACE "data"');
 325                 echo "Reanalysing database...\n";
 326                 pgsqlRunScript('ANALYSE');
 327         }
 328
 329         if ($aCMDResult['create-roads'])
 330         {
 331                 $bDidSomething = true;
 332
 333                 $oDB =& getDB();
 334                 $aDBInstances = array();
 335                 for($i = 0; $i < $iInstances; $i++)
 336                 {
 337                         $aDBInstances[$i] =& getDB(true);
 338                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 339                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, country_code, geometry) from ';
 340                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 341                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 342                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 343                 }
 344                 $bAnyBusy = true;
 345                 while($bAnyBusy)
 346                 {
 347                         $bAnyBusy = false;
 348                         for($i = 0; $i < $iInstances; $i++)
 349                         {
 350                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 351                         }
 352                         sleep(1);
 353                         echo '.';
 354                 }
 355                 echo "\n";
 356         }
 357
 358         if ($aCMDResult['import-tiger-data'])
 359         {
 360                 $bDidSomething = true;
 361
 362                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql');
 363
 364                 $aDBInstances = array();
 365                 for($i = 0; $i < $iInstances; $i++)
 366                 {
 367                         $aDBInstances[$i] =& getDB(true);
 368                 }
 369
 370                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 371                 {
 372                         echo $sFile.': ';
 373                         $hFile = fopen($sFile, "r");
 374                         $sSQL = fgets($hFile, 100000);
 375                         $iLines = 0;
 376
 377                         while(true)
 378                         {
 379                                 for($i = 0; $i < $iInstances; $i++)
 380                                 {
 381                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 382                                         {
 383                                                 while(pg_get_result($aDBInstances[$i]->connection));
 384                                                 $sSQL = fgets($hFile, 100000);
 385                                                 if (!$sSQL) break 2;
 386                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 387                                                 $iLines++;
 388                                                 if ($iLines == 1000)
 389                                                 {
 390                                                         echo ".";
 391                                                         $iLines = 0;
 392                                                 }
 393                                         }
 394                                 }
 395                                 usleep(10);
 396                         }
 397
 398                         fclose($hFile);
 399
 400                         $bAnyBusy = true;
 401                         while($bAnyBusy)
 402                         {
 403                                 $bAnyBusy = false;
 404                                 for($i = 0; $i < $iInstances; $i++)
 405                                 {
 406                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 407                                 }
 408                                 usleep(10);
 409                         }
 410                         echo "\n";
 411                 }
 412
 413                 echo "Creating indexes\n";
 414                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql');
 415         }
 416
 417         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 418         {
 419                 $bDidSomething = true;
 420                 $oDB =& getDB();
 421                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 422                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 423                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,country_code,";
 424                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select country_code,postcode,";
 425                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 426                 $sSQL .= "from placex where postcode is not null group by country_code,postcode) as x";
 427                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 428
 429                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 430                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 431                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 432                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 433         }
 434
 435         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
 436         {
 437                 $bDidSomething = true;
 438                 $oDB =& getDB();
 439
 440                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 441                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
 442                 else passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 443
 444                 $sDate = $aCMDResult['osmosis-init-date'];
 445                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
 446                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
 447                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
 448                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
 449                 $sURL .= '&stream=minute';
 450                 echo "Getting state file: $sURL\n";
 451                 $sStateFile = file_get_contents($sURL);
 452                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 453                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 454                 echo "Updating DB status\n";
 455                 pg_query($oDB->connection, 'TRUNCATE import_status');
 456                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
 457                 pg_query($oDB->connection, $sSQL);
 458
 459         }
 460
 461         if ($aCMDResult['index'] || $aCMDResult['all'])
 462         {
 463                 $bDidSomething = true;
 464                 $sOutputFile = '';
 465                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 466                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 467                 passthruCheckReturn($sBaseCmd.' -R 4');
 468                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 469                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 470                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 471                 passthruCheckReturn($sBaseCmd.' -r 26');
 472         }
 473
 474         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 475         {
 476                 echo "Search indices\n";
 477                 $bDidSomething = true;
 478                 $oDB =& getDB();
 479                 $sSQL = 'select partition from country_name order by country_code';
 480                 $aPartitions = $oDB->getCol($sSQL);
 481                 if (PEAR::isError($aPartitions))
 482                 {
 483                         fail($aPartitions->getMessage());
 484                 }
 485                 $aPartitions[] = 0;
 486
 487                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 488                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 489                 foreach($aMatches as $aMatch)
 490                 {
 491                         $sResult = '';
 492                         foreach($aPartitions as $sPartitionName)
 493                         {
 494                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 495                         }
 496                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 497                 }
 498
 499                 pgsqlRunScript($sTemplate);
 500         }
 501
 502         if (isset($aCMDResult['create-website']))
 503         {
 504                 $bDidSomething = true;
 505                 $sTargetDir = $aCMDResult['create-website'];
 506                 if (!is_dir($sTargetDir))
 507                 {
 508                         echo "You must create the website directory before calling this function.\n";
 509                         fail("Target directory does not exist.");
 510                 }
 511
 512                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 513                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 514                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 515                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 516                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 517                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 518                 echo "Symlinks created\n";
 519         }
 520
 521         if (!$bDidSomething)
 522         {
 523                 showUsage($aCMDOptions, true);
 524         }
 525
 526         function pgsqlRunScriptFile($sFilename)
 527         {
 528                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 529
 530                 // Convert database DSN to psql parameters
 531                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 532                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 533                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 534
 535                 $aDescriptors = array(
 536                         0 => array('pipe', 'r'),
 537                         1 => array('pipe', 'w'),
 538                         2 => array('file', '/dev/null', 'a')
 539                 );
 540                 $ahPipes = null;
 541                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 542                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 543
 544                 fclose($ahPipes[0]);
 545
 546                 // TODO: error checking
 547                 while(!feof($ahPipes[1]))
 548                 {
 549                         echo fread($ahPipes[1], 4096);
 550                 }
 551                 fclose($ahPipes[1]);
 552
 553                 proc_close($hProcess);
 554         }
 555
 556         function pgsqlRunScript($sScript)
 557         {
 558                 // Convert database DSN to psql parameters
 559                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 560                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 561                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 562                 $aDescriptors = array(
 563                         0 => array('pipe', 'r'),
 564                         1 => STDOUT,
 565                         2 => STDERR
 566                 );
 567                 $ahPipes = null;
 568                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 569                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 570
 571                 while(strlen($sScript))
 572                 {
 573                         $written = fwrite($ahPipes[0], $sScript);
 574                         $sScript = substr($sScript, $written);
 575                 }
 576                 fclose($ahPipes[0]);
 577                 proc_close($hProcess);
 578         }
 579
 580         function pgsqlRunRestoreData($sDumpFile)
 581         {
 582                 // Convert database DSN to psql parameters
 583                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 584                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 585                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 586
 587                 $aDescriptors = array(
 588                         0 => array('pipe', 'r'),
 589                         1 => array('pipe', 'w'),
 590                         2 => array('file', '/dev/null', 'a')
 591                 );
 592                 $ahPipes = null;
 593                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 594                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 595
 596                 fclose($ahPipes[0]);
 597
 598                 // TODO: error checking
 599                 while(!feof($ahPipes[1]))
 600                 {
 601                         echo fread($ahPipes[1], 4096);
 602                 }
 603                 fclose($ahPipes[1]);
 604
 605                 proc_close($hProcess);
 606         }
 607
 608         function pgsqlRunDropAndRestore($sDumpFile)
 609         {
 610                 // Convert database DSN to psql parameters
 611                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 612                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 613                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 614
 615                 $aDescriptors = array(
 616                         0 => array('pipe', 'r'),
 617                         1 => array('pipe', 'w'),
 618                         2 => array('file', '/dev/null', 'a')
 619                 );
 620                 $ahPipes = null;
 621                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 622                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 623
 624                 fclose($ahPipes[0]);
 625
 626                 // TODO: error checking
 627                 while(!feof($ahPipes[1]))
 628                 {
 629                         echo fread($ahPipes[1], 4096);
 630                 }
 631                 fclose($ahPipes[1]);
 632
 633                 proc_close($hProcess);
 634         }
 635
 636         function passthruCheckReturn($cmd)
 637         {
 638                 $result = -1;
 639                 passthru($cmd, $result);
 640                 if ($result != 0) fail('Error executing external command: '.$cmd);
 641         }