utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
   5         require_once(CONST_BasePath.'/lib/init-cmd.php');
   6         ini_set('memory_limit', '800M');
   7
   8         $aCMDOptions = array(
   9                 "Create and setup nominatim search system",
  10                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  11                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  12                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  13
  14                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  15                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  16
  17                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  18
  19                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  20                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  21                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  22                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  23                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  24                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
  25                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
  26                 array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'),
  27                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  28                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  29                 array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
  30                 array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
  31                 array('no-partitions', '', 0, 1, 0, 0, 'bool', "Do not partition search indices (speeds up import of single country extracts)"),
  32                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  33                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  34                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
  35                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  36                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  37                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  38                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  39                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
  40                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  41                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  42                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  43                 array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'),
  44         );
  45         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  46
  47         $bDidSomething = false;
  48
  49         // Check if osm-file is set and points to a valid file if --all or --import-data is given
  50         if ($aCMDResult['import-data'] || $aCMDResult['all'])
  51         {
  52                 if (!isset($aCMDResult['osm-file']))
  53                 {
  54                         fail('missing --osm-file for data import');
  55                 }
  56
  57                 if (!file_exists($aCMDResult['osm-file']))
  58                 {
  59                         fail('the path supplied to --osm-file does not exist');
  60                 }
  61
  62                 if (!is_readable($aCMDResult['osm-file']))
  63                 {
  64                         fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
  65                 }
  66         }
  67
  68
  69         // This is a pretty hard core default - the number of processors in the box - 1
  70         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  71         if ($iInstances < 1)
  72         {
  73                 $iInstances = 1;
  74                 echo "WARNING: resetting threads to $iInstances\n";
  75         }
  76         if ($iInstances > getProcessorCount())
  77         {
  78                 $iInstances = getProcessorCount();
  79                 echo "WARNING: resetting threads to $iInstances\n";
  80         }
  81
  82         // Assume we can steal all the cache memory in the box (unless told otherwise)
  83         if (isset($aCMDResult['osm2pgsql-cache']))
  84         {
  85                 $iCacheMemory = $aCMDResult['osm2pgsql-cache'];
  86         }
  87         else
  88         {
  89                 $iCacheMemory = getCacheMemoryMB();
  90         }
  91
  92         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  93         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  94
  95         $fPostgisVersion = (float) CONST_Postgis_Version;
  96
  97         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  98         {
  99                 echo "Create DB\n";
 100                 $bDidSomething = true;
 101                 $oDB =& DB::connect(CONST_Database_DSN, false);
 102                 if (!PEAR::isError($oDB))
 103                 {
 104                         fail('database already exists ('.CONST_Database_DSN.')');
 105                 }
 106                 passthruCheckReturn('createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 107         }
 108
 109         if ($aCMDResult['setup-db'] || $aCMDResult['all'])
 110         {
 111                 echo "Setup DB\n";
 112                 $bDidSomething = true;
 113                 // TODO: path detection, detection memory, etc.
 114
 115                 $oDB =& getDB();
 116
 117                 $sVersionString = $oDB->getOne('select version()');
 118                 preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[^0-9]#', $sVersionString, $aMatches);
 119                 if (CONST_Postgresql_Version != $aMatches[1].'.'.$aMatches[2])
 120                 {
 121                         echo "ERROR: PostgreSQL version is not correct.  Expected ".CONST_Postgresql_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 122                         exit;
 123                 }
 124
 125                 passthru('createlang plpgsql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 126                 $pgver = (float) CONST_Postgresql_Version;
 127                 if ($pgver < 9.1) {
 128                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
 129                         pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
 130                 } else {
 131                         pgsqlRunScript('CREATE EXTENSION hstore');
 132                 }
 133
 134                 if ($fPostgisVersion < 2.0) {
 135                         pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 136                         pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 137                 } else {
 138                         pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis');
 139                 }
 140                 if ($fPostgisVersion < 2.1) {
 141                         // Function was renamed in 2.1 and throws an annoying deprecation warning
 142                         pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint');
 143                 }
 144                 $sVersionString = $oDB->getOne('select postgis_full_version()');
 145                 preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
 146                 if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
 147                 {
 148                         echo "ERROR: PostGIS version is not correct.  Expected ".CONST_Postgis_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
 149                         exit;
 150                 }
 151
 152                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 153                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 154                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 155                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql');
 156                 if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz'))
 157                 {
 158                         pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz');
 159                 }
 160                 else
 161                 {
 162                         echo "WARNING: external UK postcode table not found.\n";
 163                 }
 164                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 165                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 166                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 167
 168                 if ($aCMDResult['no-partitions'])
 169                 {
 170                         pgsqlRunScript('update country_name set partition = 0');
 171                 }
 172
 173                 // the following will be needed by create_functions later but
 174                 // is only defined in the subsequently called create_tables.
 175                 // Create dummies here that will be overwritten by the proper
 176                 // versions in create-tables.
 177                 pgsqlRunScript('CREATE TABLE place_boundingbox ()');
 178                 pgsqlRunScript('create type wikipedia_article_match as ()');
 179         }
 180
 181         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 182         {
 183                 echo "Import\n";
 184                 $bDidSomething = true;
 185
 186                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 187                 if (!file_exists($osm2pgsql))
 188                 {
 189                         echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 190                         fail("osm2pgsql not found in '$osm2pgsql'");
 191                 }
 192
 193                 if (!is_null(CONST_Osm2pgsql_Flatnode_File))
 194                 {
 195                         $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
 196                 }
 197                 if (CONST_Tablespace_Osm2pgsql_Data)
 198                         $osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data;
 199                 if (CONST_Tablespace_Osm2pgsql_Index)
 200                         $osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index;
 201                 if (CONST_Tablespace_Place_Data)
 202                         $osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data;
 203                 if (CONST_Tablespace_Place_Index)
 204                         $osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index;
 205                 $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1';
 206                 $osm2pgsql .= ' -C '.$iCacheMemory;
 207                 $osm2pgsql .= ' -P '.$aDSNInfo['port'];
 208                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 209                 passthruCheckReturn($osm2pgsql);
 210
 211                 $oDB =& getDB();
 212                 $x = $oDB->getRow('select * from place limit 1');
 213                 if (PEAR::isError($x)) {
 214                         fail($x->getMessage());
 215                 }
 216                 if (!$x) fail('No Data');
 217         }
 218
 219         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 220         {
 221                 echo "Functions\n";
 222                 $bDidSomething = true;
 223                 if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) fail("nominatim module not built");
 224                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 225                 $sTemplate = str_replace('{modulepath}', CONST_InstallPath.'/module', $sTemplate);
 226                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 227                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
 228                 if (CONST_Limit_Reindexing) $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
 229                 pgsqlRunScript($sTemplate);
 230
 231                 if ($fPostgisVersion < 2.0) {
 232                         echo "Helper functions for postgis < 2.0\n";
 233                         $sTemplate = file_get_contents(CONST_BasePath.'/sql/postgis_15_aux.sql');
 234                 } else {
 235                         echo "Helper functions for postgis >= 2.0\n";
 236                         $sTemplate = file_get_contents(CONST_BasePath.'/sql/postgis_20_aux.sql');
 237                 }
 238                 pgsqlRunScript($sTemplate);
 239         }
 240
 241         if ($aCMDResult['create-minimal-tables'])
 242         {
 243                 echo "Minimal Tables\n";
 244                 $bDidSomething = true;
 245                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 246
 247                 $sScript = '';
 248
 249                 // Backstop the import process - easliest possible import id
 250                 $sScript .= "insert into import_npi_log values (18022);\n";
 251
 252                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 253                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 254
 255                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 256                 {
 257                         list($sClass, $sType) = explode(' ', trim($sLine));
 258                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 259                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 260
 261                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 262                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 263
 264                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 265                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 266                 }
 267                 fclose($hFile);
 268                 pgsqlRunScript($sScript);
 269         }
 270
 271         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 272         {
 273                 $bDidSomething = true;
 274
 275                 echo "Tables\n";
 276                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');
 277                 $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
 278                 $sTemplate = replace_tablespace('{ts:address-data}',
 279                                                 CONST_Tablespace_Address_Data, $sTemplate);
 280                 $sTemplate = replace_tablespace('{ts:address-index}',
 281                                                 CONST_Tablespace_Address_Index, $sTemplate);
 282                 $sTemplate = replace_tablespace('{ts:search-data}',
 283                                                 CONST_Tablespace_Search_Data, $sTemplate);
 284                 $sTemplate = replace_tablespace('{ts:search-index}',
 285                                                 CONST_Tablespace_Search_Index, $sTemplate);
 286                 $sTemplate = replace_tablespace('{ts:aux-data}',
 287                                                 CONST_Tablespace_Aux_Data, $sTemplate);
 288                 $sTemplate = replace_tablespace('{ts:aux-index}',
 289                                                 CONST_Tablespace_Aux_Index, $sTemplate);
 290                 pgsqlRunScript($sTemplate, false);
 291
 292                 // re-run the functions
 293                 echo "Functions\n";
 294                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 295                 $sTemplate = str_replace('{modulepath}',
 296                                              CONST_InstallPath.'/module', $sTemplate);
 297                 pgsqlRunScript($sTemplate);
 298         }
 299
 300         if ($aCMDResult['create-partition-tables'] || $aCMDResult['all'])
 301         {
 302                 echo "Partition Tables\n";
 303                 $bDidSomething = true;
 304                 $oDB =& getDB();
 305                 $sSQL = 'select distinct partition from country_name';
 306                 $aPartitions = $oDB->getCol($sSQL);
 307                 if (PEAR::isError($aPartitions))
 308                 {
 309                         fail($aPartitions->getMessage());
 310                 }
 311                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 312
 313                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
 314                 $sTemplate = replace_tablespace('{ts:address-data}',
 315                                                 CONST_Tablespace_Address_Data, $sTemplate);
 316                 $sTemplate = replace_tablespace('{ts:address-index}',
 317                                                 CONST_Tablespace_Address_Index, $sTemplate);
 318                 $sTemplate = replace_tablespace('{ts:search-data}',
 319                                                 CONST_Tablespace_Search_Data, $sTemplate);
 320                 $sTemplate = replace_tablespace('{ts:search-index}',
 321                                                 CONST_Tablespace_Search_Index, $sTemplate);
 322                 $sTemplate = replace_tablespace('{ts:aux-data}',
 323                                                 CONST_Tablespace_Aux_Data, $sTemplate);
 324                 $sTemplate = replace_tablespace('{ts:aux-index}',
 325                                                 CONST_Tablespace_Aux_Index, $sTemplate);
 326                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 327                 foreach($aMatches as $aMatch)
 328                 {
 329                         $sResult = '';
 330                         foreach($aPartitions as $sPartitionName)
 331                         {
 332                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 333                         }
 334                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 335                 }
 336
 337                 pgsqlRunScript($sTemplate);
 338         }
 339
 340
 341         if ($aCMDResult['create-partition-functions'] || $aCMDResult['all'])
 342         {
 343                 echo "Partition Functions\n";
 344                 $bDidSomething = true;
 345                 $oDB =& getDB();
 346                 $sSQL = 'select distinct partition from country_name';
 347                 $aPartitions = $oDB->getCol($sSQL);
 348                 if (PEAR::isError($aPartitions))
 349                 {
 350                         fail($aPartitions->getMessage());
 351                 }
 352                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 353
 354                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
 355                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 356                 foreach($aMatches as $aMatch)
 357                 {
 358                         $sResult = '';
 359                         foreach($aPartitions as $sPartitionName)
 360                         {
 361                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 362                         }
 363                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 364                 }
 365
 366                 pgsqlRunScript($sTemplate);
 367         }
 368
 369         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 370         {
 371                 $bDidSomething = true;
 372                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 373                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 374                 if (file_exists($sWikiArticlesFile))
 375                 {
 376                         echo "Importing wikipedia articles...";
 377                         pgsqlRunDropAndRestore($sWikiArticlesFile);
 378                         echo "...done\n";
 379                 }
 380                 else
 381                 {
 382                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 383                 }
 384                 if (file_exists($sWikiRedirectsFile))
 385                 {
 386                         echo "Importing wikipedia redirects...";
 387                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
 388                         echo "...done\n";
 389                 }
 390                 else
 391                 {
 392                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 393                 }
 394         }
 395
 396
 397         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 398         {
 399                 echo "Drop old Data\n";
 400                 $bDidSomething = true;
 401
 402                 $oDB =& getDB();
 403                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 404                 echo '.';
 405                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 406                 echo '.';
 407                 if (!pg_query($oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($oDB->connection));
 408                 echo '.';
 409                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 410                 echo '.';
 411                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 412                 echo '.';
 413                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 414                 echo '.';
 415                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 416                 echo '.';
 417                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 418                 echo '.';
 419                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 420                 echo '.';
 421                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 422                 echo '.';
 423
 424                 $sSQL = 'select distinct partition from country_name';
 425                 $aPartitions = $oDB->getCol($sSQL);
 426                 if (PEAR::isError($aPartitions))
 427                 {
 428                         fail($aPartitions->getMessage());
 429                 }
 430                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 431                 foreach($aPartitions as $sPartition)
 432                 {
 433                         if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
 434                         echo '.';
 435                 }
 436
 437                 // used by getorcreate_word_id to ignore frequent partial words
 438                 if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
 439                 echo ".\n";
 440
 441                 // pre-create the word list
 442                 if (!$aCMDResult['disable-token-precalc'])
 443                 {
 444                         echo "Loading word list\n";
 445                         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
 446                 }
 447
 448                 echo "Load Data\n";
 449                 $aDBInstances = array();
 450
 451         $aQueriesPlacex = array();
 452         $aQueriesOsmline = array();
 453         // the query is divided into parcels, so that the work between the processes, i.e. the DBInstances, will be evenly distributed
 454         $iNumberOfParcels = 100;
 455         for($i = 0; $i < $iNumberOfParcels; $i++)
 456         {
 457                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 458                         $sSQL .= 'housenumber, street, addr_place, isin, postcode, country_code, extratags, ';
 459                         $sSQL .= 'geometry) select * from place where osm_id % '.$iNumberOfParcels.' = '.$i.' and not ';
 460                         $sSQL .= '(class=\'place\' and type=\'houses\' and osm_type=\'W\' and ST_GeometryType(geometry) = \'ST_LineString\');';
 461                         array_push($aQueriesPlacex, $sSQL);
 462                         $sSQL = 'select insert_osmline (osm_id, housenumber, street, addr_place, postcode, country_code, ';
 463                         $sSQL .= 'geometry) from place where osm_id % '.$iNumberOfParcels.' = '.$i.' and ';
 464                         $sSQL .= 'class=\'place\' and type=\'houses\' and osm_type=\'W\' and ST_GeometryType(geometry) = \'ST_LineString\'';
 465                         array_push($aQueriesOsmline, $sSQL);
 466                 }
 467
 468                 for($i = 0; $i < $iInstances; $i++)
 469                 {
 470                         $aDBInstances[$i] =& getDB(true);
 471                 }
 472                 // now execute the query blocks, in the first round for placex, then for osmline,
 473                 // because insert_osmline depends on the placex table
 474                 echo 'Inserting from place to placex.';
 475                 $aQueries = $aQueriesPlacex;
 476                 for($j = 0; $j < 2; $j++)
 477                 {
 478                         $bAnyBusy = true;
 479                         while($bAnyBusy)
 480                         {
 481                                 $bAnyBusy = false;
 482
 483                                 for($i = 0; $i < $iInstances; $i++)
 484                                 {
 485                                         if (pg_connection_busy($aDBInstances[$i]->connection))
 486                                         {
 487                                                 $bAnyBusy = true;
 488                                         }
 489                                         else if (count($aQueries) > 0)
 490                                         {
 491                                                 $query = array_pop($aQueries);
 492                                                 if (!pg_send_query($aDBInstances[$i]->connection, $query))
 493                                                 {
 494                                                         fail(pg_last_error($oDB->connection));
 495                                                 }
 496                                                 else
 497                                                 {
 498                                                         pg_get_result($aDBInstances[$i]->connection);
 499                                                         $bAnyBusy = true;
 500                                                 }
 501                                         }
 502                                 }
 503                                 sleep(1);
 504                                 echo '.';
 505                         }
 506                         echo "\n";
 507                         if ($j == 0)  //for the second round with osmline
 508                         {
 509                                 echo 'Inserting from place to osmline.';
 510                                 $aQueries = $aQueriesOsmline;
 511                         }
 512                 }
 513
 514                 echo "Reanalysing database...\n";
 515                 pgsqlRunScript('ANALYSE');
 516         }
 517
 518         if ($aCMDResult['import-tiger-data'])
 519         {
 520                 $bDidSomething = true;
 521
 522                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
 523                 $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
 524                 $sTemplate = replace_tablespace('{ts:aux-data}',
 525                                                 CONST_Tablespace_Aux_Data, $sTemplate);
 526                 $sTemplate = replace_tablespace('{ts:aux-index}',
 527                                                 CONST_Tablespace_Aux_Index, $sTemplate);
 528                 pgsqlRunScript($sTemplate, false);
 529
 530                 $aDBInstances = array();
 531                 for($i = 0; $i < $iInstances; $i++)
 532                 {
 533                         $aDBInstances[$i] =& getDB(true);
 534                 }
 535
 536                 foreach(glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile)
 537                 {
 538                         echo $sFile.': ';
 539                         $hFile = fopen($sFile, "r");
 540                         $sSQL = fgets($hFile, 100000);
 541                         $iLines = 0;
 542
 543                         while(true)
 544                         {
 545                                 for($i = 0; $i < $iInstances; $i++)
 546                                 {
 547                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 548                                         {
 549                                                 while(pg_get_result($aDBInstances[$i]->connection));
 550                                                 $sSQL = fgets($hFile, 100000);
 551                                                 if (!$sSQL) break 2;
 552                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 553                                                 $iLines++;
 554                                                 if ($iLines == 1000)
 555                                                 {
 556                                                         echo ".";
 557                                                         $iLines = 0;
 558                                                 }
 559                                         }
 560                                 }
 561                                 usleep(10);
 562                         }
 563
 564                         fclose($hFile);
 565
 566                         $bAnyBusy = true;
 567                         while($bAnyBusy)
 568                         {
 569                                 $bAnyBusy = false;
 570                                 for($i = 0; $i < $iInstances; $i++)
 571                                 {
 572                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 573                                 }
 574                                 usleep(10);
 575                         }
 576                         echo "\n";
 577                 }
 578
 579                 echo "Creating indexes\n";
 580                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
 581                 $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
 582                 $sTemplate = replace_tablespace('{ts:aux-data}',
 583                                                 CONST_Tablespace_Aux_Data, $sTemplate);
 584                 $sTemplate = replace_tablespace('{ts:aux-index}',
 585                                                 CONST_Tablespace_Aux_Index, $sTemplate);
 586                 pgsqlRunScript($sTemplate, false);
 587         }
 588
 589         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 590         {
 591                 $bDidSomething = true;
 592                 $oDB =& getDB();
 593                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 594                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 595                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
 596                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
 597                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 598                 $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
 599                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 600                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
 601                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 602                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 603                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 604         }
 605
 606         if ($aCMDResult['osmosis-init'] || ($aCMDResult['all'] && !$aCMDResult['drop'])) // no use doing osmosis-init when dropping update tables
 607         {
 608                 $bDidSomething = true;
 609                 $oDB =& getDB();
 610
 611                 if (!file_exists(CONST_Osmosis_Binary))
 612                 {
 613                         echo "Please download osmosis.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 614                         if (!$aCMDResult['all'])
 615                         {
 616                                 fail("osmosis not found in '".CONST_Osmosis_Binary."'");
 617                         }
 618                 }
 619                 else
 620                 {
 621                         if (file_exists(CONST_InstallPath.'/settings/configuration.txt'))
 622                         {
 623                                 echo "settings/configuration.txt already exists\n";
 624                         }
 625                         else
 626                         {
 627                                 passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_InstallPath.'/settings');
 628                                 // update osmosis configuration.txt with our settings
 629                                 passthru("sed -i 's!baseUrl=.*!baseUrl=".CONST_Replication_Url."!' ".CONST_InstallPath.'/settings/configuration.txt');
 630                                 passthru("sed -i 's:maxInterval = .*:maxInterval = ".CONST_Replication_MaxInterval.":' ".CONST_InstallPath.'/settings/configuration.txt');
 631                         }
 632
 633                         // Find the last node in the DB
 634                         $iLastOSMID = $oDB->getOne("select max(osm_id) from place where osm_type = 'N'");
 635
 636                         // Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
 637                         $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID."/1";
 638                         $sLastNodeXML = file_get_contents($sLastNodeURL);
 639                         preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 640                         $iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
 641
 642                         // Search for the correct state file - uses file timestamps so need to sort by date descending
 643                         $sRepURL = CONST_Replication_Url."/";
 644                         $sRep = file_get_contents($sRepURL."?C=M;O=D;F=1");
 645                         // download.geofabrik.de:    <a href="000/">000/</a></td><td align="right">26-Feb-2013 11:53  </td>
 646                         // planet.openstreetmap.org: <a href="273/">273/</a>                    2013-03-11 07:41    -
 647                         preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>\s*([-0-9a-zA-Z]+ [0-9]{2}:[0-9]{2})#', $sRep, $aRepMatches, PREG_SET_ORDER);
 648                         if ($aRepMatches)
 649                         {
 650                                 $aPrevRepMatch = false;
 651                                 foreach($aRepMatches as $aRepMatch)
 652                                 {
 653                                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 654                                         $aPrevRepMatch = $aRepMatch;
 655                                 }
 656                                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 657
 658                                 $sRepURL .= $aRepMatch[1];
 659                                 $sRep = file_get_contents($sRepURL."?C=M;O=D;F=1");
 660                                 preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>\s*([-0-9a-zA-Z]+ [0-9]{2}:[0-9]{2})#', $sRep, $aRepMatches, PREG_SET_ORDER);
 661                                 $aPrevRepMatch = false;
 662                                 foreach($aRepMatches as $aRepMatch)
 663                                 {
 664                                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 665                                         $aPrevRepMatch = $aRepMatch;
 666                                 }
 667                                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 668
 669                                 $sRepURL .= $aRepMatch[1];
 670                                 $sRep = file_get_contents($sRepURL."?C=M;O=D;F=1");
 671                                 preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a>\s*([-0-9a-zA-Z]+ [0-9]{2}:[0-9]{2})#', $sRep, $aRepMatches, PREG_SET_ORDER);
 672                                 $aPrevRepMatch = false;
 673                                 foreach($aRepMatches as $aRepMatch)
 674                                 {
 675                                         if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
 676                                         $aPrevRepMatch = $aRepMatch;
 677                                 }
 678                                 if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
 679
 680                                 $sRepURL .= $aRepMatch[1].'.state.txt';
 681                                 echo "Getting state file: $sRepURL\n";
 682                                 $sStateFile = file_get_contents($sRepURL);
 683                                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 684                                 file_put_contents(CONST_InstallPath.'/settings/state.txt', $sStateFile);
 685                                 echo "Updating DB status\n";
 686                                 pg_query($oDB->connection, 'TRUNCATE import_status');
 687                                 $sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
 688                                 pg_query($oDB->connection, $sSQL);
 689                         }
 690                         else
 691                         {
 692                                 if (!$aCMDResult['all'])
 693                                 {
 694                                         fail("Cannot read state file directory.");
 695                                 }
 696                         }
 697                 }
 698         }
 699
 700         if ($aCMDResult['index'] || $aCMDResult['all'])
 701         {
 702                 $bDidSomething = true;
 703                 $sOutputFile = '';
 704                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 705                 $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
 706                 passthruCheckReturn($sBaseCmd.' -R 4');
 707                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 708                 passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
 709                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 710                 passthruCheckReturn($sBaseCmd.' -r 26');
 711         }
 712
 713         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 714         {
 715                 echo "Search indices\n";
 716                 $bDidSomething = true;
 717                 $oDB =& getDB();
 718                 $sSQL = 'select distinct partition from country_name';
 719                 $aPartitions = $oDB->getCol($sSQL);
 720                 if (PEAR::isError($aPartitions))
 721                 {
 722                         fail($aPartitions->getMessage());
 723                 }
 724                 if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 725
 726                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 727                 $sTemplate = replace_tablespace('{ts:address-index}',
 728                                                 CONST_Tablespace_Address_Index, $sTemplate);
 729                 $sTemplate = replace_tablespace('{ts:search-index}',
 730                                                 CONST_Tablespace_Search_Index, $sTemplate);
 731                 $sTemplate = replace_tablespace('{ts:aux-index}',
 732                                                 CONST_Tablespace_Aux_Index, $sTemplate);
 733                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 734                 foreach($aMatches as $aMatch)
 735                 {
 736                         $sResult = '';
 737                         foreach($aPartitions as $sPartitionName)
 738                         {
 739                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 740                         }
 741                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 742                 }
 743
 744                 pgsqlRunScript($sTemplate);
 745         }
 746
 747         if (isset($aCMDResult['create-website']))
 748         {
 749                 $bDidSomething = true;
 750                 $sTargetDir = $aCMDResult['create-website'];
 751                 if (!is_dir($sTargetDir))
 752                 {
 753                         echo "You must create the website directory before calling this function.\n";
 754                         fail("Target directory does not exist.");
 755                 }
 756
 757                 @symlink(CONST_InstallPath.'/website/details.php', $sTargetDir.'/details.php');
 758                 @symlink(CONST_InstallPath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 759                 @symlink(CONST_InstallPath.'/website/search.php', $sTargetDir.'/search.php');
 760                 @symlink(CONST_InstallPath.'/website/search.php', $sTargetDir.'/index.php');
 761                 @symlink(CONST_InstallPath.'/website/lookup.php', $sTargetDir.'/lookup.php');
 762                 @symlink(CONST_InstallPath.'/website/deletable.php', $sTargetDir.'/deletable.php');
 763                 @symlink(CONST_InstallPath.'/website/polygons.php', $sTargetDir.'/polygons.php');
 764                 @symlink(CONST_InstallPath.'/website/status.php', $sTargetDir.'/status.php');
 765                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 766                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 767                 @symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css');
 768                 echo "Symlinks created\n";
 769
 770                 $sTestFile = @file_get_contents(CONST_Website_BaseURL.'js/tiles.js');
 771                 if (!$sTestFile)
 772                 {
 773                         echo "\nWARNING: Unable to access the website at ".CONST_Website_BaseURL."\n";
 774                         echo "You may want to update settings/local.php with @define('CONST_Website_BaseURL', 'http://[HOST]/[PATH]/');\n";
 775                 }
 776         }
 777
 778         if ($aCMDResult['drop'])
 779         {
 780                 // The implementation is potentially a bit dangerous because it uses
 781                 // a positive selection of tables to keep, and deletes everything else.
 782                 // Including any tables that the unsuspecting user might have manually
 783                 // created. USE AT YOUR OWN PERIL.
 784                 $bDidSomething = true;
 785
 786                 // tables we want to keep. everything else goes.
 787                 $aKeepTables = array(
 788                    "*columns",
 789                    "import_polygon_*",
 790                    "import_status",
 791                    "place_addressline",
 792                    "location_property*",
 793                    "placex",
 794                    "search_name",
 795                    "seq_*",
 796                    "word",
 797                    "query_log",
 798                    "new_query_log",
 799                    "gb_postcode",
 800                    "spatial_ref_sys",
 801                    "country_name",
 802                    "place_classtype_*"
 803                 );
 804
 805                 $oDB =& getDB();
 806                 $aDropTables = array();
 807                 $aHaveTables = $oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'");
 808                 if (PEAR::isError($aHaveTables))
 809                 {
 810                         fail($aPartitions->getMessage());
 811                 }
 812                 foreach($aHaveTables as $sTable)
 813                 {
 814                         $bFound = false;
 815                         foreach ($aKeepTables as $sKeep)
 816                         {
 817                                 if (fnmatch($sKeep, $sTable))
 818                                 {
 819                                         $bFound = true;
 820                                         break;
 821                                 }
 822                         }
 823                         if (!$bFound) array_push($aDropTables, $sTable);
 824                 }
 825
 826                 foreach ($aDropTables as $sDrop)
 827                 {
 828                         if ($aCMDResult['verbose']) echo "dropping table $sDrop\n";
 829                         @pg_query($oDB->connection, "DROP TABLE $sDrop CASCADE");
 830                         // ignore warnings/errors as they might be caused by a table having
 831                         // been deleted already by CASCADE
 832                 }
 833
 834                 if (!is_null(CONST_Osm2pgsql_Flatnode_File))
 835                 {
 836                         if ($aCMDResult['verbose']) echo "deleting ".CONST_Osm2pgsql_Flatnode_File."\n";
 837                         unlink(CONST_Osm2pgsql_Flatnode_File);
 838                 }
 839         }
 840
 841         if (!$bDidSomething)
 842         {
 843                 showUsage($aCMDOptions, true);
 844         }
 845         else
 846         {
 847                 echo "Setup finished.\n";
 848         }
 849
 850         function pgsqlRunScriptFile($sFilename)
 851         {
 852                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 853
 854                 // Convert database DSN to psql parameters
 855                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 856                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 857                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 858
 859                 $ahGzipPipes = null;
 860                 if (preg_match('/\\.gz$/', $sFilename))
 861                 {
 862                         $aDescriptors = array(
 863                                 0 => array('pipe', 'r'),
 864                                 1 => array('pipe', 'w'),
 865                                 2 => array('file', '/dev/null', 'a')
 866                         );
 867                         $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes);
 868                         if (!is_resource($hGzipProcess)) fail('unable to start zcat');
 869                         $aReadPipe = $ahGzipPipes[1];
 870                         fclose($ahGzipPipes[0]);
 871                 }
 872                 else
 873                 {
 874                         $sCMD .= ' -f '.$sFilename;
 875                         $aReadPipe = array('pipe', 'r');
 876                 }
 877
 878                 $aDescriptors = array(
 879                         0 => $aReadPipe,
 880                         1 => array('pipe', 'w'),
 881                         2 => array('file', '/dev/null', 'a')
 882                 );
 883                 $ahPipes = null;
 884                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 885                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 886
 887
 888                 // TODO: error checking
 889                 while(!feof($ahPipes[1]))
 890                 {
 891                         echo fread($ahPipes[1], 4096);
 892                 }
 893                 fclose($ahPipes[1]);
 894
 895                 $iReturn = proc_close($hProcess);
 896                 if ($iReturn > 0)
 897                 {
 898                         fail("pgsql returned with error code ($iReturn)");
 899                 }
 900                 if ($ahGzipPipes)
 901                 {
 902                         fclose($ahGzipPipes[1]);
 903                         proc_close($hGzipProcess);
 904                 }
 905
 906         }
 907
 908         function pgsqlRunScript($sScript, $bfatal = true)
 909         {
 910                 global $aCMDResult;
 911                 // Convert database DSN to psql parameters
 912                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 913                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 914                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 915                 if ($bfatal && !$aCMDResult['ignore-errors'])
 916                         $sCMD .= ' -v ON_ERROR_STOP=1';
 917                 $aDescriptors = array(
 918                         0 => array('pipe', 'r'),
 919                         1 => STDOUT,
 920                         2 => STDERR
 921                 );
 922                 $ahPipes = null;
 923                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 924                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 925
 926                 while(strlen($sScript))
 927                 {
 928                         $written = fwrite($ahPipes[0], $sScript);
 929                         if ($written <= 0) break;
 930                         $sScript = substr($sScript, $written);
 931                 }
 932                 fclose($ahPipes[0]);
 933                 $iReturn = proc_close($hProcess);
 934                 if ($bfatal && $iReturn > 0)
 935                 {
 936                         fail("pgsql returned with error code ($iReturn)");
 937                 }
 938         }
 939
 940         function pgsqlRunRestoreData($sDumpFile)
 941         {
 942                 // Convert database DSN to psql parameters
 943                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 944                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 945                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 946
 947                 $aDescriptors = array(
 948                         0 => array('pipe', 'r'),
 949                         1 => array('pipe', 'w'),
 950                         2 => array('file', '/dev/null', 'a')
 951                 );
 952                 $ahPipes = null;
 953                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 954                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 955
 956                 fclose($ahPipes[0]);
 957
 958                 // TODO: error checking
 959                 while(!feof($ahPipes[1]))
 960                 {
 961                         echo fread($ahPipes[1], 4096);
 962                 }
 963                 fclose($ahPipes[1]);
 964
 965                 $iReturn = proc_close($hProcess);
 966         }
 967
 968         function pgsqlRunDropAndRestore($sDumpFile)
 969         {
 970                 // Convert database DSN to psql parameters
 971                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 972                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 973                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
 974
 975                 $aDescriptors = array(
 976                         0 => array('pipe', 'r'),
 977                         1 => array('pipe', 'w'),
 978                         2 => array('file', '/dev/null', 'a')
 979                 );
 980                 $ahPipes = null;
 981                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 982                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 983
 984                 fclose($ahPipes[0]);
 985
 986                 // TODO: error checking
 987                 while(!feof($ahPipes[1]))
 988                 {
 989                         echo fread($ahPipes[1], 4096);
 990                 }
 991                 fclose($ahPipes[1]);
 992
 993                 $iReturn = proc_close($hProcess);
 994         }
 995
 996         function passthruCheckReturn($cmd)
 997         {
 998                 $result = -1;
 999                 passthru($cmd, $result);
1000                 if ($result != 0) fail('Error executing external command: '.$cmd);
1001         }
1002
1003         function replace_tablespace($sTemplate, $sTablespace, $sSql)
1004         {
1005                 if ($sTablespace)
1006                         $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"',
1007                                             $sSql);
1008                 else
1009                         $sSql = str_replace($sTemplate, '', $sSql);
1010
1011                 return $sSql;
1012         }
1013