utils/setup.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
   5         ini_set('memory_limit', '800M');
   6
   7         $aCMDOptions = array(
   8                 "Create and setup nominatim search system",
   9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  12
  13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
  14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
  15
  16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
  17
  18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
  19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
  20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
  21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
  22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
  23                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
  24                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
  25                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
  26                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
  27                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
  28                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
  29                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  30                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
  31                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
  32                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
  33                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
  34                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
  35                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
  36                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
  37         );
  38         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  39
  40         $bDidSomething = false;
  41
  42         // This is a pretty hard core defult - the number of processors in the box - 1
  43         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
  44         if ($iInstances < 1)
  45         {
  46                 $iInstances = 1;
  47                 echo "WARNING: resetting threads to $iInstances\n";
  48         }
  49         if ($iInstances > getProcessorCount())
  50         {
  51                 $iInstances = getProcessorCount();
  52                 echo "WARNING: resetting threads to $iInstances\n";
  53         }
  54
  55         // Assume we can steal all the cache memory in the box (unless told otherwise)
  56         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
  57         if ($iCacheMemory > getTotalMemoryMB())
  58         {
  59                 $iCacheMemory = getCacheMemoryMB();
  60                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
  61         }
  62
  63         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
  64         {
  65                 $sBaseFile = basename($aCMDResult['osm-file']);
  66                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
  67                 {
  68                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
  69                         $iTime -= (60*60*24);
  70                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
  71                 }
  72         }
  73         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
  74         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
  75
  76         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  77         {
  78                 echo "Create DB\n";
  79                 $bDidSomething = true;
  80                 $oDB =& DB::connect(CONST_Database_DSN, false);
  81                 if (!PEAR::isError($oDB))
  82                 {
  83                         fail('database already exists ('.CONST_Database_DSN.')');
  84                 }
  85                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
  86         }
  87
  88         if ($aCMDResult['create-db'] || $aCMDResult['all'])
  89         {
  90                 echo "Create DB (2)\n";
  91                 $bDidSomething = true;
  92                 // TODO: path detection, detection memory, etc.
  93
  94                 $oDB =& getDB();
  95                 passthru('createlang plpgsql '.$aDSNInfo['database']);
  96                 $pgver = (float) CONST_Postgresql_Version;
  97                 if ($pgver < 9.1) {
  98                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
  99                 } else {
 100                         pgsqlRunScript('CREATE EXTENSION hstore');
 101                 }
 102                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 103                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 104                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 105                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 106                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
 107                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
 108                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 109                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 110                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
 111                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 112         }
 113
 114         if ($aCMDResult['import-data'] || $aCMDResult['all'])
 115         {
 116                 echo "Import\n";
 117                 $bDidSomething = true;
 118
 119                 $osm2pgsql = CONST_Osm2pgsql_Binary;
 120                 if (!file_exists($osm2pgsql)) fail("please download and build osm2pgsql");
 121                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
 122                 $osm2pgsql .= ' -C '.$iCacheMemory;
 123                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 124                 passthru($osm2pgsql);
 125
 126                 $oDB =& getDB();
 127                 $x = $oDB->getRow('select * from place limit 1');
 128                 if (PEAR::isError($x)) {
 129                         fail($x->getMessage());
 130                 }
 131                 if (!$x) fail('No Data');
 132         }
 133
 134         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 135         {
 136                 echo "Functions\n";
 137                 $bDidSomething = true;
 138                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 139                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 140                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 141                 pgsqlRunScript($sTemplate);
 142         }
 143
 144         if ($aCMDResult['create-minimal-tables'])
 145         {
 146                 echo "Minimal Tables\n";
 147                 $bDidSomething = true;
 148                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
 149
 150                 $sScript = '';
 151
 152                 // Backstop the import process - easliest possible import id
 153                 $sScript .= "insert into import_npi_log values (18022);\n";
 154
 155                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
 156                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
 157
 158                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
 159                 {
 160                         list($sClass, $sType) = explode(' ', trim($sLine));
 161                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
 162                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
 163
 164                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
 165                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
 166
 167                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
 168                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
 169                 }
 170                 fclose($hFile);
 171                 pgsqlRunScript($sScript);
 172         }
 173
 174         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 175         {
 176                 echo "Tables\n";
 177                 $bDidSomething = true;
 178                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
 179
 180                 // re-run the functions
 181                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 182                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 183                 pgsqlRunScript($sTemplate);
 184         }
 185
 186         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 187         {
 188                 echo "Partitions\n";
 189                 $bDidSomething = true;
 190                 $oDB =& getDB();
 191                 $sSQL = 'select partition from country_name order by country_code';
 192                 $aPartitions = $oDB->getCol($sSQL);
 193                 if (PEAR::isError($aPartitions))
 194                 {
 195                         fail($aPartitions->getMessage());
 196                 }
 197                 $aPartitions[] = 0;
 198
 199                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
 200                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 201                 foreach($aMatches as $aMatch)
 202                 {
 203                         $sResult = '';
 204                         foreach($aPartitions as $sPartitionName)
 205                         {
 206                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 207                         }
 208                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 209                 }
 210
 211                 pgsqlRunScript($sTemplate);
 212         }
 213
 214         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
 215         {
 216                 $bDidSomething = true;
 217                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
 218                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
 219                 if (file_exists($sWikiArticlesFile))
 220                 {
 221                         echo "Importing wikipedia articles...";
 222                         pgsqlRunRestoreData($sWikiArticlesFile);
 223                         echo "...done\n";
 224                 }
 225                 else
 226                 {
 227                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
 228                 }
 229                 if (file_exists($sWikiRedirectsFile))
 230                 {
 231                         echo "Importing wikipedia redirects...";
 232                         pgsqlRunRestoreData($sWikiRedirectsFile);
 233                         echo "...done\n";
 234                 }
 235                 else
 236                 {
 237                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
 238                 }
 239         }
 240
 241
 242         if ($aCMDResult['load-data'] || $aCMDResult['all'])
 243         {
 244                 echo "Load Data\n";
 245                 $bDidSomething = true;
 246
 247                 $oDB =& getDB();
 248                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
 249                 echo '.';
 250                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
 251                 echo '.';
 252                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
 253                 echo '.';
 254                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
 255                 echo '.';
 256                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
 257                 echo '.';
 258                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
 259                 echo '.';
 260                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
 261                 echo '.';
 262                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
 263                 echo '.';
 264                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 265                 echo '.';
 266
 267                 // pre-create the word list
 268                 if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 269                 echo '.';
 270                 if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct postcode as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
 271                 echo '.';
 272                 if (!pg_query($oDB->connection, 'select count(getorcreate_housenumber_id(v)) from (select distinct housenumber as v from place where housenumber is not null) as w;')) fail(pg_last_error($oDB->connection));
 273                 echo '.';
 274
 275                 $aDBInstances = array();
 276                 for($i = 0; $i < $iInstances; $i++)
 277                 {
 278                         $aDBInstances[$i] =& getDB(true);
 279                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 280                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 281                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 282                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 283                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 284                 }
 285                 $bAnyBusy = true;
 286                 while($bAnyBusy)
 287                 {
 288                         $bAnyBusy = false;
 289                         for($i = 0; $i < $iInstances; $i++)
 290                         {
 291                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 292                         }
 293                         sleep(1);
 294                         echo '.';
 295                 }
 296                 echo "\n";
 297                 echo "Reanalysing database...\n";
 298                 pgsqlRunScript('ANALYSE');
 299         }
 300
 301         if ($aCMDResult['create-roads'])
 302         {
 303                 $bDidSomething = true;
 304
 305                 $oDB =& getDB();
 306                 $aDBInstances = array();
 307                 for($i = 0; $i < $iInstances; $i++)
 308                 {
 309                         $aDBInstances[$i] =& getDB(true);
 310                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
 311                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, country_code, geometry) from ';
 312                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
 313                         if ($aCMDResult['verbose']) echo "$sSQL\n";
 314                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 315                 }
 316                 $bAnyBusy = true;
 317                 while($bAnyBusy)
 318                 {
 319                         $bAnyBusy = false;
 320                         for($i = 0; $i < $iInstances; $i++)
 321                         {
 322                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 323                         }
 324                         sleep(1);
 325                         echo '.';
 326                 }
 327                 echo "\n";
 328         }
 329
 330         if ($aCMDResult['import-tiger-data'])
 331         {
 332                 $bDidSomething = true;
 333
 334                 $aDBInstances = array();
 335                 for($i = 0; $i < $iInstances; $i++)
 336                 {
 337                         $aDBInstances[$i] =& getDB(true);
 338                 }
 339
 340                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
 341                 {
 342                         echo $sFile.': ';
 343                         $hFile = fopen($sFile, "r");
 344                         $sSQL = fgets($hFile, 100000);
 345                         $iLines = 0;
 346
 347                         while(true)
 348                         {
 349                                 for($i = 0; $i < $iInstances; $i++)
 350                                 {
 351                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
 352                                         {
 353                                                 while(pg_get_result($aDBInstances[$i]->connection));
 354                                                 $sSQL = fgets($hFile, 100000);
 355                                                 if (!$sSQL) break 2;
 356                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 357                                                 $iLines++;
 358                                                 if ($iLines == 1000)
 359                                                 {
 360                                                         echo ".";
 361                                                         $iLines = 0;
 362                                                 }
 363                                         }
 364                                 }
 365                                 usleep(10);
 366                         }
 367
 368                         fclose($hFile);
 369
 370                         $bAnyBusy = true;
 371                         while($bAnyBusy)
 372                         {
 373                                 $bAnyBusy = false;
 374                                 for($i = 0; $i < $iInstances; $i++)
 375                                 {
 376                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
 377                                 }
 378                                 usleep(10);
 379                         }
 380                         echo "\n";
 381                 }
 382         }
 383
 384         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
 385         {
 386                 $bDidSomething = true;
 387                 $oDB =& getDB();
 388                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
 389                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 390                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,country_code,";
 391                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select country_code,postcode,";
 392                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 393                 $sSQL .= "from placex where postcode is not null group by country_code,postcode) as x";
 394                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 395
 396                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
 397                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
 398                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
 399                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 400         }
 401
 402         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
 403         {
 404                 $bDidSomething = true;
 405                 $oDB =& getDB();
 406
 407                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
 408                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
 409                 else passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
 410
 411                 $sDate = $aCMDResult['osmosis-init-date'];
 412                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
 413                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
 414                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
 415                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
 416                 $sURL .= '&stream=minute';
 417                 echo "Getting state file: $sURL\n";
 418                 $sStateFile = file_get_contents($sURL);
 419                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 420                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 421                 echo "Updating DB status\n";
 422                 pg_query($oDB->connection, 'TRUNCATE import_status');
 423                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
 424                 pg_query($oDB->connection, $sSQL);
 425
 426         }
 427
 428         if ($aCMDResult['index'] || $aCMDResult['all'])
 429         {
 430                 $bDidSomething = true;
 431                 $sOutputFile = '';
 432                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
 433                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
 434                 passthru($sBaseCmd.' -R 4');
 435                 pgsqlRunScript('ANALYSE');
 436                 passthru($sBaseCmd.' -r 5 -R 25');
 437                 pgsqlRunScript('ANALYSE');
 438                 passthru($sBaseCmd.' -r 26');
 439         }
 440
 441         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
 442         {
 443                 echo "Search indices\n";
 444                 $bDidSomething = true;
 445                 $oDB =& getDB();
 446                 $sSQL = 'select partition from country_name order by country_code';
 447                 $aPartitions = $oDB->getCol($sSQL);
 448                 if (PEAR::isError($aPartitions))
 449                 {
 450                         fail($aPartitions->getMessage());
 451                 }
 452                 $aPartitions[] = 0;
 453
 454                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 455                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 456                 foreach($aMatches as $aMatch)
 457                 {
 458                         $sResult = '';
 459                         foreach($aPartitions as $sPartitionName)
 460                         {
 461                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
 462                         }
 463                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
 464                 }
 465
 466                 pgsqlRunScript($sTemplate);
 467         }
 468
 469         if (isset($aCMDResult['create-website']))
 470         {
 471                 $bDidSomething = true;
 472                 $sTargetDir = $aCMDResult['create-website'];
 473                 if (!is_dir($sTargetDir)) fail('please specify a directory to setup');
 474                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
 475                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 476                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 477                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
 478                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 479                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
 480                 echo "Symlinks created\n";
 481         }
 482
 483         if (!$bDidSomething)
 484         {
 485                 showUsage($aCMDOptions, true);
 486         }
 487
 488         function pgsqlRunScriptFile($sFilename)
 489         {
 490                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 491
 492                 // Convert database DSN to psql paramaters
 493                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 494                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 495                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
 496
 497                 $aDescriptors = array(
 498                         0 => array('pipe', 'r'),
 499                         1 => array('pipe', 'w'),
 500                         2 => array('file', '/dev/null', 'a')
 501                 );
 502                 $ahPipes = null;
 503                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 504                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 505
 506                 fclose($ahPipes[0]);
 507
 508                 // TODO: error checking
 509                 while(!feof($ahPipes[1]))
 510                 {
 511                         echo fread($ahPipes[1], 4096);
 512                 }
 513                 fclose($ahPipes[1]);
 514
 515                 proc_close($hProcess);
 516         }
 517
 518         function pgsqlRunScript($sScript)
 519         {
 520                 // Convert database DSN to psql paramaters
 521                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 522                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 523                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
 524                 $aDescriptors = array(
 525                         0 => array('pipe', 'r'),
 526                         1 => STDOUT,
 527                         2 => STDERR
 528                 );
 529                 $ahPipes = null;
 530                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
 531                 if (!is_resource($hProcess)) fail('unable to start pgsql');
 532
 533                 while(strlen($sScript))
 534                 {
 535                         $written = fwrite($ahPipes[0], $sScript);
 536                         $sScript = substr($sScript, $written);
 537                 }
 538                 fclose($ahPipes[0]);
 539                 proc_close($hProcess);
 540         }
 541
 542         function pgsqlRunRestoreData($sDumpFile)
 543         {
 544                 // Convert database DSN to psql paramaters
 545                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 546                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 547                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
 548
 549                 $aDescriptors = array(
 550                         0 => array('pipe', 'r'),
 551                         1 => array('pipe', 'w'),
 552                         2 => array('file', '/dev/null', 'a')
 553                 );
 554                 $ahPipes = null;
 555                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 556                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
 557
 558                 fclose($ahPipes[0]);
 559
 560                 // TODO: error checking
 561                 while(!feof($ahPipes[1]))
 562                 {
 563                         echo fread($ahPipes[1], 4096);
 564                 }
 565                 fclose($ahPipes[1]);
 566
 567                 proc_close($hProcess);
 568         }