]> git.openstreetmap.org Git - nominatim.git/commitdiff
Merge pull request #1754 from mtmail/nominatim-db-tests-against-postgres
authorSarah Hoffmann <lonvia@denofr.de>
Sun, 26 Apr 2020 08:20:30 +0000 (10:20 +0200)
committerGitHub <noreply@github.com>
Sun, 26 Apr 2020 08:20:30 +0000 (10:20 +0200)
Nominatim::DB tests against separate postgresql database

1  2 
lib/setup/SetupClass.php

diff --combined lib/setup/SetupClass.php
index 56d9f3451e02701643e67dca36706a34f432274a,7c1c628e0e70171b08a60d103b17cdaf152fdf60..385eff70eb6cda7160cb84f7a6b5c5ff0c96b08e
@@@ -3,7 -3,6 +3,7 @@@
  namespace Nominatim\Setup;
  
  require_once(CONST_BasePath.'/lib/setup/AddressLevelParser.php');
 +require_once(CONST_BasePath.'/lib/Shell.php');
  
  class SetupFunctions
  {
@@@ -52,7 -51,7 +52,7 @@@
          }
  
          // setting member variables based on command line options stored in $aCMDResult
 -        $this->bQuiet = $aCMDResult['quiet'];
 +        $this->bQuiet = isset($aCMDResult['quiet']) && $aCMDResult['quiet'];
          $this->bVerbose = $aCMDResult['verbose'];
  
          //setting default values which are not set by the update.php array
@@@ -77,7 -76,7 +77,7 @@@
              $this->bEnableDiffUpdates = false;
          }
  
 -        $this->bDrop = $aCMDResult['drop'];
 +        $this->bDrop = isset($aCMDResult['drop']) && $aCMDResult['drop'];
      }
  
      public function createDB()
          info('Create DB');
          $oDB = new \Nominatim\DB;
  
-         if ($oDB->databaseExists()) {
+         if ($oDB->checkConnection()) {
              fail('database already exists ('.CONST_Database_DSN.')');
          }
  
 -        $sCreateDBCmd = 'createdb -E UTF-8'
 -            .' -p '.escapeshellarg($this->aDSNInfo['port'])
 -            .' '.escapeshellarg($this->aDSNInfo['database']);
 +        $oCmd = (new \Nominatim\Shell('createdb'))
 +                ->addParams('-E', 'UTF-8')
 +                ->addParams('-p', $this->aDSNInfo['port']);
 +
          if (isset($this->aDSNInfo['username'])) {
 -            $sCreateDBCmd .= ' -U '.escapeshellarg($this->aDSNInfo['username']);
 +            $oCmd->addParams('-U', $this->aDSNInfo['username']);
 +        }
 +        if (isset($this->aDSNInfo['password'])) {
 +            $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
          }
 -
          if (isset($this->aDSNInfo['hostspec'])) {
 -            $sCreateDBCmd .= ' -h '.escapeshellarg($this->aDSNInfo['hostspec']);
 +            $oCmd->addParams('-h', $this->aDSNInfo['hostspec']);
          }
 +        $oCmd->addParams($this->aDSNInfo['database']);
  
 -        $result = $this->runWithPgEnv($sCreateDBCmd);
 -        if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd);
 +        $result = $oCmd->run();
 +        if ($result != 0) fail('Error executing external command: '.$oCmd->escapedCmd());
      }
  
      public function connect()
      {
          info('Import data');
  
 -        $osm2pgsql = CONST_Osm2pgsql_Binary;
 -        if (!file_exists($osm2pgsql)) {
 +        if (!file_exists(CONST_Osm2pgsql_Binary)) {
              echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n";
              echo "Normally you should not need to set this manually.\n";
 -            fail("osm2pgsql not found in '$osm2pgsql'");
 +            fail("osm2pgsql not found in '".CONST_Osm2pgsql_Binary."'");
          }
  
 -        $osm2pgsql .= ' -S '.escapeshellarg(CONST_Import_Style);
 +        $oCmd = new \Nominatim\Shell(CONST_Osm2pgsql_Binary);
 +        $oCmd->addParams('--style', CONST_Import_Style);
  
          if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
 -            $osm2pgsql .= ' --flat-nodes '.escapeshellarg(CONST_Osm2pgsql_Flatnode_File);
 -        }
 -
 -        if (CONST_Tablespace_Osm2pgsql_Data)
 -            $osm2pgsql .= ' --tablespace-slim-data '.escapeshellarg(CONST_Tablespace_Osm2pgsql_Data);
 -        if (CONST_Tablespace_Osm2pgsql_Index)
 -            $osm2pgsql .= ' --tablespace-slim-index '.escapeshellarg(CONST_Tablespace_Osm2pgsql_Index);
 -        if (CONST_Tablespace_Place_Data)
 -            $osm2pgsql .= ' --tablespace-main-data '.escapeshellarg(CONST_Tablespace_Place_Data);
 -        if (CONST_Tablespace_Place_Index)
 -            $osm2pgsql .= ' --tablespace-main-index '.escapeshellarg(CONST_Tablespace_Place_Index);
 -        $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1';
 -        $osm2pgsql .= ' -C '.escapeshellarg($this->iCacheMemory);
 -        $osm2pgsql .= ' -P '.escapeshellarg($this->aDSNInfo['port']);
 +            $oCmd->addParams('--flat-nodes', CONST_Osm2pgsql_Flatnode_File);
 +        }
 +        if (CONST_Tablespace_Osm2pgsql_Data) {
 +            $oCmd->addParams('--tablespace-slim-data', CONST_Tablespace_Osm2pgsql_Data);
 +        }
 +        if (CONST_Tablespace_Osm2pgsql_Index) {
 +            $oCmd->addParams('--tablespace-slim-index', CONST_Tablespace_Osm2pgsql_Index);
 +        }
 +        if (CONST_Tablespace_Place_Data) {
 +            $oCmd->addParams('--tablespace-main-data', CONST_Tablespace_Place_Data);
 +        }
 +        if (CONST_Tablespace_Place_Index) {
 +            $oCmd->addParams('--tablespace-main-index', CONST_Tablespace_Place_Index);
 +        }
 +        $oCmd->addParams('--latlong', '--slim', '--create');
 +        $oCmd->addParams('--output', 'gazetteer');
 +        $oCmd->addParams('--hstore');
 +        $oCmd->addParams('--number-processes', 1);
 +        $oCmd->addParams('--cache', $this->iCacheMemory);
 +        $oCmd->addParams('--port', $this->aDSNInfo['port']);
 +
          if (isset($this->aDSNInfo['username'])) {
 -            $osm2pgsql .= ' -U '.escapeshellarg($this->aDSNInfo['username']);
 +            $oCmd->addParams('--username', $this->aDSNInfo['username']);
 +        }
 +        if (isset($this->aDSNInfo['password'])) {
 +            $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
          }
          if (isset($this->aDSNInfo['hostspec'])) {
 -            $osm2pgsql .= ' -H '.escapeshellarg($this->aDSNInfo['hostspec']);
 +            $oCmd->addParams('--host', $this->aDSNInfo['hostspec']);
          }
 -        $osm2pgsql .= ' -d '.escapeshellarg($this->aDSNInfo['database']).' '.escapeshellarg($sOSMFile);
 -
 -        $this->runWithPgEnv($osm2pgsql);
 +        $oCmd->addParams('--database', $this->aDSNInfo['database']);
 +        $oCmd->addParams($sOSMFile);
 +        $oCmd->run();
  
          if (!$this->sIgnoreErrors && !$this->oDB->getRow('select * from place limit 1')) {
              fail('No Data');
  
      public function index($bIndexNoanalyse)
      {
 -        $sBaseCmd = CONST_BasePath.'/nominatim/nominatim.py'
 -            .' -d '.escapeshellarg($this->aDSNInfo['database'])
 -            .' -P '.escapeshellarg($this->aDSNInfo['port'])
 -            .' -t '.escapeshellarg($this->iInstances);
 +        $oBaseCmd = (new \Nominatim\Shell(CONST_BasePath.'/nominatim/nominatim.py'))
 +                    ->addParams('--database', $this->aDSNInfo['database'])
 +                    ->addParams('--port', $this->aDSNInfo['port'])
 +                    ->addParams('--threads', $this->iInstances);
 +
          if (!$this->bQuiet) {
 -            $sBaseCmd .= ' -v';
 +            $oBaseCmd->addParams('-v');
          }
          if ($this->bVerbose) {
 -            $sBaseCmd .= ' -v';
 +            $oBaseCmd->addParams('-v');
          }
          if (isset($this->aDSNInfo['hostspec'])) {
 -            $sBaseCmd .= ' -H '.escapeshellarg($this->aDSNInfo['hostspec']);
 +            $oBaseCmd->addParams('--host', $this->aDSNInfo['hostspec']);
          }
          if (isset($this->aDSNInfo['username'])) {
 -            $sBaseCmd .= ' -U '.escapeshellarg($this->aDSNInfo['username']);
 +            $oBaseCmd->addParams('--user', $this->aDSNInfo['username']);
 +        }
 +        if (isset($this->aDSNInfo['password'])) {
 +            $oBaseCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
          }
  
          info('Index ranks 0 - 4');
 -        $iStatus = $this->runWithPgEnv($sBaseCmd.' -R 4');
 +        $oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4);
 +        echo $oCmd->escapedCmd();
 +        
 +        $iStatus = $oCmd->run();
          if ($iStatus != 0) {
              fail('error status ' . $iStatus . ' running nominatim!');
          }
          if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
  
          info('Index ranks 5 - 25');
 -        $iStatus = $this->runWithPgEnv($sBaseCmd.' -r 5 -R 25');
 +        $oCmd = (clone $oBaseCmd)->addParams('--minrank', 5, '--maxrank', 25);
 +        $iStatus = $oCmd->run();
          if ($iStatus != 0) {
              fail('error status ' . $iStatus . ' running nominatim!');
          }
          if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
  
          info('Index ranks 26 - 30');
 -        $iStatus = $this->runWithPgEnv($sBaseCmd.' -r 26');
 +        $oCmd = (clone $oBaseCmd)->addParams('--minrank', 26);
 +        $iStatus = $oCmd->run();
          if ($iStatus != 0) {
              fail('error status ' . $iStatus . ' running nominatim!');
          }
                         );
  
          $aDropTables = array();
-         $aHaveTables = $this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'");
+         $aHaveTables = $this->oDB->getListOfTables();
  
          foreach ($aHaveTables as $sTable) {
              $bFound = false;
      {
          if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
  
 -        $sCMD = 'psql'
 -            .' -p '.escapeshellarg($this->aDSNInfo['port'])
 -            .' -d '.escapeshellarg($this->aDSNInfo['database']);
 +        $oCmd = (new \Nominatim\Shell('psql'))
 +                ->addParams('--port', $this->aDSNInfo['port'])
 +                ->addParams('--dbname', $this->aDSNInfo['database']);
 +
          if (!$this->bVerbose) {
 -            $sCMD .= ' -q';
 +            $oCmd->addParams('--quiet');
          }
          if (isset($this->aDSNInfo['hostspec'])) {
 -            $sCMD .= ' -h '.escapeshellarg($this->aDSNInfo['hostspec']);
 +            $oCmd->addParams('--host', $this->aDSNInfo['hostspec']);
          }
          if (isset($this->aDSNInfo['username'])) {
 -            $sCMD .= ' -U '.escapeshellarg($this->aDSNInfo['username']);
 +            $oCmd->addParams('--username', $this->aDSNInfo['username']);
          }
 -        $aProcEnv = null;
          if (isset($this->aDSNInfo['password'])) {
 -            $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
 +            $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
          }
          $ahGzipPipes = null;
          if (preg_match('/\\.gz$/', $sFilename)) {
                               1 => array('pipe', 'w'),
                               2 => array('file', '/dev/null', 'a')
                              );
 -            $hGzipProcess = proc_open('zcat '.escapeshellarg($sFilename), $aDescriptors, $ahGzipPipes);
 +            $oZcatCmd = new \Nominatim\Shell('zcat', $sFilename);
 +
 +            $hGzipProcess = proc_open($oZcatCmd->escapedCmd(), $aDescriptors, $ahGzipPipes);
              if (!is_resource($hGzipProcess)) fail('unable to start zcat');
              $aReadPipe = $ahGzipPipes[1];
              fclose($ahGzipPipes[0]);
          } else {
 -            $sCMD .= ' -f '.escapeshellarg($sFilename);
 +            $oCmd->addParams('--file', $sFilename);
              $aReadPipe = array('pipe', 'r');
          }
          $aDescriptors = array(
                           2 => array('file', '/dev/null', 'a')
                          );
          $ahPipes = null;
 -        $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv);
 +
 +        $hProcess = proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv);
          if (!is_resource($hProcess)) fail('unable to start pgsql');
          // TODO: error checking
          while (!feof($ahPipes[1])) {
          return $sSql;
      }
  
 -    private function runWithPgEnv($sCmd)
 -    {
 -        if ($this->bVerbose) {
 -            echo "Execute: $sCmd\n";
 -        }
 -
 -        $aProcEnv = null;
 -
 -        if (isset($this->aDSNInfo['password'])) {
 -            $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
 -        }
 -
 -        return runWithEnv($sCmd, $aProcEnv);
 -    }
 -
      /**
       * Drop table with the given name if it exists.
       *
      private function dropTable($sName)
      {
          if ($this->bVerbose) echo "Dropping table $sName\n";
-         $this->oDB->exec('DROP TABLE IF EXISTS '.$sName.' CASCADE');
+         $this->oDB->deleteTable($sName);
      }
  
      /**