From: Sarah Hoffmann Date: Sun, 17 Jan 2021 20:02:50 +0000 (+0100) Subject: remove old nominatim.py in favour of 'nominatim index' X-Git-Tag: v3.7.0~52^2 X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/504922ffbecd42eed01dfb9da6bbf2c7aae9a094 remove old nominatim.py in favour of 'nominatim index' The PHP scripts need to know the position of the nominatim tool in order to call it. This is handed in as environment variable, so it can be set by the Python script. --- diff --git a/cmake/script.tmpl b/cmake/script.tmpl index 30b8717b..aa25a124 100755 --- a/cmake/script.tmpl +++ b/cmake/script.tmpl @@ -8,5 +8,6 @@ require('@CMAKE_SOURCE_DIR@/lib/dotenv_loader.php'); @define('CONST_DataDir', '@CMAKE_SOURCE_DIR@'); loadDotEnv(); +$_SERVER['NOMINATIM_NOMINATIM_TOOL'] = '@CMAKE_BINARY_DIR@/nominatim'; require_once('@CMAKE_SOURCE_DIR@/lib/admin/@script_source@'); diff --git a/cmake/tool.tmpl b/cmake/tool.tmpl index 40f2b8ea..43646792 100755 --- a/cmake/tool.tmpl +++ b/cmake/tool.tmpl @@ -1,8 +1,11 @@ #!/usr/bin/env python3 import sys +import os sys.path.insert(1, '@CMAKE_SOURCE_DIR@') +os.environ['NOMINATIM_NOMINATIM_TOOL'] = __file__ + from nominatim import cli exit(cli.nominatim(module_dir='@CMAKE_BINARY_DIR@/module', diff --git a/lib/Shell.php b/lib/Shell.php index 59c4473b..72f90735 100644 --- a/lib/Shell.php +++ b/lib/Shell.php @@ -7,7 +7,7 @@ class Shell public function __construct($sBaseCmd, ...$aParams) { if (!$sBaseCmd) { - throw new Exception('Command missing in new() call'); + throw new \Exception('Command missing in new() call'); } $this->baseCmd = $sBaseCmd; $this->aParams = array(); diff --git a/lib/admin/update.php b/lib/admin/update.php index 50f611d7..fe9658b5 100644 --- a/lib/admin/update.php +++ b/lib/admin/update.php @@ -105,25 +105,14 @@ if ($fPostgresVersion >= 11.0) { } -$oIndexCmd = (new \Nominatim\Shell(CONST_DataDir.'/nominatim/nominatim.py')) - ->addParams('--database', $aDSNInfo['database']) - ->addParams('--port', $aDSNInfo['port']) - ->addParams('--threads', $aResult['index-instances']); -if (!$aResult['quiet']) { - $oIndexCmd->addParams('--verbose'); +$oIndexCmd = (new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'))) + ->addParams('index'); +if ($aResult['quiet']) { + $oIndexCmd->addParams('--quiet'); } if ($aResult['verbose']) { $oIndexCmd->addParams('--verbose'); } -if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $oIndexCmd->addParams('--host', $aDSNInfo['hostspec']); -} -if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $oIndexCmd->addParams('--username', $aDSNInfo['username']); -} -if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $oIndexCmd->addEnvPair('PGPASSWORD', $aDSNInfo['password']); -} $sPyosmiumBin = getSetting('PYOSMIUM_BINARY'); $sBaseURL = getSetting('REPLICATION_URL'); @@ -288,15 +277,9 @@ if ($aResult['recompute-word-counts']) { } if ($aResult['index']) { - $oCmd = (clone $oIndexCmd) - ->addParams('--minrank', $aResult['index-rank'], '-b'); - $oCmd->run(); - $oCmd = (clone $oIndexCmd) ->addParams('--minrank', $aResult['index-rank']); $oCmd->run(); - - $oDB->exec('update import_status set indexed = true'); } if ($aResult['update-address-levels']) { @@ -438,15 +421,6 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { if (!$aResult['no-index']) { $fCMDStartTime = time(); - $oThisIndexCmd = clone($oIndexCmd); - $oThisIndexCmd->addParams('-b'); - echo $oThisIndexCmd->escapedCmd()."\n"; - $iErrorLevel = $oThisIndexCmd->run(); - if ($iErrorLevel) { - echo "Error: $iErrorLevel\n"; - exit($iErrorLevel); - } - $oThisIndexCmd = clone($oIndexCmd); echo $oThisIndexCmd->escapedCmd()."\n"; $iErrorLevel = $oThisIndexCmd->run(); @@ -463,9 +437,6 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { var_Dump($sSQL); $oDB->exec($sSQL); echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n"; - - $sSQL = 'update import_status set indexed = true'; - $oDB->exec($sSQL); } else { if ($aResult['import-osmosis-all']) { echo "Error: --no-index cannot be used with continuous imports (--import-osmosis-all).\n"; diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php index 77b14a8a..d17fdca7 100755 --- a/lib/setup/SetupClass.php +++ b/lib/setup/SetupClass.php @@ -549,26 +549,15 @@ class SetupFunctions { $this->checkModulePresence(); // raises exception on failure - $oBaseCmd = (new \Nominatim\Shell(CONST_DataDir.'/nominatim/nominatim.py')) - ->addParams('--database', $this->aDSNInfo['database']) - ->addParams('--port', $this->aDSNInfo['port']) - ->addParams('--threads', $this->iInstances); + $oBaseCmd = (new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'))) + ->addParams('index'); - if (!$this->bQuiet) { - $oBaseCmd->addParams('-v'); + if ($this->bQuiet) { + $oBaseCmd->addParams('-q'); } if ($this->bVerbose) { $oBaseCmd->addParams('-v'); } - if (isset($this->aDSNInfo['hostspec'])) { - $oBaseCmd->addParams('--host', $this->aDSNInfo['hostspec']); - } - if (isset($this->aDSNInfo['username'])) { - $oBaseCmd->addParams('--user', $this->aDSNInfo['username']); - } - if (isset($this->aDSNInfo['password'])) { - $oBaseCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']); - } info('Index ranks 0 - 4'); $oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4); @@ -581,14 +570,14 @@ class SetupFunctions if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); info('Index administrative boundaries'); - $oCmd = (clone $oBaseCmd)->addParams('-b'); + $oCmd = (clone $oBaseCmd)->addParams('--boundaries-only'); $iStatus = $oCmd->run(); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); } info('Index ranks 5 - 25'); - $oCmd = (clone $oBaseCmd)->addParams('--minrank', 5, '--maxrank', 25); + $oCmd = (clone $oBaseCmd)->addParams('--no-boundaries', '--minrank', 5, '--maxrank', 25); $iStatus = $oCmd->run(); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); @@ -597,7 +586,7 @@ class SetupFunctions if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); info('Index ranks 26 - 30'); - $oCmd = (clone $oBaseCmd)->addParams('--minrank', 26); + $oCmd = (clone $oBaseCmd)->addParams('--no-boundaries', '--minrank', 26); $iStatus = $oCmd->run(); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); diff --git a/nominatim/cli.py b/nominatim/cli.py index acb6839f..65ea90bb 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -78,7 +78,7 @@ class CommandlineParser: args.project_dir = Path(args.project_dir) logging.basicConfig(stream=sys.stderr, - format='%(asctime)s %(levelname)s: %(message)s', + format='%(asctime)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=max(4 - args.verbose, 1) * 10) @@ -328,6 +328,9 @@ class UpdateIndex: if not args.boundaries_only: indexer.index_by_rank(args.minrank, args.maxrank) + if not args.no_boundaries and not args.boundaries_only: + indexer.update_status_table() + return 0 diff --git a/nominatim/indexer/indexer.py b/nominatim/indexer/indexer.py index d86303c4..094d1279 100644 --- a/nominatim/indexer/indexer.py +++ b/nominatim/indexer/indexer.py @@ -124,6 +124,13 @@ class Indexer: else: self.index(RankRunner(maxrank)) + def update_status_table(self): + """ Update the status in the status table to 'indexed'. + """ + with self.conn.cursor() as cur: + cur.execute('UPDATE import_status SET indexed = true') + self.conn.commit() + def index(self, obj, batch=1): """ Index a single rank or table. `obj` describes the SQL to use for indexing. `batch` describes the number of objects that diff --git a/nominatim/indexer/progress.py b/nominatim/indexer/progress.py index 99120673..c9d8816b 100644 --- a/nominatim/indexer/progress.py +++ b/nominatim/indexer/progress.py @@ -26,7 +26,7 @@ class ProgressLogger: self.done_places = 0 self.rank_start_time = datetime.now() self.log_interval = log_interval - self.next_info = INITIAL_PROGRESS if LOG.isEnabledFor(logging.INFO) else total + 1 + self.next_info = INITIAL_PROGRESS if LOG.isEnabledFor(logging.WARNING) else total + 1 def add(self, num=1): """ Mark `num` places as processed. Print a log message if the @@ -47,9 +47,9 @@ class ProgressLogger: places_per_sec = self.done_places / done_time eta = (self.total_places - self.done_places) / places_per_sec - LOG.info("Done %d in %d @ %.3f per second - %s ETA (seconds): %.2f", - self.done_places, int(done_time), - places_per_sec, self.name, eta) + LOG.warning("Done %d in %d @ %.3f per second - %s ETA (seconds): %.2f", + self.done_places, int(done_time), + places_per_sec, self.name, eta) self.next_info += int(places_per_sec) * self.log_interval diff --git a/nominatim/nominatim.py b/nominatim/nominatim.py deleted file mode 100755 index fdc2bcba..00000000 --- a/nominatim/nominatim.py +++ /dev/null @@ -1,84 +0,0 @@ -#! /usr/bin/env python3 -#----------------------------------------------------------------------------- -# nominatim - [description] -#----------------------------------------------------------------------------- -# -# Indexing tool for the Nominatim database. -# -# Based on C version by Brian Quinion -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -from argparse import ArgumentParser, RawDescriptionHelpFormatter -import logging -import sys -import getpass - -from indexer.indexer import Indexer - -def nominatim_arg_parser(): - """ Setup the command-line parser for the tool. - """ - parser = ArgumentParser(description="Indexing tool for Nominatim.", - formatter_class=RawDescriptionHelpFormatter) - - parser.add_argument('-d', '--database', - dest='dbname', action='store', default='nominatim', - help='Name of the PostgreSQL database to connect to.') - parser.add_argument('-U', '--username', - dest='user', action='store', - help='PostgreSQL user name.') - parser.add_argument('-W', '--password', - dest='password_prompt', action='store_true', - help='Force password prompt.') - parser.add_argument('-H', '--host', - dest='host', action='store', - help='PostgreSQL server hostname or socket location.') - parser.add_argument('-P', '--port', - dest='port', action='store', - help='PostgreSQL server port') - parser.add_argument('-b', '--boundary-only', - dest='boundary_only', action='store_true', - help='Only index administrative boundaries (ignores min/maxrank).') - parser.add_argument('-r', '--minrank', - dest='minrank', type=int, metavar='RANK', default=0, - help='Minimum/starting rank.') - parser.add_argument('-R', '--maxrank', - dest='maxrank', type=int, metavar='RANK', default=30, - help='Maximum/finishing rank.') - parser.add_argument('-t', '--threads', - dest='threads', type=int, metavar='NUM', default=1, - help='Number of threads to create for indexing.') - parser.add_argument('-v', '--verbose', - dest='loglevel', action='count', default=0, - help='Increase verbosity') - - return parser - -if __name__ == '__main__': - OPTIONS = nominatim_arg_parser().parse_args(sys.argv[1:]) - - logging.basicConfig(stream=sys.stderr, format='%(levelname)s: %(message)s', - level=max(3 - OPTIONS.loglevel, 0) * 10) - - OPTIONS.password = None - if OPTIONS.password_prompt: - PASSWORD = getpass.getpass("Database password: ") - OPTIONS.password = PASSWORD - - if OPTIONS.boundary_only: - Indexer(OPTIONS).index_boundaries() - else: - Indexer(OPTIONS).index_by_rank() diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index 68d7b2f4..0ee92137 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -91,6 +91,7 @@ class NominatimEnvironment: self.test_env['NOMINATIM_BINDIR'] = self.src_dir / 'utils' self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module' self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql' + self.test_env['NOMINATIM_NOMINATIM_TOOL'] = self.build_dir / 'nominatim' if self.server_module_path: self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path