From: Sarah Hoffmann Date: Wed, 13 Jan 2021 17:25:15 +0000 (+0100) Subject: implement warming in new cli tool X-Git-Tag: v3.7.0~57^2~13 X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/04690ad8c4734c15efc920bc0dab59cf78e3e7cb implement warming in new cli tool Adds infrastructure for calling the legacy PHP scripts. As the CONST_* values cannot be set from the python script, hand the values in via secret environment variables instead. These are all temporary hacks for the transition phase to python code. --- diff --git a/lib/lib.php b/lib/lib.php index b4c415d9..5d4038d6 100644 --- a/lib/lib.php +++ b/lib/lib.php @@ -5,6 +5,11 @@ require('Symfony/Component/Dotenv/autoload.php'); function loadSettings($sProjectDir) { @define('CONST_InstallDir', $sProjectDir); + // Temporary hack to set the direcory via environment instead of + // the installed scripts. Neither setting is part of the official + // set of settings. + defined('CONST_DataDir') or define('CONST_DataDir', $_SERVER['NOMINATIM_DATADIR']); + defined('CONST_BinDir') or define('CONST_BinDir', $_SERVER['NOMINATIM_BINDIR']); $dotenv = new \Symfony\Component\Dotenv\Dotenv(); $dotenv->load(CONST_DataDir.'/settings/env.defaults'); diff --git a/nominatim/admin/__init__.py b/nominatim/admin/__init__.py new file mode 100644 index 00000000..cab6fb8b --- /dev/null +++ b/nominatim/admin/__init__.py @@ -0,0 +1,4 @@ +""" +Module with functions for importing, updating Nominatim databases +as well as general maintenance helpers. +""" diff --git a/nominatim/admin/exec_utils.py b/nominatim/admin/exec_utils.py new file mode 100644 index 00000000..91225b7a --- /dev/null +++ b/nominatim/admin/exec_utils.py @@ -0,0 +1,21 @@ +""" +Helper functions for executing external programs. +""" +from pathlib import Path +import subprocess + +def run_legacy_script(script, *args, nominatim_env=None): + """ Run a Nominatim PHP script with the given arguments. + """ + cmd = ['/usr/bin/env', 'php', '-Cq', + nominatim_env.phplib_dir / 'admin' / script] + cmd.extend(args) + + env = nominatim_env.config.get_os_env() + env['NOMINATIM_DATADIR'] = str(nominatim_env.data_dir) + env['NOMINATIM_BINDIR'] = str(nominatim_env.data_dir / 'utils') + + proc = subprocess.run(cmd, cwd=str(nominatim_env.project_dir), env=env) + + return proc.returncode + diff --git a/nominatim/cli.py b/nominatim/cli.py index fcdea418..a40dc07e 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -5,6 +5,10 @@ database administration and querying. import sys import argparse import logging +from pathlib import Path + +from .config import Configuration +from .admin.exec_utils import run_legacy_script class CommandlineParser: """ Wraps some of the common functions for parsing the command line @@ -29,7 +33,7 @@ class CommandlineParser: help='Print only error messages') group.add_argument('-v', '--verbose', action='count', default=1, help='Increase verboseness of output') - group.add_argument('--project-dir', metavar='DIR', + group.add_argument('--project-dir', metavar='DIR', default='.', help='Base directory of the Nominatim installation (default:.)') group.add_argument('-j', '--threads', metavar='NUM', type=int, help='Number of parallel threads to use') @@ -48,20 +52,27 @@ class CommandlineParser: parser.set_defaults(command=cmd) cmd.add_args(parser) - def run(self): + def run(self, **kwargs): """ Parse the command line arguments of the program and execute the appropriate subcommand. """ args = self.parser.parse_args() if args.subcommand is None: - self.parser.print_help() - else: - logging.basicConfig(stream=sys.stderr, - format='%(asctime)s %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - level=max(4 - args.verbose, 1) * 10) - args.command.run(args) + return self.parser.print_help() + + for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir'): + setattr(args, arg, Path(kwargs[arg])) + args.project_dir = Path(args.project_dir) + + logging.basicConfig(stream=sys.stderr, + format='%(asctime)s %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + level=max(4 - args.verbose, 1) * 10) + + args.config = Configuration(args.project_dir, args.data_dir / 'settings') + + args.command.run(args) class SetupAll: @@ -290,7 +301,12 @@ class AdminWarm: @staticmethod def run(args): - print("TODO: ./utils/warm.php", args) + params = ['warm.php'] + if args.target == 'reverse': + params.append('--reverse-only') + if args.target == 'search': + params.append('--search-only') + return run_legacy_script(*params, nominatim_env=args) class QueryExport: @@ -372,4 +388,4 @@ def nominatim(**kwargs): parser.add_subcommand('details', QueryTodo) parser.add_subcommand('status', QueryTodo) - parser.run() + parser.run(**kwargs) diff --git a/nominatim/config.py b/nominatim/config.py new file mode 100644 index 00000000..4fa887a1 --- /dev/null +++ b/nominatim/config.py @@ -0,0 +1,39 @@ +""" +Nominatim configuration accessor. +""" +import sys +import os + +from dotenv import dotenv_values + +class Configuration: + """ Load and manage the project configuration. + + Nominatim uses dotenv to configure the software. Configuration options + are resolved in the following order: + + * from the OS environment + * from the .env file in the project directory of the installation + * from the default installation in the configuration directory + + All Nominatim configuration options are prefixed with 'NOMINATIM_' to + avoid conflicts with other environment variables. + """ + + def __init__(self, project_dir, config_dir): + self._config = dotenv_values(str((config_dir / 'env.defaults').resolve())) + self._config.update(dotenv_values(str((project_dir / '.env').resolve()))) + + def __getattr__(self, name): + name = 'NOMINATIM_' + name + + return os.environ.get(name) or self._config[name] + + def get_os_env(self): + """ Return a copy of the OS environment with the Nominatim configuration + merged in. + """ + env = dict(os.environ) + env.update(self._config) + + return env