implement warming in new cli tool

Adds infrastructure for calling the legacy PHP scripts. As the
CONST_* values cannot be set from the python script, hand the values
in via secret environment variables instead. These are all
temporary hacks for the transition phase to python code.

diff --git a/lib/lib.php b/lib/lib.php
index b4c415d9d228d9db553fe678a3b0c34ca7838b4f..5d4038d6db9eb3570dfee4ba9c614046fd55e01f 100644 (file)
--- a/lib/lib.php
+++ b/lib/lib.php
@@ -5,6 +5,11 @@ require('Symfony/Component/Dotenv/autoload.php');
 function loadSettings($sProjectDir)
 {
     @define('CONST_InstallDir', $sProjectDir);
+    // Temporary hack to set the direcory via environment instead of
+    // the installed scripts. Neither setting is part of the official
+    // set of settings.
+    defined('CONST_DataDir') or define('CONST_DataDir', $_SERVER['NOMINATIM_DATADIR']);
+    defined('CONST_BinDir') or define('CONST_BinDir', $_SERVER['NOMINATIM_BINDIR']);
 
     $dotenv = new \Symfony\Component\Dotenv\Dotenv();
     $dotenv->load(CONST_DataDir.'/settings/env.defaults');

diff --git a/nominatim/admin/__init__.py b/nominatim/admin/__init__.py
new file mode 100644 (file)
index 0000000..cab6fb8
--- /dev/null
+++ b/nominatim/admin/__init__.py
@@ -0,0 +1,4 @@
+"""
+Module with functions for importing, updating Nominatim databases
+as well as general maintenance helpers.
+"""

diff --git a/nominatim/admin/exec_utils.py b/nominatim/admin/exec_utils.py
new file mode 100644 (file)
index 0000000..91225b7
--- /dev/null
+++ b/nominatim/admin/exec_utils.py
@@ -0,0 +1,21 @@
+"""
+Helper functions for executing external programs.
+"""
+from pathlib import Path
+import subprocess
+
+def run_legacy_script(script, *args, nominatim_env=None):
+        """ Run a Nominatim PHP script with the given arguments.
+        """
+        cmd = ['/usr/bin/env', 'php', '-Cq',
+               nominatim_env.phplib_dir / 'admin' / script]
+        cmd.extend(args)
+
+        env = nominatim_env.config.get_os_env()
+        env['NOMINATIM_DATADIR'] = str(nominatim_env.data_dir)
+        env['NOMINATIM_BINDIR'] = str(nominatim_env.data_dir / 'utils')
+
+        proc = subprocess.run(cmd, cwd=str(nominatim_env.project_dir), env=env)
+
+        return proc.returncode
+

diff --git a/nominatim/cli.py b/nominatim/cli.py
index fcdea4187c6705886a1f64d9c753f1819a539cf3..a40dc07ea07f0375b956f5ca2267448fd84216f1 100644 (file)
--- a/nominatim/cli.py
+++ b/nominatim/cli.py
@@ -5,6 +5,10 @@ database administration and querying.
 import sys
 import argparse
 import logging
+from pathlib import Path
+
+from .config import Configuration
+from .admin.exec_utils import run_legacy_script
 
 class CommandlineParser:
     """ Wraps some of the common functions for parsing the command line
@@ -29,7 +33,7 @@ class CommandlineParser:
                            help='Print only error messages')
         group.add_argument('-v', '--verbose', action='count', default=1,
                            help='Increase verboseness of output')
-        group.add_argument('--project-dir', metavar='DIR',
+        group.add_argument('--project-dir', metavar='DIR', default='.',
                            help='Base directory of the Nominatim installation (default:.)')
         group.add_argument('-j', '--threads', metavar='NUM', type=int,
                            help='Number of parallel threads to use')
@@ -48,20 +52,27 @@ class CommandlineParser:
         parser.set_defaults(command=cmd)
         cmd.add_args(parser)
 
-    def run(self):
+    def run(self, **kwargs):
         """ Parse the command line arguments of the program and execute the
             appropriate subcommand.
         """
         args = self.parser.parse_args()
 
         if args.subcommand is None:
-            self.parser.print_help()
-        else:
-            logging.basicConfig(stream=sys.stderr,
-                                format='%(asctime)s %(levelname)s: %(message)s',
-                                datefmt='%Y-%m-%d %H:%M:%S',
-                                level=max(4 - args.verbose, 1) * 10)
-            args.command.run(args)
+            return self.parser.print_help()
+
+        for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir'):
+            setattr(args, arg, Path(kwargs[arg]))
+        args.project_dir = Path(args.project_dir)
+
+        logging.basicConfig(stream=sys.stderr,
+                            format='%(asctime)s %(levelname)s: %(message)s',
+                            datefmt='%Y-%m-%d %H:%M:%S',
+                            level=max(4 - args.verbose, 1) * 10)
+
+        args.config = Configuration(args.project_dir, args.data_dir / 'settings')
+
+        args.command.run(args)
 
 
 class SetupAll:
@@ -290,7 +301,12 @@ class AdminWarm:
 
     @staticmethod
     def run(args):
-        print("TODO: ./utils/warm.php", args)
+        params = ['warm.php']
+        if args.target == 'reverse':
+            params.append('--reverse-only')
+        if args.target == 'search':
+            params.append('--search-only')
+        return run_legacy_script(*params, nominatim_env=args)
 
 
 class QueryExport:
@@ -372,4 +388,4 @@ def nominatim(**kwargs):
     parser.add_subcommand('details', QueryTodo)
     parser.add_subcommand('status', QueryTodo)
 
-    parser.run()
+    parser.run(**kwargs)

diff --git a/nominatim/config.py b/nominatim/config.py
new file mode 100644 (file)
index 0000000..4fa887a
--- /dev/null
+++ b/nominatim/config.py
@@ -0,0 +1,39 @@
+"""
+Nominatim configuration accessor.
+"""
+import sys
+import os
+
+from dotenv import dotenv_values
+
+class Configuration:
+    """ Load and manage the project configuration.
+
+        Nominatim uses dotenv to configure the software. Configuration options
+        are resolved in the following order:
+
+         * from the OS environment
+         * from the .env file in the project directory of the installation
+         * from the default installation in the configuration directory
+
+        All Nominatim configuration options are prefixed with 'NOMINATIM_' to
+        avoid conflicts with other environment variables.
+    """
+
+    def __init__(self, project_dir, config_dir):
+        self._config = dotenv_values(str((config_dir / 'env.defaults').resolve()))
+        self._config.update(dotenv_values(str((project_dir / '.env').resolve())))
+
+    def __getattr__(self, name):
+        name = 'NOMINATIM_' + name
+
+        return os.environ.get(name) or self._config[name]
+
+    def get_os_env(self):
+        """ Return a copy of the OS environment with the Nominatim configuration
+            merged in.
+        """
+        env = dict(os.environ)
+        env.update(self._config)
+
+        return env