add migration for new lookup index

[nominatim.git] / nominatim / config.py

diff --git a/nominatim/config.py b/nominatim/config.py
index d1df17b726bac3db70d7c8a788c98a9abbf4401b..785f4acda14d0f0c7f344f79718a8d150bc56001 100644 (file)
--- a/nominatim/config.py
+++ b/nominatim/config.py
@@ -1,9 +1,17 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Nominatim configuration accessor.
 """
 import logging
 import os
 from pathlib import Path
 """
 Nominatim configuration accessor.
 """
 import logging
 import os
 from pathlib import Path

+import json
+import yaml

 
 from dotenv import dotenv_values
 
 
 from dotenv import dotenv_values
 


@@ -11,6 +19,27 @@ from nominatim.errors import UsageError
 
 LOG = logging.getLogger()
 
 
 LOG = logging.getLogger()
 

+
+def flatten_config_list(content, section=''):
+    """ Flatten YAML configuration lists that contain include sections
+        which are lists themselves.
+    """
+    if not content:
+        return []
+
+    if not isinstance(content, list):
+        raise UsageError(f"List expected in section '{section}'.")
+
+    output = []
+    for ele in content:
+        if isinstance(ele, list):
+            output.extend(flatten_config_list(ele, section))
+        else:
+            output.append(ele)
+
+    return output
+
+

 class Configuration:
     """ Load and manage the project configuration.
 
 class Configuration:
     """ Load and manage the project configuration.
 


@@ -30,16 +59,10 @@ class Configuration:
         self.project_dir = project_dir
         self.config_dir = config_dir
         self._config = dotenv_values(str((config_dir / 'env.defaults').resolve()))
         self.project_dir = project_dir
         self.config_dir = config_dir
         self._config = dotenv_values(str((config_dir / 'env.defaults').resolve()))

-        if project_dir is not None:
+        if project_dir is not None and (project_dir / '.env').is_file():

             self._config.update(dotenv_values(str((project_dir / '.env').resolve())))
 
             self._config.update(dotenv_values(str((project_dir / '.env').resolve())))
 

-        # Add defaults for variables that are left empty to set the default.
-        # They may still be overwritten by environment variables.
-        if not self._config['NOMINATIM_ADDRESS_LEVEL_CONFIG']:
-            self._config['NOMINATIM_ADDRESS_LEVEL_CONFIG'] = \
-                str(config_dir / 'address-levels.json')
-
-        class _LibDirs: # pylint: disable=too-few-public-methods
+        class _LibDirs:

             pass
 
         self.lib_dir = _LibDirs()
             pass
 
         self.lib_dir = _LibDirs()


@@ -53,7 +76,10 @@ class Configuration:
     def __getattr__(self, name):
         name = 'NOMINATIM_' + name
 
     def __getattr__(self, name):
         name = 'NOMINATIM_' + name
 

-        return self.environ.get(name) or self._config[name]
+        if name in self.environ:
+            return self.environ[name]
+
+        return self._config[name]

 
     def get_bool(self, name):
         """ Return the given configuration parameter as a boolean.
 
     def get_bool(self, name):
         """ Return the given configuration parameter as a boolean.


@@ -68,10 +94,27 @@ class Configuration:
         """
         try:
             return int(self.__getattr__(name))
         """
         try:
             return int(self.__getattr__(name))

-        except ValueError:
+        except ValueError as exp:

             LOG.fatal("Invalid setting NOMINATIM_%s. Needs to be a number.", name)
             LOG.fatal("Invalid setting NOMINATIM_%s. Needs to be a number.", name)

-            raise UsageError("Configuration error.")
+            raise UsageError("Configuration error.") from exp
+
+
+    def get_path(self, name):
+        """ Return the given configuration parameter as a Path.
+            If a relative path is configured, then the function converts this
+            into an absolute path with the project directory as root path.
+            If the configuration is unset, a falsy value is returned.
+        """
+        value = self.__getattr__(name)
+        if value:
+            value = Path(value)
+
+            if not value.is_absolute():
+                value = self.project_dir / value
+
+            value = value.resolve()

 
 

+        return value

 
     def get_libpq_dsn(self):
         """ Get configured database DSN converted into the key/value format
 
     def get_libpq_dsn(self):
         """ Get configured database DSN converted into the key/value format


@@ -103,7 +146,7 @@ class Configuration:
         if style in ('admin', 'street', 'address', 'full', 'extratags'):
             return self.config_dir / 'import-{}.style'.format(style)
 
         if style in ('admin', 'street', 'address', 'full', 'extratags'):
             return self.config_dir / 'import-{}.style'.format(style)
 

-        return Path(style)
+        return self.find_config_file('', 'IMPORT_STYLE')

 
 
     def get_os_env(self):
 
 
     def get_os_env(self):


@@ -114,3 +157,98 @@ class Configuration:
         env.update(self.environ)
 
         return env
         env.update(self.environ)
 
         return env

+
+
+    def load_sub_configuration(self, filename, config=None):
+        """ Load additional configuration from a file. `filename` is the name
+            of the configuration file. The file is first searched in the
+            project directory and then in the global settings dirctory.
+
+            If `config` is set, then the name of the configuration file can
+            be additionally given through a .env configuration option. When
+            the option is set, then the file will be exclusively loaded as set:
+            if the name is an absolute path, the file name is taken as is,
+            if the name is relative, it is taken to be relative to the
+            project directory.
+
+            The format of the file is determined from the filename suffix.
+            Currently only files with extension '.yaml' are supported.
+
+            YAML files support a special '!include' construct. When the
+            directive is given, the value is taken to be a filename, the file
+            is loaded using this function and added at the position in the
+            configuration tree.
+        """
+        configfile = self.find_config_file(filename, config)
+
+        if configfile.suffix in ('.yaml', '.yml'):
+            return self._load_from_yaml(configfile)
+
+        if configfile.suffix == '.json':
+            with configfile.open('r') as cfg:
+                return json.load(cfg)
+
+        raise UsageError(f"Config file '{configfile}' has unknown format.")
+
+
+    def find_config_file(self, filename, config=None):
+        """ Resolve the location of a configuration file given a filename and
+            an optional configuration option with the file name.
+            Raises a UsageError when the file cannot be found or is not
+            a regular file.
+        """
+        if config is not None:
+            cfg_filename = self.__getattr__(config)
+            if cfg_filename:
+                cfg_filename = Path(cfg_filename)
+
+                if cfg_filename.is_absolute():
+                    cfg_filename = cfg_filename.resolve()
+
+                    if not cfg_filename.is_file():
+                        LOG.fatal("Cannot find config file '%s'.", cfg_filename)
+                        raise UsageError("Config file not found.")
+
+                    return cfg_filename
+
+                filename = cfg_filename
+
+
+        search_paths = [self.project_dir, self.config_dir]
+        for path in search_paths:
+            if path is not None and (path / filename).is_file():
+                return path / filename
+
+        LOG.fatal("Configuration file '%s' not found.\nDirectories searched: %s",
+                  filename, search_paths)
+        raise UsageError("Config file not found.")
+
+
+    def _load_from_yaml(self, cfgfile):
+        """ Load a YAML configuration file. This installs a special handler that
+            allows to include other YAML files using the '!include' operator.
+        """
+        yaml.add_constructor('!include', self._yaml_include_representer,
+                             Loader=yaml.SafeLoader)
+        return yaml.safe_load(cfgfile.read_text(encoding='utf-8'))
+
+
+    def _yaml_include_representer(self, loader, node):
+        """ Handler for the '!include' operator in YAML files.
+
+            When the filename is relative, then the file is first searched in the
+            project directory and then in the global settings dirctory.
+        """
+        fname = loader.construct_scalar(node)
+
+        if Path(fname).is_absolute():
+            configfile = Path(fname)
+        else:
+            configfile = self.find_config_file(loader.construct_scalar(node))
+
+        if configfile.suffix != '.yaml':
+            LOG.fatal("Format error while reading '%s': only YAML format supported.",
+                      configfile)
+            raise UsageError("Cannot handle config file format.")
+
+        return yaml.safe_load(configfile.read_text(encoding='utf-8'))