move module installation to legacy tokenizer

diff --git a/nominatim/clicmd/setup.py b/nominatim/clicmd/setup.py
index 687279726a303036e43f08beea8f50529fe7393e..066c296037153cc73f8fe126db4b6d79ec6750f5 100644 (file)
--- a/nominatim/clicmd/setup.py
+++ b/nominatim/clicmd/setup.py
@@ -68,12 +68,6 @@ class SetupAll:
                                                     args.no_partitions,
                                                     rouser=args.config.DATABASE_WEBUSER)
 
                                                     args.no_partitions,
                                                     rouser=args.config.DATABASE_WEBUSER)
 

-            LOG.warning('Installing database module')
-            with connect(args.config.get_libpq_dsn()) as conn:
-                database_import.install_module(args.module_dir, args.project_dir,
-                                               args.config.DATABASE_MODULE_PATH,
-                                               conn=conn)
-

             LOG.warning('Importing OSM data file')
             database_import.import_osm_data(Path(args.osm_file),
                                             args.osm2pgsql_options(0, 1),
             LOG.warning('Importing OSM data file')
             database_import.import_osm_data(Path(args.osm_file),
                                             args.osm2pgsql_options(0, 1),

diff --git a/nominatim/tokenizer/legacy_tokenizer.py b/nominatim/tokenizer/legacy_tokenizer.py
index ab3e320e2dfb576a7dcfff9d4340f6adc27bf157..bd7b770921544b002c3a78a5fb5fcf078692c522 100644 (file)
--- a/nominatim/tokenizer/legacy_tokenizer.py
+++ b/nominatim/tokenizer/legacy_tokenizer.py
@@ -1,16 +1,61 @@
 """
 Tokenizer implementing normalisation as used before Nominatim 4.
 """
 """
 Tokenizer implementing normalisation as used before Nominatim 4.
 """

+import logging
+import shutil
+
+import psycopg2
+

 from nominatim.db.connection import connect
 from nominatim.db import properties
 from nominatim.db.connection import connect
 from nominatim.db import properties

+from nominatim.errors import UsageError

 
 DBCFG_NORMALIZATION = "tokenizer_normalization"
 
 
 DBCFG_NORMALIZATION = "tokenizer_normalization"
 

+LOG = logging.getLogger()
+

 def create(dsn, data_dir):
     """ Create a new instance of the tokenizer provided by this module.
     """
     return LegacyTokenizer(dsn, data_dir)
 
 def create(dsn, data_dir):
     """ Create a new instance of the tokenizer provided by this module.
     """
     return LegacyTokenizer(dsn, data_dir)
 

+
+def _install_module(src_dir, module_dir):
+    """ Copies the PostgreSQL normalisation module into the project
+        directory if necessary. For historical reasons the module is
+        saved in the '/module' subdirectory and not with the other tokenizer
+        data.
+
+        The function detects when the installation is run from the
+        build directory. It doesn't touch the module in that case.
+    """
+    if module_dir.exists() and src_dir.samefile(module_dir):
+        LOG.info('Running from build directory. Leaving database module as is.')
+        return
+
+    if not module_dir.exists():
+        module_dir.mkdir()
+
+    destfile = module_dir / 'nominatim.so'
+    shutil.copy(str(src_dir / 'nominatim.so'), str(destfile))
+    destfile.chmod(0o755)
+
+    LOG.info('Database module installed at %s', str(destfile))
+
+
+def _check_module(module_dir, conn):
+    with conn.cursor() as cur:
+        try:
+            cur.execute("""CREATE FUNCTION nominatim_test_import_func(text)
+                           RETURNS text AS '{}/nominatim.so', 'transliteration'
+                           LANGUAGE c IMMUTABLE STRICT;
+                           DROP FUNCTION nominatim_test_import_func(text)
+                        """.format(module_dir))
+        except psycopg2.DatabaseError as err:
+            LOG.fatal("Error accessing database module: %s", err)
+            raise UsageError("Database module cannot be accessed.") from err
+
+

 class LegacyTokenizer:
     """ The legacy tokenizer uses a special PostgreSQL module to normalize
         names and queries. The tokenizer thus implements normalization through
 class LegacyTokenizer:
     """ The legacy tokenizer uses a special PostgreSQL module to normalize
         names and queries. The tokenizer thus implements normalization through


@@ -29,12 +74,24 @@ class LegacyTokenizer:
             This copies all necessary data in the project directory to make
             sure the tokenizer remains stable even over updates.
         """
             This copies all necessary data in the project directory to make
             sure the tokenizer remains stable even over updates.
         """

+        # Find and optionally install the PsotgreSQL normalization module.
+        if config.DATABASE_MODULE_PATH:
+            LOG.info("Using custom path for database module at '%s'",
+                     config.DATABASE_MODULE_PATH)
+            module_dir = config.DATABASE_MODULE_PATH
+        else:
+            _install_module(config.lib_dir.module, config.project_dir / 'module')
+            module_dir = config.project_dir / 'module'
+

         self.normalization = config.TERM_NORMALIZATION
 
         self.normalization = config.TERM_NORMALIZATION
 

-        # Stable configuration is saved in the database.

         with connect(self.dsn) as conn:
         with connect(self.dsn) as conn:

-            properties.set_property(conn, DBCFG_NORMALIZATION,
-                                    self.normalization)
+            _check_module(module_dir, conn)
+
+            # Stable configuration is saved in the database.
+            properties.set_property(conn, DBCFG_NORMALIZATION, self.normalization)
+
+            conn.commit()

 
 
     def init_from_project(self):
 
 
     def init_from_project(self):

diff --git a/nominatim/tools/database_import.py b/nominatim/tools/database_import.py
index 3618ed28e14416d412f4033d6130bc4593d3aa38..324a82cf00fbb39b277aa57e4c6441092e9cf907 100644 (file)
--- a/nominatim/tools/database_import.py
+++ b/nominatim/tools/database_import.py
@@ -5,11 +5,9 @@ import logging
 import os
 import selectors
 import subprocess
 import os
 import selectors
 import subprocess

-import shutil

 from pathlib import Path
 
 import psutil
 from pathlib import Path
 
 import psutil

-import psycopg2

 
 from nominatim.db.connection import connect, get_pg_env
 from nominatim.db import utils as db_utils
 
 from nominatim.db.connection import connect, get_pg_env
 from nominatim.db import utils as db_utils


@@ -89,49 +87,6 @@ def setup_extensions(conn):
         raise UsageError('PostGIS version is too old.')
 
 
         raise UsageError('PostGIS version is too old.')
 
 

-def install_module(src_dir, project_dir, module_dir, conn=None):
-    """ Copy the normalization module from src_dir into the project
-        directory under the '/module' directory. If 'module_dir' is set, then
-        use the module from there instead and check that it is accessible
-        for Postgresql.
-
-        The function detects when the installation is run from the
-        build directory. It doesn't touch the module in that case.
-
-        If 'conn' is given, then the function also tests if the module
-        can be access via the given database.
-    """
-    if not module_dir:
-        module_dir = project_dir / 'module'
-
-        if not module_dir.exists() or not src_dir.samefile(module_dir):
-
-            if not module_dir.exists():
-                module_dir.mkdir()
-
-            destfile = module_dir / 'nominatim.so'
-            shutil.copy(str(src_dir / 'nominatim.so'), str(destfile))
-            destfile.chmod(0o755)
-
-            LOG.info('Database module installed at %s', str(destfile))
-        else:
-            LOG.info('Running from build directory. Leaving database module as is.')
-    else:
-        LOG.info("Using custom path for database module at '%s'", module_dir)
-
-    if conn is not None:
-        with conn.cursor() as cur:
-            try:
-                cur.execute("""CREATE FUNCTION nominatim_test_import_func(text)
-                               RETURNS text AS '{}/nominatim.so', 'transliteration'
-                               LANGUAGE c IMMUTABLE STRICT;
-                               DROP FUNCTION nominatim_test_import_func(text)
-                            """.format(module_dir))
-            except psycopg2.DatabaseError as err:
-                LOG.fatal("Error accessing database module: %s", err)
-                raise UsageError("Database module cannot be accessed.") from err
-
-

 def import_base_data(dsn, sql_dir, ignore_partitions=False):
     """ Create and populate the tables with basic static data that provides
         the background for geocoding. Data is assumed to not yet exist.
 def import_base_data(dsn, sql_dir, ignore_partitions=False):
     """ Create and populate the tables with basic static data that provides
         the background for geocoding. Data is assumed to not yet exist.

diff --git a/test/python/test_cli.py b/test/python/test_cli.py
index 6b8bfc55eff9612231dcc4e2054adb28b53962a9..10a31542045e66522c4f125591138d207300195e 100644 (file)
--- a/test/python/test_cli.py
+++ b/test/python/test_cli.py
@@ -88,7 +88,6 @@ def test_import_bad_file(temp_db):
 def test_import_full(temp_db, mock_func_factory):
     mocks = [
         mock_func_factory(nominatim.tools.database_import, 'setup_database_skeleton'),
 def test_import_full(temp_db, mock_func_factory):
     mocks = [
         mock_func_factory(nominatim.tools.database_import, 'setup_database_skeleton'),

-        mock_func_factory(nominatim.tools.database_import, 'install_module'),

         mock_func_factory(nominatim.tools.database_import, 'import_osm_data'),
         mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'),
         mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
         mock_func_factory(nominatim.tools.database_import, 'import_osm_data'),
         mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'),
         mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),

diff --git a/test/python/test_tokenizer_legacy.py b/test/python/test_tokenizer_legacy.py
index cb6fb00bb85f67eadf0a8af8fdad63be5e1b7498..44937904b508c0f84aa65ec12198dd65bc0a390a 100644 (file)
--- a/test/python/test_tokenizer_legacy.py
+++ b/test/python/test_tokenizer_legacy.py
@@ -5,24 +5,78 @@ import pytest
 
 from nominatim.tokenizer import legacy_tokenizer
 from nominatim.db import properties
 
 from nominatim.tokenizer import legacy_tokenizer
 from nominatim.db import properties

+from nominatim.errors import UsageError

 
 @pytest.fixture
 
 @pytest.fixture

-def tokenizer(dsn, tmp_path, def_config, property_table):
-    tok = legacy_tokenizer.create(dsn, tmp_path)
-    tok.init_new_db(def_config)
+def test_config(def_config, tmp_path):
+    def_config.project_dir = tmp_path / 'project'
+    def_config.project_dir.mkdir()

 
 

-    return tok
+    module_dir = tmp_path / 'module_src'
+    module_dir.mkdir()
+    (module_dir / 'nominatim.so').write_text('TEST nomiantim.so')

 
 

-def test_init_new(dsn, tmp_path, def_config, property_table, monkeypatch, temp_db_conn):
+    def_config.lib_dir.module = module_dir
+
+    return def_config
+
+
+@pytest.fixture
+def tokenizer_factory(dsn, tmp_path, monkeypatch):
+
+    def _maker():
+        return legacy_tokenizer.create(dsn, tmp_path / 'tokenizer')
+
+    return _maker
+
+@pytest.fixture
+def tokenizer_setup(tokenizer_factory, test_config, property_table, monkeypatch):
+    monkeypatch.setattr(legacy_tokenizer, '_check_module' , lambda m, c: None)
+    tok = tokenizer_factory()
+    tok.init_new_db(test_config)
+
+
+def test_init_new(tokenizer_factory, test_config, property_table, monkeypatch, temp_db_conn):

     monkeypatch.setenv('NOMINATIM_TERM_NORMALIZATION', 'xxvv')
     monkeypatch.setenv('NOMINATIM_TERM_NORMALIZATION', 'xxvv')

+    monkeypatch.setattr(legacy_tokenizer, '_check_module' , lambda m, c: None)

 
 

-    tok = legacy_tokenizer.create(dsn, tmp_path)
-    tok.init_new_db(def_config)
+    tok = tokenizer_factory()
+    tok.init_new_db(test_config)

 
     assert properties.get_property(temp_db_conn, legacy_tokenizer.DBCFG_NORMALIZATION) == 'xxvv'
 
 
     assert properties.get_property(temp_db_conn, legacy_tokenizer.DBCFG_NORMALIZATION) == 'xxvv'
 

+    outfile = test_config.project_dir / 'module' / 'nominatim.so'
+
+    assert outfile.exists()
+    assert outfile.read_text() == 'TEST nomiantim.so'
+    assert outfile.stat().st_mode == 33261
+
+
+def test_init_module_load_failed(tokenizer_factory, test_config, property_table, monkeypatch, temp_db_conn):
+    tok = tokenizer_factory()
+
+    with pytest.raises(UsageError):
+        tok.init_new_db(test_config)
+
+
+def test_init_module_custom(tokenizer_factory, test_config, property_table,
+                            monkeypatch, tmp_path):
+    module_dir = (tmp_path / 'custom').resolve()
+    module_dir.mkdir()
+    (module_dir/ 'nominatim.so').write_text('CUSTOM nomiantim.so')
+
+    monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', str(module_dir))
+    monkeypatch.setattr(legacy_tokenizer, '_check_module' , lambda m, c: None)
+
+    tok = tokenizer_factory()
+    tok.init_new_db(test_config)
+
+    assert not (test_config.project_dir / 'module').exists()
+
+
+def test_init_from_project(tokenizer_setup, tokenizer_factory):
+    tok = tokenizer_factory()

 
 

-def test_init_from_project(tokenizer):
-    tokenizer.init_from_project()
+    tok.init_from_project()

 
 

-    assert tokenizer.normalization is not None
+    assert tok.normalization is not None

diff --git a/test/python/test_tools_database_import.py b/test/python/test_tools_database_import.py
index 280ca704ce7c8419b6b745390401d285ef29bf16..7ae19e2a5f0d23abedebad1008fc7c9aa2035e7c 100644 (file)
--- a/test/python/test_tools_database_import.py
+++ b/test/python/test_tools_database_import.py
@@ -80,39 +80,6 @@ def test_setup_extensions_old_postgis(temp_db_conn, monkeypatch):
         database_import.setup_extensions(temp_db_conn)
 
 
         database_import.setup_extensions(temp_db_conn)
 
 

-def test_install_module(tmp_path):
-    src_dir = tmp_path / 'source'
-    src_dir.mkdir()
-    (src_dir / 'nominatim.so').write_text('TEST nomiantim.so')
-
-    project_dir = tmp_path / 'project'
-    project_dir.mkdir()
-
-    database_import.install_module(src_dir, project_dir, '')
-
-    outfile = project_dir / 'module' / 'nominatim.so'
-
-    assert outfile.exists()
-    assert outfile.read_text() == 'TEST nomiantim.so'
-    assert outfile.stat().st_mode == 33261
-
-
-def test_install_module_custom(tmp_path):
-    (tmp_path / 'nominatim.so').write_text('TEST nomiantim.so')
-
-    database_import.install_module(tmp_path, tmp_path, str(tmp_path.resolve()))
-
-    assert not (tmp_path / 'module').exists()
-
-
-def test_install_module_fail_access(temp_db_conn, tmp_path):
-    (tmp_path / 'nominatim.so').write_text('TEST nomiantim.so')
-
-    with pytest.raises(UsageError, match='.*module cannot be accessed.*'):
-        database_import.install_module(tmp_path, tmp_path, '',
-                                       conn=temp_db_conn)
-
-

 def test_import_base_data(src_dir, temp_db, temp_db_cursor):
     temp_db_cursor.execute('CREATE EXTENSION hstore')
     temp_db_cursor.execute('CREATE EXTENSION postgis')
 def test_import_base_data(src_dir, temp_db, temp_db_cursor):
     temp_db_cursor.execute('CREATE EXTENSION hstore')
     temp_db_cursor.execute('CREATE EXTENSION postgis')