bdd: run all setup via nominatim Python library

Drops all calls to PHP utility functions. nominatim cli functions
are used where possible, to stay as close to the final code as
possible with the tests.

By removing the PHP calls, the test code now only uses osm2pgsql and
the database module from the build directory.

diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py
index 168334b1659d381b1ea98875d0ce90a14f8e412e..170dd6392ba1d8b86569c59bea277013348b79c4 100644 (file)
--- a/test/bdd/steps/nominatim_environment.py
+++ b/test/bdd/steps/nominatim_environment.py
@@ -109,6 +109,22 @@ class NominatimEnvironment:
         cfg = Configuration(None, self.src_dir / 'settings', environ=self.test_env)
         refresh.setup_website(Path(self.website_dir.name) / 'website', self.src_dir / 'lib-php', cfg)
 
+    def get_libpq_dsn(self):
+        dsn = self.test_env['NOMINATIM_DATABASE_DSN']
+
+        def quote_param(param):
+            key, val = param.split('=')
+            val = val.replace('\\', '\\\\').replace("'", "\\'")
+            if ' ' in val:
+                val = "'" + val + "'"
+            return key + '=' + val
+
+        if dsn.startswith('pgsql:'):
+            # Old PHP DSN format. Convert before returning.
+            return ' '.join([quote_param(p) for p in dsn[6:].split(';')])
+
+        return dsn
+
 
     def db_drop_database(self, name):
         """ Drop the database with the given name.
@@ -132,34 +148,16 @@ class NominatimEnvironment:
         if self._reuse_or_drop_db(self.template_db):
             return
 
-        try:
-            # call the first part of database setup
-            self.write_nominatim_config(self.template_db)
-            self.run_setup_script('create-db', 'setup-db')
-            # remove external data to speed up indexing for tests
-            conn = self.connect_database(self.template_db)
-            cur = conn.cursor()
-            cur.execute("""select tablename from pg_tables
-                           where tablename in ('gb_postcode', 'us_postcode')""")
-            for t in cur:
-                conn.cursor().execute('TRUNCATE TABLE {}'.format(t[0]))
-            conn.commit()
-            conn.close()
+        self.write_nominatim_config(self.template_db)
 
-            # execute osm2pgsql import on an empty file to get the right tables
+        try:
+            # execute nominatim import on an empty file to get the right tables
             with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
                 fd.write(b'<osm version="0.6"></osm>')
                 fd.flush()
-                self.run_setup_script('import-data',
-                                      'ignore-errors',
-                                      'create-functions',
-                                      'create-tables',
-                                      'create-partition-tables',
-                                      'create-partition-functions',
-                                      'load-data',
-                                      'create-search-indices',
-                                      osm_file=fd.name,
-                                      osm2pgsql_cache='200')
+                self.run_nominatim('import', '--osm-file', fd.name,
+                                             '--osm2pgsql-cache', '1',
+                                             '--ignore-errors')
         except:
             self.db_drop_database(self.template_db)
             raise
@@ -179,12 +177,11 @@ class NominatimEnvironment:
             return
 
         testdata = Path('__file__') / '..' / '..' / 'testdb'
-        self.test_env['NOMINATIM_TIGER_DATA_PATH'] = str((testdata / 'tiger').resolve())
         self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
 
         try:
             self.run_nominatim('import', '--osm-file', str(self.api_test_file))
-            self.run_setup_script('import-tiger-data')
+            self.run_nominatim('add-data', '--tiger-data', str((testdata / 'tiger').resolve()))
             self.run_nominatim('freeze')
 
             phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
@@ -259,6 +256,9 @@ class NominatimEnvironment:
     def run_nominatim(self, *cmdline):
         """ Run the nominatim command-line tool via the library.
         """
+        if self.website_dir is not None:
+            cmdline = list(cmdline) + ['--project-dir', self.website_dir.name]
+
         cli.nominatim(module_dir='',
                       osm2pgsql_path=str(self.build_dir / 'osm2pgsql' / 'osm2pgsql'),
                       phplib_dir=str(self.src_dir / 'lib-php'),
@@ -269,31 +269,6 @@ class NominatimEnvironment:
                       phpcgi_path='',
                       environ=self.test_env)
 
-    def run_setup_script(self, *args, **kwargs):
-        """ Run the Nominatim setup script with the given arguments.
-        """
-        self.run_nominatim_script('setup', *args, **kwargs)
-
-    def run_update_script(self, *args, **kwargs):
-        """ Run the Nominatim update script with the given arguments.
-        """
-        self.run_nominatim_script('update', *args, **kwargs)
-
-    def run_nominatim_script(self, script, *args, **kwargs):
-        """ Run one of the Nominatim utility scripts with the given arguments.
-        """
-        cmd = ['/usr/bin/env', 'php', '-Cq']
-        cmd.append((Path(self.src_dir) / 'lib-php' / 'admin' / '{}.php'.format(script)).resolve())
-        cmd.extend(['--' + x for x in args])
-        for k, v in kwargs.items():
-            cmd.extend(('--' + k.replace('_', '-'), str(v)))
-
-        if self.website_dir is not None:
-            cwd = self.website_dir.name
-        else:
-            cwd = None
-
-        run_script(cmd, cwd=cwd, env=self.test_env)
 
     def copy_from_place(self, db):
         """ Copy data from place to the placex and location_property_osmline

diff --git a/test/bdd/steps/steps_db_ops.py b/test/bdd/steps/steps_db_ops.py
index 9d443b434da263654ed0f40c4f5e843bf2a0d433..465eed6589f013b523a043fa71949b40f6db2837 100644 (file)
--- a/test/bdd/steps/steps_db_ops.py
+++ b/test/bdd/steps/steps_db_ops.py
@@ -86,10 +86,24 @@ def import_and_index_data_from_place_table(context):
     """ Import data previously set up in the place table.
     """
     context.nominatim.copy_from_place(context.db)
-    context.nominatim.run_setup_script('calculate-postcodes')
+
+    # XXX use tool function as soon as it is ported
+    with context.db.cursor() as cur:
+        with (context.nominatim.src_dir / 'lib-sql' / 'postcode_tables.sql').open('r') as fd:
+            cur.execute(fd.read())
+        cur.execute("""
+            INSERT INTO location_postcode
+             (place_id, indexed_status, country_code, postcode, geometry)
+            SELECT nextval('seq_place'), 1, country_code,
+                   upper(trim (both ' ' from address->'postcode')) as pc,
+                   ST_Centroid(ST_Collect(ST_Centroid(geometry)))
+              FROM placex
+             WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'
+                   AND geometry IS NOT null
+             GROUP BY country_code, pc""")
 
     # Call directly as the refresh function does not include postcodes.
-    indexer = Indexer(context.nominatim.test_env['NOMINATIM_DATABASE_DSN'][6:], 1)
+    indexer = Indexer(context.nominatim.get_libpq_dsn(), 1)
     indexer.index_full(analyse=False)
 
     check_database_integrity(context)

diff --git a/test/bdd/steps/steps_osm_data.py b/test/bdd/steps/steps_osm_data.py
index 844fb27484f984a51873e3268315f9d94756174b..fb4591bfb53504d45ba8805f592b28a629ad4110 100644 (file)
--- a/test/bdd/steps/steps_osm_data.py
+++ b/test/bdd/steps/steps_osm_data.py
@@ -1,6 +1,23 @@
 import tempfile
 import random
 import os
+from pathlib import Path
+
+from nominatim.tools.exec_utils import run_osm2pgsql
+
+def get_osm2pgsql_options(nominatim_env, fname, append):
+    return dict(import_file=fname,
+                osm2pgsql=str(nominatim_env.build_dir / 'osm2pgsql' / 'osm2pgsql'),
+                osm2pgsql_cache=50,
+                osm2pgsql_style=str(nominatim_env.src_dir / 'settings' / 'import-extratags.style'),
+                threads=1,
+                dsn=nominatim_env.get_libpq_dsn(),
+                flatnode_file='',
+                tablespaces=dict(slim_data='', slim_index='',
+                                 main_data='', main_index=''),
+                append=append
+               )
+
 
 def write_opl_file(opl, grid):
     """ Create a temporary OSM file from OPL and return the file name. It is
@@ -52,9 +69,10 @@ def load_osm_file(context):
     """
     # create an OSM file and import it
     fname = write_opl_file(context.text, context.osm)
-    context.nominatim.run_setup_script('import-data', osm_file=fname,
-                                       osm2pgsql_cache=300)
-    os.remove(fname)
+    try:
+        run_osm2pgsql(get_osm2pgsql_options(context.nominatim, fname, append=False))
+    finally:
+        os.remove(fname)
 
     ### reintroduce the triggers/indexes we've lost by having osm2pgsql set up place again
     cur = context.db.cursor()
@@ -80,5 +98,7 @@ def update_from_osm_file(context):
 
     # create an OSM file and import it
     fname = write_opl_file(context.text, context.osm)
-    context.nominatim.run_update_script(import_diff=fname)
-    os.remove(fname)
+    try:
+        run_osm2pgsql(get_osm2pgsql_options(context.nominatim, fname, append=True))
+    finally:
+        os.remove(fname)