test/bdd/steps/nominatim_environment.py

   1 from pathlib import Path
   2 import sys
   3 import tempfile
   4
   5 import psycopg2
   6 import psycopg2.extras
   7
   8 sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..').resolve()))
   9
  10 from nominatim.config import Configuration
  11 from steps.utils import run_script
  12
  13 class NominatimEnvironment:
  14     """ Collects all functions for the execution of Nominatim functions.
  15     """
  16
  17     def __init__(self, config):
  18         self.build_dir = Path(config['BUILDDIR']).resolve()
  19         self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve()
  20         self.db_host = config['DB_HOST']
  21         self.db_port = config['DB_PORT']
  22         self.db_user = config['DB_USER']
  23         self.db_pass = config['DB_PASS']
  24         self.template_db = config['TEMPLATE_DB']
  25         self.test_db = config['TEST_DB']
  26         self.api_test_db = config['API_TEST_DB']
  27         self.api_test_file = config['API_TEST_FILE']
  28         self.server_module_path = config['SERVER_MODULE_PATH']
  29         self.reuse_template = not config['REMOVE_TEMPLATE']
  30         self.keep_scenario_db = config['KEEP_TEST_DB']
  31         self.code_coverage_path = config['PHPCOV']
  32         self.code_coverage_id = 1
  33
  34         self.default_config = Configuration(None, self.src_dir / 'settings').get_os_env()
  35         self.test_env = None
  36         self.template_db_done = False
  37         self.api_db_done = False
  38         self.website_dir = None
  39
  40     def connect_database(self, dbname):
  41         """ Return a connection to the database with the given name.
  42             Uses configured host, user and port.
  43         """
  44         dbargs = {'database': dbname}
  45         if self.db_host:
  46             dbargs['host'] = self.db_host
  47         if self.db_port:
  48             dbargs['port'] = self.db_port
  49         if self.db_user:
  50             dbargs['user'] = self.db_user
  51         if self.db_pass:
  52             dbargs['password'] = self.db_pass
  53         conn = psycopg2.connect(**dbargs)
  54         return conn
  55
  56     def next_code_coverage_file(self):
  57         """ Generate the next name for a coverage file.
  58         """
  59         fn = Path(self.code_coverage_path) / "{:06d}.cov".format(self.code_coverage_id)
  60         self.code_coverage_id += 1
  61
  62         return fn.resolve()
  63
  64     def write_nominatim_config(self, dbname):
  65         """ Set up a custom test configuration that connects to the given
  66             database. This sets up the environment variables so that they can
  67             be picked up by dotenv and creates a project directory with the
  68             appropriate website scripts.
  69         """
  70         dsn = 'pgsql:dbname={}'.format(dbname)
  71         if self.db_host:
  72             dsn += ';host=' + self.db_host
  73         if self.db_port:
  74             dsn += ';port=' + self.db_port
  75         if self.db_user:
  76             dsn += ';user=' + self.db_user
  77         if self.db_pass:
  78             dsn += ';password=' + self.db_pass
  79
  80         if self.website_dir is not None \
  81            and self.test_env is not None \
  82            and dsn == self.test_env['NOMINATIM_DATABASE_DSN']:
  83             return # environment already set uo
  84
  85         self.test_env = dict(self.default_config)
  86         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
  87         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
  88         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
  89         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
  90         self.test_env['NOMINATIM_DATADIR'] = self.src_dir
  91         self.test_env['NOMINATIM_SQLDIR'] = self.src_dir / 'lib-sql'
  92         self.test_env['NOMINATIM_CONFIGDIR'] = self.src_dir / 'settings'
  93         self.test_env['NOMINATIM_BINDIR'] = self.src_dir / 'utils'
  94         self.test_env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = self.build_dir / 'module'
  95         self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql'
  96         self.test_env['NOMINATIM_NOMINATIM_TOOL'] = self.build_dir / 'nominatim'
  97
  98         if self.server_module_path:
  99             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
 100         else:
 101             # avoid module being copied into the temporary environment
 102             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module'
 103
 104         if self.website_dir is not None:
 105             self.website_dir.cleanup()
 106
 107         self.website_dir = tempfile.TemporaryDirectory()
 108         self.run_setup_script('setup-website')
 109
 110
 111     def db_drop_database(self, name):
 112         """ Drop the database with the given name.
 113         """
 114         conn = self.connect_database('postgres')
 115         conn.set_isolation_level(0)
 116         cur = conn.cursor()
 117         cur.execute('DROP DATABASE IF EXISTS {}'.format(name))
 118         conn.close()
 119
 120     def setup_template_db(self):
 121         """ Setup a template database that already contains common test data.
 122             Having a template database speeds up tests considerably but at
 123             the price that the tests sometimes run with stale data.
 124         """
 125         if self.template_db_done:
 126             return
 127
 128         self.template_db_done = True
 129
 130         if self._reuse_or_drop_db(self.template_db):
 131             return
 132
 133         try:
 134             # call the first part of database setup
 135             self.write_nominatim_config(self.template_db)
 136             self.run_setup_script('create-db', 'setup-db')
 137             # remove external data to speed up indexing for tests
 138             conn = self.connect_database(self.template_db)
 139             cur = conn.cursor()
 140             cur.execute("""select tablename from pg_tables
 141                            where tablename in ('gb_postcode', 'us_postcode')""")
 142             for t in cur:
 143                 conn.cursor().execute('TRUNCATE TABLE {}'.format(t[0]))
 144             conn.commit()
 145             conn.close()
 146
 147             # execute osm2pgsql import on an empty file to get the right tables
 148             with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
 149                 fd.write(b'<osm version="0.6"></osm>')
 150                 fd.flush()
 151                 self.run_setup_script('import-data',
 152                                       'ignore-errors',
 153                                       'create-functions',
 154                                       'create-tables',
 155                                       'create-partition-tables',
 156                                       'create-partition-functions',
 157                                       'load-data',
 158                                       'create-search-indices',
 159                                       osm_file=fd.name,
 160                                       osm2pgsql_cache='200')
 161         except:
 162             self.db_drop_database(self.template_db)
 163             raise
 164
 165
 166     def setup_api_db(self):
 167         """ Setup a test against the API test database.
 168         """
 169         self.write_nominatim_config(self.api_test_db)
 170
 171         if self.api_db_done:
 172             return
 173
 174         self.api_db_done = True
 175
 176         if self._reuse_or_drop_db(self.api_test_db):
 177             return
 178
 179         testdata = Path('__file__') / '..' / '..' / 'testdb'
 180         self.test_env['NOMINATIM_TIGER_DATA_PATH'] = str((testdata / 'tiger').resolve())
 181         self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
 182
 183         try:
 184             self.run_setup_script('all', osm_file=self.api_test_file)
 185             self.run_setup_script('import-tiger-data')
 186
 187             phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
 188             run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
 189         except:
 190             self.db_drop_database(self.api_test_db)
 191             raise
 192
 193
 194     def setup_unknown_db(self):
 195         """ Setup a test against a non-existing database.
 196         """
 197         self.write_nominatim_config('UNKNOWN_DATABASE_NAME')
 198
 199     def setup_db(self, context):
 200         """ Setup a test against a fresh, empty test database.
 201         """
 202         self.setup_template_db()
 203         self.write_nominatim_config(self.test_db)
 204         conn = self.connect_database(self.template_db)
 205         conn.set_isolation_level(0)
 206         cur = conn.cursor()
 207         cur.execute('DROP DATABASE IF EXISTS {}'.format(self.test_db))
 208         cur.execute('CREATE DATABASE {} TEMPLATE = {}'.format(self.test_db, self.template_db))
 209         conn.close()
 210         context.db = self.connect_database(self.test_db)
 211         context.db.autocommit = True
 212         psycopg2.extras.register_hstore(context.db, globally=False)
 213
 214     def teardown_db(self, context):
 215         """ Remove the test database, if it exists.
 216         """
 217         if 'db' in context:
 218             context.db.close()
 219
 220         if not self.keep_scenario_db:
 221             self.db_drop_database(self.test_db)
 222
 223     def _reuse_or_drop_db(self, name):
 224         """ Check for the existance of the given DB. If reuse is enabled,
 225             then the function checks for existance and returns True if the
 226             database is already there. Otherwise an existing database is
 227             dropped and always false returned.
 228         """
 229         if self.reuse_template:
 230             conn = self.connect_database('postgres')
 231             with conn.cursor() as cur:
 232                 cur.execute('select count(*) from pg_database where datname = %s',
 233                             (name,))
 234                 if cur.fetchone()[0] == 1:
 235                     return True
 236             conn.close()
 237         else:
 238             self.db_drop_database(name)
 239
 240         return False
 241
 242     def reindex_placex(self, db):
 243         """ Run the indexing step until all data in the placex has
 244             been processed. Indexing during updates can produce more data
 245             to index under some circumstances. That is why indexing may have
 246             to be run multiple times.
 247         """
 248         with db.cursor() as cur:
 249             while True:
 250                 self.run_update_script('index')
 251
 252                 cur.execute("SELECT 'a' FROM placex WHERE indexed_status != 0 LIMIT 1")
 253                 if cur.rowcount == 0:
 254                     return
 255
 256     def run_setup_script(self, *args, **kwargs):
 257         """ Run the Nominatim setup script with the given arguments.
 258         """
 259         self.run_nominatim_script('setup', *args, **kwargs)
 260
 261     def run_update_script(self, *args, **kwargs):
 262         """ Run the Nominatim update script with the given arguments.
 263         """
 264         self.run_nominatim_script('update', *args, **kwargs)
 265
 266     def run_nominatim_script(self, script, *args, **kwargs):
 267         """ Run one of the Nominatim utility scripts with the given arguments.
 268         """
 269         cmd = ['/usr/bin/env', 'php', '-Cq']
 270         cmd.append((Path(self.src_dir) / 'lib-php' / 'admin' / '{}.php'.format(script)).resolve())
 271         cmd.extend(['--' + x for x in args])
 272         for k, v in kwargs.items():
 273             cmd.extend(('--' + k.replace('_', '-'), str(v)))
 274
 275         if self.website_dir is not None:
 276             cwd = self.website_dir.name
 277         else:
 278             cwd = None
 279
 280         run_script(cmd, cwd=cwd, env=self.test_env)
 281
 282     def copy_from_place(self, db):
 283         """ Copy data from place to the placex and location_property_osmline
 284             tables invoking the appropriate triggers.
 285         """
 286         self.run_setup_script('create-functions', 'create-partition-functions')
 287
 288         with db.cursor() as cur:
 289             cur.execute("""INSERT INTO placex (osm_type, osm_id, class, type,
 290                                                name, admin_level, address,
 291                                                extratags, geometry)
 292                              SELECT osm_type, osm_id, class, type,
 293                                     name, admin_level, address,
 294                                     extratags, geometry
 295                                FROM place
 296                                WHERE not (class='place' and type='houses' and osm_type='W')""")
 297             cur.execute("""INSERT INTO location_property_osmline (osm_id, address, linegeo)
 298                              SELECT osm_id, address, geometry
 299                                FROM place
 300                               WHERE class='place' and type='houses'
 301                                     and osm_type='W'
 302                                     and ST_GeometryType(geometry) = 'ST_LineString'""")