test/bdd/steps/nominatim_environment.py

   1 import os
   2 from pathlib import Path
   3 import tempfile
   4
   5 import psycopg2
   6 import psycopg2.extras
   7
   8 from steps.utils import run_script
   9
  10 class NominatimEnvironment:
  11     """ Collects all functions for the execution of Nominatim functions.
  12     """
  13
  14     def __init__(self, config):
  15         self.build_dir = Path(config['BUILDDIR']).resolve()
  16         self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve()
  17         self.db_host = config['DB_HOST']
  18         self.db_port = config['DB_PORT']
  19         self.db_user = config['DB_USER']
  20         self.db_pass = config['DB_PASS']
  21         self.template_db = config['TEMPLATE_DB']
  22         self.test_db = config['TEST_DB']
  23         self.api_test_db = config['API_TEST_DB']
  24         self.api_test_file = config['API_TEST_FILE']
  25         self.server_module_path = config['SERVER_MODULE_PATH']
  26         self.reuse_template = not config['REMOVE_TEMPLATE']
  27         self.keep_scenario_db = config['KEEP_TEST_DB']
  28         self.code_coverage_path = config['PHPCOV']
  29         self.code_coverage_id = 1
  30
  31         self.test_env = None
  32         self.template_db_done = False
  33         self.api_db_done = False
  34         self.website_dir = None
  35
  36     def connect_database(self, dbname):
  37         """ Return a connection to the database with the given name.
  38             Uses configured host, user and port.
  39         """
  40         dbargs = {'database': dbname}
  41         if self.db_host:
  42             dbargs['host'] = self.db_host
  43         if self.db_port:
  44             dbargs['port'] = self.db_port
  45         if self.db_user:
  46             dbargs['user'] = self.db_user
  47         if self.db_pass:
  48             dbargs['password'] = self.db_pass
  49         conn = psycopg2.connect(**dbargs)
  50         return conn
  51
  52     def next_code_coverage_file(self):
  53         """ Generate the next name for a coverage file.
  54         """
  55         fn = Path(self.code_coverage_path) / "{:06d}.cov".format(self.code_coverage_id)
  56         self.code_coverage_id += 1
  57
  58         return fn.resolve()
  59
  60     def write_nominatim_config(self, dbname):
  61         """ Set up a custom test configuration that connects to the given
  62             database. This sets up the environment variables so that they can
  63             be picked up by dotenv and creates a project directory with the
  64             appropriate website scripts.
  65         """
  66         dsn = 'pgsql:dbname={}'.format(dbname)
  67         if self.db_host:
  68             dsn += ';host=' + self.db_host
  69         if self.db_port:
  70             dsn += ';port=' + self.db_port
  71         if self.db_user:
  72             dsn += ';user=' + self.db_user
  73         if self.db_pass:
  74             dsn += ';password=' + self.db_pass
  75
  76         if self.website_dir is not None \
  77            and self.test_env is not None \
  78            and dsn == self.test_env['NOMINATIM_DATABASE_DSN']:
  79             return # environment already set uo
  80
  81         self.test_env = os.environ
  82         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
  83         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
  84         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
  85         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
  86
  87         if self.server_module_path:
  88             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
  89
  90         if self.website_dir is not None:
  91             self.website_dir.cleanup()
  92
  93         self.website_dir = tempfile.TemporaryDirectory()
  94         self.run_setup_script('setup-website')
  95
  96
  97     def db_drop_database(self, name):
  98         """ Drop the database with the given name.
  99         """
 100         conn = self.connect_database('postgres')
 101         conn.set_isolation_level(0)
 102         cur = conn.cursor()
 103         cur.execute('DROP DATABASE IF EXISTS {}'.format(name))
 104         conn.close()
 105
 106     def setup_template_db(self):
 107         """ Setup a template database that already contains common test data.
 108             Having a template database speeds up tests considerably but at
 109             the price that the tests sometimes run with stale data.
 110         """
 111         if self.template_db_done:
 112             return
 113
 114         self.template_db_done = True
 115
 116         if self._reuse_or_drop_db(self.template_db):
 117             return
 118
 119         try:
 120             # call the first part of database setup
 121             self.write_nominatim_config(self.template_db)
 122             self.run_setup_script('create-db', 'setup-db')
 123             # remove external data to speed up indexing for tests
 124             conn = self.connect_database(self.template_db)
 125             cur = conn.cursor()
 126             cur.execute("""select tablename from pg_tables
 127                            where tablename in ('gb_postcode', 'us_postcode')""")
 128             for t in cur:
 129                 conn.cursor().execute('TRUNCATE TABLE {}'.format(t[0]))
 130             conn.commit()
 131             conn.close()
 132
 133             # execute osm2pgsql import on an empty file to get the right tables
 134             with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
 135                 fd.write(b'<osm version="0.6"></osm>')
 136                 fd.flush()
 137                 self.run_setup_script('import-data',
 138                                       'ignore-errors',
 139                                       'create-functions',
 140                                       'create-tables',
 141                                       'create-partition-tables',
 142                                       'create-partition-functions',
 143                                       'load-data',
 144                                       'create-search-indices',
 145                                       osm_file=fd.name,
 146                                       osm2pgsql_cache='200')
 147         except:
 148             self.db_drop_database(self.template_db)
 149             raise
 150
 151
 152     def setup_api_db(self):
 153         """ Setup a test against the API test database.
 154         """
 155         self.write_nominatim_config(self.api_test_db)
 156
 157         if self.api_db_done:
 158             return
 159
 160         self.api_db_done = True
 161
 162         if self._reuse_or_drop_db(self.api_test_db):
 163             return
 164
 165         testdata = Path('__file__') / '..' / '..' / 'testdb'
 166         self.test_env['NOMINATIM_TIGER_DATA_PATH'] = str((testdata / 'tiger').resolve())
 167         self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
 168
 169         try:
 170             self.run_setup_script('all', osm_file=self.api_test_file)
 171             self.run_setup_script('import-tiger-data')
 172
 173             phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
 174             run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
 175         except:
 176             self.db_drop_database(self.api_test_db)
 177             raise
 178
 179
 180     def setup_unknown_db(self):
 181         """ Setup a test against a non-existing database.
 182         """
 183         self.write_nominatim_config('UNKNOWN_DATABASE_NAME')
 184
 185     def setup_db(self, context):
 186         """ Setup a test against a fresh, empty test database.
 187         """
 188         self.setup_template_db()
 189         self.write_nominatim_config(self.test_db)
 190         conn = self.connect_database(self.template_db)
 191         conn.set_isolation_level(0)
 192         cur = conn.cursor()
 193         cur.execute('DROP DATABASE IF EXISTS {}'.format(self.test_db))
 194         cur.execute('CREATE DATABASE {} TEMPLATE = {}'.format(self.test_db, self.template_db))
 195         conn.close()
 196         context.db = self.connect_database(self.test_db)
 197         context.db.autocommit = True
 198         psycopg2.extras.register_hstore(context.db, globally=False)
 199
 200     def teardown_db(self, context):
 201         """ Remove the test database, if it exists.
 202         """
 203         if 'db' in context:
 204             context.db.close()
 205
 206         if not self.keep_scenario_db:
 207             self.db_drop_database(self.test_db)
 208
 209     def _reuse_or_drop_db(self, name):
 210         """ Check for the existance of the given DB. If reuse is enabled,
 211             then the function checks for existance and returns True if the
 212             database is already there. Otherwise an existing database is
 213             dropped and always false returned.
 214         """
 215         if self.reuse_template:
 216             conn = self.connect_database('postgres')
 217             with conn.cursor() as cur:
 218                 cur.execute('select count(*) from pg_database where datname = %s',
 219                             (name,))
 220                 if cur.fetchone()[0] == 1:
 221                     return True
 222             conn.close()
 223         else:
 224             self.db_drop_database(name)
 225
 226         return False
 227
 228     def reindex_placex(self, db):
 229         """ Run the indexing step until all data in the placex has
 230             been processed. Indexing during updates can produce more data
 231             to index under some circumstances. That is why indexing may have
 232             to be run multiple times.
 233         """
 234         with db.cursor() as cur:
 235             while True:
 236                 self.run_update_script('index')
 237
 238                 cur.execute("SELECT 'a' FROM placex WHERE indexed_status != 0 LIMIT 1")
 239                 if cur.rowcount == 0:
 240                     return
 241
 242     def run_setup_script(self, *args, **kwargs):
 243         """ Run the Nominatim setup script with the given arguments.
 244         """
 245         self.run_nominatim_script('setup', *args, **kwargs)
 246
 247     def run_update_script(self, *args, **kwargs):
 248         """ Run the Nominatim update script with the given arguments.
 249         """
 250         self.run_nominatim_script('update', *args, **kwargs)
 251
 252     def run_nominatim_script(self, script, *args, **kwargs):
 253         """ Run one of the Nominatim utility scripts with the given arguments.
 254         """
 255         cmd = ['/usr/bin/env', 'php', '-Cq']
 256         cmd.append((Path(self.build_dir) / 'utils' / '{}.php'.format(script)).resolve())
 257         cmd.extend(['--' + x for x in args])
 258         for k, v in kwargs.items():
 259             cmd.extend(('--' + k.replace('_', '-'), str(v)))
 260
 261         if self.website_dir is not None:
 262             cwd = self.website_dir.name
 263         else:
 264             cwd = self.build_dir
 265
 266         run_script(cmd, cwd=cwd, env=self.test_env)
 267
 268     def copy_from_place(self, db):
 269         """ Copy data from place to the placex and location_property_osmline
 270             tables invoking the appropriate triggers.
 271         """
 272         self.run_setup_script('create-functions', 'create-partition-functions')
 273
 274         with db.cursor() as cur:
 275             cur.execute("""INSERT INTO placex (osm_type, osm_id, class, type,
 276                                                name, admin_level, address,
 277                                                extratags, geometry)
 278                              SELECT osm_type, osm_id, class, type,
 279                                     name, admin_level, address,
 280                                     extratags, geometry
 281                                FROM place
 282                                WHERE not (class='place' and type='houses' and osm_type='W')""")
 283             cur.execute("""INSERT INTO location_property_osmline (osm_id, address, linegeo)
 284                              SELECT osm_id, address, geometry
 285                                FROM place
 286                               WHERE class='place' and type='houses'
 287                                     and osm_type='W'
 288                                     and ST_GeometryType(geometry) = 'ST_LineString'""")