test/bdd/steps/nominatim_environment.py

   1 import os
   2 from pathlib import Path
   3 import tempfile
   4
   5 import psycopg2
   6 import psycopg2.extras
   7
   8 from steps.utils import run_script
   9
  10 class NominatimEnvironment:
  11     """ Collects all functions for the execution of Nominatim functions.
  12     """
  13
  14     def __init__(self, config):
  15         self.build_dir = Path(config['BUILDDIR']).resolve()
  16         self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve()
  17         self.db_host = config['DB_HOST']
  18         self.db_port = config['DB_PORT']
  19         self.db_user = config['DB_USER']
  20         self.db_pass = config['DB_PASS']
  21         self.template_db = config['TEMPLATE_DB']
  22         self.test_db = config['TEST_DB']
  23         self.api_test_db = config['API_TEST_DB']
  24         self.api_test_file = config['API_TEST_FILE']
  25         self.server_module_path = config['SERVER_MODULE_PATH']
  26         self.reuse_template = not config['REMOVE_TEMPLATE']
  27         self.keep_scenario_db = config['KEEP_TEST_DB']
  28         self.code_coverage_path = config['PHPCOV']
  29         self.code_coverage_id = 1
  30
  31         self.test_env = None
  32         self.template_db_done = False
  33         self.api_db_done = False
  34         self.website_dir = None
  35
  36     def connect_database(self, dbname):
  37         """ Return a connection to the database with the given name.
  38             Uses configured host, user and port.
  39         """
  40         dbargs = {'database': dbname}
  41         if self.db_host:
  42             dbargs['host'] = self.db_host
  43         if self.db_port:
  44             dbargs['port'] = self.db_port
  45         if self.db_user:
  46             dbargs['user'] = self.db_user
  47         if self.db_pass:
  48             dbargs['password'] = self.db_pass
  49         conn = psycopg2.connect(**dbargs)
  50         return conn
  51
  52     def next_code_coverage_file(self):
  53         """ Generate the next name for a coverage file.
  54         """
  55         fn = Path(self.code_coverage_path) / "{:06d}.cov".format(self.code_coverage_id)
  56         self.code_coverage_id += 1
  57
  58         return fn.resolve()
  59
  60     def write_nominatim_config(self, dbname):
  61         """ Set up a custom test configuration that connects to the given
  62             database. This sets up the environment variables so that they can
  63             be picked up by dotenv and creates a project directory with the
  64             appropriate website scripts.
  65         """
  66         dsn = 'pgsql:dbname={}'.format(dbname)
  67         if self.db_host:
  68             dsn += ';host=' + self.db_host
  69         if self.db_port:
  70             dsn += ';port=' + self.db_port
  71         if self.db_user:
  72             dsn += ';user=' + self.db_user
  73         if self.db_pass:
  74             dsn += ';password=' + self.db_pass
  75
  76         if self.website_dir is not None \
  77            and self.test_env is not None \
  78            and dsn == self.test_env['NOMINATIM_DATABASE_DSN']:
  79             return # environment already set uo
  80
  81         self.test_env = os.environ
  82         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
  83         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
  84         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
  85         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
  86
  87         if self.server_module_path:
  88             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
  89
  90         if self.website_dir is not None:
  91             self.website_dir.cleanup()
  92
  93         self.website_dir = tempfile.TemporaryDirectory()
  94         self.run_setup_script('setup-website')
  95
  96
  97     def db_drop_database(self, name):
  98         """ Drop the database with the given name.
  99         """
 100         conn = self.connect_database('postgres')
 101         conn.set_isolation_level(0)
 102         cur = conn.cursor()
 103         cur.execute('DROP DATABASE IF EXISTS {}'.format(name))
 104         conn.close()
 105
 106     def setup_template_db(self):
 107         """ Setup a template database that already contains common test data.
 108             Having a template database speeds up tests considerably but at
 109             the price that the tests sometimes run with stale data.
 110         """
 111         if self.template_db_done:
 112             return
 113
 114         self.template_db_done = True
 115
 116         if self._reuse_or_drop_db(self.template_db):
 117             return
 118
 119         try:
 120             # call the first part of database setup
 121             self.write_nominatim_config(self.template_db)
 122             self.run_setup_script('create-db', 'setup-db')
 123             # remove external data to speed up indexing for tests
 124             conn = self.connect_database(self.template_db)
 125             cur = conn.cursor()
 126             cur.execute("""select tablename from pg_tables
 127                            where tablename in ('gb_postcode', 'us_postcode')""")
 128             for t in cur:
 129                 conn.cursor().execute('TRUNCATE TABLE {}'.format(t[0]))
 130             conn.commit()
 131             conn.close()
 132
 133             # execute osm2pgsql import on an empty file to get the right tables
 134             with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
 135                 fd.write(b'<osm version="0.6"></osm>')
 136                 fd.flush()
 137                 self.run_setup_script('import-data',
 138                                       'ignore-errors',
 139                                       'create-functions',
 140                                       'create-tables',
 141                                       'create-partition-tables',
 142                                       'create-partition-functions',
 143                                       'load-data',
 144                                       'create-search-indices',
 145                                       osm_file=fd.name,
 146                                       osm2pgsql_cache='200')
 147         except:
 148             self.db_drop_database(self.template_db)
 149             raise
 150
 151
 152     def setup_api_db(self):
 153         """ Setup a test against the API test database.
 154         """
 155         self.write_nominatim_config(self.api_test_db)
 156
 157         if self.api_db_done:
 158             return
 159
 160         self.api_db_done = True
 161
 162         if self._reuse_or_drop_db(self.api_test_db):
 163             return
 164
 165         testdata = Path('__file__') / '..' / '..' / 'testdb'
 166         self.test_env['NOMINATIM_TIGER_DATA_PATH'] = str((testdata / 'tiger').resolve())
 167         self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
 168
 169         try:
 170             self.run_setup_script('all', 'import-tiger-data',
 171                                   osm_file=self.api_test_file)
 172         except:
 173             self.db_drop_database(self.api_test_db)
 174             raise
 175
 176
 177     def setup_unknown_db(self):
 178         """ Setup a test against a non-existing database.
 179         """
 180         self.write_nominatim_config('UNKNOWN_DATABASE_NAME')
 181
 182     def setup_db(self, context):
 183         """ Setup a test against a fresh, empty test database.
 184         """
 185         self.setup_template_db()
 186         self.write_nominatim_config(self.test_db)
 187         conn = self.connect_database(self.template_db)
 188         conn.set_isolation_level(0)
 189         cur = conn.cursor()
 190         cur.execute('DROP DATABASE IF EXISTS {}'.format(self.test_db))
 191         cur.execute('CREATE DATABASE {} TEMPLATE = {}'.format(self.test_db, self.template_db))
 192         conn.close()
 193         context.db = self.connect_database(self.test_db)
 194         context.db.autocommit = True
 195         psycopg2.extras.register_hstore(context.db, globally=False)
 196
 197     def teardown_db(self, context):
 198         """ Remove the test database, if it exists.
 199         """
 200         if 'db' in context:
 201             context.db.close()
 202
 203         if not self.keep_scenario_db:
 204             self.db_drop_database(self.test_db)
 205
 206     def _reuse_or_drop_db(self, name):
 207         """ Check for the existance of the given DB. If reuse is enabled,
 208             then the function checks for existance and returns True if the
 209             database is already there. Otherwise an existing database is
 210             dropped and always false returned.
 211         """
 212         if self.reuse_template:
 213             conn = self.connect_database('postgres')
 214             with conn.cursor() as cur:
 215                 cur.execute('select count(*) from pg_database where datname = %s',
 216                             (name,))
 217                 if cur.fetchone()[0] == 1:
 218                     return True
 219             conn.close()
 220         else:
 221             self.db_drop_database(name)
 222
 223         return False
 224
 225     def reindex_placex(self, db):
 226         """ Run the indexing step until all data in the placex has
 227             been processed. Indexing during updates can produce more data
 228             to index under some circumstances. That is why indexing may have
 229             to be run multiple times.
 230         """
 231         with db.cursor() as cur:
 232             while True:
 233                 self.run_update_script('index')
 234
 235                 cur.execute("SELECT 'a' FROM placex WHERE indexed_status != 0 LIMIT 1")
 236                 if cur.rowcount == 0:
 237                     return
 238
 239     def run_setup_script(self, *args, **kwargs):
 240         """ Run the Nominatim setup script with the given arguments.
 241         """
 242         self.run_nominatim_script('setup', *args, **kwargs)
 243
 244     def run_update_script(self, *args, **kwargs):
 245         """ Run the Nominatim update script with the given arguments.
 246         """
 247         self.run_nominatim_script('update', *args, **kwargs)
 248
 249     def run_nominatim_script(self, script, *args, **kwargs):
 250         """ Run one of the Nominatim utility scripts with the given arguments.
 251         """
 252         cmd = ['/usr/bin/env', 'php', '-Cq']
 253         cmd.append((Path(self.build_dir) / 'utils' / '{}.php'.format(script)).resolve())
 254         cmd.extend(['--' + x for x in args])
 255         for k, v in kwargs.items():
 256             cmd.extend(('--' + k.replace('_', '-'), str(v)))
 257
 258         if self.website_dir is not None:
 259             cwd = self.website_dir.name
 260         else:
 261             cwd = self.build_dir
 262
 263         run_script(cmd, cwd=cwd, env=self.test_env)
 264
 265     def copy_from_place(self, db):
 266         """ Copy data from place to the placex and location_property_osmline
 267             tables invoking the appropriate triggers.
 268         """
 269         self.run_setup_script('create-functions', 'create-partition-functions')
 270
 271         with db.cursor() as cur:
 272             cur.execute("""INSERT INTO placex (osm_type, osm_id, class, type,
 273                                                name, admin_level, address,
 274                                                extratags, geometry)
 275                              SELECT osm_type, osm_id, class, type,
 276                                     name, admin_level, address,
 277                                     extratags, geometry
 278                                FROM place
 279                                WHERE not (class='place' and type='houses' and osm_type='W')""")
 280             cur.execute("""INSERT INTO location_property_osmline (osm_id, address, linegeo)
 281                              SELECT osm_id, address, geometry
 282                                FROM place
 283                               WHERE class='place' and type='houses'
 284                                     and osm_type='W'
 285                                     and ST_GeometryType(geometry) = 'ST_LineString'""")