test/bdd/steps/nominatim_environment.py

   1 from pathlib import Path
   2 import sys
   3 import tempfile
   4
   5 import psycopg2
   6 import psycopg2.extras
   7
   8 sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..').resolve()))
   9
  10 from nominatim.config import Configuration
  11 from steps.utils import run_script
  12
  13 class NominatimEnvironment:
  14     """ Collects all functions for the execution of Nominatim functions.
  15     """
  16
  17     def __init__(self, config):
  18         self.build_dir = Path(config['BUILDDIR']).resolve()
  19         self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve()
  20         self.db_host = config['DB_HOST']
  21         self.db_port = config['DB_PORT']
  22         self.db_user = config['DB_USER']
  23         self.db_pass = config['DB_PASS']
  24         self.template_db = config['TEMPLATE_DB']
  25         self.test_db = config['TEST_DB']
  26         self.api_test_db = config['API_TEST_DB']
  27         self.api_test_file = config['API_TEST_FILE']
  28         self.server_module_path = config['SERVER_MODULE_PATH']
  29         self.reuse_template = not config['REMOVE_TEMPLATE']
  30         self.keep_scenario_db = config['KEEP_TEST_DB']
  31         self.code_coverage_path = config['PHPCOV']
  32         self.code_coverage_id = 1
  33
  34         self.default_config = Configuration(None, self.src_dir / 'settings').get_os_env()
  35         self.test_env = None
  36         self.template_db_done = False
  37         self.api_db_done = False
  38         self.website_dir = None
  39
  40     def connect_database(self, dbname):
  41         """ Return a connection to the database with the given name.
  42             Uses configured host, user and port.
  43         """
  44         dbargs = {'database': dbname}
  45         if self.db_host:
  46             dbargs['host'] = self.db_host
  47         if self.db_port:
  48             dbargs['port'] = self.db_port
  49         if self.db_user:
  50             dbargs['user'] = self.db_user
  51         if self.db_pass:
  52             dbargs['password'] = self.db_pass
  53         conn = psycopg2.connect(**dbargs)
  54         return conn
  55
  56     def next_code_coverage_file(self):
  57         """ Generate the next name for a coverage file.
  58         """
  59         fn = Path(self.code_coverage_path) / "{:06d}.cov".format(self.code_coverage_id)
  60         self.code_coverage_id += 1
  61
  62         return fn.resolve()
  63
  64     def write_nominatim_config(self, dbname):
  65         """ Set up a custom test configuration that connects to the given
  66             database. This sets up the environment variables so that they can
  67             be picked up by dotenv and creates a project directory with the
  68             appropriate website scripts.
  69         """
  70         dsn = 'pgsql:dbname={}'.format(dbname)
  71         if self.db_host:
  72             dsn += ';host=' + self.db_host
  73         if self.db_port:
  74             dsn += ';port=' + self.db_port
  75         if self.db_user:
  76             dsn += ';user=' + self.db_user
  77         if self.db_pass:
  78             dsn += ';password=' + self.db_pass
  79
  80         if self.website_dir is not None \
  81            and self.test_env is not None \
  82            and dsn == self.test_env['NOMINATIM_DATABASE_DSN']:
  83             return # environment already set uo
  84
  85         self.test_env = dict(self.default_config)
  86         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
  87         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
  88         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
  89         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
  90         self.test_env['NOMINATIM_DATADIR'] = self.src_dir
  91         self.test_env['NOMINATIM_BINDIR'] = self.src_dir / 'utils'
  92         self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module'
  93         self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql'
  94         self.test_env['NOMINATIM_NOMINATIM_TOOL'] = self.build_dir / 'nominatim'
  95
  96         if self.server_module_path:
  97             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
  98
  99         if self.website_dir is not None:
 100             self.website_dir.cleanup()
 101
 102         self.website_dir = tempfile.TemporaryDirectory()
 103         self.run_setup_script('setup-website')
 104
 105
 106     def db_drop_database(self, name):
 107         """ Drop the database with the given name.
 108         """
 109         conn = self.connect_database('postgres')
 110         conn.set_isolation_level(0)
 111         cur = conn.cursor()
 112         cur.execute('DROP DATABASE IF EXISTS {}'.format(name))
 113         conn.close()
 114
 115     def setup_template_db(self):
 116         """ Setup a template database that already contains common test data.
 117             Having a template database speeds up tests considerably but at
 118             the price that the tests sometimes run with stale data.
 119         """
 120         if self.template_db_done:
 121             return
 122
 123         self.template_db_done = True
 124
 125         if self._reuse_or_drop_db(self.template_db):
 126             return
 127
 128         try:
 129             # call the first part of database setup
 130             self.write_nominatim_config(self.template_db)
 131             self.run_setup_script('create-db', 'setup-db')
 132             # remove external data to speed up indexing for tests
 133             conn = self.connect_database(self.template_db)
 134             cur = conn.cursor()
 135             cur.execute("""select tablename from pg_tables
 136                            where tablename in ('gb_postcode', 'us_postcode')""")
 137             for t in cur:
 138                 conn.cursor().execute('TRUNCATE TABLE {}'.format(t[0]))
 139             conn.commit()
 140             conn.close()
 141
 142             # execute osm2pgsql import on an empty file to get the right tables
 143             with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
 144                 fd.write(b'<osm version="0.6"></osm>')
 145                 fd.flush()
 146                 self.run_setup_script('import-data',
 147                                       'ignore-errors',
 148                                       'create-functions',
 149                                       'create-tables',
 150                                       'create-partition-tables',
 151                                       'create-partition-functions',
 152                                       'load-data',
 153                                       'create-search-indices',
 154                                       osm_file=fd.name,
 155                                       osm2pgsql_cache='200')
 156         except:
 157             self.db_drop_database(self.template_db)
 158             raise
 159
 160
 161     def setup_api_db(self):
 162         """ Setup a test against the API test database.
 163         """
 164         self.write_nominatim_config(self.api_test_db)
 165
 166         if self.api_db_done:
 167             return
 168
 169         self.api_db_done = True
 170
 171         if self._reuse_or_drop_db(self.api_test_db):
 172             return
 173
 174         testdata = Path('__file__') / '..' / '..' / 'testdb'
 175         self.test_env['NOMINATIM_TIGER_DATA_PATH'] = str((testdata / 'tiger').resolve())
 176         self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
 177
 178         try:
 179             self.run_setup_script('all', osm_file=self.api_test_file)
 180             self.run_setup_script('import-tiger-data')
 181
 182             phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
 183             run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
 184         except:
 185             self.db_drop_database(self.api_test_db)
 186             raise
 187
 188
 189     def setup_unknown_db(self):
 190         """ Setup a test against a non-existing database.
 191         """
 192         self.write_nominatim_config('UNKNOWN_DATABASE_NAME')
 193
 194     def setup_db(self, context):
 195         """ Setup a test against a fresh, empty test database.
 196         """
 197         self.setup_template_db()
 198         self.write_nominatim_config(self.test_db)
 199         conn = self.connect_database(self.template_db)
 200         conn.set_isolation_level(0)
 201         cur = conn.cursor()
 202         cur.execute('DROP DATABASE IF EXISTS {}'.format(self.test_db))
 203         cur.execute('CREATE DATABASE {} TEMPLATE = {}'.format(self.test_db, self.template_db))
 204         conn.close()
 205         context.db = self.connect_database(self.test_db)
 206         context.db.autocommit = True
 207         psycopg2.extras.register_hstore(context.db, globally=False)
 208
 209     def teardown_db(self, context):
 210         """ Remove the test database, if it exists.
 211         """
 212         if 'db' in context:
 213             context.db.close()
 214
 215         if not self.keep_scenario_db:
 216             self.db_drop_database(self.test_db)
 217
 218     def _reuse_or_drop_db(self, name):
 219         """ Check for the existance of the given DB. If reuse is enabled,
 220             then the function checks for existance and returns True if the
 221             database is already there. Otherwise an existing database is
 222             dropped and always false returned.
 223         """
 224         if self.reuse_template:
 225             conn = self.connect_database('postgres')
 226             with conn.cursor() as cur:
 227                 cur.execute('select count(*) from pg_database where datname = %s',
 228                             (name,))
 229                 if cur.fetchone()[0] == 1:
 230                     return True
 231             conn.close()
 232         else:
 233             self.db_drop_database(name)
 234
 235         return False
 236
 237     def reindex_placex(self, db):
 238         """ Run the indexing step until all data in the placex has
 239             been processed. Indexing during updates can produce more data
 240             to index under some circumstances. That is why indexing may have
 241             to be run multiple times.
 242         """
 243         with db.cursor() as cur:
 244             while True:
 245                 self.run_update_script('index')
 246
 247                 cur.execute("SELECT 'a' FROM placex WHERE indexed_status != 0 LIMIT 1")
 248                 if cur.rowcount == 0:
 249                     return
 250
 251     def run_setup_script(self, *args, **kwargs):
 252         """ Run the Nominatim setup script with the given arguments.
 253         """
 254         self.run_nominatim_script('setup', *args, **kwargs)
 255
 256     def run_update_script(self, *args, **kwargs):
 257         """ Run the Nominatim update script with the given arguments.
 258         """
 259         self.run_nominatim_script('update', *args, **kwargs)
 260
 261     def run_nominatim_script(self, script, *args, **kwargs):
 262         """ Run one of the Nominatim utility scripts with the given arguments.
 263         """
 264         cmd = ['/usr/bin/env', 'php', '-Cq']
 265         cmd.append((Path(self.src_dir) / 'lib' / 'admin' / '{}.php'.format(script)).resolve())
 266         cmd.extend(['--' + x for x in args])
 267         for k, v in kwargs.items():
 268             cmd.extend(('--' + k.replace('_', '-'), str(v)))
 269
 270         if self.website_dir is not None:
 271             cwd = self.website_dir.name
 272         else:
 273             cwd = None
 274
 275         run_script(cmd, cwd=cwd, env=self.test_env)
 276
 277     def copy_from_place(self, db):
 278         """ Copy data from place to the placex and location_property_osmline
 279             tables invoking the appropriate triggers.
 280         """
 281         self.run_setup_script('create-functions', 'create-partition-functions')
 282
 283         with db.cursor() as cur:
 284             cur.execute("""INSERT INTO placex (osm_type, osm_id, class, type,
 285                                                name, admin_level, address,
 286                                                extratags, geometry)
 287                              SELECT osm_type, osm_id, class, type,
 288                                     name, admin_level, address,
 289                                     extratags, geometry
 290                                FROM place
 291                                WHERE not (class='place' and type='houses' and osm_type='W')""")
 292             cur.execute("""INSERT INTO location_property_osmline (osm_id, address, linegeo)
 293                              SELECT osm_id, address, geometry
 294                                FROM place
 295                               WHERE class='place' and type='houses'
 296                                     and osm_type='W'
 297                                     and ST_GeometryType(geometry) = 'ST_LineString'""")