test/bdd/steps/nominatim_environment.py

   1 from pathlib import Path
   2 import sys
   3 import tempfile
   4
   5 import psycopg2
   6 import psycopg2.extras
   7
   8 sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..').resolve()))
   9
  10 from nominatim.config import Configuration
  11 from steps.utils import run_script
  12
  13 class NominatimEnvironment:
  14     """ Collects all functions for the execution of Nominatim functions.
  15     """
  16
  17     def __init__(self, config):
  18         self.build_dir = Path(config['BUILDDIR']).resolve()
  19         self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve()
  20         self.db_host = config['DB_HOST']
  21         self.db_port = config['DB_PORT']
  22         self.db_user = config['DB_USER']
  23         self.db_pass = config['DB_PASS']
  24         self.template_db = config['TEMPLATE_DB']
  25         self.test_db = config['TEST_DB']
  26         self.api_test_db = config['API_TEST_DB']
  27         self.api_test_file = config['API_TEST_FILE']
  28         self.server_module_path = config['SERVER_MODULE_PATH']
  29         self.reuse_template = not config['REMOVE_TEMPLATE']
  30         self.keep_scenario_db = config['KEEP_TEST_DB']
  31         self.code_coverage_path = config['PHPCOV']
  32         self.code_coverage_id = 1
  33
  34         self.default_config = Configuration(None, self.src_dir / 'settings').get_os_env()
  35         self.test_env = None
  36         self.template_db_done = False
  37         self.api_db_done = False
  38         self.website_dir = None
  39
  40     def connect_database(self, dbname):
  41         """ Return a connection to the database with the given name.
  42             Uses configured host, user and port.
  43         """
  44         dbargs = {'database': dbname}
  45         if self.db_host:
  46             dbargs['host'] = self.db_host
  47         if self.db_port:
  48             dbargs['port'] = self.db_port
  49         if self.db_user:
  50             dbargs['user'] = self.db_user
  51         if self.db_pass:
  52             dbargs['password'] = self.db_pass
  53         conn = psycopg2.connect(**dbargs)
  54         return conn
  55
  56     def next_code_coverage_file(self):
  57         """ Generate the next name for a coverage file.
  58         """
  59         fn = Path(self.code_coverage_path) / "{:06d}.cov".format(self.code_coverage_id)
  60         self.code_coverage_id += 1
  61
  62         return fn.resolve()
  63
  64     def write_nominatim_config(self, dbname):
  65         """ Set up a custom test configuration that connects to the given
  66             database. This sets up the environment variables so that they can
  67             be picked up by dotenv and creates a project directory with the
  68             appropriate website scripts.
  69         """
  70         dsn = 'pgsql:dbname={}'.format(dbname)
  71         if self.db_host:
  72             dsn += ';host=' + self.db_host
  73         if self.db_port:
  74             dsn += ';port=' + self.db_port
  75         if self.db_user:
  76             dsn += ';user=' + self.db_user
  77         if self.db_pass:
  78             dsn += ';password=' + self.db_pass
  79
  80         if self.website_dir is not None \
  81            and self.test_env is not None \
  82            and dsn == self.test_env['NOMINATIM_DATABASE_DSN']:
  83             return # environment already set uo
  84
  85         self.test_env = dict(self.default_config)
  86         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
  87         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
  88         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
  89         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
  90         self.test_env['NOMINATIM_DATADIR'] = self.src_dir
  91         self.test_env['NOMINATIM_SQLDIR'] = self.src_dir / 'lib-sql'
  92         self.test_env['NOMINATIM_CONFIGDIR'] = self.src_dir / 'settings'
  93         self.test_env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = self.build_dir / 'module'
  94         self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql'
  95         self.test_env['NOMINATIM_NOMINATIM_TOOL'] = self.build_dir / 'nominatim'
  96
  97         if self.server_module_path:
  98             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
  99         else:
 100             # avoid module being copied into the temporary environment
 101             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module'
 102
 103         if self.website_dir is not None:
 104             self.website_dir.cleanup()
 105
 106         self.website_dir = tempfile.TemporaryDirectory()
 107         self.run_setup_script('setup-website')
 108
 109
 110     def db_drop_database(self, name):
 111         """ Drop the database with the given name.
 112         """
 113         conn = self.connect_database('postgres')
 114         conn.set_isolation_level(0)
 115         cur = conn.cursor()
 116         cur.execute('DROP DATABASE IF EXISTS {}'.format(name))
 117         conn.close()
 118
 119     def setup_template_db(self):
 120         """ Setup a template database that already contains common test data.
 121             Having a template database speeds up tests considerably but at
 122             the price that the tests sometimes run with stale data.
 123         """
 124         if self.template_db_done:
 125             return
 126
 127         self.template_db_done = True
 128
 129         if self._reuse_or_drop_db(self.template_db):
 130             return
 131
 132         try:
 133             # call the first part of database setup
 134             self.write_nominatim_config(self.template_db)
 135             self.run_setup_script('create-db', 'setup-db')
 136             # remove external data to speed up indexing for tests
 137             conn = self.connect_database(self.template_db)
 138             cur = conn.cursor()
 139             cur.execute("""select tablename from pg_tables
 140                            where tablename in ('gb_postcode', 'us_postcode')""")
 141             for t in cur:
 142                 conn.cursor().execute('TRUNCATE TABLE {}'.format(t[0]))
 143             conn.commit()
 144             conn.close()
 145
 146             # execute osm2pgsql import on an empty file to get the right tables
 147             with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
 148                 fd.write(b'<osm version="0.6"></osm>')
 149                 fd.flush()
 150                 self.run_setup_script('import-data',
 151                                       'ignore-errors',
 152                                       'create-functions',
 153                                       'create-tables',
 154                                       'create-partition-tables',
 155                                       'create-partition-functions',
 156                                       'load-data',
 157                                       'create-search-indices',
 158                                       osm_file=fd.name,
 159                                       osm2pgsql_cache='200')
 160         except:
 161             self.db_drop_database(self.template_db)
 162             raise
 163
 164
 165     def setup_api_db(self):
 166         """ Setup a test against the API test database.
 167         """
 168         self.write_nominatim_config(self.api_test_db)
 169
 170         if self.api_db_done:
 171             return
 172
 173         self.api_db_done = True
 174
 175         if self._reuse_or_drop_db(self.api_test_db):
 176             return
 177
 178         testdata = Path('__file__') / '..' / '..' / 'testdb'
 179         self.test_env['NOMINATIM_TIGER_DATA_PATH'] = str((testdata / 'tiger').resolve())
 180         self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
 181
 182         try:
 183             self.run_setup_script('all', osm_file=self.api_test_file)
 184             self.run_setup_script('import-tiger-data')
 185
 186             phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
 187             run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
 188         except:
 189             self.db_drop_database(self.api_test_db)
 190             raise
 191
 192
 193     def setup_unknown_db(self):
 194         """ Setup a test against a non-existing database.
 195         """
 196         self.write_nominatim_config('UNKNOWN_DATABASE_NAME')
 197
 198     def setup_db(self, context):
 199         """ Setup a test against a fresh, empty test database.
 200         """
 201         self.setup_template_db()
 202         self.write_nominatim_config(self.test_db)
 203         conn = self.connect_database(self.template_db)
 204         conn.set_isolation_level(0)
 205         cur = conn.cursor()
 206         cur.execute('DROP DATABASE IF EXISTS {}'.format(self.test_db))
 207         cur.execute('CREATE DATABASE {} TEMPLATE = {}'.format(self.test_db, self.template_db))
 208         conn.close()
 209         context.db = self.connect_database(self.test_db)
 210         context.db.autocommit = True
 211         psycopg2.extras.register_hstore(context.db, globally=False)
 212
 213     def teardown_db(self, context):
 214         """ Remove the test database, if it exists.
 215         """
 216         if 'db' in context:
 217             context.db.close()
 218
 219         if not self.keep_scenario_db:
 220             self.db_drop_database(self.test_db)
 221
 222     def _reuse_or_drop_db(self, name):
 223         """ Check for the existance of the given DB. If reuse is enabled,
 224             then the function checks for existance and returns True if the
 225             database is already there. Otherwise an existing database is
 226             dropped and always false returned.
 227         """
 228         if self.reuse_template:
 229             conn = self.connect_database('postgres')
 230             with conn.cursor() as cur:
 231                 cur.execute('select count(*) from pg_database where datname = %s',
 232                             (name,))
 233                 if cur.fetchone()[0] == 1:
 234                     return True
 235             conn.close()
 236         else:
 237             self.db_drop_database(name)
 238
 239         return False
 240
 241     def reindex_placex(self, db):
 242         """ Run the indexing step until all data in the placex has
 243             been processed. Indexing during updates can produce more data
 244             to index under some circumstances. That is why indexing may have
 245             to be run multiple times.
 246         """
 247         with db.cursor() as cur:
 248             while True:
 249                 self.run_update_script('index')
 250
 251                 cur.execute("SELECT 'a' FROM placex WHERE indexed_status != 0 LIMIT 1")
 252                 if cur.rowcount == 0:
 253                     return
 254
 255     def run_setup_script(self, *args, **kwargs):
 256         """ Run the Nominatim setup script with the given arguments.
 257         """
 258         self.run_nominatim_script('setup', *args, **kwargs)
 259
 260     def run_update_script(self, *args, **kwargs):
 261         """ Run the Nominatim update script with the given arguments.
 262         """
 263         self.run_nominatim_script('update', *args, **kwargs)
 264
 265     def run_nominatim_script(self, script, *args, **kwargs):
 266         """ Run one of the Nominatim utility scripts with the given arguments.
 267         """
 268         cmd = ['/usr/bin/env', 'php', '-Cq']
 269         cmd.append((Path(self.src_dir) / 'lib-php' / 'admin' / '{}.php'.format(script)).resolve())
 270         cmd.extend(['--' + x for x in args])
 271         for k, v in kwargs.items():
 272             cmd.extend(('--' + k.replace('_', '-'), str(v)))
 273
 274         if self.website_dir is not None:
 275             cwd = self.website_dir.name
 276         else:
 277             cwd = None
 278
 279         run_script(cmd, cwd=cwd, env=self.test_env)
 280
 281     def copy_from_place(self, db):
 282         """ Copy data from place to the placex and location_property_osmline
 283             tables invoking the appropriate triggers.
 284         """
 285         self.run_setup_script('create-functions', 'create-partition-functions')
 286
 287         with db.cursor() as cur:
 288             cur.execute("""INSERT INTO placex (osm_type, osm_id, class, type,
 289                                                name, admin_level, address,
 290                                                extratags, geometry)
 291                              SELECT osm_type, osm_id, class, type,
 292                                     name, admin_level, address,
 293                                     extratags, geometry
 294                                FROM place
 295                                WHERE not (class='place' and type='houses' and osm_type='W')""")
 296             cur.execute("""INSERT INTO location_property_osmline (osm_id, address, linegeo)
 297                              SELECT osm_id, address, geometry
 298                                FROM place
 299                               WHERE class='place' and type='houses'
 300                                     and osm_type='W'
 301                                     and ST_GeometryType(geometry) = 'ST_LineString'""")