]> git.openstreetmap.org Git - nominatim.git/blob - test/bdd/steps/nominatim_environment.py
bdd: directly call utility scripts in lib
[nominatim.git] / test / bdd / steps / nominatim_environment.py
1 from pathlib import Path
2 import sys
3 import tempfile
4
5 import psycopg2
6 import psycopg2.extras
7
8 sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..').resolve()))
9
10 from nominatim.config import Configuration
11 from steps.utils import run_script
12
13 class NominatimEnvironment:
14     """ Collects all functions for the execution of Nominatim functions.
15     """
16
17     def __init__(self, config):
18         self.build_dir = Path(config['BUILDDIR']).resolve()
19         self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve()
20         self.db_host = config['DB_HOST']
21         self.db_port = config['DB_PORT']
22         self.db_user = config['DB_USER']
23         self.db_pass = config['DB_PASS']
24         self.template_db = config['TEMPLATE_DB']
25         self.test_db = config['TEST_DB']
26         self.api_test_db = config['API_TEST_DB']
27         self.api_test_file = config['API_TEST_FILE']
28         self.server_module_path = config['SERVER_MODULE_PATH']
29         self.reuse_template = not config['REMOVE_TEMPLATE']
30         self.keep_scenario_db = config['KEEP_TEST_DB']
31         self.code_coverage_path = config['PHPCOV']
32         self.code_coverage_id = 1
33
34         self.default_config = Configuration(None, self.src_dir / 'settings').get_os_env()
35         self.test_env = None
36         self.template_db_done = False
37         self.api_db_done = False
38         self.website_dir = None
39
40     def connect_database(self, dbname):
41         """ Return a connection to the database with the given name.
42             Uses configured host, user and port.
43         """
44         dbargs = {'database': dbname}
45         if self.db_host:
46             dbargs['host'] = self.db_host
47         if self.db_port:
48             dbargs['port'] = self.db_port
49         if self.db_user:
50             dbargs['user'] = self.db_user
51         if self.db_pass:
52             dbargs['password'] = self.db_pass
53         conn = psycopg2.connect(**dbargs)
54         return conn
55
56     def next_code_coverage_file(self):
57         """ Generate the next name for a coverage file.
58         """
59         fn = Path(self.code_coverage_path) / "{:06d}.cov".format(self.code_coverage_id)
60         self.code_coverage_id += 1
61
62         return fn.resolve()
63
64     def write_nominatim_config(self, dbname):
65         """ Set up a custom test configuration that connects to the given
66             database. This sets up the environment variables so that they can
67             be picked up by dotenv and creates a project directory with the
68             appropriate website scripts.
69         """
70         dsn = 'pgsql:dbname={}'.format(dbname)
71         if self.db_host:
72             dsn += ';host=' + self.db_host
73         if self.db_port:
74             dsn += ';port=' + self.db_port
75         if self.db_user:
76             dsn += ';user=' + self.db_user
77         if self.db_pass:
78             dsn += ';password=' + self.db_pass
79
80         if self.website_dir is not None \
81            and self.test_env is not None \
82            and dsn == self.test_env['NOMINATIM_DATABASE_DSN']:
83             return # environment already set uo
84
85         self.test_env = dict(self.default_config)
86         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
87         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
88         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
89         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
90         self.test_env['NOMINATIM_DATADIR'] = self.src_dir
91         self.test_env['NOMINATIM_BINDIR'] = self.src_dir / 'utils'
92         self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module'
93         self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql'
94
95         if self.server_module_path:
96             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
97
98         if self.website_dir is not None:
99             self.website_dir.cleanup()
100
101         self.website_dir = tempfile.TemporaryDirectory()
102         self.run_setup_script('setup-website')
103
104
105     def db_drop_database(self, name):
106         """ Drop the database with the given name.
107         """
108         conn = self.connect_database('postgres')
109         conn.set_isolation_level(0)
110         cur = conn.cursor()
111         cur.execute('DROP DATABASE IF EXISTS {}'.format(name))
112         conn.close()
113
114     def setup_template_db(self):
115         """ Setup a template database that already contains common test data.
116             Having a template database speeds up tests considerably but at
117             the price that the tests sometimes run with stale data.
118         """
119         if self.template_db_done:
120             return
121
122         self.template_db_done = True
123
124         if self._reuse_or_drop_db(self.template_db):
125             return
126
127         try:
128             # call the first part of database setup
129             self.write_nominatim_config(self.template_db)
130             self.run_setup_script('create-db', 'setup-db')
131             # remove external data to speed up indexing for tests
132             conn = self.connect_database(self.template_db)
133             cur = conn.cursor()
134             cur.execute("""select tablename from pg_tables
135                            where tablename in ('gb_postcode', 'us_postcode')""")
136             for t in cur:
137                 conn.cursor().execute('TRUNCATE TABLE {}'.format(t[0]))
138             conn.commit()
139             conn.close()
140
141             # execute osm2pgsql import on an empty file to get the right tables
142             with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
143                 fd.write(b'<osm version="0.6"></osm>')
144                 fd.flush()
145                 self.run_setup_script('import-data',
146                                       'ignore-errors',
147                                       'create-functions',
148                                       'create-tables',
149                                       'create-partition-tables',
150                                       'create-partition-functions',
151                                       'load-data',
152                                       'create-search-indices',
153                                       osm_file=fd.name,
154                                       osm2pgsql_cache='200')
155         except:
156             self.db_drop_database(self.template_db)
157             raise
158
159
160     def setup_api_db(self):
161         """ Setup a test against the API test database.
162         """
163         self.write_nominatim_config(self.api_test_db)
164
165         if self.api_db_done:
166             return
167
168         self.api_db_done = True
169
170         if self._reuse_or_drop_db(self.api_test_db):
171             return
172
173         testdata = Path('__file__') / '..' / '..' / 'testdb'
174         self.test_env['NOMINATIM_TIGER_DATA_PATH'] = str((testdata / 'tiger').resolve())
175         self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
176
177         try:
178             self.run_setup_script('all', osm_file=self.api_test_file)
179             self.run_setup_script('import-tiger-data')
180
181             phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
182             run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
183         except:
184             self.db_drop_database(self.api_test_db)
185             raise
186
187
188     def setup_unknown_db(self):
189         """ Setup a test against a non-existing database.
190         """
191         self.write_nominatim_config('UNKNOWN_DATABASE_NAME')
192
193     def setup_db(self, context):
194         """ Setup a test against a fresh, empty test database.
195         """
196         self.setup_template_db()
197         self.write_nominatim_config(self.test_db)
198         conn = self.connect_database(self.template_db)
199         conn.set_isolation_level(0)
200         cur = conn.cursor()
201         cur.execute('DROP DATABASE IF EXISTS {}'.format(self.test_db))
202         cur.execute('CREATE DATABASE {} TEMPLATE = {}'.format(self.test_db, self.template_db))
203         conn.close()
204         context.db = self.connect_database(self.test_db)
205         context.db.autocommit = True
206         psycopg2.extras.register_hstore(context.db, globally=False)
207
208     def teardown_db(self, context):
209         """ Remove the test database, if it exists.
210         """
211         if 'db' in context:
212             context.db.close()
213
214         if not self.keep_scenario_db:
215             self.db_drop_database(self.test_db)
216
217     def _reuse_or_drop_db(self, name):
218         """ Check for the existance of the given DB. If reuse is enabled,
219             then the function checks for existance and returns True if the
220             database is already there. Otherwise an existing database is
221             dropped and always false returned.
222         """
223         if self.reuse_template:
224             conn = self.connect_database('postgres')
225             with conn.cursor() as cur:
226                 cur.execute('select count(*) from pg_database where datname = %s',
227                             (name,))
228                 if cur.fetchone()[0] == 1:
229                     return True
230             conn.close()
231         else:
232             self.db_drop_database(name)
233
234         return False
235
236     def reindex_placex(self, db):
237         """ Run the indexing step until all data in the placex has
238             been processed. Indexing during updates can produce more data
239             to index under some circumstances. That is why indexing may have
240             to be run multiple times.
241         """
242         with db.cursor() as cur:
243             while True:
244                 self.run_update_script('index')
245
246                 cur.execute("SELECT 'a' FROM placex WHERE indexed_status != 0 LIMIT 1")
247                 if cur.rowcount == 0:
248                     return
249
250     def run_setup_script(self, *args, **kwargs):
251         """ Run the Nominatim setup script with the given arguments.
252         """
253         self.run_nominatim_script('setup', *args, **kwargs)
254
255     def run_update_script(self, *args, **kwargs):
256         """ Run the Nominatim update script with the given arguments.
257         """
258         self.run_nominatim_script('update', *args, **kwargs)
259
260     def run_nominatim_script(self, script, *args, **kwargs):
261         """ Run one of the Nominatim utility scripts with the given arguments.
262         """
263         cmd = ['/usr/bin/env', 'php', '-Cq']
264         cmd.append((Path(self.src_dir) / 'lib' / 'admin' / '{}.php'.format(script)).resolve())
265         cmd.extend(['--' + x for x in args])
266         for k, v in kwargs.items():
267             cmd.extend(('--' + k.replace('_', '-'), str(v)))
268
269         if self.website_dir is not None:
270             cwd = self.website_dir.name
271         else:
272             cwd = None
273
274         run_script(cmd, cwd=cwd, env=self.test_env)
275
276     def copy_from_place(self, db):
277         """ Copy data from place to the placex and location_property_osmline
278             tables invoking the appropriate triggers.
279         """
280         self.run_setup_script('create-functions', 'create-partition-functions')
281
282         with db.cursor() as cur:
283             cur.execute("""INSERT INTO placex (osm_type, osm_id, class, type,
284                                                name, admin_level, address,
285                                                extratags, geometry)
286                              SELECT osm_type, osm_id, class, type,
287                                     name, admin_level, address,
288                                     extratags, geometry
289                                FROM place
290                                WHERE not (class='place' and type='houses' and osm_type='W')""")
291             cur.execute("""INSERT INTO location_property_osmline (osm_id, address, linegeo)
292                              SELECT osm_id, address, geometry
293                                FROM place
294                               WHERE class='place' and type='houses'
295                                     and osm_type='W'
296                                     and ST_GeometryType(geometry) = 'ST_LineString'""")