]> git.openstreetmap.org Git - nominatim.git/blob - test/bdd/steps/nominatim_environment.py
Merge remote-tracking branch 'upstream/master'
[nominatim.git] / test / bdd / steps / nominatim_environment.py
1 from pathlib import Path
2 import sys
3 import tempfile
4
5 import psycopg2
6 import psycopg2.extras
7
8 sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..').resolve()))
9
10 from nominatim.config import Configuration
11 from steps.utils import run_script
12
13 class NominatimEnvironment:
14     """ Collects all functions for the execution of Nominatim functions.
15     """
16
17     def __init__(self, config):
18         self.build_dir = Path(config['BUILDDIR']).resolve()
19         self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve()
20         self.db_host = config['DB_HOST']
21         self.db_port = config['DB_PORT']
22         self.db_user = config['DB_USER']
23         self.db_pass = config['DB_PASS']
24         self.template_db = config['TEMPLATE_DB']
25         self.test_db = config['TEST_DB']
26         self.api_test_db = config['API_TEST_DB']
27         self.api_test_file = config['API_TEST_FILE']
28         self.server_module_path = config['SERVER_MODULE_PATH']
29         self.reuse_template = not config['REMOVE_TEMPLATE']
30         self.keep_scenario_db = config['KEEP_TEST_DB']
31         self.code_coverage_path = config['PHPCOV']
32         self.code_coverage_id = 1
33
34         self.default_config = Configuration(None, self.src_dir / 'settings').get_os_env()
35         self.test_env = None
36         self.template_db_done = False
37         self.api_db_done = False
38         self.website_dir = None
39
40     def connect_database(self, dbname):
41         """ Return a connection to the database with the given name.
42             Uses configured host, user and port.
43         """
44         dbargs = {'database': dbname}
45         if self.db_host:
46             dbargs['host'] = self.db_host
47         if self.db_port:
48             dbargs['port'] = self.db_port
49         if self.db_user:
50             dbargs['user'] = self.db_user
51         if self.db_pass:
52             dbargs['password'] = self.db_pass
53         conn = psycopg2.connect(**dbargs)
54         return conn
55
56     def next_code_coverage_file(self):
57         """ Generate the next name for a coverage file.
58         """
59         fn = Path(self.code_coverage_path) / "{:06d}.cov".format(self.code_coverage_id)
60         self.code_coverage_id += 1
61
62         return fn.resolve()
63
64     def write_nominatim_config(self, dbname):
65         """ Set up a custom test configuration that connects to the given
66             database. This sets up the environment variables so that they can
67             be picked up by dotenv and creates a project directory with the
68             appropriate website scripts.
69         """
70         dsn = 'pgsql:dbname={}'.format(dbname)
71         if self.db_host:
72             dsn += ';host=' + self.db_host
73         if self.db_port:
74             dsn += ';port=' + self.db_port
75         if self.db_user:
76             dsn += ';user=' + self.db_user
77         if self.db_pass:
78             dsn += ';password=' + self.db_pass
79
80         if self.website_dir is not None \
81            and self.test_env is not None \
82            and dsn == self.test_env['NOMINATIM_DATABASE_DSN']:
83             return # environment already set uo
84
85         self.test_env = dict(self.default_config)
86         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
87         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
88         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
89         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
90         self.test_env['NOMINATIM_DATADIR'] = self.src_dir
91         self.test_env['NOMINATIM_BINDIR'] = self.src_dir / 'utils'
92         self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module'
93         self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql'
94         self.test_env['NOMINATIM_NOMINATIM_TOOL'] = self.build_dir / 'nominatim'
95
96         if self.server_module_path:
97             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
98
99         if self.website_dir is not None:
100             self.website_dir.cleanup()
101
102         self.website_dir = tempfile.TemporaryDirectory()
103         self.run_setup_script('setup-website')
104
105
106     def db_drop_database(self, name):
107         """ Drop the database with the given name.
108         """
109         conn = self.connect_database('postgres')
110         conn.set_isolation_level(0)
111         cur = conn.cursor()
112         cur.execute('DROP DATABASE IF EXISTS {}'.format(name))
113         conn.close()
114
115     def setup_template_db(self):
116         """ Setup a template database that already contains common test data.
117             Having a template database speeds up tests considerably but at
118             the price that the tests sometimes run with stale data.
119         """
120         if self.template_db_done:
121             return
122
123         self.template_db_done = True
124
125         if self._reuse_or_drop_db(self.template_db):
126             return
127
128         try:
129             # call the first part of database setup
130             self.write_nominatim_config(self.template_db)
131             self.run_setup_script('create-db', 'setup-db')
132             # remove external data to speed up indexing for tests
133             conn = self.connect_database(self.template_db)
134             cur = conn.cursor()
135             cur.execute("""select tablename from pg_tables
136                            where tablename in ('gb_postcode', 'us_postcode')""")
137             for t in cur:
138                 conn.cursor().execute('TRUNCATE TABLE {}'.format(t[0]))
139             conn.commit()
140             conn.close()
141
142             # execute osm2pgsql import on an empty file to get the right tables
143             with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
144                 fd.write(b'<osm version="0.6"></osm>')
145                 fd.flush()
146                 self.run_setup_script('import-data',
147                                       'ignore-errors',
148                                       'create-functions',
149                                       'create-tables',
150                                       'create-partition-tables',
151                                       'create-partition-functions',
152                                       'load-data',
153                                       'create-search-indices',
154                                       osm_file=fd.name,
155                                       osm2pgsql_cache='200')
156         except:
157             self.db_drop_database(self.template_db)
158             raise
159
160
161     def setup_api_db(self):
162         """ Setup a test against the API test database.
163         """
164         self.write_nominatim_config(self.api_test_db)
165
166         if self.api_db_done:
167             return
168
169         self.api_db_done = True
170
171         if self._reuse_or_drop_db(self.api_test_db):
172             return
173
174         testdata = Path('__file__') / '..' / '..' / 'testdb'
175         self.test_env['NOMINATIM_TIGER_DATA_PATH'] = str((testdata / 'tiger').resolve())
176         self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
177
178         try:
179             self.run_setup_script('all', osm_file=self.api_test_file)
180             self.run_setup_script('import-tiger-data')
181
182             phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
183             run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
184         except:
185             self.db_drop_database(self.api_test_db)
186             raise
187
188
189     def setup_unknown_db(self):
190         """ Setup a test against a non-existing database.
191         """
192         self.write_nominatim_config('UNKNOWN_DATABASE_NAME')
193
194     def setup_db(self, context):
195         """ Setup a test against a fresh, empty test database.
196         """
197         self.setup_template_db()
198         self.write_nominatim_config(self.test_db)
199         conn = self.connect_database(self.template_db)
200         conn.set_isolation_level(0)
201         cur = conn.cursor()
202         cur.execute('DROP DATABASE IF EXISTS {}'.format(self.test_db))
203         cur.execute('CREATE DATABASE {} TEMPLATE = {}'.format(self.test_db, self.template_db))
204         conn.close()
205         context.db = self.connect_database(self.test_db)
206         context.db.autocommit = True
207         psycopg2.extras.register_hstore(context.db, globally=False)
208
209     def teardown_db(self, context):
210         """ Remove the test database, if it exists.
211         """
212         if 'db' in context:
213             context.db.close()
214
215         if not self.keep_scenario_db:
216             self.db_drop_database(self.test_db)
217
218     def _reuse_or_drop_db(self, name):
219         """ Check for the existance of the given DB. If reuse is enabled,
220             then the function checks for existance and returns True if the
221             database is already there. Otherwise an existing database is
222             dropped and always false returned.
223         """
224         if self.reuse_template:
225             conn = self.connect_database('postgres')
226             with conn.cursor() as cur:
227                 cur.execute('select count(*) from pg_database where datname = %s',
228                             (name,))
229                 if cur.fetchone()[0] == 1:
230                     return True
231             conn.close()
232         else:
233             self.db_drop_database(name)
234
235         return False
236
237     def reindex_placex(self, db):
238         """ Run the indexing step until all data in the placex has
239             been processed. Indexing during updates can produce more data
240             to index under some circumstances. That is why indexing may have
241             to be run multiple times.
242         """
243         with db.cursor() as cur:
244             while True:
245                 self.run_update_script('index')
246
247                 cur.execute("SELECT 'a' FROM placex WHERE indexed_status != 0 LIMIT 1")
248                 if cur.rowcount == 0:
249                     return
250
251     def run_setup_script(self, *args, **kwargs):
252         """ Run the Nominatim setup script with the given arguments.
253         """
254         self.run_nominatim_script('setup', *args, **kwargs)
255
256     def run_update_script(self, *args, **kwargs):
257         """ Run the Nominatim update script with the given arguments.
258         """
259         self.run_nominatim_script('update', *args, **kwargs)
260
261     def run_nominatim_script(self, script, *args, **kwargs):
262         """ Run one of the Nominatim utility scripts with the given arguments.
263         """
264         cmd = ['/usr/bin/env', 'php', '-Cq']
265         cmd.append((Path(self.src_dir) / 'lib' / 'admin' / '{}.php'.format(script)).resolve())
266         cmd.extend(['--' + x for x in args])
267         for k, v in kwargs.items():
268             cmd.extend(('--' + k.replace('_', '-'), str(v)))
269
270         if self.website_dir is not None:
271             cwd = self.website_dir.name
272         else:
273             cwd = None
274
275         run_script(cmd, cwd=cwd, env=self.test_env)
276
277     def copy_from_place(self, db):
278         """ Copy data from place to the placex and location_property_osmline
279             tables invoking the appropriate triggers.
280         """
281         self.run_setup_script('create-functions', 'create-partition-functions')
282
283         with db.cursor() as cur:
284             cur.execute("""INSERT INTO placex (osm_type, osm_id, class, type,
285                                                name, admin_level, address,
286                                                extratags, geometry)
287                              SELECT osm_type, osm_id, class, type,
288                                     name, admin_level, address,
289                                     extratags, geometry
290                                FROM place
291                                WHERE not (class='place' and type='houses' and osm_type='W')""")
292             cur.execute("""INSERT INTO location_property_osmline (osm_id, address, linegeo)
293                              SELECT osm_id, address, geometry
294                                FROM place
295                               WHERE class='place' and type='houses'
296                                     and osm_type='W'
297                                     and ST_GeometryType(geometry) = 'ST_LineString'""")