]> git.openstreetmap.org Git - nominatim.git/blob - test/bdd/environment.py
Merge pull request #1318 from mtmail/php-pdo
[nominatim.git] / test / bdd / environment.py
1 from behave import *
2 import logging
3 import os
4 import psycopg2
5 import psycopg2.extras
6 import subprocess
7 import tempfile
8 from nose.tools import * # for assert functions
9 from sys import version_info as python_version
10
11 logger = logging.getLogger(__name__)
12
13 userconfig = {
14     'BUILDDIR' : os.path.join(os.path.split(__file__)[0], "../../build"),
15     'REMOVE_TEMPLATE' : False,
16     'KEEP_TEST_DB' : False,
17     'DB_HOST' : None,
18     'DB_PORT' : None,
19     'DB_USER' : None,
20     'DB_PASS' : None,
21     'TEMPLATE_DB' : 'test_template_nominatim',
22     'TEST_DB' : 'test_nominatim',
23     'API_TEST_DB' : 'test_api_nominatim',
24     'TEST_SETTINGS_FILE' : '/tmp/nominatim_settings.php',
25     'SERVER_MODULE_PATH' : None,
26     'PHPCOV' : False, # set to output directory to enable code coverage
27 }
28
29 use_step_matcher("re")
30
31 class NominatimEnvironment(object):
32     """ Collects all functions for the execution of Nominatim functions.
33     """
34
35     def __init__(self, config):
36         self.build_dir = os.path.abspath(config['BUILDDIR'])
37         self.src_dir = os.path.abspath(os.path.join(os.path.split(__file__)[0], "../.."))
38         self.db_host = config['DB_HOST']
39         self.db_port = config['DB_PORT']
40         self.db_user = config['DB_USER']
41         self.db_pass = config['DB_PASS']
42         self.template_db = config['TEMPLATE_DB']
43         self.test_db = config['TEST_DB']
44         self.api_test_db = config['API_TEST_DB']
45         self.server_module_path = config['SERVER_MODULE_PATH']
46         self.local_settings_file = config['TEST_SETTINGS_FILE']
47         self.reuse_template = not config['REMOVE_TEMPLATE']
48         self.keep_scenario_db = config['KEEP_TEST_DB']
49         self.code_coverage_path = config['PHPCOV']
50         self.code_coverage_id = 1
51         os.environ['NOMINATIM_SETTINGS'] = self.local_settings_file
52
53         self.template_db_done = False
54
55     def connect_database(self, dbname):
56         dbargs = {'database': dbname}
57         if self.db_host:
58             dbargs['host'] = self.db_host
59         if self.db_port:
60             dbargs['port'] = self.db_port
61         if self.db_user:
62             dbargs['user'] = self.db_user
63         if self.db_pass:
64             dbargs['password'] = self.db_pass
65         conn = psycopg2.connect(**dbargs)
66         return conn
67
68     def next_code_coverage_file(self):
69         fn = os.path.join(self.code_coverage_path, "%06d.cov" % self.code_coverage_id)
70         self.code_coverage_id += 1
71
72         return fn
73
74     def write_nominatim_config(self, dbname):
75         f = open(self.local_settings_file, 'w')
76         # https://secure.php.net/manual/en/ref.pdo-pgsql.connection.php
77         f.write("<?php\n  @define('CONST_Database_DSN', 'pgsql:dbname=%s%s%s%s%s');\n" %
78                 (dbname,
79                  (';host=' + self.db_host) if self.db_host else '',
80                  (';port=' + self.db_port) if self.db_port else '',
81                  (';user=' + self.db_user) if self.db_user else '',
82                  (';password=' + self.db_pass) if self.db_pass else ''
83                  ))
84         f.write("@define('CONST_Osm2pgsql_Flatnode_File', null);\n")
85         f.close()
86
87     def cleanup(self):
88         try:
89             os.remove(self.local_settings_file)
90         except OSError:
91             pass # ignore missing file
92
93     def db_drop_database(self, name):
94         conn = self.connect_database('postgres')
95         conn.set_isolation_level(0)
96         cur = conn.cursor()
97         cur.execute('DROP DATABASE IF EXISTS %s' % (name, ))
98         conn.close()
99
100     def setup_template_db(self):
101         if self.template_db_done:
102             return
103
104         self.template_db_done = True
105
106         if self.reuse_template:
107             # check that the template is there
108             conn = self.connect_database('postgres')
109             cur = conn.cursor()
110             cur.execute('select count(*) from pg_database where datname = %s',
111                         (self.template_db,))
112             if cur.fetchone()[0] == 1:
113                 return
114             conn.close()
115         else:
116             # just in case... make sure a previous table has been dropped
117             self.db_drop_database(self.template_db)
118
119         try:
120             # call the first part of database setup
121             self.write_nominatim_config(self.template_db)
122             self.run_setup_script('create-db', 'setup-db')
123             # remove external data to speed up indexing for tests
124             conn = self.connect_database(self.template_db)
125             cur = conn.cursor()
126             cur.execute("""select tablename from pg_tables
127                            where tablename in ('gb_postcode', 'us_postcode')""")
128             for t in cur:
129                 conn.cursor().execute('TRUNCATE TABLE %s' % (t[0],))
130             conn.commit()
131             conn.close()
132
133             # execute osm2pgsql import on an empty file to get the right tables
134             with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
135                 fd.write(b'<osm version="0.6"></osm>')
136                 fd.flush()
137                 self.run_setup_script('import-data',
138                                       'ignore-errors',
139                                       'create-functions',
140                                       'create-tables',
141                                       'create-partition-tables',
142                                       'create-partition-functions',
143                                       'load-data',
144                                       'create-search-indices',
145                                       osm_file=fd.name,
146                                       osm2pgsql_cache='200')
147         except:
148             self.db_drop_database(self.template_db)
149             raise
150
151
152     def setup_api_db(self, context):
153         self.write_nominatim_config(self.api_test_db)
154
155     def setup_unknown_db(self, context):
156         self.write_nominatim_config('UNKNOWN_DATABASE_NAME')
157
158     def setup_db(self, context):
159         self.setup_template_db()
160         self.write_nominatim_config(self.test_db)
161         conn = self.connect_database(self.template_db)
162         conn.set_isolation_level(0)
163         cur = conn.cursor()
164         cur.execute('DROP DATABASE IF EXISTS %s' % (self.test_db, ))
165         cur.execute('CREATE DATABASE %s TEMPLATE = %s' % (self.test_db, self.template_db))
166         conn.close()
167         context.db = self.connect_database(self.test_db)
168         if python_version[0] < 3:
169             psycopg2.extras.register_hstore(context.db, globally=False, unicode=True)
170         else:
171             psycopg2.extras.register_hstore(context.db, globally=False)
172
173     def teardown_db(self, context):
174         if 'db' in context:
175             context.db.close()
176
177         if not self.keep_scenario_db:
178             self.db_drop_database(self.test_db)
179
180     def run_setup_script(self, *args, **kwargs):
181         if self.server_module_path:
182             kwargs = dict(kwargs)
183             kwargs['module_path'] = self.server_module_path
184         self.run_nominatim_script('setup', *args, **kwargs)
185
186     def run_update_script(self, *args, **kwargs):
187         self.run_nominatim_script('update', *args, **kwargs)
188
189     def run_nominatim_script(self, script, *args, **kwargs):
190         cmd = ['/usr/bin/env', 'php', '-Cq']
191         cmd.append(os.path.join(self.build_dir, 'utils', '%s.php' % script))
192         cmd.extend(['--%s' % x for x in args])
193         for k, v in kwargs.items():
194             cmd.extend(('--' + k.replace('_', '-'), str(v)))
195         proc = subprocess.Popen(cmd, cwd=self.build_dir,
196                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
197         (outp, outerr) = proc.communicate()
198         logger.debug("run_nominatim_script: %s\n%s\n%s" % (cmd, outp, outerr))
199         assert (proc.returncode == 0), "Script '%s' failed:\n%s\n%s\n" % (script, outp, outerr)
200
201
202 class OSMDataFactory(object):
203
204     def __init__(self):
205         scriptpath = os.path.dirname(os.path.abspath(__file__))
206         self.scene_path = os.environ.get('SCENE_PATH',
207                            os.path.join(scriptpath, '..', 'scenes', 'data'))
208         self.scene_cache = {}
209         self.clear_grid()
210
211     def parse_geometry(self, geom, scene):
212         if geom.find(':') >= 0:
213             return "ST_SetSRID(%s, 4326)" % self.get_scene_geometry(scene, geom)
214
215         if geom.find(',') < 0:
216             out = "POINT(%s)" % self.mk_wkt_point(geom)
217         elif geom.find('(') < 0:
218             line = ','.join([self.mk_wkt_point(x) for x in geom.split(',')])
219             out = "LINESTRING(%s)" % line
220         else:
221             inner = geom.strip('() ')
222             line = ','.join([self.mk_wkt_point(x) for x in inner.split(',')])
223             out = "POLYGON((%s))" % line
224
225         return "ST_SetSRID('%s'::geometry, 4326)" % out
226
227     def mk_wkt_point(self, point):
228         geom = point.strip()
229         if geom.find(' ') >= 0:
230             return geom
231         else:
232             pt = self.grid_node(int(geom))
233             assert_is_not_none(pt, "Point not found in grid")
234             return "%f %f" % pt
235
236     def get_scene_geometry(self, default_scene, name):
237         geoms = []
238         for obj in name.split('+'):
239             oname = obj.strip()
240             if oname.startswith(':'):
241                 assert_is_not_none(default_scene, "You need to set a scene")
242                 defscene = self.load_scene(default_scene)
243                 wkt = defscene[oname[1:]]
244             else:
245                 scene, obj = oname.split(':', 2)
246                 scene_geoms = self.load_scene(scene)
247                 wkt = scene_geoms[obj]
248
249             geoms.append("'%s'::geometry" % wkt)
250
251         if len(geoms) == 1:
252             return geoms[0]
253         else:
254             return 'ST_LineMerge(ST_Collect(ARRAY[%s]))' % ','.join(geoms)
255
256     def load_scene(self, name):
257         if name in self.scene_cache:
258             return self.scene_cache[name]
259
260         scene = {}
261         with open(os.path.join(self.scene_path, "%s.wkt" % name), 'r') as fd:
262             for line in fd:
263                 if line.strip():
264                     obj, wkt = line.split('|', 2)
265                     scene[obj.strip()] = wkt.strip()
266             self.scene_cache[name] = scene
267
268         return scene
269
270     def clear_grid(self):
271         self.grid = {}
272
273     def add_grid_node(self, nodeid, x, y):
274         self.grid[nodeid] = (x, y)
275
276     def grid_node(self, nodeid):
277         return self.grid.get(nodeid)
278
279
280 def before_all(context):
281     # logging setup
282     context.config.setup_logging()
283     # set up -D options
284     for k,v in userconfig.items():
285         context.config.userdata.setdefault(k, v)
286     logging.debug('User config: %s' %(str(context.config.userdata)))
287     # Nominatim test setup
288     context.nominatim = NominatimEnvironment(context.config.userdata)
289     context.osm = OSMDataFactory()
290
291 def after_all(context):
292     context.nominatim.cleanup()
293
294
295 def before_scenario(context, scenario):
296     if 'DB' in context.tags:
297         context.nominatim.setup_db(context)
298     elif 'APIDB' in context.tags:
299         context.nominatim.setup_api_db(context)
300     elif 'UNKNOWNDB' in context.tags:
301         context.nominatim.setup_unknown_db(context)
302     context.scene = None
303
304 def after_scenario(context, scenario):
305     if 'DB' in context.tags:
306         context.nominatim.teardown_db(context)