test/bdd/steps/osm_data.py

   1 import subprocess
   2 import tempfile
   3 import random
   4 import os
   5 from nose.tools import * # for assert functions
   6
   7 @given(u'the (\d+ )?grid')
   8 def define_node_grid(context, grid_step):
   9     """
  10     Define a grid of node positions.
  11     """
  12     if grid_step is not None:
  13         grid_step = int(grd_step.strip())
  14     else:
  15         grid_step = 0.00001
  16
  17     context.osm.clear_grid()
  18
  19     i = 0
  20     for h in context.table.headings:
  21         if h.isdigit():
  22             context.osm.add_grid_node(int(h), 0, i)
  23         i += grid_step
  24
  25     x = grid_step
  26     for r in context.table:
  27         y = 0
  28         for h in r:
  29             if h.isdigit():
  30                 context.osm.add_grid_node(int(h), x, y)
  31             y += grid_step
  32         x += grid_step
  33
  34
  35 @when(u'loading osm data')
  36 def load_osm_file(context):
  37     """
  38     Load the given data into a freshly created test data using osm2pgsql.
  39     No further indexing is done.
  40
  41     The data is expected as attached text in OPL format.
  42     """
  43     # create a OSM file in /tmp and import it
  44     with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.opl', delete=False) as fd:
  45         fname = fd.name
  46         for line in context.text.splitlines():
  47             if line.startswith('n') and line.find(' x') < 0:
  48                 coord = context.osm.grid_node(int(line[1:].split(' ')[0]))
  49                 if coord is None:
  50                     coord = (random.random() * 360 - 180,
  51                              random.random() * 180 - 90)
  52                 line += " x%f y%f" % coord
  53             fd.write(line.encode('utf-8'))
  54             fd.write(b'\n')
  55
  56     context.nominatim.run_setup_script('import-data', osm_file=fname,
  57                                        osm2pgsql_cache=300)
  58
  59     ### reintroduce the triggers/indexes we've lost by having osm2pgsql set up place again
  60     cur = context.db.cursor()
  61     cur.execute("""CREATE TRIGGER place_before_delete BEFORE DELETE ON place
  62                     FOR EACH ROW EXECUTE PROCEDURE place_delete()""")
  63     cur.execute("""CREATE TRIGGER place_before_insert BEFORE INSERT ON place
  64                    FOR EACH ROW EXECUTE PROCEDURE place_insert()""")
  65     cur.execute("""CREATE UNIQUE INDEX idx_place_osm_unique on place using btree(osm_id,osm_type,class,type)""")
  66     context.db.commit()
  67
  68     os.remove(fname)
  69
  70 @when(u'updating osm data')
  71 def update_from_osm_file(context):
  72     """
  73     Update a database previously populated with 'loading osm data'.
  74     Needs to run indexing on the existing data first to yield the correct result.
  75
  76     The data is expected as attached text in OPL format.
  77     """
  78     context.nominatim.run_setup_script('create-functions', 'create-partition-functions')
  79
  80     cur = context.db.cursor()
  81     cur.execute("""insert into placex (osm_type, osm_id, class, type, name,
  82                    admin_level,  housenumber, street, addr_place, isin, postcode,
  83                    country_code, extratags, geometry) select * from place""")
  84     cur.execute(
  85         """insert into location_property_osmline
  86                (osm_id, interpolationtype, street, addr_place,
  87                 postcode, calculated_country_code, linegeo)
  88              SELECT osm_id, housenumber, street, addr_place,
  89                     postcode, country_code, geometry from place
  90               WHERE class='place' and type='houses' and osm_type='W'
  91                     and ST_GeometryType(geometry) = 'ST_LineString'""")
  92     context.db.commit()
  93     context.nominatim.run_setup_script('index', 'index-noanalyse')
  94     context.nominatim.run_setup_script('create-functions', 'create-partition-functions',
  95                                        'enable-diff-updates')
  96
  97     # create a OSM file in /tmp and import it
  98     with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.opl', delete=False) as fd:
  99         fname = fd.name
 100         for line in context.text.splitlines():
 101             if line.startswith('n') and line.find(' x') < 0:
 102                     line += " x%d y%d" % (random.random() * 360 - 180,
 103                                           random.random() * 180 - 90)
 104             fd.write(line.encode('utf-8'))
 105             fd.write(b'\n')
 106
 107     context.nominatim.run_update_script(import_diff=fname)
 108     os.remove(fname)