From: Sarah Hoffmann Date: Sun, 11 Jan 2015 14:40:37 +0000 (+0100) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~492 X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/d6fa3ce8ff49fe56f24d10975e83c8b4b65e5f79?hp=f1c4615c32532cdb03845c017f660fa5c5a06331 Merge remote-tracking branch 'upstream/master' Conflicts: sql/tiger_import_finish.sql --- diff --git a/sql/functions.sql b/sql/functions.sql index 5e95d0f4..577e02bf 100644 --- a/sql/functions.sql +++ b/sql/functions.sql @@ -1362,17 +1362,17 @@ BEGIN -- waterway ways are linked when they are part of a relation and have the same class/type IF NEW.osm_type = 'R' and NEW.class = 'waterway' THEN - FOR relation IN select * from planet_osm_rels r where r.id = NEW.osm_id and r.parts != array[]::bigint[] + FOR relation_members IN select members from planet_osm_rels r where r.id = NEW.osm_id and r.parts != array[]::bigint[] LOOP - FOR i IN 1..array_upper(relation.members, 1) BY 2 LOOP - IF relation.members[i+1] in ('', 'main_stream', 'side_stream') AND substring(relation.members[i],1,1) = 'w' THEN - --DEBUG: RAISE WARNING 'waterway parent %, child %/%', NEW.osm_id, i, relation.parts[i]; - FOR location IN SELECT * FROM placex - WHERE osm_type = 'W' and osm_id = substring(relation.members[i],2,200)::bigint + FOR i IN 1..array_upper(relation_members, 1) BY 2 LOOP + IF relation_members[i+1] in ('', 'main_stream', 'side_stream') AND substring(relation_members[i],1,1) = 'w' THEN + --DEBUG: RAISE WARNING 'waterway parent %, child %/%', NEW.osm_id, i, relation.members[i]; + FOR linked_node_id IN SELECT place_id FROM placex + WHERE osm_type = 'W' and osm_id = substring(relation_members[i],2,200)::bigint and class = NEW.class and type = NEW.type - and ( relation.members[i+1] != 'side_stream' or NEW.name->'name' = name->'name') + and ( relation_members[i+1] != 'side_stream' or NEW.name->'name' = name->'name') LOOP - UPDATE placex SET linked_place_id = NEW.place_id WHERE place_id = location.place_id; + UPDATE placex SET linked_place_id = NEW.place_id WHERE place_id = linked_node_id; END LOOP; END IF; END LOOP; diff --git a/sql/tiger_import_finish.sql b/sql/tiger_import_finish.sql index d6ec1833..a7d837f4 100644 --- a/sql/tiger_import_finish.sql +++ b/sql/tiger_import_finish.sql @@ -1,9 +1,9 @@ -CREATE INDEX idx_location_property_tiger_housenumber_parent_place_id_imp ON location_property_tiger_import (parent_place_id, housenumber); -CREATE UNIQUE INDEX idx_location_property_tiger_place_id_imp ON location_property_tiger_import (place_id); +CREATE INDEX idx_location_property_tiger_housenumber_parent_place_id_imp ON location_property_tiger_import (parent_place_id, housenumber) {ts:aux-index}; +CREATE UNIQUE INDEX idx_location_property_tiger_place_id_imp ON location_property_tiger_import (place_id) {ts:aux-index}; -GRANT SELECT ON location_property_tiger_import TO "www-data"; +GRANT SELECT ON location_property_tiger_import TO "{www-user}"; ---DROP TABLE location_property_tiger; +--DROP TABLE IF EXISTS location_property_tiger; --ALTER TABLE location_property_tiger_import RENAME TO location_property_tiger; --ALTER INDEX idx_location_property_tiger_housenumber_parent_place_id_imp RENAME TO idx_location_property_tiger_housenumber_parent_place_id; diff --git a/sql/tiger_import_start.sql b/sql/tiger_import_start.sql index a6f417f2..9846f609 100644 --- a/sql/tiger_import_start.sql +++ b/sql/tiger_import_start.sql @@ -1,5 +1,5 @@ DROP TABLE IF EXISTS location_property_tiger_import; -CREATE TABLE location_property_tiger_import () INHERITS (location_property); +CREATE TABLE location_property_tiger_import () INHERITS (location_property) {ts:aux-data}; CREATE OR REPLACE FUNCTION tigger_create_interpolation(linegeo GEOMETRY, in_startnumber INTEGER, in_endnumber INTEGER, interpolationtype TEXT, diff --git a/tests/features/osm2pgsql/import/tags.feature b/tests/features/osm2pgsql/import/tags.feature new file mode 100644 index 00000000..37940756 --- /dev/null +++ b/tests/features/osm2pgsql/import/tags.feature @@ -0,0 +1,79 @@ +@DB +Feature: Tag evaluation + Tests if tags are correctly imported into the place table + + Scenario Outline: Name tags + Given the osm nodes: + | id | tags + | 1 | 'highway' : 'yes', '' : 'Foo' + When loading osm data + Then table place contains + | object | name + | N1 | '' : 'Foo' + + Examples: + | nametag + | ref + | int_ref + | nat_ref + | reg_ref + | loc_ref + | old_ref + | iata + | icao + | pcode:1 + | pcode:2 + | pcode:3 + | name + | name:de + | name:bt-BR + | int_name + | int_name:xxx + | nat_name + | nat_name:fr + | reg_name + | reg_name:1 + | loc_name + | loc_name:DE + | old_name + | old_name:v1 + | alt_name + | alt_name:dfe + | alt_name_1 + | official_name + | common_name + | common_name:pot + | short_name + | short_name:CH + | operator + + Scenario Outline: Ignored name tags + Given the osm nodes: + | id | tags + | 1 | 'highway' : 'yes', '' : 'Foo', 'name' : 'real' + When loading osm data + Then table place contains + | object | name + | N1 | 'name' : 'real' + + Examples: + | nametag + | name_de + | Name + | ref:de + | ref_de + | my:ref + : br:name + + Scenario: Special character in name tag + Given the osm nodes: + | id | tags + | 1 | 'highway' : 'yes', 'name: de' : 'Foo', 'name' : 'real' + | 2 | 'highway' : 'yes', 'name:\nde' : 'Foo', 'name' : 'real' + | 3 | 'highway' : 'yes', 'name:\tde' : 'Foo', 'name:\\' : 'real' + When loading osm data + Then table place contains + | object | name + | N1 | 'name:_de' : 'Foo', 'name' : 'real' + | N2 | 'name:_de' : 'Foo', 'name' : 'real' + | N3 | 'name:_de' : 'Foo', 'name:\\\\' : 'real' diff --git a/tests/steps/db_results.py b/tests/steps/db_results.py index f65e9924..2566418e 100644 --- a/tests/steps/db_results.py +++ b/tests/steps/db_results.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) @step(u'table placex contains as names for (N|R|W)(\d+)') def check_placex_names(step, osmtyp, osmid): - """ Check for the exact content of the name hstaore in placex. + """ Check for the exact content of the name hstore in placex. """ cur = world.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) cur.execute('SELECT name FROM placex where osm_type = %s and osm_id =%s', (osmtyp, int(osmid))) @@ -43,47 +43,55 @@ def check_placex_content(step, tablename): given columns are tested. If there is more than one line for an OSM object, they must match in these columns. """ - cur = world.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) - for line in step.hashes: - osmtype, osmid, cls = world.split_id(line['object']) - q = 'SELECT *' - if tablename == 'placex': - q = q + ", ST_X(centroid) as clat, ST_Y(centroid) as clon" - q = q + ", ST_GeometryType(geometry) as geometrytype" - q = q + ' FROM %s where osm_type = %%s and osm_id = %%s' % (tablename,) - if cls is None: - params = (osmtype, osmid) - else: - q = q + ' and class = %s' - params = (osmtype, osmid, cls) - cur.execute(q, params) - assert(cur.rowcount > 0) - for res in cur: - for k,v in line.iteritems(): - if not k == 'object': - assert_in(k, res) - if type(res[k]) is dict: - val = world.make_hash(v) - assert_equals(res[k], val) - elif k in ('parent_place_id', 'linked_place_id'): - pid = world.get_placeid(v) - assert_equals(pid, res[k], "Results for '%s'/'%s' differ: '%s' != '%s'" % (line['object'], k, pid, res[k])) - elif k == 'centroid': - world.match_geometry((res['clat'], res['clon']), v) - else: - assert_equals(str(res[k]), v, "Results for '%s'/'%s' differ: '%s' != '%s'" % (line['object'], k, str(res[k]), v)) + try: + cur = world.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + for line in step.hashes: + osmtype, osmid, cls = world.split_id(line['object']) + q = 'SELECT *' + if tablename == 'placex': + q = q + ", ST_X(centroid) as clat, ST_Y(centroid) as clon" + q = q + ", ST_GeometryType(geometry) as geometrytype" + q = q + ' FROM %s where osm_type = %%s and osm_id = %%s' % (tablename,) + if cls is None: + params = (osmtype, osmid) + else: + q = q + ' and class = %s' + params = (osmtype, osmid, cls) + cur.execute(q, params) + assert(cur.rowcount > 0) + for res in cur: + for k,v in line.iteritems(): + if not k == 'object': + assert_in(k, res) + if type(res[k]) is dict: + val = world.make_hash(v) + assert_equals(res[k], val) + elif k in ('parent_place_id', 'linked_place_id'): + pid = world.get_placeid(v) + assert_equals(pid, res[k], "Results for '%s'/'%s' differ: '%s' != '%s'" % (line['object'], k, pid, res[k])) + elif k == 'centroid': + world.match_geometry((res['clat'], res['clon']), v) + else: + assert_equals(str(res[k]), v, "Results for '%s'/'%s' differ: '%s' != '%s'" % (line['object'], k, str(res[k]), v)) + finally: + cur.close() + world.conn.commit() @step(u'table (placex?) has no entry for (N|R|W)(\d+)(:\w+)?') def check_placex_missing(step, tablename, osmtyp, osmid, placeclass): cur = world.conn.cursor() - q = 'SELECT count(*) FROM %s where osm_type = %%s and osm_id = %%s' % (tablename, ) - args = [osmtyp, int(osmid)] - if placeclass is not None: - q = q + ' and class = %s' - args.append(placeclass[1:]) - cur.execute(q, args) - numres = cur.fetchone()[0] - assert_equals (numres, 0) + try: + q = 'SELECT count(*) FROM %s where osm_type = %%s and osm_id = %%s' % (tablename, ) + args = [osmtyp, int(osmid)] + if placeclass is not None: + q = q + ' and class = %s' + args.append(placeclass[1:]) + cur.execute(q, args) + numres = cur.fetchone()[0] + assert_equals (numres, 0) + finally: + cur.close() + world.conn.commit() @step(u'search_name table contains$') def check_search_name_content(step): diff --git a/tests/steps/terrain.py b/tests/steps/terrain.py index cfd243db..746485ac 100644 --- a/tests/steps/terrain.py +++ b/tests/steps/terrain.py @@ -27,7 +27,7 @@ class NominatimConfig: self.template_db = os.environ.get('TEMPLATE_DB', 'test_template_nominatim') self.test_db = os.environ.get('TEST_DB', 'test_nominatim') self.local_settings_file = os.environ.get('NOMINATIM_SETTINGS', '/tmp/nominatim_settings.php') - self.reuse_template = 'NOMINATIM_REUSE_TEMPLATE' in os.environ + self.reuse_template = 'NOMINATIM_REMOVE_TEMPLATE' not in os.environ self.keep_scenario_db = 'NOMINATIM_KEEP_SCENARIO_DB' in os.environ os.environ['NOMINATIM_SETTINGS'] = '/tmp/nominatim_settings.php' diff --git a/utils/setup.php b/utils/setup.php index a31fbadf..c45cc857 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -500,7 +500,13 @@ { $bDidSomething = true; - pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_start.sql'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = replace_tablespace('{ts:aux-data}', + CONST_Tablespace_Aux_Data, $sTemplate); + $sTemplate = replace_tablespace('{ts:aux-index}', + CONST_Tablespace_Aux_Index, $sTemplate); + pgsqlRunScript($sTemplate, false); $aDBInstances = array(); for($i = 0; $i < $iInstances; $i++) @@ -552,7 +558,13 @@ } echo "Creating indexes\n"; - pgsqlRunScriptFile(CONST_BasePath.'/sql/tiger_import_finish.sql'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = replace_tablespace('{ts:aux-data}', + CONST_Tablespace_Aux_Data, $sTemplate); + $sTemplate = replace_tablespace('{ts:aux-index}', + CONST_Tablespace_Aux_Index, $sTemplate); + pgsqlRunScript($sTemplate, false); } if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])