X-Git-Url: https://git.openstreetmap.org/nominatim.git/blobdiff_plain/31273a42bd925fa6baad86504c760c93e051106c..71ae7f10f74251ebef13abe8d186276e6255b5d4:/nominatim/import.c diff --git a/nominatim/import.c b/nominatim/import.c index 03cf6fa3..419ab6a5 100644 --- a/nominatim/import.c +++ b/nominatim/import.c @@ -50,6 +50,9 @@ struct feature xmlChar * rankAddress; xmlChar * rankSearch; xmlChar * countryCode; + xmlChar * parentPlaceID; + xmlChar * parentType; + xmlChar * parentID; xmlChar * adminLevel; xmlChar * houseNumber; xmlChar * geometry; @@ -67,9 +70,12 @@ int featureNameLines = 0; int featureExtraTagLines = 0; int featureCount = 0; xmlHashTablePtr partionTableTagsHash; +xmlHashTablePtr partionTableTagsHashDelete; char featureNameString[MAX_FEATURENAMESTRING]; char featureExtraTagString[MAX_FEATUREEXTRATAGSTRING]; +extern int verbose; + void StartElement(xmlTextReaderPtr reader, const xmlChar *name) { char * value; @@ -135,13 +141,24 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name) feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank"); feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance"); + feature.parentPlaceID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_place_id"); +/* + if (strlen(feature.parentPlaceID) == 0) + { + xmlFree(feature.parentPlaceID); + feature.parentPlaceID = NULL; + } +*/ + feature.parentType = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_type"); + feature.parentID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_id"); + feature.countryCode = NULL; feature.adminLevel = NULL; feature.houseNumber = NULL; feature.geometry = NULL; featureAddressLines = 0; featureNameLines = 0; - featureExtraTagLines = 0; + featureExtraTagLines = 0; return; } @@ -312,8 +329,7 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name) void EndElement(xmlTextReaderPtr reader, const xmlChar *name) { PGresult * res; - PGresult * resPlaceID; - const char * paramValues[11]; + const char * paramValues[14]; char * place_id; char * partionQueryName; int i, namePos, lineTypeLen, lineValueLen; @@ -348,11 +364,12 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) } } */ - place_id = feature.placeID; + place_id = (char *)feature.placeID; - if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE) + if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD) { paramValues[0] = (const char *)place_id; + if (verbose) fprintf(stderr, "placex_delete: %s\n", paramValues[0]); res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0); if (PQresultStatus(res) != PGRES_COMMAND_OK) { @@ -362,6 +379,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) } PQclear(res); + if (verbose) fprintf(stderr, "search_name_delete: %s\n", paramValues[0]); res = PQexecPrepared(conn, "search_name_delete", 1, paramValues, NULL, NULL, 0); if (PQresultStatus(res) != PGRES_COMMAND_OK) { @@ -371,6 +389,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) } PQclear(res); + if (verbose) fprintf(stderr, "place_addressline_delete: %s\n", paramValues[0]); res = PQexecPrepared(conn, "place_addressline_delete", 1, paramValues, NULL, NULL, 0); if (PQresultStatus(res) != PGRES_COMMAND_OK) { @@ -379,6 +398,19 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) exit(EXIT_FAILURE); } PQclear(res); + + partionQueryName = xmlHashLookup2(partionTableTagsHashDelete, feature.key, feature.value); + if (partionQueryName) + { + res = PQexecPrepared(conn, partionQueryName, 1, paramValues, NULL, NULL, 0); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + fprintf(stderr, "%s: DELETE failed: %s", partionQueryName, PQerrorMessage(conn)); + PQclear(res); + exit(EXIT_FAILURE); + } + PQclear(res); + } } if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD) @@ -398,8 +430,8 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) lineValueLen = 0; for (i = 0; i < featureNameLines; i++) { - lineTypeLen = strlen(BAD_CAST featureName[i].type); - lineValueLen = strlen(BAD_CAST featureName[i].value); + lineTypeLen = (int)strlen((char *) featureName[i].type); + lineValueLen = (int)strlen((char *) featureName[i].value); if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING) { fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value); @@ -407,17 +439,22 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) } if (namePos) strcpy(featureNameString+(namePos++), ","); strcpy(featureNameString+(namePos++), "\""); - strcpy(featureNameString+namePos, BAD_CAST featureName[i].type); + strcpy(featureNameString+namePos, (char*) featureName[i].type); namePos += lineTypeLen; strcpy(featureNameString+namePos, "\"=>\""); namePos += 4; - strcpy(featureNameString+namePos, BAD_CAST featureName[i].value); + strcpy(featureNameString+namePos, (char *) featureName[i].value); namePos += lineValueLen; strcpy(featureNameString+(namePos++), "\""); + + xmlFree(featureName[i].type); + xmlFree(featureName[i].value); } } paramValues[5] = (const char *)featureNameString; + paramValues[6] = (const char *)feature.countryCode; + featureExtraTagString[0] = 0; if (featureExtraTagLines) { @@ -426,8 +463,8 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) lineValueLen = 0; for (i = 0; i < featureExtraTagLines; i++) { - lineTypeLen = strlen(BAD_CAST featureExtraTag[i].type); - lineValueLen = strlen(BAD_CAST featureExtraTag[i].value); + lineTypeLen = strlen((char *) featureExtraTag[i].type); + lineValueLen = strlen((char *) featureExtraTag[i].value); if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING) { fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value); @@ -435,45 +472,61 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) } if (namePos) strcpy(featureExtraTagString+(namePos++),","); strcpy(featureExtraTagString+(namePos++), "\""); - strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].type); + strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].type); namePos += lineTypeLen; strcpy(featureExtraTagString+namePos, "\"=>\""); namePos += 4; - strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].value); + strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value); namePos += lineValueLen; strcpy(featureExtraTagString+(namePos++), "\""); + + xmlFree(featureExtraTag[i].type); + xmlFree(featureExtraTag[i].value); } } - paramValues[6] = (const char *)featureExtraTagString; - - paramValues[7] = (const char *)feature.adminLevel; - paramValues[8] = (const char *)feature.houseNumber; - paramValues[9] = (const char *)feature.rankAddress; - paramValues[10] = (const char *)feature.rankSearch; - paramValues[11] = (const char *)feature.geometry; - res = PQexecPrepared(conn, "placex_insert", 12, paramValues, NULL, NULL, 0); - if (PQresultStatus(res) != PGRES_COMMAND_OK) + paramValues[7] = (const char *)featureExtraTagString; + + if (xmlStrlen(feature.parentPlaceID) == 0) + paramValues[8] = "0"; + else + paramValues[8] = (const char *)feature.parentPlaceID; + + paramValues[9] = (const char *)feature.adminLevel; + paramValues[10] = (const char *)feature.houseNumber; + paramValues[11] = (const char *)feature.rankAddress; + paramValues[12] = (const char *)feature.rankSearch; + paramValues[13] = (const char *)feature.geometry; + if (strlen(paramValues[3]) && strlen(paramValues[13])) { - fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn)); - PQclear(res); - exit(EXIT_FAILURE); + if (verbose) fprintf(stderr, "placex_insert: %s\n", paramValues[0]); + res = PQexecPrepared(conn, "placex_insert", 14, paramValues, NULL, NULL, 0); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn)); + fprintf(stderr, "index_placex: INSERT failed: %s %s %s", paramValues[0], paramValues[1], paramValues[2]); + PQclear(res); + exit(EXIT_FAILURE); + } + PQclear(res); } - PQclear(res); for (i = 0; i < featureAddressLines; i++) { // insert into place_address paramValues[0] = (const char *)place_id; paramValues[1] = (const char *)featureAddress[i].distance; + if (paramValues[1] == NULL || strlen(paramValues[1]) == 0) paramValues[1] = "0"; paramValues[2] = (const char *)featureAddress[i].type; paramValues[3] = (const char *)featureAddress[i].id; paramValues[4] = (const char *)featureAddress[i].key; paramValues[5] = (const char *)featureAddress[i].value; paramValues[6] = (const char *)featureAddress[i].isAddress; + if (verbose) fprintf(stderr, "placex_insert: %s %s\n", paramValues[2], paramValues[3]); res = PQexecPrepared(conn, "place_addressline_insert", 7, paramValues, NULL, NULL, 0); if (PQresultStatus(res) != PGRES_COMMAND_OK) { fprintf(stderr, "place_addressline_insert: INSERT failed: %s", PQerrorMessage(conn)); + fprintf(stderr, "(%s,%s,%s,%s,%s,%s,%s)",paramValues[0],paramValues[1],paramValues[2],paramValues[3],paramValues[4],paramValues[5],paramValues[6]); PQclear(res); exit(EXIT_FAILURE); } @@ -488,15 +541,33 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) if (featureNameLines) { - paramValues[0] = (const char *)place_id; - res = PQexecPrepared(conn, "search_name_insert", 1, paramValues, NULL, NULL, 0); - if (PQresultStatus(res) != PGRES_COMMAND_OK) - { - fprintf(stderr, "search_name_insert: INSERT failed: %s", PQerrorMessage(conn)); + if (xmlStrlen(feature.parentPlaceID) > 0 && featureAddressLines == 0) + { + paramValues[0] = (const char *)place_id; + paramValues[1] = (const char *)feature.parentPlaceID; + if (verbose) fprintf(stderr, "search_name_from_parent_insert: INSERT %s %s\n", paramValues[0], paramValues[1]); + res = PQexecPrepared(conn, "search_name_from_parent_insert", 2, paramValues, NULL, NULL, 0); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + fprintf(stderr, "search_name_from_parent_insert: INSERT failed: %s", PQerrorMessage(conn)); + PQclear(res); + exit(EXIT_FAILURE); + } + PQclear(res); + } + else + { + paramValues[0] = (const char *)place_id; + if (verbose) fprintf(stderr, "search_name_insert: INSERT %s\n", paramValues[0]); + res = PQexecPrepared(conn, "search_name_insert", 1, paramValues, NULL, NULL, 0); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + fprintf(stderr, "search_name_insert: INSERT failed: %s", PQerrorMessage(conn)); + PQclear(res); + exit(EXIT_FAILURE); + } PQclear(res); - exit(EXIT_FAILURE); } - PQclear(res); } partionQueryName = xmlHashLookup2(partionTableTagsHash, feature.key, feature.value); @@ -513,7 +584,6 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) exit(EXIT_FAILURE); } PQclear(res); - } } @@ -536,8 +606,11 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) xmlFree(feature.value); xmlFree(feature.rankAddress); xmlFree(feature.rankSearch); -// if (feature.name) xmlFree(feature.name); if (feature.countryCode) xmlFree(feature.countryCode); + if (feature.parentPlaceID) xmlFree(feature.parentPlaceID); + if (feature.parentType) xmlFree(feature.parentType); + if (feature.parentID) xmlFree(feature.parentID); +// if (feature.name) xmlFree(feature.name); if (feature.adminLevel) xmlFree(feature.adminLevel); if (feature.houseNumber) xmlFree(feature.houseNumber); if (feature.geometry) xmlFree(feature.geometry); @@ -595,6 +668,7 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons } partionTableTagsHash = xmlHashCreate(200); + partionTableTagsHashDelete = xmlHashCreate(200); partionTagsFile = fopen(partionTagsFilename, "rt"); if (!partionTagsFile) @@ -636,6 +710,27 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons } xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName); + + partionQueryName = malloc(strlen("partition_delete_")+strlen(osmkey)+strlen(osmvalue)+2); + strcpy(partionQueryName, "partition_delete_"); + strcat(partionQueryName, osmkey); + strcat(partionQueryName, "_"); + strcat(partionQueryName, osmvalue); + + strcpy(partionQuerySQL, "delete from place_classtype_"); + strcat(partionQuerySQL, osmkey); + strcat(partionQuerySQL, "_"); + strcat(partionQuerySQL, osmvalue); + strcat(partionQuerySQL, " where place_id = $1::integer"); + + res = PQprepare(conn, partionQueryName, partionQuerySQL, 1, NULL); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn)); + exit(EXIT_FAILURE); + } + + xmlHashAddEntry2(partionTableTagsHashDelete, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName); } res = PQprepare(conn, "get_new_place_id", @@ -657,8 +752,8 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons } res = PQprepare(conn, "placex_insert", - "insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,admin_level,housenumber,rank_address,rank_search,geometry) " - "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, st_setsrid($12, 4326))", + "insert into placex (place_id,osm_type,osm_id,class,type,name,country_code,extratags,parent_place_id,admin_level,housenumber,rank_address,rank_search,geometry) " + "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, st_setsrid($14, 4326))", 12, NULL); if (PQresultStatus(res) != PGRES_COMMAND_OK) { @@ -668,11 +763,26 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons res = PQprepare(conn, "search_name_insert", "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) " - "select place_id, rank_address, rank_search, country_code, make_keywords(name), " - "(select uniq(sort(array_agg(name_vector))) from place_addressline join search_name on " - "(address_place_id = search_name.place_id) where place_addressline.place_id = $1 ), st_centroid(geometry) from placex " + "select place_id, rank_search, rank_address, country_code, make_keywords(name), " + "(select uniq(sort(array_agg(parent_search_name.name_vector))) from search_name as parent_search_name where place_id in " + "(select distinct address_place_id from place_addressline where place_addressline.place_id = $1 limit 1000)" + "), st_centroid(geometry) from placex " "where place_id = $1", 1, NULL); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn)); + exit(EXIT_FAILURE); + } + + res = PQprepare(conn, "search_name_from_parent_insert", + "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) " + "select place_id, rank_search, rank_address, country_code, make_keywords(name), " + "(select uniq(sort(name_vector+nameaddress_vector)) from search_name as parent_search_name " + "where parent_search_name.place_id = $2 ), st_centroid(geometry) from placex " + "where place_id = $1", + 2, NULL); if (PQresultStatus(res) != PGRES_COMMAND_OK) { fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn)); @@ -740,6 +850,7 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons xmlFreeTextReader(reader); xmlHashFree(partionTableTagsHash, NULL); + xmlHashFree(partionTableTagsHashDelete, NULL); return 0; }