15 #include "nominatim.h"
 
  17 #include "postgresql.h"
 
  23 void nominatim_export(int rank_min, int rank_max, const char *conninfo, const char *structuredoutputfile)
 
  25     xmlTextWriterPtr writer;
 
  31     PGresult * resSectors;
 
  39     const char *paramValues[2];
 
  46     Oid pg_prepare_params[2];
 
  48     conn = PQconnectdb(conninfo);
 
  49     if (PQstatus(conn) != CONNECTION_OK)
 
  51         fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
 
  55     pg_prepare_params[0] = PG_OID_INT4;
 
  56     res = PQprepare(conn, "index_sectors",
 
  57                     "select geometry_sector,count(*) from placex where rank_search = $1 and indexed_status = 0 group by geometry_sector order by geometry_sector",
 
  58                     1, pg_prepare_params);
 
  59     if (PQresultStatus(res) != PGRES_COMMAND_OK) exit(EXIT_FAILURE);
 
  62     pg_prepare_params[0] = PG_OID_INT4;
 
  63     pg_prepare_params[1] = PG_OID_INT4;
 
  64     res = PQprepare(conn, "index_sector_places",
 
  65                     "select place_id from placex where rank_search = $1 and geometry_sector = $2",
 
  66                     2, pg_prepare_params);
 
  67     if (PQresultStatus(res) != PGRES_COMMAND_OK) exit(EXIT_FAILURE);
 
  70     nominatim_exportCreatePreparedQueries(conn);
 
  72     // Create the output file
 
  73     writer = nominatim_exportXMLStart(structuredoutputfile);
 
  75     for (rank = rank_min; rank <= rank_max; rank++)
 
  77         printf("Starting rank %d\n", rank);
 
  79         paramRank = PGint32(rank);
 
  80         paramValues[0] = (char *)¶mRank;
 
  81         paramLengths[0] = sizeof(paramRank);
 
  83         resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
 
  84         if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
 
  86             fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
 
  90         if (PQftype(resSectors, 0) != PG_OID_INT4)
 
  92             fprintf(stderr, "Sector value has unexpected type\n");
 
  96         if (PQftype(resSectors, 1) != PG_OID_INT8)
 
  98             fprintf(stderr, "Sector value has unexpected type\n");
 
 104         for (iSector = 0; iSector < PQntuples(resSectors); iSector++)
 
 106             sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
 
 108             // Get all the place_id's for this sector
 
 109             paramRank = PGint32(rank);
 
 110             paramValues[0] = (char *)¶mRank;
 
 111             paramLengths[0] = sizeof(paramRank);
 
 113             paramSector = PGint32(sector);
 
 114             paramValues[1] = (char *)¶mSector;
 
 115             paramLengths[1] = sizeof(paramSector);
 
 117             resPlaces = PQexecPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
 
 118             if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
 
 120                 fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
 
 124             if (PQftype(resPlaces, 0) != PG_OID_INT8)
 
 126                 fprintf(stderr, "Place_id value has unexpected type\n");
 
 131             tuples = PQntuples(resPlaces);
 
 132             for (i = 0; i < tuples; i++)
 
 134                 nominatim_exportPlace(PGint64(*((uint64_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL, NULL);
 
 136                 if (rankTotalDone%1000 == 0) printf("Done %i (k)\n", rankTotalDone/1000);
 
 143     nominatim_exportXMLEnd(writer);
 
 148 void nominatim_exportCreatePreparedQueries(PGconn * conn)
 
 150     Oid pg_prepare_params[2];
 
 153     pg_prepare_params[0] = PG_OID_INT8;
 
 154     res = PQprepare(conn, "placex_details",
 
 155                     "select placex.osm_type, placex.osm_id, placex.class, placex.type, placex.name, placex.housenumber, placex.country_code, ST_AsText(placex.geometry), placex.admin_level, placex.rank_address, placex.rank_search, placex.parent_place_id, parent.osm_type, parent.osm_id, placex.indexed_status from placex left outer join placex as parent on (placex.parent_place_id = parent.place_id) where placex.place_id = $1",
 
 156                     1, pg_prepare_params);
 
 157     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 
 159         fprintf(stderr, "Error preparing placex_details: %s", PQerrorMessage(conn));
 
 164     pg_prepare_params[0] = PG_OID_INT8;
 
 165     res = PQprepare(conn, "placex_address",
 
 166                     "select osm_type,osm_id,class,type,distance,cached_rank_address,isaddress from place_addressline join placex on (address_place_id = placex.place_id) where place_addressline.place_id = $1 and address_place_id != place_addressline.place_id order by cached_rank_address asc,osm_type,osm_id",
 
 167                     1, pg_prepare_params);
 
 168     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 
 170         fprintf(stderr, "Error preparing placex_address: %s", PQerrorMessage(conn));
 
 175     pg_prepare_params[0] = PG_OID_INT8;
 
 176     res = PQprepare(conn, "placex_names",
 
 177                     "select (each(name)).key,(each(name)).value from (select name from placex where place_id = $1) as x order by (each(name)).key",
 
 178                     1, pg_prepare_params);
 
 179     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 
 181         fprintf(stderr, "Error preparing placex_names: %s", PQerrorMessage(conn));
 
 186     pg_prepare_params[0] = PG_OID_INT8;
 
 187     res = PQprepare(conn, "placex_extratags",
 
 188                     "select (each(extratags)).key,(each(extratags)).value from (select extratags from placex where place_id = $1) as x order by (each(extratags)).key",
 
 189                     1, pg_prepare_params);
 
 190     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 
 192         fprintf(stderr, "Error preparing placex_extratags: %s", PQerrorMessage(conn));
 
 198 xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile)
 
 200     xmlTextWriterPtr writer;
 
 202     writer = xmlNewTextWriterFilename(structuredoutputfile, 0);
 
 205         fprintf(stderr, "Unable to open %s\n", structuredoutputfile);
 
 208     xmlTextWriterSetIndent(writer, 1);
 
 209     if (xmlTextWriterStartDocument(writer, NULL, "UTF8", NULL) < 0)
 
 211         fprintf(stderr, "xmlTextWriterStartDocument failed\n");
 
 214     if (xmlTextWriterStartElement(writer, BAD_CAST "osmStructured") < 0)
 
 216         fprintf(stderr, "xmlTextWriterStartElement failed\n");
 
 219     if (xmlTextWriterWriteAttribute(writer, BAD_CAST "version", BAD_CAST "0.1") < 0)
 
 221         fprintf(stderr, "xmlTextWriterWriteAttribute failed\n");
 
 224     if (xmlTextWriterWriteAttribute(writer, BAD_CAST "generator", BAD_CAST "Nominatim") < 0)
 
 226         fprintf(stderr, "xmlTextWriterWriteAttribute failed\n");
 
 235 void nominatim_exportXMLEnd(xmlTextWriterPtr writer)
 
 237     nominatim_exportEndMode(writer);
 
 239     // End <osmStructured>
 
 240     if (xmlTextWriterEndElement(writer) < 0)
 
 242         fprintf(stderr, "xmlTextWriterEndElement failed\n");
 
 245     if (xmlTextWriterEndDocument(writer) < 0)
 
 247         fprintf(stderr, "xmlTextWriterEndDocument failed\n");
 
 250     xmlFreeTextWriter(writer);
 
 253 void nominatim_exportStartMode(xmlTextWriterPtr writer, int newMode)
 
 255     if (mode == newMode) return;
 
 257     nominatim_exportEndMode(writer);
 
 265         if (xmlTextWriterStartElement(writer, BAD_CAST "add") < 0)
 
 267             fprintf(stderr, "xmlTextWriterStartElement failed\n");
 
 273         if (xmlTextWriterStartElement(writer, BAD_CAST "update") < 0)
 
 275             fprintf(stderr, "xmlTextWriterStartElement failed\n");
 
 281         if (xmlTextWriterStartElement(writer, BAD_CAST "delete") < 0)
 
 283             fprintf(stderr, "xmlTextWriterStartElement failed\n");
 
 291 void nominatim_exportEndMode(xmlTextWriterPtr writer)
 
 295     if (xmlTextWriterEndElement(writer) < 0)
 
 297         fprintf(stderr, "xmlTextWriterEndElement failed\n");
 
 302 void nominatim_exportPlaceQueries(uint64_t place_id, PGconn * conn, struct export_data * querySet)
 
 304     const char *        paramValues[1];
 
 307     uint64_t            paramPlaceID;
 
 309     paramPlaceID = PGint64(place_id);
 
 310     paramValues[0] = (char *)¶mPlaceID;
 
 311     paramLengths[0] = sizeof(paramPlaceID);
 
 314     querySet->res = PQexecPrepared(conn, "placex_details", 1, paramValues, paramLengths, paramFormats, 0);
 
 315     if (PQresultStatus(querySet->res) != PGRES_TUPLES_OK)
 
 317         fprintf(stderr, "placex_details: SELECT failed: %s", PQerrorMessage(conn));
 
 318         PQclear(querySet->res);
 
 322     querySet->resNames = PQexecPrepared(conn, "placex_names", 1, paramValues, paramLengths, paramFormats, 0);
 
 323     if (PQresultStatus(querySet->resNames) != PGRES_TUPLES_OK)
 
 325         fprintf(stderr, "placex_names: SELECT failed: %s", PQerrorMessage(conn));
 
 326         PQclear(querySet->resNames);
 
 330     querySet->resAddress = PQexecPrepared(conn, "placex_address", 1, paramValues, paramLengths, paramFormats, 0);
 
 331     if (PQresultStatus(querySet->resAddress) != PGRES_TUPLES_OK)
 
 333         fprintf(stderr, "placex_address: SELECT failed: %s", PQerrorMessage(conn));
 
 334         PQclear(querySet->resAddress);
 
 338     querySet->resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0);
 
 339     if (PQresultStatus(querySet->resExtraTags) != PGRES_TUPLES_OK)
 
 341         fprintf(stderr, "placex_extratags: SELECT failed: %s", PQerrorMessage(conn));
 
 342         PQclear(querySet->resExtraTags);
 
 347 void nominatim_exportFreeQueries(struct export_data * querySet)
 
 349     PQclear(querySet->res);
 
 350     PQclear(querySet->resNames);
 
 351     PQclear(querySet->resAddress);
 
 352     PQclear(querySet->resExtraTags);
 
 356  * Requirements: the prepared queries must exist
 
 358 void nominatim_exportPlace(uint64_t place_id, PGconn * conn, 
 
 359   xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex, struct export_data * prevQuerySet)
 
 361     struct export_data          querySet;
 
 365     nominatim_exportPlaceQueries(place_id, conn, &querySet);
 
 367     // Add, modify or delete?
 
 370         if ((PQgetvalue(prevQuerySet->res, 0, 14) && strcmp(PQgetvalue(prevQuerySet->res, 0, 14), "100") == 0) || PQntuples(querySet.res) == 0)
 
 373             if (writer_mutex) pthread_mutex_lock( writer_mutex );
 
 374             nominatim_exportStartMode(writer, 3);
 
 375             xmlTextWriterStartElement(writer, BAD_CAST "feature");
 
 376             xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id);
 
 377             xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 0));
 
 378             xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 1));
 
 379             xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 2));
 
 380             xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 3));
 
 381             xmlTextWriterEndElement(writer);
 
 382             if (writer_mutex) pthread_mutex_unlock( writer_mutex );
 
 383             nominatim_exportFreeQueries(&querySet);
 
 386         if (PQgetvalue(prevQuerySet->res, 0, 14) && strcmp(PQgetvalue(prevQuerySet->res, 0, 14), "1") == 0)
 
 389             if (writer_mutex) pthread_mutex_lock( writer_mutex );
 
 390             nominatim_exportStartMode(writer, 1);  
 
 394             // Update, but only if something has changed
 
 396             // TODO: detect changes
 
 398             if (writer_mutex) pthread_mutex_lock( writer_mutex );
 
 399             nominatim_exportStartMode(writer, 2);  
 
 405        if (writer_mutex) pthread_mutex_lock( writer_mutex );
 
 406        nominatim_exportStartMode(writer, 1);  
 
 409     xmlTextWriterStartElement(writer, BAD_CAST "feature");
 
 410     xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id);
 
 411     xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.res, 0, 0));
 
 412     xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(querySet.res, 0, 1));
 
 413     xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(querySet.res, 0, 2));
 
 414     xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(querySet.res, 0, 3));
 
 415     xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(querySet.res, 0, 9));
 
 416     xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(querySet.res, 0, 10));
 
 417     xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_place_id", BAD_CAST PQgetvalue(querySet.res, 0, 11));
 
 418     xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_type", BAD_CAST PQgetvalue(querySet.res, 0, 12));
 
 419     xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_id", BAD_CAST PQgetvalue(querySet.res, 0, 13));
 
 421     if (PQntuples(querySet.resNames))
 
 423         xmlTextWriterStartElement(writer, BAD_CAST "names");
 
 425         for (i = 0; i < PQntuples(querySet.resNames); i++)
 
 427             xmlTextWriterStartElement(writer, BAD_CAST "name");
 
 428             xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resNames, i, 0));
 
 429             xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.resNames, i, 1));
 
 430             xmlTextWriterEndElement(writer);
 
 433         xmlTextWriterEndElement(writer);
 
 436     if (PQgetvalue(querySet.res, 0, 5) && strlen(PQgetvalue(querySet.res, 0, 5)))
 
 438         xmlTextWriterStartElement(writer, BAD_CAST "houseNumber");
 
 439         xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 5));
 
 440         xmlTextWriterEndElement(writer);
 
 443     if (PQgetvalue(querySet.res, 0, 8) && strlen(PQgetvalue(querySet.res, 0, 8)))
 
 445         xmlTextWriterStartElement(writer, BAD_CAST "adminLevel");
 
 446         xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 8));
 
 447         xmlTextWriterEndElement(writer);
 
 450     if (PQgetvalue(querySet.res, 0, 6) && strlen(PQgetvalue(querySet.res, 0, 6)))
 
 452         xmlTextWriterStartElement(writer, BAD_CAST "countryCode");
 
 453         xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 6));
 
 454         xmlTextWriterEndElement(writer);
 
 457     if (PQntuples(querySet.resAddress) > 0)
 
 459         xmlTextWriterStartElement(writer, BAD_CAST "address");
 
 460         for (i = 0; i < PQntuples(querySet.resAddress); i++)
 
 462             xmlTextWriterStartElement(writer, BAD_CAST getRankLabel(atoi(PQgetvalue(querySet.resAddress, i, 5))));
 
 463             xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(querySet.resAddress, i, 5));
 
 464             xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resAddress, i, 0));
 
 465             xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(querySet.resAddress, i, 1));
 
 466             xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(querySet.resAddress, i, 2));
 
 467             xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(querySet.resAddress, i, 3));
 
 468             xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(querySet.resAddress, i, 4));
 
 469             xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(querySet.resAddress, i, 6));
 
 470             xmlTextWriterEndElement(writer);
 
 472         xmlTextWriterEndElement(writer);
 
 475     if (PQntuples(querySet.resExtraTags))
 
 477         xmlTextWriterStartElement(writer, BAD_CAST "tags");
 
 479         for (i = 0; i < PQntuples(querySet.resExtraTags); i++)
 
 481             xmlTextWriterStartElement(writer, BAD_CAST "tag");
 
 482             xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resExtraTags, i, 0));
 
 483             xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.resExtraTags, i, 1));
 
 484             xmlTextWriterEndElement(writer);
 
 487         xmlTextWriterEndElement(writer);
 
 491     xmlTextWriterStartElement(writer, BAD_CAST "osmGeometry");
 
 492     xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 7));
 
 493     xmlTextWriterEndElement(writer);
 
 495     xmlTextWriterEndElement(writer); // </feature>
 
 497     if (writer_mutex) pthread_mutex_unlock( writer_mutex );
 
 499     nominatim_exportFreeQueries(&querySet);
 
 502 const char * getRankLabel(int rank)
 
 540         return "neighborhood";