nominatim/import.c

   1 /*
   2 */
   3 #include <stdlib.h>
   4 #include <string.h>
   5
   6 #include <libpq-fe.h>
   7
   8 #include <libxml/xmlstring.h>
   9 #include <libxml/xmlreader.h>
  10 #include <libxml/hash.h>
  11
  12 #include "nominatim.h"
  13 #include "import.h"
  14 #include "input.h"
  15
  16 typedef enum { FILETYPE_NONE, FILETYPE_STRUCTUREDV0P1 } filetypes_t;
  17 typedef enum { FILEMODE_NONE, FILEMODE_ADD, FILEMODE_UPDATE, FILEMODE_DELETE } filemodes_t;
  18
  19 #define MAX_FEATUREADDRESS 5000
  20 #define MAX_FEATURENAMES 10000
  21 #define MAX_FEATUREEXTRATAGS 10000
  22 #define MAX_FEATURENAMESTRING 1000000
  23 #define MAX_FEATUREEXTRATAGSTRING 500000
  24
  25 struct feature_address
  26 {
  27     int                 place_id;
  28     int                 rankAddress;
  29     char                        isAddress[2];
  30     xmlChar *   type;
  31     xmlChar *   id;
  32     xmlChar *   key;
  33     xmlChar *   value;
  34     xmlChar *   distance;
  35 };
  36
  37 struct feature_tag
  38 {
  39     xmlChar *   type;
  40     xmlChar *   value;
  41 };
  42
  43 struct feature
  44 {
  45     xmlChar *   placeID;
  46     xmlChar *   type;
  47     xmlChar *   id;
  48     xmlChar *   key;
  49     xmlChar *   value;
  50     xmlChar *   rankAddress;
  51     xmlChar *   rankSearch;
  52     xmlChar *   countryCode;
  53     xmlChar *   parentPlaceID;
  54     xmlChar *   parentType;
  55     xmlChar *   parentID;
  56     xmlChar *   adminLevel;
  57     xmlChar *   houseNumber;
  58     xmlChar *   geometry;
  59 } feature;
  60
  61 int                                     fileType = FILETYPE_NONE;
  62 int                                     fileMode = FILEMODE_ADD;
  63 PGconn *                                conn;
  64 struct feature_address  featureAddress[MAX_FEATUREADDRESS];
  65 struct feature_tag              featureName[MAX_FEATURENAMES];
  66 struct feature_tag              featureExtraTag[MAX_FEATUREEXTRATAGS];
  67 struct feature                  feature;
  68 int                                     featureAddressLines = 0;
  69 int                                     featureNameLines = 0;
  70 int                                     featureExtraTagLines = 0;
  71 int                                     featureCount = 0;
  72 xmlHashTablePtr                 partionTableTagsHash;
  73 xmlHashTablePtr                 partionTableTagsHashDelete;
  74 char                                    featureNameString[MAX_FEATURENAMESTRING];
  75 char                                    featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
  76
  77 void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
  78 {
  79     char * value;
  80     float version;
  81     int isAddressLine;
  82
  83     if (fileType == FILETYPE_NONE)
  84     {
  85         // Potential to handle other file types in the future / versions
  86         if (xmlStrEqual(name, BAD_CAST "osmStructured"))
  87         {
  88             value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "version");
  89             version = strtof(value, NULL);
  90             xmlFree(value);
  91
  92             if (version == (float)0.1)
  93             {
  94                 fileType = FILETYPE_STRUCTUREDV0P1;
  95                 fileMode = FILEMODE_ADD;
  96             }
  97             else
  98             {
  99                 fprintf( stderr, "Unknown osmStructured version %f (%s)\n", version, value );
 100                 exit_nicely();
 101             }
 102         }
 103         else
 104         {
 105             fprintf( stderr, "Unknown XML document type: %s\n", name );
 106             exit_nicely();
 107         }
 108         return;
 109     }
 110
 111     if (xmlStrEqual(name, BAD_CAST "add"))
 112     {
 113         fileMode = FILEMODE_ADD;
 114         return;
 115     }
 116     if (xmlStrEqual(name, BAD_CAST "update"))
 117     {
 118         fileMode = FILEMODE_UPDATE;
 119         return;
 120     }
 121     if (xmlStrEqual(name, BAD_CAST "delete"))
 122     {
 123         fileMode = FILEMODE_DELETE;
 124         return;
 125     }
 126     if (fileMode == FILEMODE_NONE)
 127     {
 128         fprintf( stderr, "Unknown import mode in: %s\n", name );
 129         exit_nicely();
 130     }
 131
 132     if (xmlStrEqual(name, BAD_CAST "feature"))
 133     {
 134         feature.placeID = xmlTextReaderGetAttribute(reader, BAD_CAST "place_id");
 135         feature.type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 136         feature.id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
 137         feature.key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
 138         feature.value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
 139         feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
 140         feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance");
 141
 142         feature.parentPlaceID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_place_id");
 143 /*
 144         if (strlen(feature.parentPlaceID) == 0)
 145         {
 146                 xmlFree(feature.parentPlaceID);
 147                 feature.parentPlaceID = NULL;
 148         }
 149 */
 150         feature.parentType = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_type");
 151         feature.parentID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_id");
 152
 153         feature.countryCode = NULL;
 154         feature.adminLevel = NULL;
 155         feature.houseNumber = NULL;
 156         feature.geometry = NULL;
 157         featureAddressLines = 0;
 158         featureNameLines = 0;
 159         featureExtraTagLines = 0;
 160
 161         return;
 162     }
 163     if (xmlStrEqual(name, BAD_CAST "names")) return;
 164     if (xmlStrEqual(name, BAD_CAST "name"))
 165     {
 166         if (featureNameLines < MAX_FEATURENAMES)
 167         {
 168                 featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 169             featureName[featureNameLines].value = xmlTextReaderReadString(reader);
 170                 featureNameLines++;
 171                 }
 172                 else
 173                 {
 174             fprintf( stderr, "Too many name elements (%s%s)\n", feature.type, feature.id);
 175 //            exit_nicely();
 176         }
 177         return;
 178     }
 179     if (xmlStrEqual(name, BAD_CAST "tags")) return;
 180     if (xmlStrEqual(name, BAD_CAST "tag"))
 181     {
 182         if (featureExtraTagLines < MAX_FEATUREEXTRATAGS)
 183                 {
 184                 featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 185             featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
 186                 featureExtraTagLines++;
 187                 }
 188                 else
 189         {
 190             fprintf( stderr, "Too many extra tag elements (%s%s)\n", feature.type, feature.id);
 191 //            exit_nicely();
 192         }
 193         return;
 194     }
 195     if (xmlStrEqual(name, BAD_CAST "osmGeometry"))
 196     {
 197         feature.geometry = xmlTextReaderReadString(reader);
 198         return;
 199     }
 200     if (xmlStrEqual(name, BAD_CAST "adminLevel"))
 201     {
 202         feature.adminLevel = xmlTextReaderReadString(reader);
 203         return;
 204     }
 205     if (xmlStrEqual(name, BAD_CAST "countryCode"))
 206     {
 207         feature.countryCode = xmlTextReaderReadString(reader);
 208         return;
 209     }
 210     if (xmlStrEqual(name, BAD_CAST "houseNumber"))
 211     {
 212         feature.houseNumber = xmlTextReaderReadString(reader);
 213         return;
 214     }
 215     if (xmlStrEqual(name, BAD_CAST "address"))
 216     {
 217         featureAddressLines = 0;
 218         return;
 219     }
 220     isAddressLine = 0;
 221     if (xmlStrEqual(name, BAD_CAST "continent"))
 222     {
 223         isAddressLine = 1;
 224     }
 225     else if (xmlStrEqual(name, BAD_CAST "sea"))
 226     {
 227         isAddressLine = 1;
 228     }
 229     else if (xmlStrEqual(name, BAD_CAST "country"))
 230     {
 231         isAddressLine = 1;
 232     }
 233     else if (xmlStrEqual(name, BAD_CAST "state"))
 234     {
 235         isAddressLine = 1;
 236     }
 237     else if (xmlStrEqual(name, BAD_CAST "county"))
 238     {
 239         isAddressLine = 1;
 240     }
 241     else if (xmlStrEqual(name, BAD_CAST "city"))
 242     {
 243         isAddressLine = 1;
 244     }
 245     else if (xmlStrEqual(name, BAD_CAST "town"))
 246     {
 247         isAddressLine = 1;
 248     }
 249     else if (xmlStrEqual(name, BAD_CAST "village"))
 250     {
 251         isAddressLine = 1;
 252     }
 253     else if (xmlStrEqual(name, BAD_CAST "unknown"))
 254     {
 255         isAddressLine = 1;
 256     }
 257     else if (xmlStrEqual(name, BAD_CAST "suburb"))
 258     {
 259         isAddressLine = 1;
 260     }
 261     else if (xmlStrEqual(name, BAD_CAST "postcode"))
 262     {
 263         isAddressLine = 1;
 264     }
 265     else if (xmlStrEqual(name, BAD_CAST "neighborhood"))
 266     {
 267         isAddressLine = 1;
 268     }
 269     else if (xmlStrEqual(name, BAD_CAST "street"))
 270     {
 271         isAddressLine = 1;
 272     }
 273     else if (xmlStrEqual(name, BAD_CAST "access"))
 274     {
 275         isAddressLine = 1;
 276     }
 277     else if (xmlStrEqual(name, BAD_CAST "building"))
 278     {
 279         isAddressLine = 1;
 280     }
 281     else if (xmlStrEqual(name, BAD_CAST "other"))
 282     {
 283         isAddressLine = 1;
 284     }
 285     if (isAddressLine)
 286     {
 287         if (featureAddressLines < MAX_FEATUREADDRESS)
 288                 {
 289                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
 290             if (!value)
 291                 {
 292                 fprintf( stderr, "Address element missing rank\n");
 293                     exit_nicely();
 294                 }
 295             featureAddress[featureAddressLines].rankAddress =  atoi(value);
 296                 xmlFree(value);
 297
 298                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
 299             if (!value)
 300                 {
 301                     fprintf( stderr, "Address element missing rank\n");
 302                 exit_nicely();
 303                 }
 304                 if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
 305             else strcpy(featureAddress[featureAddressLines].isAddress, "f");
 306                 xmlFree(value);
 307
 308                 featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 309             featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
 310                 featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
 311                 featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
 312             featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance");
 313
 314             featureAddressLines++;
 315                 }
 316                 else
 317         {
 318             fprintf( stderr, "Too many address elements (%s%s)\n", feature.type, feature.id);
 319 //            exit_nicely();
 320         }
 321
 322         return;
 323     }
 324     fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
 325 }
 326
 327 void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
 328 {
 329     PGresult *          res;
 330     const char *        paramValues[14];
 331     char *                      place_id;
 332     char *                      partionQueryName;
 333     int i, namePos, lineTypeLen, lineValueLen;
 334
 335     if (xmlStrEqual(name, BAD_CAST "feature"))
 336     {
 337         featureCount++;
 338         if (featureCount % 1000 == 0) printf("feature %i(k)\n", featureCount/1000);
 339 /*
 340         if (fileMode == FILEMODE_ADD)
 341         {
 342             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 0, NULL, NULL, NULL, 0);
 343             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
 344             {
 345                 fprintf(stderr, "get_place_id: INSERT failed: %s", PQerrorMessage(conn));
 346                 PQclear(resPlaceID);
 347                 exit(EXIT_FAILURE);
 348             }
 349         }
 350         else
 351         {
 352             paramValues[0] = (const char *)feature.type;
 353             paramValues[1] = (const char *)feature.id;
 354             paramValues[2] = (const char *)feature.key;
 355             paramValues[3] = (const char *)feature.value;
 356             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 4, paramValues, NULL, NULL, 0);
 357             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
 358             {
 359                 fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
 360                 PQclear(resPlaceID);
 361                 exit(EXIT_FAILURE);
 362             }
 363         }
 364 */
 365         place_id = (char *)feature.placeID;
 366
 367         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD)
 368         {
 369             paramValues[0] = (const char *)place_id;
 370             res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0);
 371             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 372             {
 373                 fprintf(stderr, "placex_delete: DELETE failed: %s", PQerrorMessage(conn));
 374                 PQclear(res);
 375                 exit(EXIT_FAILURE);
 376             }
 377             PQclear(res);
 378
 379             res = PQexecPrepared(conn, "search_name_delete", 1, paramValues, NULL, NULL, 0);
 380             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 381             {
 382                 fprintf(stderr, "search_name_delete: DELETE failed: %s", PQerrorMessage(conn));
 383                 PQclear(res);
 384                 exit(EXIT_FAILURE);
 385             }
 386             PQclear(res);
 387
 388             res = PQexecPrepared(conn, "place_addressline_delete", 1, paramValues, NULL, NULL, 0);
 389             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 390             {
 391                 fprintf(stderr, "place_addressline_delete: DELETE failed: %s", PQerrorMessage(conn));
 392                 PQclear(res);
 393                 exit(EXIT_FAILURE);
 394             }
 395             PQclear(res);
 396
 397             partionQueryName = xmlHashLookup2(partionTableTagsHashDelete, feature.key, feature.value);
 398             if (partionQueryName)
 399             {
 400                 res = PQexecPrepared(conn, partionQueryName, 1, paramValues, NULL, NULL, 0);
 401                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 402                 {
 403                     fprintf(stderr, "%s: DELETE failed: %s", partionQueryName, PQerrorMessage(conn));
 404                     PQclear(res);
 405                     exit(EXIT_FAILURE);
 406                 }
 407                 PQclear(res);
 408             }
 409         }
 410
 411         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD)
 412         {
 413             // Insert into placex
 414             paramValues[0] = (const char *)place_id;
 415             paramValues[1] = (const char *)feature.type;
 416             paramValues[2] = (const char *)feature.id;
 417             paramValues[3] = (const char *)feature.key;
 418             paramValues[4] = (const char *)feature.value;
 419
 420             featureNameString[0] = 0;
 421             if (featureNameLines)
 422             {
 423                 namePos = 0;
 424                 lineTypeLen = 0;
 425                 lineValueLen = 0;
 426                 for (i = 0; i < featureNameLines; i++)
 427                 {
 428                     lineTypeLen = (int)strlen((char *) featureName[i].type);
 429                     lineValueLen = (int)strlen((char *) featureName[i].value);
 430                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING)
 431                     {
 432                         fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value);
 433                         break;
 434                     }
 435                     if (namePos) strcpy(featureNameString+(namePos++), ",");
 436                     strcpy(featureNameString+(namePos++), "\"");
 437                     strcpy(featureNameString+namePos, (char*) featureName[i].type);
 438                     namePos += lineTypeLen;
 439                     strcpy(featureNameString+namePos, "\"=>\"");
 440                     namePos += 4;
 441                     strcpy(featureNameString+namePos, (char *) featureName[i].value);
 442                     namePos += lineValueLen;
 443                     strcpy(featureNameString+(namePos++), "\"");
 444                 }
 445             }
 446             paramValues[5] = (const char *)featureNameString;
 447
 448             paramValues[6] = (const char *)feature.countryCode;
 449
 450             featureExtraTagString[0] = 0;
 451             if (featureExtraTagLines)
 452             {
 453                 namePos = 0;
 454                 lineTypeLen = 0;
 455                 lineValueLen = 0;
 456                 for (i = 0; i < featureExtraTagLines; i++)
 457                 {
 458                     lineTypeLen = strlen((char *) featureExtraTag[i].type);
 459                     lineValueLen = strlen((char *) featureExtraTag[i].value);
 460                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING)
 461                     {
 462                         fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value);
 463                         break;
 464                     }
 465                     if (namePos) strcpy(featureExtraTagString+(namePos++),",");
 466                     strcpy(featureExtraTagString+(namePos++), "\"");
 467                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].type);
 468                     namePos += lineTypeLen;
 469                     strcpy(featureExtraTagString+namePos, "\"=>\"");
 470                     namePos += 4;
 471                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value);
 472                     namePos += lineValueLen;
 473                     strcpy(featureExtraTagString+(namePos++), "\"");
 474                 }
 475             }
 476             paramValues[7] = (const char *)featureExtraTagString;
 477
 478             if (strlen(feature.parentPlaceID) == 0)
 479                 paramValues[8] = "0";
 480             else
 481                 paramValues[8] = (const char *)feature.parentPlaceID;
 482
 483             paramValues[9] = (const char *)feature.adminLevel;
 484             paramValues[10] = (const char *)feature.houseNumber;
 485             paramValues[11] = (const char *)feature.rankAddress;
 486             paramValues[12] = (const char *)feature.rankSearch;
 487             paramValues[13] = (const char *)feature.geometry;
 488             if (strlen(paramValues[3]))
 489             {
 490                 res = PQexecPrepared(conn, "placex_insert", 14, paramValues, NULL, NULL, 0);
 491                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 492                 {
 493                     fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
 494                     fprintf(stderr, "index_placex: INSERT failed: %s %s %s", paramValues[0], paramValues[1], paramValues[2]);
 495                     PQclear(res);
 496                     exit(EXIT_FAILURE);
 497                }
 498                PQclear(res);
 499             }
 500
 501             for (i = 0; i < featureAddressLines; i++)
 502             {
 503                 // insert into place_address
 504                 paramValues[0] = (const char *)place_id;
 505                 paramValues[1] = (const char *)featureAddress[i].distance;
 506                 paramValues[2] = (const char *)featureAddress[i].type;
 507                 paramValues[3] = (const char *)featureAddress[i].id;
 508                 paramValues[4] = (const char *)featureAddress[i].key;
 509                 paramValues[5] = (const char *)featureAddress[i].value;
 510                 paramValues[6] = (const char *)featureAddress[i].isAddress;
 511                 res = PQexecPrepared(conn, "place_addressline_insert", 7, paramValues, NULL, NULL, 0);
 512                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 513                 {
 514                     fprintf(stderr, "place_addressline_insert: INSERT failed: %s", PQerrorMessage(conn));
 515                     PQclear(res);
 516                     exit(EXIT_FAILURE);
 517                 }
 518                 PQclear(res);
 519
 520                 xmlFree(featureAddress[i].type);
 521                 xmlFree(featureAddress[i].id);
 522                 xmlFree(featureAddress[i].key);
 523                 xmlFree(featureAddress[i].value);
 524                 xmlFree(featureAddress[i].distance);
 525             }
 526
 527             if (featureNameLines)
 528             {
 529                 paramValues[0] = (const char *)place_id;
 530                 res = PQexecPrepared(conn, "search_name_insert", 1, paramValues, NULL, NULL, 0);
 531                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 532                 {
 533                     fprintf(stderr, "search_name_insert: INSERT failed: %s", PQerrorMessage(conn));
 534                     PQclear(res);
 535                     exit(EXIT_FAILURE);
 536                 }
 537                 PQclear(res);
 538             }
 539
 540             partionQueryName = xmlHashLookup2(partionTableTagsHash, feature.key, feature.value);
 541             if (partionQueryName)
 542             {
 543                 // insert into partition table
 544                 paramValues[0] = (const char *)place_id;
 545                 paramValues[1] = (const char *)feature.geometry;
 546                 res = PQexecPrepared(conn, partionQueryName, 2, paramValues, NULL, NULL, 0);
 547                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 548                 {
 549                     fprintf(stderr, "%s: INSERT failed: %s", partionQueryName, PQerrorMessage(conn));
 550                     PQclear(res);
 551                     exit(EXIT_FAILURE);
 552                 }
 553                 PQclear(res);
 554             }
 555
 556         }
 557         else
 558         {
 559             for (i = 0; i < featureAddressLines; i++)
 560             {
 561                 xmlFree(featureAddress[i].type);
 562                 xmlFree(featureAddress[i].id);
 563                 xmlFree(featureAddress[i].key);
 564                 xmlFree(featureAddress[i].value);
 565                 xmlFree(featureAddress[i].distance);
 566             }
 567         }
 568
 569         xmlFree(feature.placeID);
 570         xmlFree(feature.type);
 571         xmlFree(feature.id);
 572         xmlFree(feature.key);
 573         xmlFree(feature.value);
 574         xmlFree(feature.rankAddress);
 575         xmlFree(feature.rankSearch);
 576         if (feature.parentPlaceID) xmlFree(feature.parentPlaceID);
 577         if (feature.parentType) xmlFree(feature.parentType);
 578         if (feature.parentID) xmlFree(feature.parentID);
 579 //              if (feature.name) xmlFree(feature.name);
 580         if (feature.countryCode) xmlFree(feature.countryCode);
 581         if (feature.adminLevel) xmlFree(feature.adminLevel);
 582         if (feature.houseNumber) xmlFree(feature.houseNumber);
 583         if (feature.geometry) xmlFree(feature.geometry);
 584
 585 //        PQclear(resPlaceID);
 586     }
 587 }
 588
 589 static void processNode(xmlTextReaderPtr reader)
 590 {
 591     xmlChar *name;
 592     name = xmlTextReaderName(reader);
 593     if (name == NULL)
 594     {
 595         name = xmlStrdup(BAD_CAST "--");
 596     }
 597
 598     switch (xmlTextReaderNodeType(reader))
 599     {
 600     case XML_READER_TYPE_ELEMENT:
 601         StartElement(reader, name);
 602         if (xmlTextReaderIsEmptyElement(reader))
 603             EndElement(reader, name); /* No end_element for self closing tags! */
 604         break;
 605     case XML_READER_TYPE_END_ELEMENT:
 606         EndElement(reader, name);
 607         break;
 608     case XML_READER_TYPE_TEXT:
 609     case XML_READER_TYPE_CDATA:
 610     case XML_READER_TYPE_SIGNIFICANT_WHITESPACE:
 611         /* Ignore */
 612         break;
 613     default:
 614         fprintf(stderr, "Unknown node type %d\n", xmlTextReaderNodeType(reader));
 615         break;
 616     }
 617
 618     xmlFree(name);
 619 }
 620
 621 int nominatim_import(const char *conninfo, const char *partionTagsFilename, const char *filename)
 622 {
 623     xmlTextReaderPtr    reader;
 624     int                                 ret = 0;
 625     PGresult *                  res;
 626     FILE *                              partionTagsFile;
 627     char *                              partionQueryName;
 628     char                                partionQuerySQL[1024];
 629
 630     conn = PQconnectdb(conninfo);
 631     if (PQstatus(conn) != CONNECTION_OK)
 632     {
 633         fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
 634         exit(EXIT_FAILURE);
 635     }
 636
 637     partionTableTagsHash = xmlHashCreate(200);
 638     partionTableTagsHashDelete = xmlHashCreate(200);
 639
 640     partionTagsFile = fopen(partionTagsFilename, "rt");
 641     if (!partionTagsFile)
 642     {
 643         fprintf(stderr, "Unable to read partition tags file: %s\n", partionTagsFilename);
 644         exit(EXIT_FAILURE);
 645     }
 646
 647     char buffer[1024], osmkey[256], osmvalue[256];
 648     int fields;
 649     while (fgets(buffer, sizeof(buffer), partionTagsFile) != NULL)
 650     {
 651         fields = sscanf( buffer, "%23s %63s", osmkey, osmvalue );
 652
 653         if ( fields <= 0 ) continue;
 654
 655         if ( fields != 2  )
 656         {
 657             fprintf( stderr, "Error partition file\n");
 658             exit_nicely();
 659         }
 660         partionQueryName = malloc(strlen("partition_insert_")+strlen(osmkey)+strlen(osmvalue)+2);
 661         strcpy(partionQueryName, "partition_insert_");
 662         strcat(partionQueryName, osmkey);
 663         strcat(partionQueryName, "_");
 664         strcat(partionQueryName, osmvalue);
 665
 666         strcpy(partionQuerySQL, "insert into place_classtype_");
 667         strcat(partionQuerySQL, osmkey);
 668         strcat(partionQuerySQL, "_");
 669         strcat(partionQuerySQL, osmvalue);
 670         strcat(partionQuerySQL, " (place_id, centroid) values ($1, ST_Centroid(st_setsrid($2, 4326)))");
 671
 672         res = PQprepare(conn, partionQueryName, partionQuerySQL, 2, NULL);
 673         if (PQresultStatus(res) != PGRES_COMMAND_OK)
 674         {
 675             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
 676             exit(EXIT_FAILURE);
 677         }
 678
 679         xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
 680
 681         partionQueryName = malloc(strlen("partition_delete_")+strlen(osmkey)+strlen(osmvalue)+2);
 682         strcpy(partionQueryName, "partition_delete_");
 683         strcat(partionQueryName, osmkey);
 684         strcat(partionQueryName, "_");
 685         strcat(partionQueryName, osmvalue);
 686
 687         strcpy(partionQuerySQL, "delete from place_classtype_");
 688         strcat(partionQuerySQL, osmkey);
 689         strcat(partionQuerySQL, "_");
 690         strcat(partionQuerySQL, osmvalue);
 691         strcat(partionQuerySQL, " where place_id = $1::integer");
 692
 693         res = PQprepare(conn, partionQueryName, partionQuerySQL, 1, NULL);
 694         if (PQresultStatus(res) != PGRES_COMMAND_OK)
 695         {
 696             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
 697             exit(EXIT_FAILURE);
 698         }
 699
 700         xmlHashAddEntry2(partionTableTagsHashDelete, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
 701     }
 702
 703     res = PQprepare(conn, "get_new_place_id",
 704                     "select nextval('seq_place')",
 705                     0, NULL);
 706     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 707     {
 708         fprintf(stderr, "Failed to prepare get_new_place_id: %s\n", PQerrorMessage(conn));
 709         exit(EXIT_FAILURE);
 710     }
 711
 712     res = PQprepare(conn, "get_place_id",
 713                     "select place_id from placex where osm_type = $1 and osm_id = $2 and class = $3 and type = $4",
 714                     4, NULL);
 715     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 716     {
 717         fprintf(stderr, "Failed to prepare get_place_id: %s\n", PQerrorMessage(conn));
 718         exit(EXIT_FAILURE);
 719     }
 720
 721     res = PQprepare(conn, "placex_insert",
 722                     "insert into placex (place_id,osm_type,osm_id,class,type,name,country_code,extratags,parent_place_id,admin_level,housenumber,rank_address,rank_search,geometry) "
 723                     "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, st_setsrid($14, 4326))",
 724                     12, NULL);
 725     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 726     {
 727         fprintf(stderr, "Failed to prepare placex_insert: %s\n", PQerrorMessage(conn));
 728         exit(EXIT_FAILURE);
 729     }
 730
 731     res = PQprepare(conn, "search_name_insert",
 732                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
 733                     "select place_id, rank_address, rank_search, country_code, make_keywords(name), "
 734                     "(select uniq(sort(array_agg(name_vector))) from place_addressline join search_name on "
 735                     "(address_place_id = search_name.place_id) where place_addressline.place_id = $1 ), st_centroid(geometry) from placex "
 736                     "where place_id = $1",
 737                     1, NULL);
 738     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 739     {
 740         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
 741         exit(EXIT_FAILURE);
 742     }
 743
 744     res = PQprepare(conn, "place_addressline_insert",
 745                     "insert into place_addressline (place_id, address_place_id, fromarea, isaddress, distance, cached_rank_address) "
 746                     "select $1, place_id, false, $7, $2, rank_address from placex where osm_type = $3 and osm_id = $4 and class = $5 and type = $6",
 747                     7, NULL);
 748     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 749     {
 750         fprintf(stderr, "Failed to prepare place_addressline_insert: %s\n", PQerrorMessage(conn));
 751         exit(EXIT_FAILURE);
 752     }
 753
 754     res = PQprepare(conn, "placex_delete",
 755                     "delete from placex where place_id = $1",
 756                     1, NULL);
 757     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 758     {
 759         fprintf(stderr, "Failed to prepare placex_delete: %s\n", PQerrorMessage(conn));
 760         exit(EXIT_FAILURE);
 761     }
 762
 763     res = PQprepare(conn, "search_name_delete",
 764                     "delete from search_name where place_id = $1",
 765                     1, NULL);
 766     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 767     {
 768         fprintf(stderr, "Failed to prepare search_name_delete: %s\n", PQerrorMessage(conn));
 769         exit(EXIT_FAILURE);
 770     }
 771
 772     res = PQprepare(conn, "place_addressline_delete",
 773                     "delete from place_addressline where place_id = $1",
 774                     1, NULL);
 775     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 776     {
 777         fprintf(stderr, "Failed to prepare place_addressline_delete: %s\n", PQerrorMessage(conn));
 778         exit(EXIT_FAILURE);
 779     }
 780
 781     featureCount = 0;
 782
 783     reader = inputUTF8(filename);
 784
 785     if (reader == NULL)
 786     {
 787         fprintf(stderr, "Unable to open %s\n", filename);
 788         return 1;
 789     }
 790
 791     ret = xmlTextReaderRead(reader);
 792     while (ret == 1)
 793     {
 794         processNode(reader);
 795         ret = xmlTextReaderRead(reader);
 796     }
 797     if (ret != 0)
 798     {
 799         fprintf(stderr, "%s : failed to parse\n", filename);
 800         return ret;
 801     }
 802
 803     xmlFreeTextReader(reader);
 804     xmlHashFree(partionTableTagsHash, NULL);
 805     xmlHashFree(partionTableTagsHashDelete, NULL);
 806
 807     return 0;
 808 }