nominatim/import.c

   1 /*
   2 */
   3 #include <stdlib.h>
   4 #include <string.h>
   5
   6 #include <libpq-fe.h>
   7
   8 #include <libxml/xmlstring.h>
   9 #include <libxml/xmlreader.h>
  10 #include <libxml/hash.h>
  11
  12 #include "nominatim.h"
  13 #include "import.h"
  14 #include "input.h"
  15
  16 typedef enum { FILETYPE_NONE, FILETYPE_STRUCTUREDV0P1 } filetypes_t;
  17 typedef enum { FILEMODE_NONE, FILEMODE_ADD, FILEMODE_UPDATE, FILEMODE_DELETE } filemodes_t;
  18
  19 #define MAX_FEATUREADDRESS 5000
  20 #define MAX_FEATURENAMES 10000
  21 #define MAX_FEATUREEXTRATAGS 10000
  22 #define MAX_FEATURENAMESTRING 1000000
  23 #define MAX_FEATUREEXTRATAGSTRING 500000
  24
  25 struct feature_address
  26 {
  27     int                 place_id;
  28     int                 rankAddress;
  29     char                        isAddress[2];
  30     xmlChar *   type;
  31     xmlChar *   id;
  32     xmlChar *   key;
  33     xmlChar *   value;
  34     xmlChar *   distance;
  35 };
  36
  37 struct feature_tag
  38 {
  39     xmlChar *   type;
  40     xmlChar *   value;
  41 };
  42
  43 struct feature
  44 {
  45     xmlChar *   placeID;
  46     xmlChar *   type;
  47     xmlChar *   id;
  48     xmlChar *   key;
  49     xmlChar *   value;
  50     xmlChar *   rankAddress;
  51     xmlChar *   rankSearch;
  52     xmlChar *   countryCode;
  53     xmlChar *   adminLevel;
  54     xmlChar *   houseNumber;
  55     xmlChar *   geometry;
  56 } feature;
  57
  58 int                                     fileType = FILETYPE_NONE;
  59 int                                     fileMode = FILEMODE_ADD;
  60 PGconn *                                conn;
  61 struct feature_address  featureAddress[MAX_FEATUREADDRESS];
  62 struct feature_tag              featureName[MAX_FEATURENAMES];
  63 struct feature_tag              featureExtraTag[MAX_FEATUREEXTRATAGS];
  64 struct feature                  feature;
  65 int                                     featureAddressLines = 0;
  66 int                                     featureNameLines = 0;
  67 int                                     featureExtraTagLines = 0;
  68 int                                     featureCount = 0;
  69 xmlHashTablePtr                 partionTableTagsHash;
  70 char                                    featureNameString[MAX_FEATURENAMESTRING];
  71 char                                    featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
  72
  73 void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
  74 {
  75     char * value;
  76     float version;
  77     int isAddressLine;
  78
  79     if (fileType == FILETYPE_NONE)
  80     {
  81         // Potential to handle other file types in the future / versions
  82         if (xmlStrEqual(name, BAD_CAST "osmStructured"))
  83         {
  84             value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "version");
  85             version = strtof(value, NULL);
  86             xmlFree(value);
  87
  88             if (version == (float)0.1)
  89             {
  90                 fileType = FILETYPE_STRUCTUREDV0P1;
  91                 fileMode = FILEMODE_ADD;
  92             }
  93             else
  94             {
  95                 fprintf( stderr, "Unknown osmStructured version %f (%s)\n", version, value );
  96                 exit_nicely();
  97             }
  98         }
  99         else
 100         {
 101             fprintf( stderr, "Unknown XML document type: %s\n", name );
 102             exit_nicely();
 103         }
 104         return;
 105     }
 106
 107     if (xmlStrEqual(name, BAD_CAST "add"))
 108     {
 109         fileMode = FILEMODE_ADD;
 110         return;
 111     }
 112     if (xmlStrEqual(name, BAD_CAST "update"))
 113     {
 114         fileMode = FILEMODE_UPDATE;
 115         return;
 116     }
 117     if (xmlStrEqual(name, BAD_CAST "delete"))
 118     {
 119         fileMode = FILEMODE_DELETE;
 120         return;
 121     }
 122     if (fileMode == FILEMODE_NONE)
 123     {
 124         fprintf( stderr, "Unknown import mode in: %s\n", name );
 125         exit_nicely();
 126     }
 127
 128     if (xmlStrEqual(name, BAD_CAST "feature"))
 129     {
 130         feature.placeID = xmlTextReaderGetAttribute(reader, BAD_CAST "place_id");
 131         feature.type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 132         feature.id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
 133         feature.key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
 134         feature.value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
 135         feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
 136         feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance");
 137
 138         feature.countryCode = NULL;
 139         feature.adminLevel = NULL;
 140         feature.houseNumber = NULL;
 141         feature.geometry = NULL;
 142         featureAddressLines = 0;
 143         featureNameLines = 0;
 144                 featureExtraTagLines = 0;
 145
 146         return;
 147     }
 148     if (xmlStrEqual(name, BAD_CAST "names")) return;
 149     if (xmlStrEqual(name, BAD_CAST "name"))
 150     {
 151         if (featureNameLines < MAX_FEATURENAMES)
 152         {
 153                 featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 154             featureName[featureNameLines].value = xmlTextReaderReadString(reader);
 155                 featureNameLines++;
 156                 }
 157                 else
 158                 {
 159             fprintf( stderr, "Too many name elements (%s%s)\n", feature.type, feature.id);
 160 //            exit_nicely();
 161         }
 162         return;
 163     }
 164     if (xmlStrEqual(name, BAD_CAST "tags")) return;
 165     if (xmlStrEqual(name, BAD_CAST "tag"))
 166     {
 167         if (featureExtraTagLines < MAX_FEATUREEXTRATAGS)
 168                 {
 169                 featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 170             featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
 171                 featureExtraTagLines++;
 172                 }
 173                 else
 174         {
 175             fprintf( stderr, "Too many extra tag elements (%s%s)\n", feature.type, feature.id);
 176 //            exit_nicely();
 177         }
 178         return;
 179     }
 180     if (xmlStrEqual(name, BAD_CAST "osmGeometry"))
 181     {
 182         feature.geometry = xmlTextReaderReadString(reader);
 183         return;
 184     }
 185     if (xmlStrEqual(name, BAD_CAST "adminLevel"))
 186     {
 187         feature.adminLevel = xmlTextReaderReadString(reader);
 188         return;
 189     }
 190     if (xmlStrEqual(name, BAD_CAST "countryCode"))
 191     {
 192         feature.countryCode = xmlTextReaderReadString(reader);
 193         return;
 194     }
 195     if (xmlStrEqual(name, BAD_CAST "houseNumber"))
 196     {
 197         feature.houseNumber = xmlTextReaderReadString(reader);
 198         return;
 199     }
 200     if (xmlStrEqual(name, BAD_CAST "address"))
 201     {
 202         featureAddressLines = 0;
 203         return;
 204     }
 205     isAddressLine = 0;
 206     if (xmlStrEqual(name, BAD_CAST "continent"))
 207     {
 208         isAddressLine = 1;
 209     }
 210     else if (xmlStrEqual(name, BAD_CAST "sea"))
 211     {
 212         isAddressLine = 1;
 213     }
 214     else if (xmlStrEqual(name, BAD_CAST "country"))
 215     {
 216         isAddressLine = 1;
 217     }
 218     else if (xmlStrEqual(name, BAD_CAST "state"))
 219     {
 220         isAddressLine = 1;
 221     }
 222     else if (xmlStrEqual(name, BAD_CAST "county"))
 223     {
 224         isAddressLine = 1;
 225     }
 226     else if (xmlStrEqual(name, BAD_CAST "city"))
 227     {
 228         isAddressLine = 1;
 229     }
 230     else if (xmlStrEqual(name, BAD_CAST "town"))
 231     {
 232         isAddressLine = 1;
 233     }
 234     else if (xmlStrEqual(name, BAD_CAST "village"))
 235     {
 236         isAddressLine = 1;
 237     }
 238     else if (xmlStrEqual(name, BAD_CAST "unknown"))
 239     {
 240         isAddressLine = 1;
 241     }
 242     else if (xmlStrEqual(name, BAD_CAST "suburb"))
 243     {
 244         isAddressLine = 1;
 245     }
 246     else if (xmlStrEqual(name, BAD_CAST "postcode"))
 247     {
 248         isAddressLine = 1;
 249     }
 250     else if (xmlStrEqual(name, BAD_CAST "neighborhood"))
 251     {
 252         isAddressLine = 1;
 253     }
 254     else if (xmlStrEqual(name, BAD_CAST "street"))
 255     {
 256         isAddressLine = 1;
 257     }
 258     else if (xmlStrEqual(name, BAD_CAST "access"))
 259     {
 260         isAddressLine = 1;
 261     }
 262     else if (xmlStrEqual(name, BAD_CAST "building"))
 263     {
 264         isAddressLine = 1;
 265     }
 266     else if (xmlStrEqual(name, BAD_CAST "other"))
 267     {
 268         isAddressLine = 1;
 269     }
 270     if (isAddressLine)
 271     {
 272         if (featureAddressLines < MAX_FEATUREADDRESS)
 273                 {
 274                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
 275             if (!value)
 276                 {
 277                 fprintf( stderr, "Address element missing rank\n");
 278                     exit_nicely();
 279                 }
 280             featureAddress[featureAddressLines].rankAddress =  atoi(value);
 281                 xmlFree(value);
 282
 283                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
 284             if (!value)
 285                 {
 286                     fprintf( stderr, "Address element missing rank\n");
 287                 exit_nicely();
 288                 }
 289                 if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
 290             else strcpy(featureAddress[featureAddressLines].isAddress, "f");
 291                 xmlFree(value);
 292
 293                 featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 294             featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
 295                 featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
 296                 featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
 297             featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance");
 298
 299             featureAddressLines++;
 300                 }
 301                 else
 302         {
 303             fprintf( stderr, "Too many address elements (%s%s)\n", feature.type, feature.id);
 304 //            exit_nicely();
 305         }
 306
 307         return;
 308     }
 309     fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
 310 }
 311
 312 void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
 313 {
 314     PGresult *          res;
 315     PGresult *          resPlaceID;
 316     const char *        paramValues[11];
 317     char *                      place_id;
 318     char *                      partionQueryName;
 319     int i, namePos, lineTypeLen, lineValueLen;
 320
 321     if (xmlStrEqual(name, BAD_CAST "feature"))
 322     {
 323         featureCount++;
 324         if (featureCount % 1000 == 0) printf("feature %i(k)\n", featureCount/1000);
 325 /*
 326         if (fileMode == FILEMODE_ADD)
 327         {
 328             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 0, NULL, NULL, NULL, 0);
 329             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
 330             {
 331                 fprintf(stderr, "get_place_id: INSERT failed: %s", PQerrorMessage(conn));
 332                 PQclear(resPlaceID);
 333                 exit(EXIT_FAILURE);
 334             }
 335         }
 336         else
 337         {
 338             paramValues[0] = (const char *)feature.type;
 339             paramValues[1] = (const char *)feature.id;
 340             paramValues[2] = (const char *)feature.key;
 341             paramValues[3] = (const char *)feature.value;
 342             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 4, paramValues, NULL, NULL, 0);
 343             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
 344             {
 345                 fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
 346                 PQclear(resPlaceID);
 347                 exit(EXIT_FAILURE);
 348             }
 349         }
 350 */
 351         place_id = feature.placeID;
 352
 353         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE)
 354         {
 355             paramValues[0] = (const char *)place_id;
 356             res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0);
 357             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 358             {
 359                 fprintf(stderr, "placex_delete: DELETE failed: %s", PQerrorMessage(conn));
 360                 PQclear(res);
 361                 exit(EXIT_FAILURE);
 362             }
 363             PQclear(res);
 364
 365             res = PQexecPrepared(conn, "search_name_delete", 1, paramValues, NULL, NULL, 0);
 366             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 367             {
 368                 fprintf(stderr, "search_name_delete: DELETE failed: %s", PQerrorMessage(conn));
 369                 PQclear(res);
 370                 exit(EXIT_FAILURE);
 371             }
 372             PQclear(res);
 373
 374             res = PQexecPrepared(conn, "place_addressline_delete", 1, paramValues, NULL, NULL, 0);
 375             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 376             {
 377                 fprintf(stderr, "place_addressline_delete: DELETE failed: %s", PQerrorMessage(conn));
 378                 PQclear(res);
 379                 exit(EXIT_FAILURE);
 380             }
 381             PQclear(res);
 382         }
 383
 384         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD)
 385         {
 386             // Insert into placex
 387             paramValues[0] = (const char *)place_id;
 388             paramValues[1] = (const char *)feature.type;
 389             paramValues[2] = (const char *)feature.id;
 390             paramValues[3] = (const char *)feature.key;
 391             paramValues[4] = (const char *)feature.value;
 392
 393             featureNameString[0] = 0;
 394             if (featureNameLines)
 395             {
 396                 namePos = 0;
 397                 lineTypeLen = 0;
 398                 lineValueLen = 0;
 399                 for (i = 0; i < featureNameLines; i++)
 400                 {
 401                     lineTypeLen = strlen(BAD_CAST featureName[i].type);
 402                     lineValueLen = strlen(BAD_CAST featureName[i].value);
 403                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING)
 404                     {
 405                         fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value);
 406                         break;
 407                     }
 408                     if (namePos) strcpy(featureNameString+(namePos++), ",");
 409                     strcpy(featureNameString+(namePos++), "\"");
 410                     strcpy(featureNameString+namePos, BAD_CAST featureName[i].type);
 411                     namePos += lineTypeLen;
 412                     strcpy(featureNameString+namePos, "\"=>\"");
 413                     namePos += 4;
 414                     strcpy(featureNameString+namePos, BAD_CAST featureName[i].value);
 415                     namePos += lineValueLen;
 416                     strcpy(featureNameString+(namePos++), "\"");
 417                 }
 418             }
 419             paramValues[5] = (const char *)featureNameString;
 420
 421             featureExtraTagString[0] = 0;
 422             if (featureExtraTagLines)
 423             {
 424                 namePos = 0;
 425                 lineTypeLen = 0;
 426                 lineValueLen = 0;
 427                 for (i = 0; i < featureExtraTagLines; i++)
 428                 {
 429                     lineTypeLen = strlen(BAD_CAST featureExtraTag[i].type);
 430                     lineValueLen = strlen(BAD_CAST featureExtraTag[i].value);
 431                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING)
 432                     {
 433                         fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value);
 434                         break;
 435                     }
 436                     if (namePos) strcpy(featureExtraTagString+(namePos++),",");
 437                     strcpy(featureExtraTagString+(namePos++), "\"");
 438                     strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].type);
 439                     namePos += lineTypeLen;
 440                     strcpy(featureExtraTagString+namePos, "\"=>\"");
 441                     namePos += 4;
 442                     strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].value);
 443                     namePos += lineValueLen;
 444                     strcpy(featureExtraTagString+(namePos++), "\"");
 445                 }
 446             }
 447             paramValues[6] = (const char *)featureExtraTagString;
 448
 449             paramValues[7] = (const char *)feature.adminLevel;
 450             paramValues[8] = (const char *)feature.houseNumber;
 451             paramValues[9] = (const char *)feature.rankAddress;
 452             paramValues[10] = (const char *)feature.rankSearch;
 453             paramValues[11] = (const char *)feature.geometry;
 454             res = PQexecPrepared(conn, "placex_insert", 12, paramValues, NULL, NULL, 0);
 455             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 456             {
 457                 fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
 458                 PQclear(res);
 459                 exit(EXIT_FAILURE);
 460             }
 461             PQclear(res);
 462
 463             for (i = 0; i < featureAddressLines; i++)
 464             {
 465                 // insert into place_address
 466                 paramValues[0] = (const char *)place_id;
 467                 paramValues[1] = (const char *)featureAddress[i].distance;
 468                 paramValues[2] = (const char *)featureAddress[i].type;
 469                 paramValues[3] = (const char *)featureAddress[i].id;
 470                 paramValues[4] = (const char *)featureAddress[i].key;
 471                 paramValues[5] = (const char *)featureAddress[i].value;
 472                 paramValues[6] = (const char *)featureAddress[i].isAddress;
 473                 res = PQexecPrepared(conn, "place_addressline_insert", 7, paramValues, NULL, NULL, 0);
 474                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 475                 {
 476                     fprintf(stderr, "place_addressline_insert: INSERT failed: %s", PQerrorMessage(conn));
 477                     PQclear(res);
 478                     exit(EXIT_FAILURE);
 479                 }
 480                 PQclear(res);
 481
 482                 xmlFree(featureAddress[i].type);
 483                 xmlFree(featureAddress[i].id);
 484                 xmlFree(featureAddress[i].key);
 485                 xmlFree(featureAddress[i].value);
 486                 xmlFree(featureAddress[i].distance);
 487             }
 488
 489             if (featureNameLines)
 490             {
 491                 paramValues[0] = (const char *)place_id;
 492                 res = PQexecPrepared(conn, "search_name_insert", 1, paramValues, NULL, NULL, 0);
 493                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 494                 {
 495                     fprintf(stderr, "search_name_insert: INSERT failed: %s", PQerrorMessage(conn));
 496                     PQclear(res);
 497                     exit(EXIT_FAILURE);
 498                 }
 499                 PQclear(res);
 500             }
 501
 502             partionQueryName = xmlHashLookup2(partionTableTagsHash, feature.key, feature.value);
 503             if (partionQueryName)
 504             {
 505                 // insert into partition table
 506                 paramValues[0] = (const char *)place_id;
 507                 paramValues[1] = (const char *)feature.geometry;
 508                 res = PQexecPrepared(conn, partionQueryName, 2, paramValues, NULL, NULL, 0);
 509                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 510                 {
 511                     fprintf(stderr, "%s: INSERT failed: %s", partionQueryName, PQerrorMessage(conn));
 512                     PQclear(res);
 513                     exit(EXIT_FAILURE);
 514                 }
 515                 PQclear(res);
 516
 517             }
 518
 519         }
 520         else
 521         {
 522             for (i = 0; i < featureAddressLines; i++)
 523             {
 524                 xmlFree(featureAddress[i].type);
 525                 xmlFree(featureAddress[i].id);
 526                 xmlFree(featureAddress[i].key);
 527                 xmlFree(featureAddress[i].value);
 528                 xmlFree(featureAddress[i].distance);
 529             }
 530         }
 531
 532         xmlFree(feature.placeID);
 533         xmlFree(feature.type);
 534         xmlFree(feature.id);
 535         xmlFree(feature.key);
 536         xmlFree(feature.value);
 537         xmlFree(feature.rankAddress);
 538         xmlFree(feature.rankSearch);
 539 //              if (feature.name) xmlFree(feature.name);
 540         if (feature.countryCode) xmlFree(feature.countryCode);
 541         if (feature.adminLevel) xmlFree(feature.adminLevel);
 542         if (feature.houseNumber) xmlFree(feature.houseNumber);
 543         if (feature.geometry) xmlFree(feature.geometry);
 544
 545 //        PQclear(resPlaceID);
 546     }
 547 }
 548
 549 static void processNode(xmlTextReaderPtr reader)
 550 {
 551     xmlChar *name;
 552     name = xmlTextReaderName(reader);
 553     if (name == NULL)
 554     {
 555         name = xmlStrdup(BAD_CAST "--");
 556     }
 557
 558     switch (xmlTextReaderNodeType(reader))
 559     {
 560     case XML_READER_TYPE_ELEMENT:
 561         StartElement(reader, name);
 562         if (xmlTextReaderIsEmptyElement(reader))
 563             EndElement(reader, name); /* No end_element for self closing tags! */
 564         break;
 565     case XML_READER_TYPE_END_ELEMENT:
 566         EndElement(reader, name);
 567         break;
 568     case XML_READER_TYPE_TEXT:
 569     case XML_READER_TYPE_CDATA:
 570     case XML_READER_TYPE_SIGNIFICANT_WHITESPACE:
 571         /* Ignore */
 572         break;
 573     default:
 574         fprintf(stderr, "Unknown node type %d\n", xmlTextReaderNodeType(reader));
 575         break;
 576     }
 577
 578     xmlFree(name);
 579 }
 580
 581 int nominatim_import(const char *conninfo, const char *partionTagsFilename, const char *filename)
 582 {
 583     xmlTextReaderPtr    reader;
 584     int                                 ret = 0;
 585     PGresult *                  res;
 586     FILE *                              partionTagsFile;
 587     char *                              partionQueryName;
 588     char                                partionQuerySQL[1024];
 589
 590     conn = PQconnectdb(conninfo);
 591     if (PQstatus(conn) != CONNECTION_OK)
 592     {
 593         fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
 594         exit(EXIT_FAILURE);
 595     }
 596
 597     partionTableTagsHash = xmlHashCreate(200);
 598
 599     partionTagsFile = fopen(partionTagsFilename, "rt");
 600     if (!partionTagsFile)
 601     {
 602         fprintf(stderr, "Unable to read partition tags file: %s\n", partionTagsFilename);
 603         exit(EXIT_FAILURE);
 604     }
 605
 606     char buffer[1024], osmkey[256], osmvalue[256];
 607     int fields;
 608     while (fgets(buffer, sizeof(buffer), partionTagsFile) != NULL)
 609     {
 610         fields = sscanf( buffer, "%23s %63s", osmkey, osmvalue );
 611
 612         if ( fields <= 0 ) continue;
 613
 614         if ( fields != 2  )
 615         {
 616             fprintf( stderr, "Error partition file\n");
 617             exit_nicely();
 618         }
 619         partionQueryName = malloc(strlen("partition_insert_")+strlen(osmkey)+strlen(osmvalue)+2);
 620         strcpy(partionQueryName, "partition_insert_");
 621         strcat(partionQueryName, osmkey);
 622         strcat(partionQueryName, "_");
 623         strcat(partionQueryName, osmvalue);
 624
 625         strcpy(partionQuerySQL, "insert into place_classtype_");
 626         strcat(partionQuerySQL, osmkey);
 627         strcat(partionQuerySQL, "_");
 628         strcat(partionQuerySQL, osmvalue);
 629         strcat(partionQuerySQL, " (place_id, centroid) values ($1, ST_Centroid(st_setsrid($2, 4326)))");
 630
 631         res = PQprepare(conn, partionQueryName, partionQuerySQL, 2, NULL);
 632         if (PQresultStatus(res) != PGRES_COMMAND_OK)
 633         {
 634             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
 635             exit(EXIT_FAILURE);
 636         }
 637
 638         xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
 639     }
 640
 641     res = PQprepare(conn, "get_new_place_id",
 642                     "select nextval('seq_place')",
 643                     0, NULL);
 644     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 645     {
 646         fprintf(stderr, "Failed to prepare get_new_place_id: %s\n", PQerrorMessage(conn));
 647         exit(EXIT_FAILURE);
 648     }
 649
 650     res = PQprepare(conn, "get_place_id",
 651                     "select place_id from placex where osm_type = $1 and osm_id = $2 and class = $3 and type = $4",
 652                     4, NULL);
 653     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 654     {
 655         fprintf(stderr, "Failed to prepare get_place_id: %s\n", PQerrorMessage(conn));
 656         exit(EXIT_FAILURE);
 657     }
 658
 659     res = PQprepare(conn, "placex_insert",
 660                     "insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,admin_level,housenumber,rank_address,rank_search,geometry) "
 661                     "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, st_setsrid($12, 4326))",
 662                     12, NULL);
 663     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 664     {
 665         fprintf(stderr, "Failed to prepare placex_insert: %s\n", PQerrorMessage(conn));
 666         exit(EXIT_FAILURE);
 667     }
 668
 669     res = PQprepare(conn, "search_name_insert",
 670                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
 671                     "select place_id, rank_address, rank_search, country_code, make_keywords(name), "
 672                     "(select uniq(sort(array_agg(name_vector))) from place_addressline join search_name on "
 673                     "(address_place_id = search_name.place_id) where place_addressline.place_id = $1 ), st_centroid(geometry) from placex "
 674                     "where place_id = $1",
 675                     1, NULL);
 676     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 677     {
 678         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
 679         exit(EXIT_FAILURE);
 680     }
 681
 682     res = PQprepare(conn, "place_addressline_insert",
 683                     "insert into place_addressline (place_id, address_place_id, fromarea, isaddress, distance, cached_rank_address) "
 684                     "select $1, place_id, false, $7, $2, rank_address from placex where osm_type = $3 and osm_id = $4 and class = $5 and type = $6",
 685                     7, NULL);
 686     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 687     {
 688         fprintf(stderr, "Failed to prepare place_addressline_insert: %s\n", PQerrorMessage(conn));
 689         exit(EXIT_FAILURE);
 690     }
 691
 692     res = PQprepare(conn, "placex_delete",
 693                     "delete from placex where place_id = $1",
 694                     1, NULL);
 695     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 696     {
 697         fprintf(stderr, "Failed to prepare placex_delete: %s\n", PQerrorMessage(conn));
 698         exit(EXIT_FAILURE);
 699     }
 700
 701     res = PQprepare(conn, "search_name_delete",
 702                     "delete from search_name where place_id = $1",
 703                     1, NULL);
 704     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 705     {
 706         fprintf(stderr, "Failed to prepare search_name_delete: %s\n", PQerrorMessage(conn));
 707         exit(EXIT_FAILURE);
 708     }
 709
 710     res = PQprepare(conn, "place_addressline_delete",
 711                     "delete from place_addressline where place_id = $1",
 712                     1, NULL);
 713     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 714     {
 715         fprintf(stderr, "Failed to prepare place_addressline_delete: %s\n", PQerrorMessage(conn));
 716         exit(EXIT_FAILURE);
 717     }
 718
 719     featureCount = 0;
 720
 721     reader = inputUTF8(filename);
 722
 723     if (reader == NULL)
 724     {
 725         fprintf(stderr, "Unable to open %s\n", filename);
 726         return 1;
 727     }
 728
 729     ret = xmlTextReaderRead(reader);
 730     while (ret == 1)
 731     {
 732         processNode(reader);
 733         ret = xmlTextReaderRead(reader);
 734     }
 735     if (ret != 0)
 736     {
 737         fprintf(stderr, "%s : failed to parse\n", filename);
 738         return ret;
 739     }
 740
 741     xmlFreeTextReader(reader);
 742     xmlHashFree(partionTableTagsHash, NULL);
 743
 744     return 0;
 745 }