]> git.openstreetmap.org Git - nominatim.git/blob - nominatim/import.c
Merge remote-tracking branch 'upstream/master'
[nominatim.git] / nominatim / import.c
1 /*
2 */
3 #include <stdlib.h>
4 #include <string.h>
5
6 #include <libpq-fe.h>
7
8 #include <libxml/xmlstring.h>
9 #include <libxml/xmlreader.h>
10 #include <libxml/hash.h>
11
12 #include "nominatim.h"
13 #include "import.h"
14 #include "input.h"
15
16 typedef enum { FILETYPE_NONE, FILETYPE_STRUCTUREDV0P1 } filetypes_t;
17 typedef enum { FILEMODE_NONE, FILEMODE_ADD, FILEMODE_UPDATE, FILEMODE_DELETE } filemodes_t;
18
19 #define MAX_FEATUREADDRESS 5000
20 #define MAX_FEATURENAMES 10000
21 #define MAX_FEATUREEXTRATAGS 10000
22 #define MAX_FEATURENAMESTRING 1000000
23 #define MAX_FEATUREEXTRATAGSTRING 500000
24
25 struct feature_address
26 {
27     int                 place_id;
28     int                 rankAddress;
29     char                        isAddress[2];
30     xmlChar *   type;
31     xmlChar *   id;
32     xmlChar *   key;
33     xmlChar *   value;
34     xmlChar *   distance;
35 };
36
37 struct feature_tag
38 {
39     xmlChar *   type;
40     xmlChar *   value;
41 };
42
43 struct feature
44 {
45     xmlChar *   placeID;
46     xmlChar *   type;
47     xmlChar *   id;
48     xmlChar *   key;
49     xmlChar *   value;
50     xmlChar *   rankAddress;
51     xmlChar *   rankSearch;
52     xmlChar *   countryCode;
53     xmlChar *   parentPlaceID;
54     xmlChar *   parentType;
55     xmlChar *   parentID;
56     xmlChar *   adminLevel;
57     xmlChar *   houseNumber;
58     xmlChar *   geometry;
59 } feature;
60
61 int                                     fileType = FILETYPE_NONE;
62 int                                     fileMode = FILEMODE_ADD;
63 PGconn *                                conn;
64 struct feature_address  featureAddress[MAX_FEATUREADDRESS];
65 struct feature_tag              featureName[MAX_FEATURENAMES];
66 struct feature_tag              featureExtraTag[MAX_FEATUREEXTRATAGS];
67 struct feature                  feature;
68 int                                     featureAddressLines = 0;
69 int                                     featureNameLines = 0;
70 int                                     featureExtraTagLines = 0;
71 int                                     featureCount = 0;
72 xmlHashTablePtr                 partionTableTagsHash;
73 xmlHashTablePtr                 partionTableTagsHashDelete;
74 char                                    featureNameString[MAX_FEATURENAMESTRING];
75 char                                    featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
76
77 extern int verbose;
78
79 void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
80 {
81     char * value;
82     float version;
83     int isAddressLine;
84
85     if (fileType == FILETYPE_NONE)
86     {
87         // Potential to handle other file types in the future / versions
88         if (xmlStrEqual(name, BAD_CAST "osmStructured"))
89         {
90             value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "version");
91             version = strtof(value, NULL);
92             xmlFree(value);
93
94             if (version == (float)0.1)
95             {
96                 fileType = FILETYPE_STRUCTUREDV0P1;
97                 fileMode = FILEMODE_ADD;
98             }
99             else
100             {
101                 fprintf( stderr, "Unknown osmStructured version %f (%s)\n", version, value );
102                 exit_nicely();
103             }
104         }
105         else
106         {
107             fprintf( stderr, "Unknown XML document type: %s\n", name );
108             exit_nicely();
109         }
110         return;
111     }
112
113     if (xmlStrEqual(name, BAD_CAST "add"))
114     {
115         fileMode = FILEMODE_ADD;
116         return;
117     }
118     if (xmlStrEqual(name, BAD_CAST "update"))
119     {
120         fileMode = FILEMODE_UPDATE;
121         return;
122     }
123     if (xmlStrEqual(name, BAD_CAST "delete"))
124     {
125         fileMode = FILEMODE_DELETE;
126         return;
127     }
128     if (fileMode == FILEMODE_NONE)
129     {
130         fprintf( stderr, "Unknown import mode in: %s\n", name );
131         exit_nicely();
132     }
133
134     if (xmlStrEqual(name, BAD_CAST "feature"))
135     {
136         feature.placeID = xmlTextReaderGetAttribute(reader, BAD_CAST "place_id");
137         feature.type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
138         feature.id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
139         feature.key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
140         feature.value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
141         feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
142         feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance");
143
144         feature.parentPlaceID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_place_id");
145 /*
146         if (strlen(feature.parentPlaceID) == 0)
147         {
148                 xmlFree(feature.parentPlaceID);
149                 feature.parentPlaceID = NULL;
150         }
151 */
152         feature.parentType = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_type");
153         feature.parentID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_id");
154
155         feature.countryCode = NULL;
156         feature.adminLevel = NULL;
157         feature.houseNumber = NULL;
158         feature.geometry = NULL;
159         featureAddressLines = 0;
160         featureNameLines = 0;
161         featureExtraTagLines = 0;
162
163         return;
164     }
165     if (xmlStrEqual(name, BAD_CAST "names")) return;
166     if (xmlStrEqual(name, BAD_CAST "name"))
167     {
168         if (featureNameLines < MAX_FEATURENAMES)
169         {
170                 featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
171             featureName[featureNameLines].value = xmlTextReaderReadString(reader);
172                 featureNameLines++;
173                 }
174                 else
175                 {
176             fprintf( stderr, "Too many name elements (%s%s)\n", feature.type, feature.id);
177 //            exit_nicely();
178         }
179         return;
180     }
181     if (xmlStrEqual(name, BAD_CAST "tags")) return;
182     if (xmlStrEqual(name, BAD_CAST "tag"))
183     {
184         if (featureExtraTagLines < MAX_FEATUREEXTRATAGS)
185                 {
186                 featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
187             featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
188                 featureExtraTagLines++;
189                 }
190                 else
191         {
192             fprintf( stderr, "Too many extra tag elements (%s%s)\n", feature.type, feature.id);
193 //            exit_nicely();
194         }
195         return;
196     }
197     if (xmlStrEqual(name, BAD_CAST "osmGeometry"))
198     {
199         feature.geometry = xmlTextReaderReadString(reader);
200         return;
201     }
202     if (xmlStrEqual(name, BAD_CAST "adminLevel"))
203     {
204         feature.adminLevel = xmlTextReaderReadString(reader);
205         return;
206     }
207     if (xmlStrEqual(name, BAD_CAST "countryCode"))
208     {
209         feature.countryCode = xmlTextReaderReadString(reader);
210         return;
211     }
212     if (xmlStrEqual(name, BAD_CAST "houseNumber"))
213     {
214         feature.houseNumber = xmlTextReaderReadString(reader);
215         return;
216     }
217     if (xmlStrEqual(name, BAD_CAST "address"))
218     {
219         featureAddressLines = 0;
220         return;
221     }
222     isAddressLine = 0;
223     if (xmlStrEqual(name, BAD_CAST "continent"))
224     {
225         isAddressLine = 1;
226     }
227     else if (xmlStrEqual(name, BAD_CAST "sea"))
228     {
229         isAddressLine = 1;
230     }
231     else if (xmlStrEqual(name, BAD_CAST "country"))
232     {
233         isAddressLine = 1;
234     }
235     else if (xmlStrEqual(name, BAD_CAST "state"))
236     {
237         isAddressLine = 1;
238     }
239     else if (xmlStrEqual(name, BAD_CAST "county"))
240     {
241         isAddressLine = 1;
242     }
243     else if (xmlStrEqual(name, BAD_CAST "city"))
244     {
245         isAddressLine = 1;
246     }
247     else if (xmlStrEqual(name, BAD_CAST "town"))
248     {
249         isAddressLine = 1;
250     }
251     else if (xmlStrEqual(name, BAD_CAST "village"))
252     {
253         isAddressLine = 1;
254     }
255     else if (xmlStrEqual(name, BAD_CAST "unknown"))
256     {
257         isAddressLine = 1;
258     }
259     else if (xmlStrEqual(name, BAD_CAST "suburb"))
260     {
261         isAddressLine = 1;
262     }
263     else if (xmlStrEqual(name, BAD_CAST "postcode"))
264     {
265         isAddressLine = 1;
266     }
267     else if (xmlStrEqual(name, BAD_CAST "neighborhood"))
268     {
269         isAddressLine = 1;
270     }
271     else if (xmlStrEqual(name, BAD_CAST "street"))
272     {
273         isAddressLine = 1;
274     }
275     else if (xmlStrEqual(name, BAD_CAST "access"))
276     {
277         isAddressLine = 1;
278     }
279     else if (xmlStrEqual(name, BAD_CAST "building"))
280     {
281         isAddressLine = 1;
282     }
283     else if (xmlStrEqual(name, BAD_CAST "other"))
284     {
285         isAddressLine = 1;
286     }
287     if (isAddressLine)
288     {
289         if (featureAddressLines < MAX_FEATUREADDRESS)
290                 {
291                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
292             if (!value)
293                 {
294                 fprintf( stderr, "Address element missing rank\n");
295                     exit_nicely();
296                 }
297             featureAddress[featureAddressLines].rankAddress =  atoi(value);
298                 xmlFree(value);
299
300                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
301             if (!value)
302                 {
303                     fprintf( stderr, "Address element missing rank\n");
304                 exit_nicely();
305                 }
306                 if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
307             else strcpy(featureAddress[featureAddressLines].isAddress, "f");
308                 xmlFree(value);
309
310                 featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
311             featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
312                 featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
313                 featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
314             featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance");
315         
316             featureAddressLines++;
317                 }
318                 else
319         {
320             fprintf( stderr, "Too many address elements (%s%s)\n", feature.type, feature.id);
321 //            exit_nicely();
322         }
323
324         return;
325     }
326     fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
327 }
328
329 void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
330 {
331     PGresult *          res;
332     const char *        paramValues[14];
333     char *                      place_id;
334     char *                      partionQueryName;
335     int i, namePos, lineTypeLen, lineValueLen;
336
337     if (xmlStrEqual(name, BAD_CAST "feature"))
338     {
339         featureCount++;
340         if (featureCount % 1000 == 0) printf("feature %i(k)\n", featureCount/1000);
341 /*
342         if (fileMode == FILEMODE_ADD)
343         {
344             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 0, NULL, NULL, NULL, 0);
345             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
346             {
347                 fprintf(stderr, "get_place_id: INSERT failed: %s", PQerrorMessage(conn));
348                 PQclear(resPlaceID);
349                 exit(EXIT_FAILURE);
350             }
351         }
352         else
353         {
354             paramValues[0] = (const char *)feature.type;
355             paramValues[1] = (const char *)feature.id;
356             paramValues[2] = (const char *)feature.key;
357             paramValues[3] = (const char *)feature.value;
358             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 4, paramValues, NULL, NULL, 0);
359             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
360             {
361                 fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
362                 PQclear(resPlaceID);
363                 exit(EXIT_FAILURE);
364             }
365         }
366 */
367         place_id = (char *)feature.placeID;
368
369         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD)
370         {
371             paramValues[0] = (const char *)place_id;
372             if (verbose) fprintf(stderr, "placex_delete: %s\n", paramValues[0]);
373             res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0);
374             if (PQresultStatus(res) != PGRES_COMMAND_OK)
375             {
376                 fprintf(stderr, "placex_delete: DELETE failed: %s", PQerrorMessage(conn));
377                 PQclear(res);
378                 exit(EXIT_FAILURE);
379             }
380             PQclear(res);
381
382             if (verbose) fprintf(stderr, "search_name_delete: %s\n", paramValues[0]);
383             res = PQexecPrepared(conn, "search_name_delete", 1, paramValues, NULL, NULL, 0);
384             if (PQresultStatus(res) != PGRES_COMMAND_OK)
385             {
386                 fprintf(stderr, "search_name_delete: DELETE failed: %s", PQerrorMessage(conn));
387                 PQclear(res);
388                 exit(EXIT_FAILURE);
389             }
390             PQclear(res);
391
392             if (verbose) fprintf(stderr, "place_addressline_delete: %s\n", paramValues[0]);
393             res = PQexecPrepared(conn, "place_addressline_delete", 1, paramValues, NULL, NULL, 0);
394             if (PQresultStatus(res) != PGRES_COMMAND_OK)
395             {
396                 fprintf(stderr, "place_addressline_delete: DELETE failed: %s", PQerrorMessage(conn));
397                 PQclear(res);
398                 exit(EXIT_FAILURE);
399             }
400             PQclear(res);
401
402             partionQueryName = xmlHashLookup2(partionTableTagsHashDelete, feature.key, feature.value);
403             if (partionQueryName)
404             {
405                 res = PQexecPrepared(conn, partionQueryName, 1, paramValues, NULL, NULL, 0);
406                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
407                 {
408                     fprintf(stderr, "%s: DELETE failed: %s", partionQueryName, PQerrorMessage(conn));
409                     PQclear(res);
410                     exit(EXIT_FAILURE);
411                 }
412                 PQclear(res);
413             }
414         }
415
416         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD)
417         {
418             // Insert into placex
419             paramValues[0] = (const char *)place_id;
420             paramValues[1] = (const char *)feature.type;
421             paramValues[2] = (const char *)feature.id;
422             paramValues[3] = (const char *)feature.key;
423             paramValues[4] = (const char *)feature.value;
424
425             featureNameString[0] = 0;
426             if (featureNameLines)
427             {
428                 namePos = 0;
429                 lineTypeLen = 0;
430                 lineValueLen = 0;
431                 for (i = 0; i < featureNameLines; i++)
432                 {
433                     lineTypeLen = (int)strlen((char *) featureName[i].type);
434                     lineValueLen = (int)strlen((char *) featureName[i].value);
435                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING)
436                     {
437                         fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value);
438                         break;
439                     }
440                     if (namePos) strcpy(featureNameString+(namePos++), ",");
441                     strcpy(featureNameString+(namePos++), "\"");
442                     strcpy(featureNameString+namePos, (char*) featureName[i].type);
443                     namePos += lineTypeLen;
444                     strcpy(featureNameString+namePos, "\"=>\"");
445                     namePos += 4;
446                     strcpy(featureNameString+namePos, (char *) featureName[i].value);
447                     namePos += lineValueLen;
448                     strcpy(featureNameString+(namePos++), "\"");
449
450                     xmlFree(featureName[i].type);
451                     xmlFree(featureName[i].value);
452                 }
453             }
454             paramValues[5] = (const char *)featureNameString;
455
456             paramValues[6] = (const char *)feature.countryCode;
457
458             featureExtraTagString[0] = 0;
459             if (featureExtraTagLines)
460             {
461                 namePos = 0;
462                 lineTypeLen = 0;
463                 lineValueLen = 0;
464                 for (i = 0; i < featureExtraTagLines; i++)
465                 {
466                     lineTypeLen = strlen((char *) featureExtraTag[i].type);
467                     lineValueLen = strlen((char *) featureExtraTag[i].value);
468                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING)
469                     {
470                         fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value);
471                         break;
472                     }
473                     if (namePos) strcpy(featureExtraTagString+(namePos++),",");
474                     strcpy(featureExtraTagString+(namePos++), "\"");
475                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].type);
476                     namePos += lineTypeLen;
477                     strcpy(featureExtraTagString+namePos, "\"=>\"");
478                     namePos += 4;
479                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value);
480                     namePos += lineValueLen;
481                     strcpy(featureExtraTagString+(namePos++), "\"");
482
483                     xmlFree(featureExtraTag[i].type);
484                     xmlFree(featureExtraTag[i].value);
485                 }
486             }
487             paramValues[7] = (const char *)featureExtraTagString;
488
489             if (strlen(feature.parentPlaceID) == 0)
490                 paramValues[8] = "0";
491             else
492                 paramValues[8] = (const char *)feature.parentPlaceID;
493
494             paramValues[9] = (const char *)feature.adminLevel;
495             paramValues[10] = (const char *)feature.houseNumber;
496             paramValues[11] = (const char *)feature.rankAddress;
497             paramValues[12] = (const char *)feature.rankSearch;
498             paramValues[13] = (const char *)feature.geometry;
499             if (strlen(paramValues[3]) && strlen(paramValues[13]))
500             {
501                 if (verbose) fprintf(stderr, "placex_insert: %s\n", paramValues[0]);
502                 res = PQexecPrepared(conn, "placex_insert", 14, paramValues, NULL, NULL, 0);
503                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
504                 {
505                     fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
506                     fprintf(stderr, "index_placex: INSERT failed: %s %s %s", paramValues[0], paramValues[1], paramValues[2]);
507                     PQclear(res);
508                     exit(EXIT_FAILURE);
509                }
510                PQclear(res);
511             }
512
513             for (i = 0; i < featureAddressLines; i++)
514             {
515                 // insert into place_address
516                 paramValues[0] = (const char *)place_id;
517                 paramValues[1] = (const char *)featureAddress[i].distance;
518                 if (paramValues[1] == NULL || strlen(paramValues[1]) == 0) paramValues[1] = "0";
519                 paramValues[2] = (const char *)featureAddress[i].type;
520                 paramValues[3] = (const char *)featureAddress[i].id;
521                 paramValues[4] = (const char *)featureAddress[i].key;
522                 paramValues[5] = (const char *)featureAddress[i].value;
523                 paramValues[6] = (const char *)featureAddress[i].isAddress;
524                 if (verbose) fprintf(stderr, "placex_insert: %s %s\n", paramValues[2], paramValues[3]);
525                 res = PQexecPrepared(conn, "place_addressline_insert", 7, paramValues, NULL, NULL, 0);
526                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
527                 {
528                     fprintf(stderr, "place_addressline_insert: INSERT failed: %s", PQerrorMessage(conn));
529                     fprintf(stderr, "(%s,%s,%s,%s,%s,%s,%s)",paramValues[0],paramValues[1],paramValues[2],paramValues[3],paramValues[4],paramValues[5],paramValues[6]);
530                     PQclear(res);
531                     exit(EXIT_FAILURE);
532                 }
533                 PQclear(res);
534
535                 xmlFree(featureAddress[i].type);
536                 xmlFree(featureAddress[i].id);
537                 xmlFree(featureAddress[i].key);
538                 xmlFree(featureAddress[i].value);
539                 xmlFree(featureAddress[i].distance);
540             }
541
542             if (featureNameLines)
543             {
544                 if (strlen(feature.parentPlaceID) > 0 && featureAddressLines == 0)
545                 {
546                     paramValues[0] = (const char *)place_id;
547                     paramValues[1] = feature.parentPlaceID;
548                     if (verbose) fprintf(stderr, "search_name_from_parent_insert: INSERT %s %s\n", paramValues[0], paramValues[1]);
549                     res = PQexecPrepared(conn, "search_name_from_parent_insert", 2, paramValues, NULL, NULL, 0);
550                     if (PQresultStatus(res) != PGRES_COMMAND_OK)
551                     {
552                         fprintf(stderr, "search_name_from_parent_insert: INSERT failed: %s", PQerrorMessage(conn));
553                         PQclear(res);
554                         exit(EXIT_FAILURE);
555                     }
556                     PQclear(res);
557                 }
558                 else
559                 {
560                     paramValues[0] = (const char *)place_id;
561                     if (verbose) fprintf(stderr, "search_name_insert: INSERT %s\n", paramValues[0]);
562                     res = PQexecPrepared(conn, "search_name_insert", 1, paramValues, NULL, NULL, 0);
563                     if (PQresultStatus(res) != PGRES_COMMAND_OK)
564                     {
565                         fprintf(stderr, "search_name_insert: INSERT failed: %s", PQerrorMessage(conn));
566                         PQclear(res);
567                         exit(EXIT_FAILURE);
568                     }
569                     PQclear(res);
570                 }
571             }
572
573             partionQueryName = xmlHashLookup2(partionTableTagsHash, feature.key, feature.value);
574             if (partionQueryName)
575             {
576                 // insert into partition table
577                 paramValues[0] = (const char *)place_id;
578                 paramValues[1] = (const char *)feature.geometry;
579                 res = PQexecPrepared(conn, partionQueryName, 2, paramValues, NULL, NULL, 0);
580                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
581                 {
582                     fprintf(stderr, "%s: INSERT failed: %s", partionQueryName, PQerrorMessage(conn));
583                     PQclear(res);
584                     exit(EXIT_FAILURE);
585                 }
586                 PQclear(res);
587             }
588
589         }
590         else
591         {
592             for (i = 0; i < featureAddressLines; i++)
593             {
594                 xmlFree(featureAddress[i].type);
595                 xmlFree(featureAddress[i].id);
596                 xmlFree(featureAddress[i].key);
597                 xmlFree(featureAddress[i].value);
598                 xmlFree(featureAddress[i].distance);
599             }
600         }
601
602         xmlFree(feature.placeID);
603         xmlFree(feature.type);
604         xmlFree(feature.id);
605         xmlFree(feature.key);
606         xmlFree(feature.value);
607         xmlFree(feature.rankAddress);
608         xmlFree(feature.rankSearch);
609         if (feature.countryCode) xmlFree(feature.countryCode);
610         if (feature.parentPlaceID) xmlFree(feature.parentPlaceID);
611         if (feature.parentType) xmlFree(feature.parentType);
612         if (feature.parentID) xmlFree(feature.parentID);
613 //              if (feature.name) xmlFree(feature.name);
614         if (feature.adminLevel) xmlFree(feature.adminLevel);
615         if (feature.houseNumber) xmlFree(feature.houseNumber);
616         if (feature.geometry) xmlFree(feature.geometry);
617
618 //        PQclear(resPlaceID);
619     }
620 }
621
622 static void processNode(xmlTextReaderPtr reader)
623 {
624     xmlChar *name;
625     name = xmlTextReaderName(reader);
626     if (name == NULL)
627     {
628         name = xmlStrdup(BAD_CAST "--");
629     }
630
631     switch (xmlTextReaderNodeType(reader))
632     {
633     case XML_READER_TYPE_ELEMENT:
634         StartElement(reader, name);
635         if (xmlTextReaderIsEmptyElement(reader))
636             EndElement(reader, name); /* No end_element for self closing tags! */
637         break;
638     case XML_READER_TYPE_END_ELEMENT:
639         EndElement(reader, name);
640         break;
641     case XML_READER_TYPE_TEXT:
642     case XML_READER_TYPE_CDATA:
643     case XML_READER_TYPE_SIGNIFICANT_WHITESPACE:
644         /* Ignore */
645         break;
646     default:
647         fprintf(stderr, "Unknown node type %d\n", xmlTextReaderNodeType(reader));
648         break;
649     }
650
651     xmlFree(name);
652 }
653
654 int nominatim_import(const char *conninfo, const char *partionTagsFilename, const char *filename)
655 {
656     xmlTextReaderPtr    reader;
657     int                                 ret = 0;
658     PGresult *                  res;
659     FILE *                              partionTagsFile;
660     char *                              partionQueryName;
661     char                                partionQuerySQL[1024];
662
663     conn = PQconnectdb(conninfo);
664     if (PQstatus(conn) != CONNECTION_OK)
665     {
666         fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
667         exit(EXIT_FAILURE);
668     }
669
670     partionTableTagsHash = xmlHashCreate(200);
671     partionTableTagsHashDelete = xmlHashCreate(200);
672
673     partionTagsFile = fopen(partionTagsFilename, "rt");
674     if (!partionTagsFile)
675     {
676         fprintf(stderr, "Unable to read partition tags file: %s\n", partionTagsFilename);
677         exit(EXIT_FAILURE);
678     }
679
680     char buffer[1024], osmkey[256], osmvalue[256];
681     int fields;
682     while (fgets(buffer, sizeof(buffer), partionTagsFile) != NULL)
683     {
684         fields = sscanf( buffer, "%23s %63s", osmkey, osmvalue );
685
686         if ( fields <= 0 ) continue;
687
688         if ( fields != 2  )
689         {
690             fprintf( stderr, "Error partition file\n");
691             exit_nicely();
692         }
693         partionQueryName = malloc(strlen("partition_insert_")+strlen(osmkey)+strlen(osmvalue)+2);
694         strcpy(partionQueryName, "partition_insert_");
695         strcat(partionQueryName, osmkey);
696         strcat(partionQueryName, "_");
697         strcat(partionQueryName, osmvalue);
698
699         strcpy(partionQuerySQL, "insert into place_classtype_");
700         strcat(partionQuerySQL, osmkey);
701         strcat(partionQuerySQL, "_");
702         strcat(partionQuerySQL, osmvalue);
703         strcat(partionQuerySQL, " (place_id, centroid) values ($1, ST_Centroid(st_setsrid($2, 4326)))");
704
705         res = PQprepare(conn, partionQueryName, partionQuerySQL, 2, NULL);
706         if (PQresultStatus(res) != PGRES_COMMAND_OK)
707         {
708             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
709             exit(EXIT_FAILURE);
710         }
711
712         xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
713
714         partionQueryName = malloc(strlen("partition_delete_")+strlen(osmkey)+strlen(osmvalue)+2);
715         strcpy(partionQueryName, "partition_delete_");
716         strcat(partionQueryName, osmkey);
717         strcat(partionQueryName, "_");
718         strcat(partionQueryName, osmvalue);
719
720         strcpy(partionQuerySQL, "delete from place_classtype_");
721         strcat(partionQuerySQL, osmkey);
722         strcat(partionQuerySQL, "_");
723         strcat(partionQuerySQL, osmvalue);
724         strcat(partionQuerySQL, " where place_id = $1::integer");
725
726         res = PQprepare(conn, partionQueryName, partionQuerySQL, 1, NULL);
727         if (PQresultStatus(res) != PGRES_COMMAND_OK)
728         {
729             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
730             exit(EXIT_FAILURE);
731         }
732
733         xmlHashAddEntry2(partionTableTagsHashDelete, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
734     }
735
736     res = PQprepare(conn, "get_new_place_id",
737                     "select nextval('seq_place')",
738                     0, NULL);
739     if (PQresultStatus(res) != PGRES_COMMAND_OK)
740     {
741         fprintf(stderr, "Failed to prepare get_new_place_id: %s\n", PQerrorMessage(conn));
742         exit(EXIT_FAILURE);
743     }
744
745     res = PQprepare(conn, "get_place_id",
746                     "select place_id from placex where osm_type = $1 and osm_id = $2 and class = $3 and type = $4",
747                     4, NULL);
748     if (PQresultStatus(res) != PGRES_COMMAND_OK)
749     {
750         fprintf(stderr, "Failed to prepare get_place_id: %s\n", PQerrorMessage(conn));
751         exit(EXIT_FAILURE);
752     }
753
754     res = PQprepare(conn, "placex_insert",
755                     "insert into placex (place_id,osm_type,osm_id,class,type,name,country_code,extratags,parent_place_id,admin_level,housenumber,rank_address,rank_search,geometry) "
756                     "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, st_setsrid($14, 4326))",
757                     12, NULL);
758     if (PQresultStatus(res) != PGRES_COMMAND_OK)
759     {
760         fprintf(stderr, "Failed to prepare placex_insert: %s\n", PQerrorMessage(conn));
761         exit(EXIT_FAILURE);
762     }
763
764     res = PQprepare(conn, "search_name_insert",
765                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
766                     "select place_id, rank_search, rank_address, country_code, make_keywords(name), "
767                     "(select uniq(sort(array_agg(parent_search_name.name_vector))) from search_name as parent_search_name where place_id in "
768                      "(select distinct address_place_id from place_addressline where place_addressline.place_id = $1 limit 1000)"
769                     "), st_centroid(geometry) from placex "
770                     "where place_id = $1",
771                     1, NULL);
772
773     if (PQresultStatus(res) != PGRES_COMMAND_OK)
774     {
775         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
776         exit(EXIT_FAILURE);
777     }
778
779     res = PQprepare(conn, "search_name_from_parent_insert",
780                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
781                     "select place_id, rank_search, rank_address, country_code, make_keywords(name), "
782                     "(select uniq(sort(name_vector+nameaddress_vector)) from search_name as parent_search_name "
783                     "where parent_search_name.place_id = $2 ), st_centroid(geometry) from placex "
784                     "where place_id = $1",
785                     2, NULL);
786     if (PQresultStatus(res) != PGRES_COMMAND_OK)
787     {
788         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
789         exit(EXIT_FAILURE);
790     }
791
792     res = PQprepare(conn, "place_addressline_insert",
793                     "insert into place_addressline (place_id, address_place_id, fromarea, isaddress, distance, cached_rank_address) "
794                     "select $1, place_id, false, $7, $2, rank_address from placex where osm_type = $3 and osm_id = $4 and class = $5 and type = $6",
795                     7, NULL);
796     if (PQresultStatus(res) != PGRES_COMMAND_OK)
797     {
798         fprintf(stderr, "Failed to prepare place_addressline_insert: %s\n", PQerrorMessage(conn));
799         exit(EXIT_FAILURE);
800     }
801
802     res = PQprepare(conn, "placex_delete",
803                     "delete from placex where place_id = $1",
804                     1, NULL);
805     if (PQresultStatus(res) != PGRES_COMMAND_OK)
806     {
807         fprintf(stderr, "Failed to prepare placex_delete: %s\n", PQerrorMessage(conn));
808         exit(EXIT_FAILURE);
809     }
810
811     res = PQprepare(conn, "search_name_delete",
812                     "delete from search_name where place_id = $1",
813                     1, NULL);
814     if (PQresultStatus(res) != PGRES_COMMAND_OK)
815     {
816         fprintf(stderr, "Failed to prepare search_name_delete: %s\n", PQerrorMessage(conn));
817         exit(EXIT_FAILURE);
818     }
819
820     res = PQprepare(conn, "place_addressline_delete",
821                     "delete from place_addressline where place_id = $1",
822                     1, NULL);
823     if (PQresultStatus(res) != PGRES_COMMAND_OK)
824     {
825         fprintf(stderr, "Failed to prepare place_addressline_delete: %s\n", PQerrorMessage(conn));
826         exit(EXIT_FAILURE);
827     }
828
829     featureCount = 0;
830
831     reader = inputUTF8(filename);
832
833     if (reader == NULL)
834     {
835         fprintf(stderr, "Unable to open %s\n", filename);
836         return 1;
837     }
838
839     ret = xmlTextReaderRead(reader);
840     while (ret == 1)
841     {
842         processNode(reader);
843         ret = xmlTextReaderRead(reader);
844     }
845     if (ret != 0)
846     {
847         fprintf(stderr, "%s : failed to parse\n", filename);
848         return ret;
849     }
850
851     xmlFreeTextReader(reader);
852     xmlHashFree(partionTableTagsHash, NULL);
853     xmlHashFree(partionTableTagsHashDelete, NULL);
854
855     return 0;
856 }