Merge remote-tracking branch 'upstream/master'

diff --combined sql/functions.sql
index 13d897d029fa3b8cdd49ab93bdb77caa17021cf7,e45062a8768c0827014f7ad20ea91743c691f6c2..4e77e633b54711b43024d4bb1c3c1ccb596b4fee
--- 1/sql/functions.sql
--- 2/sql/functions.sql
+++ b/sql/functions.sql
@@@ -561,14 -561,6 +561,6 @@@ BEGI
      RETURN nearcountry.country_code;
    END LOOP;
  
- -- RAISE WARNING 'natural earth: %', ST_AsText(place_centre);
- 
-   -- Natural earth data
-   FOR nearcountry IN select country_code from country_naturalearthdata where st_covers(geometry, place_centre) limit 1
-   LOOP
-     RETURN nearcountry.country_code;
-   END LOOP;
- 
  -- RAISE WARNING 'near osm fallback: %', ST_AsText(place_centre);
  
    -- 
@@@ -577,14 -569,6 +569,6 @@@
      RETURN nearcountry.country_code;
    END LOOP;
  
- -- RAISE WARNING 'near natural earth: %', ST_AsText(place_centre);
- 
-   -- Natural earth data 
-   FOR nearcountry IN select country_code from country_naturalearthdata where st_dwithin(geometry, place_centre, 0.5) limit 1
-   LOOP
-     RETURN nearcountry.country_code;
-   END LOOP;
- 
    RETURN NULL;
  END;
  $$
@@@ -817,11 -801,12 +801,12 @@@ DECLAR
    i INTEGER;
    postcode TEXT;
    result BOOLEAN;
+   is_area BOOLEAN;
    country_code VARCHAR(2);
    default_language VARCHAR(10);
    diameter FLOAT;
    classtable TEXT;
-   line RECORD;
+   classtype TEXT;
  BEGIN
    --DEBUG: RAISE WARNING '% % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
  
@@@ -848,148 -833,71 +833,71 @@@
    IF NEW.osm_type = 'X' THEN
      -- E'X'ternal records should already be in the right format so do nothing
    ELSE
-     NEW.rank_search := 30;
-     NEW.rank_address := NEW.rank_search;
- 
-     -- By doing in postgres we have the country available to us - currently only used for postcode
-     IF NEW.class in ('place','boundary') AND NEW.type in ('postcode','postal_code') THEN
+     is_area := ST_GeometryType(NEW.geometry) IN ('ST_Polygon','ST_MultiPolygon');
  
-         IF NEW.address IS NULL OR NOT NEW.address ? 'postcode' THEN
-             -- most likely just a part of a multipolygon postcode boundary, throw it away
-             RETURN NULL;
-         END IF;
+     IF NEW.class in ('place','boundary')
+        AND NEW.type in ('postcode','postal_code') THEN
  
-         NEW.name := hstore('ref', NEW.address->'postcode');
+       IF NEW.address IS NULL OR NOT NEW.address ? 'postcode' THEN
+           -- most likely just a part of a multipolygon postcode boundary, throw it away
+           RETURN NULL;
+       END IF;
  
-         SELECT * FROM get_postcode_rank(NEW.country_code, NEW.address->'postcode')
-           INTO NEW.rank_search, NEW.rank_address;
+       NEW.name := hstore('ref', NEW.address->'postcode');
  
-         IF NOT ST_GeometryType(NEW.geometry) IN ('ST_Polygon','ST_MultiPolygon') THEN
-             NEW.rank_address := 0;
-         END IF;
+       SELECT * FROM get_postcode_rank(NEW.country_code, NEW.address->'postcode')
+         INTO NEW.rank_search, NEW.rank_address;
  
-     ELSEIF NEW.class = 'place' THEN
-       IF NEW.type in ('continent', 'sea') THEN
-         NEW.rank_search := 2;
-         NEW.rank_address := 0;
-         NEW.country_code := NULL;
-       ELSEIF NEW.type in ('country') THEN
-         NEW.rank_search := 4;
-         IF ST_GeometryType(NEW.geometry) IN ('ST_Polygon','ST_MultiPolygon') THEN
-             NEW.rank_address := NEW.rank_search;
-         ELSE
-             NEW.rank_address := 0;
-         END IF;
-       ELSEIF NEW.type in ('state') THEN
-         NEW.rank_search := 8;
-         IF ST_GeometryType(NEW.geometry) IN ('ST_Polygon','ST_MultiPolygon') THEN
-             NEW.rank_address := NEW.rank_search;
-         ELSE
-             NEW.rank_address := 0;
-         END IF;
-       ELSEIF NEW.type in ('region') THEN
-         NEW.rank_search := 18; -- dropped from previous value of 10
-         NEW.rank_address := 0; -- So badly miss-used that better to just drop it!
-       ELSEIF NEW.type in ('county') THEN
-         NEW.rank_search := 12;
-         NEW.rank_address := NEW.rank_search;
-       ELSEIF NEW.type in ('city') THEN
-         NEW.rank_search := 16;
-         NEW.rank_address := NEW.rank_search;
-       ELSEIF NEW.type in ('island') THEN
-         NEW.rank_search := 17;
-         NEW.rank_address := 0;
-       ELSEIF NEW.type in ('town') THEN
-         NEW.rank_search := 18;
-         NEW.rank_address := 16;
-       ELSEIF NEW.type in ('village','hamlet','municipality','district','unincorporated_area','borough') THEN
-         NEW.rank_search := 19;
-         NEW.rank_address := 16;
-       ELSEIF NEW.type in ('suburb','croft','subdivision','isolated_dwelling') THEN
-         NEW.rank_search := 20;
-         NEW.rank_address := NEW.rank_search;
-       ELSEIF NEW.type in ('farm','locality','islet','mountain_pass') THEN
-         NEW.rank_search := 20;
-         NEW.rank_address := 0;
-         -- Irish townlands, tagged as place=locality and locality=townland
-         IF (NEW.extratags -> 'locality') = 'townland' THEN
-           NEW.rank_address := 20;
-         END IF;
-       ELSEIF NEW.type in ('neighbourhood') THEN
-         NEW.rank_search := 22;
-         NEW.rank_address := 22;
-       ELSEIF NEW.type in ('house','building') THEN
-         NEW.rank_search := 30;
-         NEW.rank_address := NEW.rank_search;
-       ELSEIF NEW.type in ('houses') THEN
-         -- can't guarantee all required nodes loaded yet due to caching in osm2pgsql
-         NEW.rank_search := 28;
-         NEW.rank_address := 0;
+       IF NOT is_area THEN
+           NEW.rank_address := 0;
        END IF;
- 
-     ELSEIF NEW.class = 'boundary' THEN
-       IF ST_GeometryType(NEW.geometry) NOT IN ('ST_Polygon','ST_MultiPolygon') THEN
- --        RAISE WARNING 'invalid boundary %',NEW.osm_id;
+     ELSEIF NEW.class = 'boundary' AND NOT is_area THEN
          return NULL;
-       END IF;
-       NEW.rank_search := NEW.admin_level * 2;
-       IF NEW.type = 'administrative' THEN
-         NEW.rank_address := NEW.rank_search;
+     ELSEIF NEW.class = 'railway' AND NEW.type in ('rail') THEN
+         return NULL;
+     ELSEIF NEW.osm_type = 'N' AND NEW.class = 'highway' THEN
+         NEW.rank_search = 30;
+         NEW.rank_address = 0;
+     ELSEIF NEW.class = 'landuse' AND NOT is_area THEN
+         NEW.rank_search = 30;
+         NEW.rank_address = 0;
+     ELSE
+       -- do table lookup stuff
+       IF NEW.class = 'boundary' and NEW.type = 'administrative' THEN
+         classtype = NEW.type || NEW.admin_level::TEXT;
        ELSE
-         NEW.rank_address := 0;
+         classtype = NEW.type;
        END IF;
-     ELSEIF NEW.class = 'landuse' AND ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') THEN
-       NEW.rank_search := 22;
-       IF NEW.type in ('residential', 'farm', 'farmyard', 'industrial', 'commercial', 'allotments', 'retail') THEN
-         NEW.rank_address := NEW.rank_search;
-       ELSE
-         NEW.rank_address := 0;
+       SELECT l.rank_search, l.rank_address FROM address_levels l
+        WHERE (l.country_code = NEW.country_code or l.country_code is NULL)
+              AND l.class = NEW.class AND (l.type = classtype or l.type is NULL)
+        ORDER BY l.country_code, l.class, l.type LIMIT 1
+         INTO NEW.rank_search, NEW.rank_address;
+ 
+       IF NEW.rank_search is NULL THEN
+         NEW.rank_search := 30;
        END IF;
-     ELSEIF NEW.class = 'leisure' and NEW.type in ('park') THEN
-       NEW.rank_search := 24;
-       NEW.rank_address := 0;
-     ELSEIF NEW.class = 'natural' and NEW.type in ('peak','volcano','mountain_range') THEN
-       NEW.rank_search := 18;
-       NEW.rank_address := 0;
-     ELSEIF NEW.class = 'natural' and NEW.type = 'sea' THEN
-       NEW.rank_search := 4;
-       NEW.rank_address := NEW.rank_search;
-     -- any feature more than 5 square miles is probably worth indexing
-     ELSEIF ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_Area(NEW.geometry) > 0.1 THEN
-       NEW.rank_search := 22;
-       NEW.rank_address := 0;
-     ELSEIF NEW.class = 'railway' AND NEW.type in ('rail') THEN
-       RETURN NULL;
-     ELSEIF NEW.class = 'waterway' THEN
-       IF NEW.osm_type = 'R' THEN
-         NEW.rank_search := 16;
-       ELSE
-         NEW.rank_search := 17;
+ 
+       IF NEW.rank_address is NULL THEN
+         NEW.rank_address := 30;
        END IF;
-       NEW.rank_address := 0;
-     ELSEIF NEW.class = 'highway' AND NEW.osm_type != 'N' AND NEW.type in ('service','cycleway','path','footway','steps','bridleway','motorway_link','primary_link','trunk_link','secondary_link','tertiary_link') THEN
-       NEW.rank_search := 27;
-       NEW.rank_address := NEW.rank_search;
-     ELSEIF NEW.class = 'highway' AND NEW.osm_type != 'N' THEN
-       NEW.rank_search := 26;
-       NEW.rank_address := NEW.rank_search;
-     ELSEIF NEW.class = 'mountain_pass' THEN
-         NEW.rank_search := 20;
-         NEW.rank_address := 0;
      END IF;
  
-   END IF;
- 
-   IF NEW.rank_search > 30 THEN
-     NEW.rank_search := 30;
-   END IF;
+     -- some postcorrections
+     IF NEW.class = 'place' THEN
+       IF NEW.type in ('continent', 'sea', 'country', 'state') AND NEW.osm_type = 'N' THEN
+         NEW.rank_address := 0;
+       END IF;
+     ELSEIF NEW.class = 'waterway' AND NEW.osm_type = 'R' THEN
+         -- Slightly promote waterway relations so that they are processed
+         -- before their members.
+         NEW.rank_search := NEW.rank_search - 1;
+     END IF;
  
-   IF NEW.rank_address > 30 THEN
-     NEW.rank_address := 30;
-   END IF;
+     IF (NEW.extratags -> 'capital') = 'yes' THEN
+       NEW.rank_search := NEW.rank_search - 1;
+     END IF;
  
-   IF (NEW.extratags -> 'capital') = 'yes' THEN
-     NEW.rank_search := NEW.rank_search - 1;
    END IF;
  
    -- a country code make no sense below rank 4 (country)
@@@ -2501,7 -2409,7 +2409,7 @@@ BEGI
      select placex.place_id, osm_type, osm_id, name,
        CASE WHEN extratags ? 'place' THEN 'place' ELSE class END as class,
        CASE WHEN extratags ? 'place' THEN extratags->'place' ELSE type END as type,
 -      admin_level, fromarea, isaddress,
 +      admin_level, fromarea, isaddress and linked_place_id is NULL as isaddress,
        CASE WHEN address_place_id = for_place_id AND rank_address = 0 THEN 100 WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address,
        distance,country_code,postcode
        from place_addressline join placex on (address_place_id = placex.place_id) 

diff --combined utils/update.php
index eebd686de62f2e47a8275923b2b0a2d86ed69daf,7cb21879a847b0611a341e8807170bad8c08e07e..a5170ae590dd9f3cc20606021237cc0a314600b4
--- 1/utils/update.php
--- 2/utils/update.php
+++ b/utils/update.php
@@@ -5,6 -5,7 +5,7 @@@ require_once(dirname(dirname(__FILE__))
  require_once(CONST_BasePath.'/lib/init-cmd.php');
  require_once(CONST_BasePath.'/lib/setup_functions.php');
  require_once(CONST_BasePath.'/lib/setup/SetupClass.php');
+ require_once(CONST_BasePath.'/lib/setup/AddressLevelParser.php');
  
  ini_set('memory_limit', '800M');
  
@@@ -42,13 -43,13 +43,14 @@@ $aCMDOption
  
     array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
     array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
+    array('update-address-levels', '', 0, 1, 0, 0, 'bool', 'Reimport address level configuration (EXPERT)'),
     array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
    );
  
  getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
  
  if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
 +
  if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
  
  date_default_timezone_set('Etc/UTC');
@@@ -308,6 -309,12 +310,12 @@@ if ($aResult['index']) 
      runWithEnv($sCmd, $aProcEnv);
  }
  
+ if ($aResult['update-address-levels']) {
+     echo 'Updating address levels from '.CONST_Address_Level_Config.".\n";
+     $oAlParser = new \Nominatim\Setup\AddressLevelParser(CONST_Address_Level_Config);
+     $oAlParser->createTable($oDB, 'address_levels');
+ }
+ 
  if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
      //
      if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {