From: Sarah Hoffmann <lonvia@denofr.de>
Date: Fri, 29 Sep 2017 17:44:09 +0000 (+0200)
Subject: Merge remote-tracking branch 'upstream/master'
X-Git-Tag: deploy~372
X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/2e6f5ba7e68d52aff93775ebffcdb5c59f7f65df?hp=115c909fdf65966774940ef957cf0bd63ba844f2

Merge remote-tracking branch 'upstream/master'
---

diff --git a/docs/Install-on-Centos-7.md b/docs/Install-on-Centos-7.md
index 35200d6f..5156fa02 100644
--- a/docs/Install-on-Centos-7.md
+++ b/docs/Install-on-Centos-7.md
@@ -160,6 +160,7 @@ download the country grid:
 The code must be built in a separate directory. Create this directory,
 then configure and build Nominatim in there:
 
+
     mkdir build
     cd build
     cmake $USERHOME/Nominatim
diff --git a/docs/Install-on-Ubuntu-16.md b/docs/Install-on-Ubuntu-16.md
index 322a476a..849e00b9 100644
--- a/docs/Install-on-Ubuntu-16.md
+++ b/docs/Install-on-Ubuntu-16.md
@@ -32,7 +32,8 @@ Now you can install all packages needed for Nominatim:
 If you want to run the test suite, you need to install the following
 additional packages:
 
-    sudo apt-get install -y python3-dev python3-pip python3-psycopg2 python3-tidylib phpunit
+    sudo apt-get install -y python3-setuptools python3-dev python3-pip \
+                            python3-psycopg2 python3-tidylib phpunit php-cgi
 
     pip3 install --user behave nose # urllib3
     sudo pear install PHP_CodeSniffer
@@ -147,6 +148,7 @@ download the country grid:
 The code must be built in a separate directory. Create this directory,
 then configure and build Nominatim in there:
 
+
     mkdir build
     cd build
     cmake $USERHOME/Nominatim
diff --git a/lib/Geocode.php b/lib/Geocode.php
index eb6152aa..5acb01d0 100644
--- a/lib/Geocode.php
+++ b/lib/Geocode.php
@@ -304,7 +304,7 @@ class Geocode
         $aViewbox = $oParams->getStringList('viewboxlbrt');
         if ($aViewbox) {
             if (count($aViewbox) != 4) {
-                userError("Bad parmater 'viewbox'. Expected 4 coordinates.");
+                userError("Bad parmater 'viewboxlbrt'. Expected 4 coordinates.");
             }
             $this->setViewbox($aViewbox);
         } else {
@@ -372,7 +372,7 @@ class Geocode
         $this->aAddressRankList = array();
 
         $this->aStructuredQuery = array();
-        $this->sAllowedTypesSQLList = '';
+        $this->sAllowedTypesSQLList = False;
 
         $this->loadStructuredAddressElement($sAmenity, 'amenity', 26, 30, false);
         $this->loadStructuredAddressElement($sStreet, 'street', 26, 30, false);
@@ -385,7 +385,7 @@ class Geocode
         if (sizeof($this->aStructuredQuery) > 0) {
             $this->sQuery = join(', ', $this->aStructuredQuery);
             if ($this->iMaxAddressRank < 30) {
-                $sAllowedTypesSQLList = '(\'place\',\'boundary\')';
+                $this->sAllowedTypesSQLList = '(\'place\',\'boundary\')';
             }
         }
     }
@@ -1021,15 +1021,24 @@ class Geocode
 
             // Any 'special' terms in the search?
             $bSpecialTerms = false;
-            preg_match_all('/\\[(.*)=(.*)\\]/', $sQuery, $aSpecialTermsRaw, PREG_SET_ORDER);
-            $aSpecialTerms = array();
+            preg_match_all('/\\[([\\w_]*)=([\\w_]*)\\]/', $sQuery, $aSpecialTermsRaw, PREG_SET_ORDER);
             foreach ($aSpecialTermsRaw as $aSpecialTerm) {
                 $sQuery = str_replace($aSpecialTerm[0], ' ', $sQuery);
-                $aSpecialTerms[strtolower($aSpecialTerm[1])] = $aSpecialTerm[2];
+                if (!$bSpecialTerms) {
+                    $aNewSearches = array();
+                    foreach ($aSearches as $aSearch) {
+                        $aNewSearch = $aSearch;
+                        $aNewSearch['sClass'] = $aSpecialTerm[1];
+                        $aNewSearch['sType'] = $aSpecialTerm[2];
+                        $aNewSearches[] = $aNewSearch;
+                    }
+
+                    $aSearches = $aNewSearches;
+                    $bSpecialTerms = true;
+                }
             }
 
             preg_match_all('/\\[([\\w ]*)\\]/u', $sQuery, $aSpecialTermsRaw, PREG_SET_ORDER);
-            $aSpecialTerms = array();
             if (isset($this->aStructuredQuery['amenity']) && $this->aStructuredQuery['amenity']) {
                 $aSpecialTermsRaw[] = array('['.$this->aStructuredQuery['amenity'].']', $this->aStructuredQuery['amenity']);
                 unset($this->aStructuredQuery['amenity']);
@@ -1037,6 +1046,10 @@ class Geocode
 
             foreach ($aSpecialTermsRaw as $aSpecialTerm) {
                 $sQuery = str_replace($aSpecialTerm[0], ' ', $sQuery);
+                if ($bSpecialTerms) {
+                    continue;
+                }
+
                 $sToken = chksql($this->oDB->getOne("SELECT make_standard_name('".$aSpecialTerm[1]."') AS string"));
                 $sSQL = 'SELECT * ';
                 $sSQL .= 'FROM ( ';
@@ -1044,25 +1057,17 @@ class Geocode
                 $sSQL .= '   FROM word ';
                 $sSQL .= '   WHERE word_token in (\' '.$sToken.'\')';
                 $sSQL .= ') AS x ';
-                $sSQL .= ' WHERE (class is not null AND class not in (\'place\')) ';
-                $sSQL .= ' OR country_code is not null';
+                $sSQL .= ' WHERE (class is not null AND class not in (\'place\'))';
                 if (CONST_Debug) var_Dump($sSQL);
                 $aSearchWords = chksql($this->oDB->getAll($sSQL));
                 $aNewSearches = array();
                 foreach ($aSearches as $aSearch) {
                     foreach ($aSearchWords as $aSearchTerm) {
                         $aNewSearch = $aSearch;
-                        if ($aSearchTerm['country_code']) {
-                            $aNewSearch['sCountryCode'] = strtolower($aSearchTerm['country_code']);
-                            $aNewSearches[] = $aNewSearch;
-                            $bSpecialTerms = true;
-                        }
-                        if ($aSearchTerm['class']) {
-                            $aNewSearch['sClass'] = $aSearchTerm['class'];
-                            $aNewSearch['sType'] = $aSearchTerm['type'];
-                            $aNewSearches[] = $aNewSearch;
-                            $bSpecialTerms = true;
-                        }
+                        $aNewSearch['sClass'] = $aSearchTerm['class'];
+                        $aNewSearch['sType'] = $aSearchTerm['type'];
+                        $aNewSearches[] = $aNewSearch;
+                        $bSpecialTerms = true;
                     }
                 }
                 $aSearches = $aNewSearches;
@@ -1269,8 +1274,8 @@ class Geocode
                     }
 
                     // No location term?
-                    if (!sizeof($aSearch['aName']) && !sizeof($aSearch['aAddress']) && !$aSearch['oNear']) {
-                        if ($aSearch['sCountryCode'] && !$aSearch['sClass'] && !$aSearch['sHouseNumber']) {
+                    if (!sizeof($aSearch['aName']) && !sizeof($aSearch['aAddress'])) {
+                        if ($aSearch['sCountryCode'] && !$aSearch['sClass'] && !$aSearch['sHouseNumber'] && !$aSearch['oNear']) {
                             // Just looking for a country by code - look it up
                             if (4 >= $this->iMinAddressRank && 4 <= $this->iMaxAddressRank) {
                                 $sSQL = "SELECT place_id FROM placex WHERE country_code='".$aSearch['sCountryCode']."' AND rank_search = 4";
@@ -1290,39 +1295,32 @@ class Geocode
                             if (chksql($this->oDB->getOne($sSQL))) {
                                 $sSQL = "SELECT place_id FROM place_classtype_".$aSearch['sClass']."_".$aSearch['sType']." ct";
                                 if ($sCountryCodesSQL) $sSQL .= " JOIN placex USING (place_id)";
-                                $sSQL .= " WHERE st_contains($this->sViewboxSmallSQL, ct.centroid)";
+                                if ($aSearch['oNear']) {
+                                    $sSQL .= " WHERE ".$aSearch['oNear']->withinSQL('ct.centroid');
+                                } else {
+                                    $sSQL .= " WHERE st_contains($this->sViewboxSmallSQL, ct.centroid)";
+                                }
                                 if ($sCountryCodesSQL) $sSQL .= " AND country_code in ($sCountryCodesSQL)";
                                 if (sizeof($this->aExcludePlaceIDs)) {
                                     $sSQL .= " AND place_id not in (".join(',', $this->aExcludePlaceIDs).")";
                                 }
-                                if ($this->sViewboxCentreSQL) $sSQL .= " ORDER BY ST_Distance($this->sViewboxCentreSQL, ct.centroid) ASC";
+                                if ($this->sViewboxCentreSQL) {
+                                    $sSQL .= " ORDER BY ST_Distance($this->sViewboxCentreSQL, ct.centroid) ASC";
+                                } elseif ($aSearch['oNear']) {
+                                    $sSQL .= " ORDER BY ".$aSearch['oNear']->distanceSQL('ct.centroid').' ASC';
+                                }
                                 $sSQL .= " limit $this->iLimit";
                                 if (CONST_Debug) var_dump($sSQL);
                                 $aPlaceIDs = chksql($this->oDB->getCol($sSQL));
-
-                                // If excluded place IDs are given, it is fair to assume that
-                                // there have been results in the small box, so no further
-                                // expansion in that case.
-                                // Also don't expand if bounded results were requested.
-                                if (!sizeof($aPlaceIDs) && !sizeof($this->aExcludePlaceIDs) && !$this->bBoundedSearch) {
-                                    $sSQL = "SELECT place_id FROM place_classtype_".$aSearch['sClass']."_".$aSearch['sType']." ct";
-                                    if ($sCountryCodesSQL) $sSQL .= " join placex using (place_id)";
-                                    $sSQL .= " WHERE ST_Contains($this->sViewboxLargeSQL, ct.centroid)";
-                                    if ($sCountryCodesSQL) $sSQL .= " AND country_code in ($sCountryCodesSQL)";
-                                    if ($this->sViewboxCentreSQL) $sSQL .= " ORDER BY ST_Distance($this->sViewboxCentreSQL, ct.centroid) ASC";
-                                    $sSQL .= " LIMIT $this->iLimit";
-                                    if (CONST_Debug) var_dump($sSQL);
-                                    $aPlaceIDs = chksql($this->oDB->getCol($sSQL));
-                                }
-                            } else {
+                            } else if ($aSearch['oNear']) {
                                 $sSQL = "SELECT place_id ";
                                 $sSQL .= "FROM placex ";
                                 $sSQL .= "WHERE class='".$aSearch['sClass']."' ";
                                 $sSQL .= "  AND type='".$aSearch['sType']."'";
-                                $sSQL .= "  AND ST_Contains($this->sViewboxSmallSQL, geometry) ";
+                                $sSQL .= "  AND ".$aSearch['oNear']->withinSQL('geometry');
                                 $sSQL .= "  AND linked_place_id is null";
                                 if ($sCountryCodesSQL) $sSQL .= " AND country_code in ($sCountryCodesSQL)";
-                                if ($this->sViewboxCentreSQL)   $sSQL .= " ORDER BY ST_Distance($this->sViewboxCentreSQL, centroid) ASC";
+                                $sSQL .= " ORDER BY ".$aSearch['oNear']->distanceSQL('centroid')." ASC";
                                 $sSQL .= " LIMIT $this->iLimit";
                                 if (CONST_Debug) var_dump($sSQL);
                                 $aPlaceIDs = chksql($this->oDB->getCol($sSQL));
@@ -1404,7 +1402,7 @@ class Geocode
                         if ($aSearch['sCountryCode']) $aTerms[] = "country_code = '".pg_escape_string($aSearch['sCountryCode'])."'";
                         if ($aSearch['sHouseNumber']) {
                             $aTerms[] = "address_rank between 16 and 27";
-                        } else {
+                        } elseif (!$aSearch['sClass'] || $aSearch['sOperator'] == 'name') {
                             if ($this->iMinAddressRank > 0) {
                                 $aTerms[] = "address_rank >= ".$this->iMinAddressRank;
                             }
@@ -1647,7 +1645,7 @@ class Geocode
                                         } elseif ($sPlaceIDs) {
                                             $sOrderBySQL = "ST_Distance(l.centroid, f.geometry)";
                                         } elseif ($sPlaceGeom) {
-                                            $sOrderBysSQL = "ST_Distance(st_centroid('".$sPlaceGeom."'), l.centroid)";
+                                            $sOrderBySQL = "ST_Distance(st_centroid('".$sPlaceGeom."'), l.centroid)";
                                         }
 
                                         $sSQL = "select distinct i.place_id".($sOrderBySQL?', i.order_term':'')." from (";
@@ -1683,7 +1681,7 @@ class Geocode
                                             $sOrderBySQL = "ST_Distance(l.geometry, f.geometry)";
                                         }
 
-                                        $sSQL = "SELECT distinct l.place_id".($sOrderBysSQL?','.$sOrderBysSQL:'');
+                                        $sSQL = "SELECT distinct l.place_id".($sOrderBySQL?','.$sOrderBySQL:'');
                                         $sSQL .= " FROM placex as l, placex as f ";
                                         $sSQL .= " WHERE f.place_id in ($sPlaceIDs) ";
                                         $sSQL .= "  AND ST_DWithin(l.geometry, f.centroid, $fRange) ";
@@ -1693,7 +1691,7 @@ class Geocode
                                             $sSQL .= " AND l.place_id not in (".join(',', $this->aExcludePlaceIDs).")";
                                         }
                                         if ($sCountryCodesSQL) $sSQL .= " AND l.country_code in ($sCountryCodesSQL)";
-                                        if ($sOrderBy) $sSQL .= "ORDER BY ".$OrderBysSQL." ASC";
+                                        if ($sOrderBySQL) $sSQL .= "ORDER BY ".$sOrderBySQL." ASC";
                                         if ($this->iOffset) $sSQL .= " OFFSET $this->iOffset";
                                         $sSQL .= " limit $this->iLimit";
                                         if (CONST_Debug) var_dump($sSQL);
diff --git a/lib/NearPoint.php b/lib/NearPoint.php
index e8d595cc..4f0531d9 100644
--- a/lib/NearPoint.php
+++ b/lib/NearPoint.php
@@ -133,8 +133,8 @@ class NearPoint
             $sFound    = $aData[0];
             $fQueryLat = ($aData[2]=='N'?1:-1) * ($aData[1]);
             $fQueryLon = ($aData[4]=='E'?1:-1) * ($aData[3]);
-        } elseif (preg_match('/(\\[|^|\\b)(-?[0-9]+[0-9]*\\.[0-9]+)[, ]+(-?[0-9]+[0-9]*\\.[0-9]+)(\\]|$|\\b)/', $sQuery, $aData)) {
-            /*                 1          2                             3                        4
+        } elseif (preg_match('/(\\[|^|\\b)?(-?[0-9]+[0-9]*\\.[0-9]+)[, ]+(-?[0-9]+[0-9]*\\.[0-9]+)(\\]|$|\\b)/', $sQuery, $aData)) {
+            /*                 1           2                             3                        4
              * degrees decimal
              * 12.34, 56.78
              * [12.456,-78.90]
diff --git a/lib/cmd.php b/lib/cmd.php
index 37ba87b8..d7ba285d 100644
--- a/lib/cmd.php
+++ b/lib/cmd.php
@@ -128,3 +128,23 @@ function chksql($oSql, $sMsg = false)
 
     return $oSql;
 }
+
+function info($sMsg)
+{
+    echo date('Y-m-d H:i:s == ').$sMsg."\n";
+}
+
+$aWarnings = [];
+
+function warn($sMsg)
+{
+    $GLOBALS['aWarnings'][] = $sMsg;
+    echo date('Y-m-d H:i:s == ').'WARNING: '.$sMsg."\n";
+}
+
+function repeatWarnings()
+{
+    foreach ($GLOBALS['aWarnings'] as $sMsg) {
+        echo '  * ',$sMsg."\n";
+    }
+}
diff --git a/lib/lib.php b/lib/lib.php
index 28b44c50..b7564c7f 100644
--- a/lib/lib.php
+++ b/lib/lib.php
@@ -545,8 +545,8 @@ function _debugDumpGroupedSearches($aData, $aTokens)
             echo "<td>".$aRow['sPostcode']."</td>";
             echo "<td>".$aRow['sHouseNumber']."</td>";
 
-            echo "<td>".$aRow['fLat']."</td>";
-            echo "<td>".$aRow['fLon']."</td>";
+            echo "<td>".$aRow['oNear']->lat()."</td>";
+            echo "<td>".$aRow['oNear']->lon()."</td>";
             echo "<td>".$aRow['fRadius']."</td>";
 
             echo "</tr>";
diff --git a/nominatim/index.c b/nominatim/index.c
index 9aa3fb86..ab87bdca 100644
--- a/nominatim/index.c
+++ b/nominatim/index.c
@@ -218,7 +218,7 @@ struct index_thread_data * thread_data, const char *structuredoutputfile)
                     usleep(1000);
 
                     // Aim for one update per second
-                    if (sleepcount++ > 500)
+                    if (sleepcount++ > 1000)
                     {
                         rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
                         if(interpolation)
diff --git a/test/README.md b/test/README.md
index f6c1ac2a..87782653 100644
--- a/test/README.md
+++ b/test/README.md
@@ -123,15 +123,17 @@ Before importing make sure to add the following to your local settings:
 #### Code Coverage
 
 The API tests also support code coverage tests. You need to install
-PHP_CodeCoverage. On Debian/Ubuntu run:
+[PHP_CodeCoverage](https://github.com/sebastianbergmann/php-code-coverage).
+On Debian/Ubuntu run:
 
-    apt-get install php-codecoverage
+    apt-get install php-codecoverage php-xdebug
 
 The run the API tests as follows:
 
     behave api -DPHPCOV=<coverage output dir>
 
-To generate reports, you can use the phpcov tool:
+The output directory must be an absolute path. To generate reports, you can use
+the [phpcov](https://github.com/sebastianbergmann/phpcov) tool:
 
     phpcov merge --html=<report output dir> <coverage output dir>
 
diff --git a/test/bdd/api/reverse/params.feature b/test/bdd/api/reverse/params.feature
index 765c91c3..1de31c9d 100644
--- a/test/bdd/api/reverse/params.feature
+++ b/test/bdd/api/reverse/params.feature
@@ -15,6 +15,15 @@ Feature: Parameters for Reverse API
       | jsonv2 |
       | xml |
 
+    Scenario Outline: Coordinates must be floating-point numbers
+        When sending reverse coordinates <coords>
+        Then a HTTP 400 is returned
+
+    Examples:
+      | coords    |
+      | -45.3,;   |
+      | gkjd,50   |
+
     Scenario Outline: Reverse Geocoding with extratags
         When sending <format> reverse coordinates 10.776234290950017,106.70425325632095
           | extratags |
diff --git a/test/bdd/api/search/params.feature b/test/bdd/api/search/params.feature
index bfc7cb15..b184fd86 100644
--- a/test/bdd/api/search/params.feature
+++ b/test/bdd/api/search/params.feature
@@ -19,6 +19,12 @@ Feature: Search queries
         And result 0 has not attributes address
         And result 0 has bounding box in 46.5,47.5,9,10
 
+    Scenario: Unknown formats returns a user error
+        When sending search query "Vaduz"
+          | format |
+          | x45    |
+        Then a HTTP 400 is returned
+
     Scenario: JSON search with addressdetails
         When sending json search query "Montevideo" with address
         Then address of result 0 is
@@ -73,6 +79,12 @@ Feature: Search queries
           | city |
           | Montevideo |
 
+    Scenario: Country search with bounded viewbox remain in the area
+        When sending json search query "" with address
+          | bounded | viewbox                                 | country |
+          | 1       | -56.16786,-34.84061,-56.12525,-34.86526 | de |
+        Then less than 1 result is returned
+
     Scenario: Search with bounded viewboxlbrt in right area
         When sending json search query "bar" with address
           | bounded | viewboxlbrt |
@@ -90,7 +102,7 @@ Feature: Search queries
           | ^[^,]*[Rr]estaurant.* |
 
     Scenario: bounded search remains within viewbox, even with no results
-         When sending json search query "restaurant"
+         When sending json search query "[restaurant]"
            | bounded | viewbox |
            | 1       | 43.5403125,-5.6563282,43.54285,-5.662003 |
         Then less than 1 result is returned
@@ -115,6 +127,38 @@ Feature: Search queries
           | ID | state |
           | 0  | Florida |
 
+    Scenario: viewboxes cannot be points
+        When sending json search query "foo"
+          | viewbox |
+          | 1.01,34.6,1.01,34.6 |
+        Then a HTTP 400 is returned
+
+    Scenario Outline: viewbox must have four coordinate numbers
+        When sending json search query "foo"
+          | viewbox |
+          | <viewbox> |
+        Then a HTTP 400 is returned
+
+    Examples:
+        | viewbox |
+        | 34      |
+        | 0.003,-84.4 |
+        | 5.2,4.5542,12.4 |
+        | 23.1,-6,0.11,44.2,9.1 |
+
+    Scenario Outline: viewboxlbrt must have four coordinate numbers
+        When sending json search query "foo"
+          | viewboxlbrt |
+          | <viewbox> |
+        Then a HTTP 400 is returned
+
+    Examples:
+        | viewbox |
+        | 34      |
+        | 0.003,-84.4 |
+        | 5.2,4.5542,12.4 |
+        | 23.1,-6,0.11,44.2,9.1 |
+
     Scenario: Overly large limit number for search results
         When sending json search query "restaurant"
           | limit |
@@ -127,6 +171,12 @@ Feature: Search queries
           | 4 |
         Then exactly 4 results are returned
 
+    Scenario: Limit parameter must be a number
+        When sending search query "Blue Laguna"
+          | limit |
+          | );    |
+        Then a HTTP 400 is returned
+
     Scenario: Restrict to feature type country
         When sending xml search query "Uruguay"
         Then results contain
@@ -298,3 +348,11 @@ Feature: Search queries
         | xml      | geojson |
         | json     | geojson |
         | jsonv2   | geojson |
+
+    Scenario: Search along a route
+        When sending json search query "restaurant" with address
+          | bounded | routewidth | route                                   |
+          | 1       | 0.1        | -103.23255,44.08198,-103.22516,44.08079 |
+        Then result addresses contain
+          | city |
+          | Rapid City |
diff --git a/test/bdd/api/search/postcode.feature b/test/bdd/api/search/postcode.feature
new file mode 100644
index 00000000..f92aff3c
--- /dev/null
+++ b/test/bdd/api/search/postcode.feature
@@ -0,0 +1,20 @@
+@APIDB
+Feature: Searches with postcodes
+    Various searches involving postcodes
+
+    Scenario: US 5+4 ZIP codes are shortened to 5 ZIP codes if not found
+        When sending json search query "57701 1111, us" with address
+        Then result addresses contain
+            | postcode |
+            | 57701    |
+
+    Scenario: Postcode search with address
+        When sending json search query "9486, mauren"
+        Then at least 1 result is returned
+
+    Scenario: Postcode search with country
+        When sending json search query "9486, li" with address
+        Then result addresses contain
+            | country_code |
+            | li           |
+
diff --git a/test/bdd/api/search/queries.feature b/test/bdd/api/search/queries.feature
index 638177fd..49e1d9c0 100644
--- a/test/bdd/api/search/queries.feature
+++ b/test/bdd/api/search/queries.feature
@@ -57,6 +57,45 @@ Feature: Search queries
           | place_rank |
           | 30 |
 
+    Scenario: Search with specific amenity
+        When sending json search query "[restaurant] Vaduz" with address
+        Then result addresses contain
+          | country |
+          | Liechtenstein |
+        And  results contain
+          | class   | type |
+          | amenity | restaurant |
+
+    Scenario: Search with key-value amenity
+        When sending json search query "[shop=hifi] hamburg"
+        Then results contain
+          | class | type |
+          | shop  | hifi |
+
+    Scenario: With multiple amenity search only the first is used
+        When sending json search query "[shop=hifi] [church] hamburg"
+        Then results contain
+          | class | type |
+          | shop  | hifi |
+
+    Scenario: With multiple amenity search only the first is used
+        When sending json search query "[church] [restaurant] hamburg"
+        Then results contain
+          | class   | type |
+          | amenity | place_of_worship |
+
+    Scenario: POI search near given coordinate
+        When sending json search query "restaurant near 47.16712,9.51100"
+        Then results contain
+          | class   | type |
+          | amenity | restaurant |
+
+    Scenario: Arbitrary key/value search near given coordinate
+        When sending json search query "[man_made=mast]  47.15739,9.61264"
+        Then results contain
+          | class    | type |
+          | man_made | mast |
+
     # https://trac.openstreetmap.org/ticket/5094
     Scenario: housenumbers are ordered by complete match first
         When sending json search query "6395 geminis, montevideo" with address
diff --git a/test/bdd/api/search/structured.feature b/test/bdd/api/search/structured.feature
index c93603d6..f45a1a6d 100644
--- a/test/bdd/api/search/structured.feature
+++ b/test/bdd/api/search/structured.feature
@@ -31,6 +31,17 @@ Feature: Structured search queries
           | attr        | value |
           | querystring | Old Palace Road, GU2 7UP, United Kingdom |
 
+    Scenario: Amenity, city
+        When sending json search query "" with address
+          | city  | amenity |
+          | Vaduz | church  |
+        Then result addresses contain
+          | country |
+          | Liechtenstein |
+        And  results contain
+          | class   | type |
+          | amenity | place_of_worship |
+
     Scenario: gihub #176
         When sending json search query "" with address
           | city |
diff --git a/test/bdd/steps/cgi-with-coverage.php b/test/bdd/steps/cgi-with-coverage.php
index 17104d47..165a1b0c 100644
--- a/test/bdd/steps/cgi-with-coverage.php
+++ b/test/bdd/steps/cgi-with-coverage.php
@@ -1,13 +1,20 @@
 <?php
 require_once 'SebastianBergmann/CodeCoverage/autoload.php';
+
+function coverage_shutdown($oCoverage)
+{
+    $oCoverage->stop();
+    $writer = new \SebastianBergmann\CodeCoverage\Report\PHP;
+    $writer->process($oCoverage, $_SERVER['PHP_CODE_COVERAGE_FILE']);
+}
+
 $covfilter = new SebastianBergmann\CodeCoverage\Filter();
 $covfilter->addDirectoryToWhitelist($_SERVER['COV_PHP_DIR']);
 $coverage = new SebastianBergmann\CodeCoverage\CodeCoverage(null, $covfilter);
 $coverage->start($_SERVER['COV_TEST_NAME']);
 
+register_shutdown_function('coverage_shutdown', $coverage);
+
 include $_SERVER['COV_SCRIPT_FILENAME'];
 
 
-$coverage->stop();
-$writer = new \SebastianBergmann\CodeCoverage\Report\PHP;
-$writer->process($coverage, $_SERVER['PHP_CODE_COVERAGE_FILE']);
diff --git a/test/bdd/steps/queries.py b/test/bdd/steps/queries.py
index 443342b0..963aad4d 100644
--- a/test/bdd/steps/queries.py
+++ b/test/bdd/steps/queries.py
@@ -349,7 +349,7 @@ def website_search_request(context, fmt, query, addr):
 
     context.response = SearchResponse(outp, outfmt, status)
 
-@when(u'sending (?P<fmt>\S+ )?reverse coordinates (?P<lat>[0-9.-]+)?,(?P<lon>[0-9.-]+)?')
+@when(u'sending (?P<fmt>\S+ )?reverse coordinates (?P<lat>.+)?,(?P<lon>.+)?')
 def website_reverse_request(context, fmt, lat, lon):
     params = {}
     if lat is not None:
diff --git a/test/php/Nominatim/NearPointTest.php b/test/php/Nominatim/NearPointTest.php
index 6fa9b515..5ad73451 100644
--- a/test/php/Nominatim/NearPointTest.php
+++ b/test/php/Nominatim/NearPointTest.php
@@ -35,6 +35,10 @@ class NearPointTest extends \PHPUnit_Framework_TestCase
         $this->assertEquals($aRes['pt']->radius(), 0.1);
         $this->assertEquals($aRes['query'], '');
 
+        $aRes = NearPoint::extractFromQuery(' -12.456,-78.90 ');
+        $this->assertEquals($aRes['pt']->lat(), -12.456);
+        $this->assertEquals($aRes['pt']->lon(), -78.90);
+
         // http://en.wikipedia.org/wiki/Geographic_coordinate_conversion
         // these all represent the same location
         $aQueries = array(
diff --git a/utils/setup.php b/utils/setup.php
index 67c7fd7f..2369b183 100755
--- a/utils/setup.php
+++ b/utils/setup.php
@@ -65,11 +65,11 @@ if ($aCMDResult['import-data'] || $aCMDResult['all']) {
 $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
 if ($iInstances < 1) {
     $iInstances = 1;
-    echo "WARNING: resetting threads to $iInstances\n";
+    warn("resetting threads to $iInstances");
 }
 if ($iInstances > getProcessorCount()) {
     $iInstances = getProcessorCount();
-    echo "WARNING: resetting threads to $iInstances\n";
+    warn("resetting threads to $iInstances");
 }
 
 // Assume we can steal all the cache memory in the box (unless told otherwise)
@@ -83,7 +83,7 @@ $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 
 if ($aCMDResult['create-db'] || $aCMDResult['all']) {
-    echo "Create DB\n";
+    info("Create DB");
     $bDidSomething = true;
     $oDB = DB::connect(CONST_Database_DSN, false);
     if (!PEAR::isError($oDB)) {
@@ -93,11 +93,9 @@ if ($aCMDResult['create-db'] || $aCMDResult['all']) {
 }
 
 if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
-    echo "Setup DB\n";
+    info("Setup DB");
     $bDidSomething = true;
 
-    // TODO: path detection, detection memory, etc.
-    //
     $oDB =& getDB();
 
     $fPostgresVersion = getPostgresVersion($oDB);
@@ -117,7 +115,7 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
 
     if ($iNumFunc == 0) {
         pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable");
-        echo "WARNING: Postgresql is too old. extratags and namedetails API not available.";
+        warn('Postgresql is too old. extratags and namedetails API not available.');
     }
 
     $fPostgisVersion = getPostgisVersion($oDB);
@@ -132,6 +130,26 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
         pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid');
     }
 
+    $i = chksql($oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'"));
+    if ($i == 0) {
+        echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n";
+        echo "\n          createuser ".CONST_Database_Web_User."\n\n";
+        exit(1);
+    }
+
+    // Try accessing the C module, so we know early if something is wrong
+    // and can simply error out.
+    $sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '";
+    $sSQL .= CONST_InstallPath."/module/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT";
+    $sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);';
+    $oResult = $oDB->query($sSQL);
+
+    if (PEAR::isError($oResult)) {
+        echo "\nERROR: Failed to load nominatim module. Reason:\n";
+        echo $oResult->userinfo."\n\n";
+        exit(1);
+    }
+
     if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) {
         echo "Error: you need to download the country_osm_grid first:";
         echo "\n    wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz http://www.nominatim.org/data/country_grid.sql.gz\n";
@@ -145,7 +163,7 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
     if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) {
         pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz');
     } else {
-        echo "WARNING: external UK postcode table not found.\n";
+        warn('external UK postcode table not found.');
     }
     if (CONST_Use_Extra_US_Postcodes) {
         pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
@@ -164,7 +182,7 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
 }
 
 if ($aCMDResult['import-data'] || $aCMDResult['all']) {
-    echo "Import\n";
+    info('Import data');
     $bDidSomething = true;
 
     $osm2pgsql = CONST_Osm2pgsql_Binary;
@@ -198,16 +216,18 @@ if ($aCMDResult['import-data'] || $aCMDResult['all']) {
 }
 
 if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
-    echo "Functions\n";
+    info('Create Functions');
     $bDidSomething = true;
-    if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) fail("nominatim module not built");
+    if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) {
+        fail("nominatim module not built");
+    }
     create_sql_functions($aCMDResult);
 }
 
 if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
+    info('Create Tables');
     $bDidSomething = true;
 
-    echo "Tables\n";
     $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');
     $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
     $sTemplate = replace_tablespace(
@@ -243,12 +263,12 @@ if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
     pgsqlRunScript($sTemplate, false);
 
     // re-run the functions
-    echo "Functions\n";
+    info('Recreate Functions');
     create_sql_functions($aCMDResult);
 }
 
 if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
-    echo "Partition Tables\n";
+    info('Create Partition Tables');
     $bDidSomething = true;
 
     $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
@@ -288,7 +308,7 @@ if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
 
 
 if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
-    echo "Partition Functions\n";
+    info('Create Partition Functions');
     $bDidSomething = true;
 
     $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
@@ -301,24 +321,22 @@ if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
     $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin';
     $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin';
     if (file_exists($sWikiArticlesFile)) {
-        echo "Importing wikipedia articles...";
+        info('Importing wikipedia articles');
         pgsqlRunDropAndRestore($sWikiArticlesFile);
-        echo "...done\n";
     } else {
-        echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
+        warn('wikipedia article dump file not found - places will have default importance');
     }
     if (file_exists($sWikiRedirectsFile)) {
-        echo "Importing wikipedia redirects...";
+        info('Importing wikipedia redirects');
         pgsqlRunDropAndRestore($sWikiRedirectsFile);
-        echo "...done\n";
     } else {
-        echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
+        warn('wikipedia redirect dump file not found - some place importance values may be missing');
     }
 }
 
 
 if ($aCMDResult['load-data'] || $aCMDResult['all']) {
-    echo "Drop old Data\n";
+    info('Drop old Data');
     $bDidSomething = true;
 
     $oDB =& getDB();
@@ -361,11 +379,11 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) {
 
     // pre-create the word list
     if (!$aCMDResult['disable-token-precalc']) {
-        echo "Loading word list\n";
+        info('Loading word list');
         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
     }
 
-    echo "Load Data\n";
+    info('Load Data');
     $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry';
 
     $aDBInstances = array();
@@ -402,13 +420,13 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) {
         echo '.';
     }
     echo "\n";
-    echo "Reanalysing database...\n";
+    info('Reanalysing database');
     pgsqlRunScript('ANALYSE');
 
     $sDatabaseDate = getDatabaseDate($oDB);
     pg_query($oDB->connection, 'TRUNCATE import_status');
     if ($sDatabaseDate === false) {
-        echo "WARNING: could not determine database date.\n";
+        warn('could not determine database date.');
     } else {
         $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
         pg_query($oDB->connection, $sSQL);
@@ -417,6 +435,7 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) {
 }
 
 if ($aCMDResult['import-tiger-data']) {
+    info('Import Tiger data');
     $bDidSomething = true;
 
     $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
@@ -474,7 +493,7 @@ if ($aCMDResult['import-tiger-data']) {
         echo "\n";
     }
 
-    echo "Creating indexes\n";
+    info('Creating indexes on Tiger data');
     $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
     $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
     $sTemplate = replace_tablespace(
@@ -491,6 +510,7 @@ if ($aCMDResult['import-tiger-data']) {
 }
 
 if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) {
+    info('Calculate Postcodes');
     $bDidSomething = true;
     $oDB =& getDB();
     if (!pg_query($oDB->connection, 'TRUNCATE location_postcode')) {
@@ -557,20 +577,23 @@ if ($aCMDResult['index'] || $aCMDResult['all']) {
     $bDidSomething = true;
     $sOutputFile = '';
     $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
+    info('Index ranks 0 - 4');
     passthruCheckReturn($sBaseCmd.' -R 4');
     if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
+    info('Index ranks 5 - 25');
     passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
     if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
+    info('Index ranks 26 - 30');
     passthruCheckReturn($sBaseCmd.' -r 26');
 
-    echo "Indexing postcodes....\n";
+    info('Index postcodes');
     $oDB =& getDB();
     $sSQL = 'UPDATE location_postcode SET indexed_status = 0';
     if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 }
 
 if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
-    echo "Search indices\n";
+    info('Create Search indices');
     $bDidSomething = true;
 
     $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
@@ -594,7 +617,7 @@ if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
 }
 
 if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
-    echo 'Creating search index for default country names';
+    info('Create search index for default country names');
     $bDidSomething = true;
 
     pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')");
@@ -620,6 +643,7 @@ if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
 }
 
 if ($aCMDResult['drop']) {
+    info('Drop tables only required for updates');
     // The implementation is potentially a bit dangerous because it uses
     // a positive selection of tables to keep, and deletes everything else.
     // Including any tables that the unsuspecting user might have manually
@@ -676,18 +700,25 @@ if ($aCMDResult['drop']) {
 if (!$bDidSomething) {
     showUsage($aCMDOptions, true);
 } else {
-    echo "Setup finished.\n";
+    echo "Summary of warnings:\n\n";
+    repeatWarnings();
+    echo "\n";
+    info('Setup finished.');
 }
 
 
 function pgsqlRunScriptFile($sFilename)
 {
+    global $aCMDResult;
     if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 
     // Convert database DSN to psql parameters
     $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
     if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
     $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
+    if (!$aCMDResult['verbose']) {
+        $sCMD .= ' -q';
+    }
 
     $ahGzipPipes = null;
     if (preg_match('/\\.gz$/', $sFilename)) {
@@ -738,6 +769,9 @@ function pgsqlRunScript($sScript, $bfatal = true)
     $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
     if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
     $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
+    if (!$aCMDResult['verbose']) {
+        $sCMD .= ' -q';
+    }
     if ($bfatal && !$aCMDResult['ignore-errors'])
         $sCMD .= ' -v ON_ERROR_STOP=1';
     $aDescriptors = array(
diff --git a/vagrant/Install-on-Ubuntu-16.sh b/vagrant/Install-on-Ubuntu-16.sh
index 28e38be9..9c58d063 100755
--- a/vagrant/Install-on-Ubuntu-16.sh
+++ b/vagrant/Install-on-Ubuntu-16.sh
@@ -33,7 +33,8 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
 # If you want to run the test suite, you need to install the following
 # additional packages:
 
-    sudo apt-get install -y python3-dev python3-pip python3-psycopg2 python3-tidylib phpunit
+    sudo apt-get install -y python3-setuptools python3-dev python3-pip \
+                            python3-psycopg2 python3-tidylib phpunit php-cgi
 
     pip3 install --user behave nose # urllib3
     sudo pear install PHP_CodeSniffer