]> git.openstreetmap.org Git - nominatim.git/commitdiff
Merge remote-tracking branch 'upstream/master'
authorSarah Hoffmann <lonvia@denofr.de>
Wed, 12 Dec 2012 17:55:03 +0000 (18:55 +0100)
committerSarah Hoffmann <lonvia@denofr.de>
Wed, 12 Dec 2012 17:55:03 +0000 (18:55 +0100)
Conflicts:
lib/init-website.php

16 files changed:
lib/init-website.php
lib/init.php
lib/leakybucket.php [new file with mode: 0644]
lib/lib.php
lib/template/address-json.php
lib/template/address-jsonv2.php
lib/template/address-xml.php
lib/template/details-html.php
lib/template/search-jsonv2.php
settings/settings.php
sql/functions.sql
utils/blocks.php [new file with mode: 0755]
utils/update.php
website/details.php
website/reverse.php
website/search.php

index 6db2ac374d995837b2aac2dcb9861ec6922945b6..ef8237fbfa7d4fbc6d968fc243d3978ff18c169b 100644 (file)
@@ -8,27 +8,44 @@
         strpos(CONST_WhitelistedIPs, ','.$_SERVER["REMOTE_ADDR"].',') === false)
     {
 
-        if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false)
-        {
-            header('HTTP/1.0 403 Forbidden');
-            header('Content-type: text/html; charset=utf-8');
-            echo "<html><body><h1>Access blocked</h1>";
-            echo "Your IP has been blocked for overusing OpenStreetMap's volunteer-run servers.<br> \n";
-            echo 'Please consult the <a href="http://wiki.openstreetmap.org/wiki/Nominatim_usage_policy">Nominatim usage policy</a> for more information.';
-            echo "\n</body></html>\n";
-            exit;
-        }
+       $aBucketKeys = array();
 
-        $sTempBlockedIP = file_get_contents(CONST_IPBanFile);
-        if (preg_match('/\b'.$_SERVER["REMOTE_ADDR"].'\b/', $sTempBlockedIP))
-        {
-            header('HTTP/1.0 503 Service Temporarily Unavailable');
-            header('Content-type: text/html; charset=utf-8');
-            echo "<html><body><h1>Access blocked</h1>";
-            echo "Your IP has been blocked temporarily for overusing OpenStreetMap's volunteer-run servers. This ban will be lifted automatically in a while. To avoid further blocks, please read the<br> \n";
-            echo '<a href="http://wiki.openstreetmap.org/wiki/Nominatim_usage_policy">Nominatim usage policy</a> carefully before you continue to use this service.';
-            echo "\n</body></html>\n";
-            exit;
-        }
+       if (isset($_SERVER["HTTP_REFERER"])) $aBucketKeys[] = str_replace('www.','',strtolower(parse_url($_SERVER["HTTP_REFERER"], PHP_URL_HOST)));
+       if (isset($_SERVER["REMOTE_ADDR"])) $aBucketKeys[] = $_SERVER["REMOTE_ADDR"];
+       if (isset($_GET["email"])) $aBucketKeys[] = $_GET["email"];
 
-    }
+       $fBucketVal = doBucket($aBucketKeys, 
+                       (defined('CONST_ConnectionBucket_PageType')?constant('CONST_ConnectionBucket_Cost_'.CONST_ConnectionBucket_PageType):1) + user_busy_cost(),
+                       CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
+
+       if ($fBucketVal > CONST_ConnectionBucket_WaitLimit && $fBucketVal < CONST_ConnectionBucket_BlockLimit)
+       {
+               $m = getBucketMemcache();
+               $iCurrentSleeping = $m->increment('sleepCounter');
+               if (false === $iCurrentSleeping)
+               {
+                       $m->add('sleepCounter', 0);
+                       $iCurrentSleeping = $m->increment('sleepCounter');
+               }
+               if ($iCurrentSleeping >= CONST_ConnectionBucket_MaxSleeping || isBucketSleeping($aBucketKeys))
+               {
+                       // Too many threads sleeping already.  This becomes a hard block.
+                       $fBucketVal = doBucket($aBucketKeys, CONST_ConnectionBucket_BlockLimit, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
+               }
+               else
+               {
+                       setBucketSleeping($aBucketKeys, true);
+                       sleep(($fBucketVal - CONST_ConnectionBucket_WaitLimit)/CONST_ConnectionBucket_LeakRate);
+                       $fBucketVal = doBucket($aBucketKeys, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
+                       setBucketSleeping($aBucketKeys, false);
+               }
+               $m->decrement('sleepCounter');
+       }
+
+       if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false || $fBucketVal >= CONST_ConnectionBucket_BlockLimit)
+       {
+               echo "Your IP has been blocked. \n";
+               echo "Please create a nominatim trac ticket (http://trac.openstreetmap.org/newticket?component=nominatim) to request this to be removed. \n";
+               echo "Information on the Nominatim usage policy can be found here: http://wiki.openstreetmap.org/wiki/Nominatim#Usage_Policy \n";
+               exit;
+       }
index 15e38a5b17fcf7862e71bacda260db87e7b03d60..67efdbfdf6edb3c8894f7d8a487adfde7d193633 100644 (file)
@@ -4,6 +4,7 @@
 
        require_once(CONST_BasePath.'/settings/settings.php');
        require_once(CONST_BasePath.'/lib/lib.php');
+       require_once(CONST_BasePath.'/lib/leakybucket.php');
        require_once(CONST_BasePath.'/lib/db.php');
 
        if (get_magic_quotes_gpc())
diff --git a/lib/leakybucket.php b/lib/leakybucket.php
new file mode 100644 (file)
index 0000000..6d4e8f2
--- /dev/null
@@ -0,0 +1,168 @@
+<?php
+
+       function getBucketMemcache()
+       {
+               static $m;
+
+               if (!CONST_ConnectionBucket_MemcacheServerAddress) return null;
+               if (!isset($m))
+               {
+                       $m = new Memcached();
+                       $m->addServer(CONST_ConnectionBucket_MemcacheServerAddress, CONST_ConnectionBucket_MemcacheServerPort);
+               }
+               return $m;
+       }
+
+       function doBucket($asKey, $iRequestCost, $iLeakPerSecond, $iThreshold)
+       {
+               $m = getBucketMemcache();
+               if (!$m) return 0;
+
+               $iMaxVal = 0;
+               $t = time();
+
+               foreach($asKey as $sKey)
+               {
+                       $aCurrentBlock = $m->get($sKey);
+                       if (!$aCurrentBlock)
+                       {
+                               $aCurrentBlock = array($iRequestCost, $t, false);
+                       }
+                       else
+                       {
+                               // add RequestCost
+                               // remove leak * the time since the last request 
+                               $aCurrentBlock[0] += $iRequestCost - ($t - $aCurrentBlock[1])*$iLeakPerSecond;
+                               $aCurrentBlock[1] = $t;
+                       }
+
+                       if ($aCurrentBlock[0] <= 0)
+                       {
+                               $m->delete($sKey);
+                       }
+                       else
+                       {
+                               // If we have hit the threshold stop and record this to the block list
+                               if ($aCurrentBlock[0] >= $iThreshold)
+                               {
+                                       $aCurrentBlock[0] = $iThreshold;
+
+                                       // Make up to 10 attempts to record this to memcache (with locking to prevent conflicts)
+                                       $i = 10;
+                                       for($i = 0; $i < 10; $i++)
+                                       {
+                                               $aBlockedList = $m->get('blockedList', null, $hCasToken);
+                                               if (!$aBlockedList)
+                                               {
+                                                       $aBlockedList = array();
+                                                       $m->add('blockedList', $aBlockedList);
+                                                       $aBlockedList = $m->get('blockedList', null, $hCasToken);
+                                               }
+                                               if (!isset($aBlockedList[$sKey]))
+                                               {
+                                                       $aBlockedList[$sKey] = array(1, $t);
+                                               }
+                                               else
+                                               {
+                                                       $aBlockedList[$sKey][0]++;
+                                                       $aBlockedList[$sKey][1] = $t;
+                                               }
+                                               if (sizeof($aBlockedList) > CONST_ConnectionBucket_MaxBlockList)
+                                               {
+                                                       uasort($aBlockedList, 'byValue1');
+                                                       $aBlockedList = array_slice($aBlockedList, 0, CONST_ConnectionBucket_MaxBlockList);
+                                               }
+                                               $x = $m->cas($hCasToken, 'blockedList', $aBlockedList);
+                                               if ($x) break;
+                                       }
+                               }
+                               // Only keep in memcache until the time it would have expired (to avoid clutering memcache)
+                               $m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/$iLeakPerSecond);
+                       }
+
+                       // Bucket result in the largest bucket we find
+                       $iMaxVal = max($iMaxVal, $aCurrentBlock[0]);
+               }
+
+               return $iMaxVal;
+        }
+
+       function isBucketSleeping($asKey)
+       {
+               $m = getBucketMemcache();
+               if (!$m) return false;
+
+               foreach($asKey as $sKey)
+               {
+                       $aCurrentBlock = $m->get($sKey);
+                       if ($aCurrentBlock[2]) return true;
+               }
+               return false;
+       }
+
+       function setBucketSleeping($asKey, $bVal)
+       {
+               $m = getBucketMemcache();
+               if (!$m) return false;
+
+               $iMaxVal = 0;
+               $t = time();
+
+               foreach($asKey as $sKey)
+               {
+                       $aCurrentBlock = $m->get($sKey);
+                       $aCurrentBlock[2] = $bVal;
+                       $m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/CONST_ConnectionBucket_LeakRate);
+               }
+               return true;
+       }
+
+       function byValue1($a, $b)
+       {
+               if ($a[1] == $b[1])
+               {
+                       return 0;
+               }
+               return ($a[1] > $b[1]) ? -1 : 1;
+       }
+
+       function byLastBlockTime($a, $b)
+       {
+               if ($a['lastBlockTimestamp'] == $b['lastBlockTimestamp'])
+               {
+                       return 0;
+               }
+               return ($a['lastBlockTimestamp'] > $b['lastBlockTimestamp']) ? -1 : 1;
+       }
+
+       function getBucketBlocks()
+       {
+               $m = getBucketMemcache();
+               if (!$m) return null;
+               $t = time();
+               $aBlockedList = $m->get('blockedList', null, $hCasToken);
+               if (!$aBlockedList) $aBlockedList = array();
+               foreach($aBlockedList as $sKey => $aDetails)
+               {
+                       $aCurrentBlock = $m->get($sKey);
+                       if (!$aCurrentBlock) $aCurrentBlock = array(0, $t);
+                       $iCurrentBucketSize = max(0, $aCurrentBlock[0] - ($t - $aCurrentBlock[1])*CONST_ConnectionBucket_LeakRate);
+                       $aBlockedList[$sKey] = array(
+                               'totalBlocks' => $aDetails[0],
+                               'lastBlockTimestamp' => $aDetails[1],
+                               'isSleeping' => (isset($aCurrentBlock[2])?$aCurrentBlock[2]:false),
+                               'currentBucketSize' => $iCurrentBucketSize,
+                               'currentlyBlocked' => $iCurrentBucketSize + (CONST_ConnectionBucket_Cost_Reverse) >= CONST_ConnectionBucket_BlockLimit,
+                               );
+               }
+               uasort($aBlockedList, 'byLastBlockTime');
+               return $aBlockedList;
+       }
+
+       function clearBucketBlocks()
+       {
+               $m = getBucketMemcache();
+               if (!$m) return false;
+               $m->delete('blockedList');
+               return true;
+       }
index f34b09b7a51fbef8285d4a19bd9fb9de5b1704ab..f0f7ecebfae64ca3187c8bbc920a4084a770841a 100644 (file)
@@ -60,7 +60,7 @@
        {
                $sLoadAverage = file_get_contents('/proc/loadavg');
                 $aLoadAverage = explode(' ',$sLoadAverage);
-               return (int)$aLoadAverage[0];
+               return (float)$aLoadAverage[0];
        }
 
        function getProcessorCount()
                }
 
                $aLanguages = array();
-               if (preg_match_all('/(([a-z]{1,8})(-[a-z]{1,8})?)\s*(;\s*q\s*=\s*(1|0\.[0-9]+))?/i', $_SERVER['HTTP_ACCEPT_LANGUAGE'], $aLanguagesParse, PREG_SET_ORDER))
-               {
-                       foreach($aLanguagesParse as $iLang => $aLanguage)
+               if (isset($_SERVER["HTTP_ACCEPT_LANGUAGE"])) {
+                       if (preg_match_all('/(([a-z]{1,8})(-[a-z]{1,8})?)\s*(;\s*q\s*=\s*(1|0\.[0-9]+))?/i', $_SERVER['HTTP_ACCEPT_LANGUAGE'], $aLanguagesParse, PREG_SET_ORDER))
                        {
-                               $aLanguages[$aLanguage[1]] = isset($aLanguage[5])?(float)$aLanguage[5]:1 - ($iLang/100);
-                               if (!isset($aLanguages[$aLanguage[2]])) $aLanguages[$aLanguage[2]] = $aLanguages[$aLanguage[1]]/10;
+                               foreach($aLanguagesParse as $iLang => $aLanguage)
+                               {
+                                       $aLanguages[$aLanguage[1]] = isset($aLanguage[5])?(float)$aLanguage[5]:1 - ($iLang/100);
+                                       if (!isset($aLanguages[$aLanguage[2]])) $aLanguages[$aLanguage[2]] = $aLanguages[$aLanguage[1]]/10;
+                               }
+                               arsort($aLanguages);
                        }
-                       arsort($aLanguages);
-               }
+        }
                if (!sizeof($aLanguages)) $aLanguages = array(CONST_Default_Language=>1);
                foreach($aLanguages as $sLangauge => $fLangauagePref)
                {
        function getClassTypes()
        {
                return array(
+ 'boundary:administrative:1' => array('label'=>'Continent','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'boundary:administrative:2' => array('label'=>'Country','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'place:country' => array('label'=>'Country','frequency'=>0,'icon'=>'poi_boundary_administrative','defzoom'=>6, 'defdiameter' => 15,),
+ 'boundary:administrative:3' => array('label'=>'State','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'boundary:administrative:4' => array('label'=>'State','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'place:state' => array('label'=>'State','frequency'=>0,'icon'=>'poi_boundary_administrative','defzoom'=>8, 'defdiameter' => 5.12,),
  'boundary:administrative:5' => array('label'=>'State District','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'boundary:administrative:6' => array('label'=>'County','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
+ 'boundary:administrative:7' => array('label'=>'County','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'place:county' => array('label'=>'County','frequency'=>108,'icon'=>'poi_boundary_administrative','defzoom'=>10, 'defdiameter' => 1.28,),
  'boundary:administrative:8' => array('label'=>'City','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'place:city' => array('label'=>'City','frequency'=>66,'icon'=>'poi_place_city','defzoom'=>12, 'defdiameter' => 0.32,),
  'boundary:administrative:9' => array('label'=>'City District','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'boundary:administrative:10' => array('label'=>'Suburb','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'boundary:administrative:11' => array('label'=>'Neighbourhood','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
- 'place:region' => array('label'=>'Region','frequency'=>0,'icon'=>'poi_boundary_administrative','defzoom'=>8, 'defdiameter' => 5.12,),
+ 'place:region' => array('label'=>'Region','frequency'=>0,'icon'=>'poi_boundary_administrative','defzoom'=>8, 'defdiameter' => 0.04,),
  'place:island' => array('label'=>'Island','frequency'=>288,'icon'=>'','defzoom'=>11, 'defdiameter' => 0.64,),
  'boundary:administrative' => array('label'=>'Administrative','frequency'=>413,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'place:town' => array('label'=>'Town','frequency'=>1497,'icon'=>'poi_place_town','defzoom'=>14, 'defdiameter' => 0.08,),
 
  'leisure:pitch' => array('label'=>'Pitch','frequency'=>762,'icon'=>'',),
  'highway:unsurfaced' => array('label'=>'Unsurfaced','frequency'=>492,'icon'=>'',),
- 'historic:ruins' => array('label'=>'Ruins','frequency'=>483,'icon'=>'shopping_jewelry',),
+ 'historic:ruins' => array('label'=>'Ruins','frequency'=>483,'icon'=>'tourist_ruin',),
  'amenity:college' => array('label'=>'College','frequency'=>473,'icon'=>'education_school',),
  'historic:monument' => array('label'=>'Monument','frequency'=>470,'icon'=>'tourist_monument',),
  'railway:subway' => array('label'=>'Subway','frequency'=>385,'icon'=>'',),
  'railway:disused_station' => array('label'=>'Disused Station','frequency'=>114,'icon'=>'',),
  'railway:abandoned' => array('label'=>'Abandoned','frequency'=>641,'icon'=>'',),
  'railway:disused' => array('label'=>'Disused','frequency'=>72,'icon'=>'',),
-                       );              
+                       );
        }
-       
+
        function getClassTypesWithImportance()
        {
                $aOrders = getClassTypes();
                }
                return $aOrders;
        }
-       
-       
 
     function javascript_renderData($xVal)
     {
                $sSQL = "select *,get_name_by_language(name,$sLanguagePrefArraySQL) as localname from get_addressdata($iPlaceID)";
                IF (!$bRaw) $sSQL .= " WHERE isaddress OR type = 'country_code'";
                $sSQL .= " order by rank_address desc,isaddress desc";
+
                $aAddressLines = $oDB->getAll($sSQL);
                if (PEAR::IsError($aAddressLines))
                {
                        $aTypeLabel = false;
                        if (isset($aClassType[$aLine['class'].':'.$aLine['type'].':'.$aLine['admin_level']])) $aTypeLabel = $aClassType[$aLine['class'].':'.$aLine['type'].':'.$aLine['admin_level']];
                        elseif (isset($aClassType[$aLine['class'].':'.$aLine['type']])) $aTypeLabel = $aClassType[$aLine['class'].':'.$aLine['type']];
+                       elseif (isset($aClassType['boundary:administrative:'.((int)($aLine['rank_address']/2))])) $aTypeLabel = $aClassType['boundary:administrative:'.((int)($aLine['rank_address']/2))];
                        else $aTypeLabel = array('simplelabel'=>$aLine['class']);
-                       if ($aTypeLabel && ($aLine['localname'] || $aLine['housenumber']))
+                       if ($aTypeLabel && ((isset($aLine['localname']) && $aLine['localname']) || (isset($aLine['housenumber']) && $aLine['housenumber'])))
                        {
                                $sTypeLabel = strtolower(isset($aTypeLabel['simplelabel'])?$aTypeLabel['simplelabel']:$aTypeLabel['label']);
                                $sTypeLabel = str_replace(' ','_',$sTypeLabel);
                                if (!isset($aAddress[$sTypeLabel]) && $aLine['localname']) $aAddress[$sTypeLabel] = $aLine['localname']?$aLine['localname']:$aLine['housenumber'];
                        }
                }
-//var_dump($aAddress);
-//exit;
+
                return $aAddress;
 
                $aHouseNumber = $oDB->getRow('select housenumber, get_name_by_language(name,ARRAY[\'addr:housename\']) as housename,rank_search,postcode from placex where place_id = '.$iPlaceID);
                 }
                 return true;
         }
-
index c05bce294e1837279815fd04ca69bf8c43787f30..0a1d074a3540e8f68aa0d323fbeff98307e1e0f0 100644 (file)
@@ -3,7 +3,7 @@
 
        if (!sizeof($aPlace))
        {
-               if ($sError)
+               if (isset($sError))
                        $aFilteredPlaces['error'] = $sError;
                else
                        $aFilteredPlaces['error'] = 'Unable to geocode';
index 4e751a0d4b458ba4f7cff9fbe7fe592863c92276..00f8750db3f97430f4975707a1ec6421762be796 100644 (file)
@@ -3,7 +3,7 @@
 
        if (!sizeof($aPlace))
        {
-               if ($sError)
+               if (isset($sError))
                        $aFilteredPlaces['error'] = $sError;
                else
                        $aFilteredPlaces['error'] = 'Unable to geocode';
index 75f69e1563c2a96cc8d27dddfb205a226680dcf8..fcd90b85e63cc6b9ea55c2accde01763244bebfa 100644 (file)
@@ -14,7 +14,7 @@
 
        if (!sizeof($aPlace))
        {
-               if ($sError)
+               if (isset($sError))
                        echo "<error>$sError</error>";
                else
                        echo "<error>Unable to geocode</error>";
index 4cacb307c1c469db86c37faed79eaa0ec73df01b..e2799539c896cfbc89d424d48d6382f879dec705 100644 (file)
@@ -9,7 +9,7 @@
        <script src="js/OpenLayers.js" type="text/javascript"></script>
        <script src="js/tiles.js" type="text/javascript"></script>
        <script type="text/javascript">
-        
+
                var map;
 
     function init() {
@@ -82,7 +82,7 @@
        echo ' <div>Last Updated: <span class="type">'.$aPointDetails['indexed_date'].'</span></div>';
        echo ' <div>Admin Level: <span class="adminlevel">'.$aPointDetails['admin_level'].'</span></div>';
        echo ' <div>Rank: <span class="rankaddress">'.$aPointDetails['rank_search_label'].'</span></div>';
-       if ($aPointDetails['importance']) echo ' <div>Importance: <span class="rankaddress">'.$aPointDetails['importance'].'</span></div>';
+       if ($aPointDetails['calculated_importance']) echo ' <div>Importance: <span class="rankaddress">'.$aPointDetails['calculated_importance'].($aPointDetails['importance']?'':' (estimated)').'</span></div>';
        echo ' <div>Coverage: <span class="area">'.($aPointDetails['isarea']=='t'?'Polygon':'Point').'</span></div>';
        $sOSMType = ($aPointDetails['osm_type'] == 'N'?'node':($aPointDetails['osm_type'] == 'W'?'way':($aPointDetails['osm_type'] == 'R'?'relation':'')));
        if ($sOSMType) echo ' <div>OSM: <span class="osm">'.$sOSMType.' <a href="http://www.openstreetmap.org/browse/'.$sOSMType.'/'.$aPointDetails['osm_id'].'">'.$aPointDetails['osm_id'].'</a></span></div>';
index 671edde4b8fcaafcbf4e3f7bb53fa9fed32bb028..e1552a155ad53ec489791c78a9daa2927820be62 100644 (file)
@@ -40,7 +40,7 @@
 
                $aPlace['category'] = $aPointDetails['class'];
                $aPlace['type'] = $aPointDetails['type'];
-               if ($aPointDetails['icon'])
+               if (isset($aPointDetails['icon']))
                {
                        $aPlace['icon'] = $aPointDetails['icon'];
                }
index 8ebd81ee6c326b84f8b85999a975e1d2b8653840..d82a33376f11c315f6ff4baf57580b5b17775c04 100644 (file)
        @define('CONST_Osm2pgsql_Binary', CONST_BasePath.'/osm2pgsql/osm2pgsql');
        @define('CONST_Osmosis_Binary', '/usr/bin/osmosis');
 
+       // Connection buckets to rate limit people being nasty
+       @define('CONST_ConnectionBucket_MemcacheServerAddress', false);
+       @define('CONST_ConnectionBucket_MemcacheServerPort', 11211);
+       @define('CONST_ConnectionBucket_MaxBlockList', 100);
+       @define('CONST_ConnectionBucket_LeakRate', 1);
+       @define('CONST_ConnectionBucket_BlockLimit', 10);
+       @define('CONST_ConnectionBucket_WaitLimit', 6);
+       @define('CONST_ConnectionBucket_MaxSleeping', 10);
+       @define('CONST_ConnectionBucket_Cost_Reverse', 1);
+       @define('CONST_ConnectionBucket_Cost_Search', 2);
+       @define('CONST_ConnectionBucket_Cost_Details', 3);
+
+       // Override this function to add an adjustment factor to the cost
+       // based on server load. e.g. getBlockingProcesses
+       if (!function_exists('user_busy_cost'))
+       {
+               function user_busy_cost()
+               {
+                       return 0;
+               }
+       }
+
        // Website settings
        @define('CONST_BlockedIPs', '');
        @define('CONST_IPBanFile', CONST_BasePath.'/settings/ip_blocks');
 
        @define('CONST_Search_AreaPolygons_Enabled', true);
        @define('CONST_Search_AreaPolygons', true);
-       @define('CONST_Search_TryDroppedAddressTerms', false);
 
        @define('CONST_Suggestions_Enabled', false);
 
+       @define('CONST_Search_TryDroppedAddressTerms', false);
+
        // Set to zero to disable polygon output
        @define('CONST_PolygonOutput_MaximumTypes', 1);
 
index 03104dd5a9f32fdf3ea51928af3b1f15189c944f..4c679f3b7e4675f3a4ffe55ba73912da34a098ab 100644 (file)
@@ -571,6 +571,20 @@ END;
 $$
 LANGUAGE plpgsql IMMUTABLE;
 
+CREATE OR REPLACE FUNCTION get_country_language_codes(search_country_code VARCHAR(2)) RETURNS TEXT[]
+  AS $$
+DECLARE
+  nearcountry RECORD;
+BEGIN
+  FOR nearcountry IN select country_default_language_codes from country_name where country_code = search_country_code limit 1
+  LOOP
+    RETURN lower(nearcountry.country_default_language_codes);
+  END LOOP;
+  RETURN NULL;
+END;
+$$
+LANGUAGE plpgsql IMMUTABLE;
+
 CREATE OR REPLACE FUNCTION get_partition(place geometry, in_country_code VARCHAR(10)) RETURNS INTEGER
   AS $$
 DECLARE
@@ -2157,7 +2171,9 @@ BEGIN
     END IF;
   END LOOP;
 
-  RETURN null;
+  -- anything will do as a fallback - just take the first name type thing there is
+  search := avals(name);
+  RETURN search[1];
 END;
 $$
 LANGUAGE plpgsql IMMUTABLE;
diff --git a/utils/blocks.php b/utils/blocks.php
new file mode 100755 (executable)
index 0000000..bd5efd0
--- /dev/null
@@ -0,0 +1,52 @@
+#!/usr/bin/php -Cq
+<?php
+
+        require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
+        ini_set('memory_limit', '800M');
+
+       $aCMDOptions = array(
+               "Manage service blocks / restrictions",
+               array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
+               array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
+               array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
+               array('list', 'l', 0, 1, 0, 0, 'bool', 'List recent blocks'),
+               array('delete', 'd', 0, 1, 0, 0, 'bool', 'Clear recent blocks list'),
+               array('flush', '', 0, 1, 0, 0, 'bool', 'Flush all blocks / stats'),
+       );
+       getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
+
+       $m = getBucketMemcache();
+        if (!$m)
+       {
+               echo "ERROR: Bucket memcache is not configured\n";
+               exit;
+       }
+
+       if ($aResult['list'])
+       {
+               $iCurrentSleeping = $m->get('sleepCounter');
+               echo "\n Sleeping blocks count: $iCurrentSleeping\n";
+
+               $aBlocks = getBucketBlocks();
+               echo "\n";
+               printf(" %-40s | %12s | %7s | %13s | %31s | %8s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Block Time", "Sleeping");
+               printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-31s-|-%'-8s\n", "", "", "", "", "", "");
+               foreach($aBlocks as $sKey => $aDetails)
+               {
+                       printf(" %-40s | %12s | %7s | %13s | %31s | %8s\n", $sKey, $aDetails['totalBlocks'], 
+                               (int)$aDetails['currentBucketSize'], $aDetails['currentlyBlocked']?'Y':'N', 
+                               date("r", $aDetails['lastBlockTimestamp']), $aDetails['isSleeping']?'Y':'N');
+               }
+               echo "\n";
+       }
+
+       if ($aResult['delete'])
+       {
+               $m->set('sleepCounter', 0);
+               clearBucketBlocks();
+       }
+
+       if ($aResult['flush'])
+       {
+               $m->flush();
+       }
index eadd385136c1f22b9d87ccfcc3a732181691c2d4..095aaa2ef578fd28a9fd0ec15c19f9f54c7cbbaa 100755 (executable)
@@ -67,6 +67,8 @@
        // Assume osm2pgsql is in the folder above
        $sBasePath = dirname(dirname(__FILE__));
 
+       date_default_timezone_set('Etc/UTC');
+
        $oDB =& getDB();
 
        $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
index 3d80ea5f7b96215f756790589c34bd65ab5c90b6..124b0f902cd9d3de58f95562c9e252058b2df790 100755 (executable)
@@ -1,4 +1,6 @@
 <?php
+       @define('CONST_ConnectionBucket_PageType', 'Details');
+
         require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
         require_once(CONST_BasePath.'/lib/log.php');
 
@@ -61,6 +63,7 @@
        $sSQL = "select place_id, osm_type, osm_id, class, type, name, admin_level, housenumber, street, isin, postcode, country_code, importance, wikipedia,";
        $sSQL .= " to_char(indexed_date, 'YYYY-MM-DD HH24:MI') as indexed_date, parent_place_id, rank_address, rank_search, get_searchrank_label(rank_search) as rank_search_label, get_name_by_language(name,$sLanguagePrefArraySQL) as localname, ";
        $sSQL .= " ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea, ";
+       $sSQL .= " case when importance = 0 OR importance IS NULL then 0.75-(rank_search::float/40) else importance end as calculated_importance, ";
        $sSQL .= " ST_AsText(CASE WHEN ST_NPoints(geometry) > 5000 THEN ST_SimplifyPreserveTopology(geometry, 0.0001) ELSE geometry END) as outlinestring";
        $sSQL .= " from placex where place_id = $iPlaceID";
        $aPointDetails = $oDB->getRow($sSQL);
@@ -68,7 +71,6 @@
        {
                failInternalError("Could not get details of place object.", $sSQL, $aPointDetails);
        }
-
         $aPointDetails['localname'] = $aPointDetails['localname']?$aPointDetails['localname']:$aPointDetails['housenumber'];
 
        $aClassType = getClassTypesWithImportance();
index 810be3c491fd2c96a8de4415a0b64c1c93b94a90..2c62948fbb7a20d1d89612ee28ba0a52b266a5cc 100755 (executable)
@@ -1,4 +1,6 @@
 <?php
+       @define('CONST_ConnectionBucket_PageType', 'Reverse');
+
        require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
        require_once(CONST_BasePath.'/lib/log.php');
 
index b7a59613e3073f75de6b4fea6ee8776c341ef402..24f21e12f31d1690aca25e4d3064d3e9b7c136a2 100755 (executable)
@@ -1,8 +1,11 @@
 <?php
+       @define('CONST_ConnectionBucket_PageType', 'Search');
+
        require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
        require_once(CONST_BasePath.'/lib/log.php');
 
        ini_set('memory_limit', '200M');
+
        $oDB =& getDB();
 
        // Display defaults
        $iOffset = isset($_GET['offset'])?(int)$_GET['offset']:0;
        $iMaxRank = 20;
        if ($iFinalLimit > 50) $iFinalLimit = 50;
-    $iLimit = $iFinalLimit + min($iFinalLimit, 10);
+       $iLimit = $iFinalLimit + min($iFinalLimit, 10);
        $iMinAddressRank = 0;
        $iMaxAddressRank = 30;
+       $sAllowedTypesSQLList = false;
 
        // Format for output
        if (isset($_GET['format']) && ($_GET['format'] == 'html' || $_GET['format'] == 'xml' || $_GET['format'] == 'json' ||  $_GET['format'] == 'jsonv2'))
                                array('postalcode', 16, 25),
                                );
        $aStructuredQuery = array();
+       $sAllowedTypesSQLList = '';
        foreach($aStructuredOptions as $aStructuredOption)
        {
                loadStructuredAddressElement($aStructuredQuery, $iMinAddressRank, $iMaxAddressRank, $_GET, $aStructuredOption[0], $aStructuredOption[1], $aStructuredOption[2]);
        }
        if (sizeof($aStructuredQuery) > 0) {
                $sQuery = join(', ', $aStructuredQuery);
+               $sAllowedTypesSQLList = '(\'place\',\'boundary\')';
        }
 
        if ($sQuery)
                                                        if ($bBoundingBoxSearch) $aTerms[] = "centroid && $sViewboxSmallSQL";
                                                        if ($sNearPointSQL) $aOrder[] = "ST_Distance($sNearPointSQL, centroid) asc";
 
-                                                       $sImportanceSQL = 'case when importance = 0 OR importance IS NULL then 0.92-(search_rank::float/33) else importance end';
+                                                       $sImportanceSQL = 'case when importance = 0 OR importance IS NULL then 0.75-(search_rank::float/40) else importance end';
 
                                                        if ($sViewboxSmallSQL) $sImportanceSQL .= " * case when ST_Contains($sViewboxSmallSQL, centroid) THEN 1 ELSE 0.5 END";
                                                        if ($sViewboxLargeSQL) $sImportanceSQL .= " * case when ST_Contains($sViewboxLargeSQL, centroid) THEN 1 ELSE 0.5 END";
                                        $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,";
                                        $sSQL .= "avg(ST_X(ST_Centroid(geometry))) as lon,avg(ST_Y(ST_Centroid(geometry))) as lat, ";
 //                                     $sSQL .= $sOrderSQL." as porder, ";
-                                       $sSQL .= "coalesce(importance,0.9-(rank_search::float/30)) as importance ";
+                                       $sSQL .= "coalesce(importance,0.75-(rank_search::float/40)) as importance ";
                                        $sSQL .= "from placex where place_id in ($sPlaceIDs) ";
                                        $sSQL .= "and placex.rank_address between $iMinAddressRank and $iMaxAddressRank ";
+                                       if ($sAllowedTypesSQLList) $sSQL .= "and placex.class in $sAllowedTypesSQLList ";
                                        $sSQL .= "and linked_place_id is null ";
                                        $sSQL .= "group by osm_type,osm_id,class,type,admin_level,rank_search,rank_address,country_code,importance";
                                        if (!$bDeDupe) $sSQL .= ",place_id";
                                        $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,";
                                        $sSQL .= "avg(ST_X(ST_Centroid(geometry))) as lon,avg(ST_Y(ST_Centroid(geometry))) as lat, ";
 //                                     $sSQL .= $sOrderSQL." as porder, ";
-                                       $sSQL .= "coalesce(importance,0.9-(rank_search::float/30)) as importance ";
+                                       $sSQL .= "coalesce(importance,0.75-(rank_search::float/40)) as importance ";
                                        $sSQL .= "from placex where place_id in ($sPlaceIDs) ";
                                        $sSQL .= "and placex.rank_address between $iMinAddressRank and $iMaxAddressRank ";
                                        $sSQL .= "group by osm_type,osm_id,class,type,admin_level,rank_search,rank_address,country_code,importance";
        }
        uasort($aSearchResults, 'byImportance');
 
-//var_dump($aSearchResults);exit;
-       
        $aOSMIDDone = array();
        $aClassTypeNameDone = array();
        $aToFilter = $aSearchResults;
                logEnd($oDB, $hLog, sizeof($aToFilter));
        }
        $sMoreURL = CONST_Website_BaseURL.'search?format='.urlencode($sOutputFormat).'&exclude_place_ids='.join(',',$aExcludePlaceIDs);
-       $sMoreURL .= '&accept-language='.$_SERVER["HTTP_ACCEPT_LANGUAGE"];
+       if (isset($_SERVER["HTTP_ACCEPT_LANGUAGE"])) $sMoreURL .= '&accept-language='.$_SERVER["HTTP_ACCEPT_LANGUAGE"];
        if ($bShowPolygons) $sMoreURL .= '&polygon=1';
        if ($bShowAddressDetails) $sMoreURL .= '&addressdetails=1';
        if (isset($_GET['viewbox']) && $_GET['viewbox']) $sMoreURL .= '&viewbox='.urlencode($_GET['viewbox']);