From: Grant Slater <git@firefishy.com>
Date: Tue, 24 Mar 2020 17:03:18 +0000 (+0000)
Subject: tilecache: rate limit Browser UA not sending a referer
X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/751317da2082df39c5fa65f432aa70d68bd8c1ac

tilecache: rate limit Browser UA not sending a referer

Rate limit Browsers which do not send a referer.
Without a referer we are unable to say who is accessing our server.

No referer could be due to some of the following options:
* https accessing http tiles (we have STS is place to avoid)
* Browser Extension blocking referer (eg: uMatrix)
* Site with Referrer-Policy set
---

diff --git a/cookbooks/tilecache/templates/default/nginx_tile.conf.erb b/cookbooks/tilecache/templates/default/nginx_tile.conf.erb
index fb03fca55..a6adb0e5b 100644
--- a/cookbooks/tilecache/templates/default/nginx_tile.conf.erb
+++ b/cookbooks/tilecache/templates/default/nginx_tile.conf.erb
@@ -54,9 +54,9 @@ map $http_user_agent $denied_scraper {
   '~^Java\/'             1; # Library Default
   '~^tiles$'             1; # Library Default
   '~^okhttp\/'           1; # Library Default
-  '~^Microsoft-ATL-Native\/' 1; #Library Default
+  '~^Microsoft-ATL-Native\/' 1; # Library Default
   '/n software IPWorks HTTP/S Component - www.nsoftware.com' 1; #Library default
-  '~^Wget\/'             1; #Library Default
+  '~^Wget\/'             1; # Library Default
   'C# TilesDownloader'   1; # Downloader
   'MapDownloader'        1; # Downloader
   '~^staticmaps'         1; # Downloader
@@ -71,6 +71,7 @@ map $http_user_agent $denied_scraper {
   'com.soft373.taptaxi'  1;
   'com.kradac.ktxcore'   1;
   'ru.crowdsystems.topcontrol.knd' 1;
+  # '~[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}' 1; # Fake UA
 }
 
 map $http_referer $denied_referer {
@@ -127,6 +128,17 @@ map $osm_referer$http_user_agent $limit_http_pragma {
   '~^osmMozilla\/5\.0\ ' $http_pragma; # Pass Header
 }
 
+# Find Browser User-Agents which are not sending a referer.
+# Browsers with no referer could be due to Browser extension or website Referrer-Policy
+map $http_referer$scheme$http_user_agent $deny_missing_referer {
+  default                             0; # Not denied
+  '~^httpsMozilla\/5\.0\ \(X11'       1;
+  '~^httpsMozilla\/5\.0\ \(Windows'   1;
+  '~^httpsMozilla\/5\.0\ \(iPhone'    1;
+  '~^httpsMozilla\/5\.0\ \(Macintosh' 1;
+  '~^httpsMozilla\/5\.0\ \(Linux'     1;
+}
+
 server {
     # IPv4
     listen       80 deferred backlog=16384 reuseport fastopen=2048 default_server;
@@ -275,6 +287,13 @@ server {
       proxy_cache_min_uses 4;
 
       add_header x-cache-status $upstream_cache_status;
+<% else %>
+      # Severely rate limit Browser UAs which are not sending a referer.
+      # With no referer we do not know who is using tiles
+      if ($deny_missing_referer) {
+        set $limit_rate 1024;
+        add_header x-cache-ratelimit "missing-referer";
+      }
 <% end -%>
 
       # Set a QoS cookie if none presented (uses nginx Map)