nominatim: add a fast lane for requests from osm.org

diff --git a/cookbooks/nominatim/attributes/default.rb b/cookbooks/nominatim/attributes/default.rb
index 1b82d5f797a86308559b7ff0f1739bc081ca834c..5c895864dc55120e82a153ff76787afa88753d6c 100644 (file)
--- a/cookbooks/nominatim/attributes/default.rb
+++ b/cookbooks/nominatim/attributes/default.rb
@@ -1,5 +1,5 @@
 default[:nominatim][:dbadmins] = []
-default[:nominatim][:dbcluster] = "14/main"
+default[:nominatim][:dbcluster] = "17/main"
 default[:nominatim][:dbname] = "nominatim"
 default[:nominatim][:postgis] = "3"
 default[:nominatim][:logdir] = "/var/log/nominatim"
@@ -9,7 +9,8 @@ default[:nominatim][:ui_repository] = "https://git.openstreetmap.org/public/nomi
 default[:nominatim][:ui_revision] = "master"
 default[:nominatim][:qa_repository] = "https://github.com/osm-search/Nominatim-Data-Analyser"
 default[:nominatim][:qa_revision] = "deploy"
-default[:nominatim][:api_workers] = 10
+default[:nominatim][:api_workers]["nominatim"] = 8
+default[:nominatim][:api_workers]["nominatim_fastlane"] = 2
 default[:nominatim][:api_pool_size] = 10
 default[:nominatim][:api_query_timeout] = 5
 default[:nominatim][:api_request_timeout] = 20

diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb
index ad17d279495af43987fc41e6e57c69b62b407942..b0e65460abd25a6b4d4e9c2c9171e690bd3035ed 100644 (file)
--- a/cookbooks/nominatim/recipes/default.rb
+++ b/cookbooks/nominatim/recipes/default.rb
@@ -191,26 +191,26 @@ end
   end
 end
 
-systemd_service "nominatim" do
-  description "Nominatim running as a gunicorn application"
-  user "www-data"
-  group "www-data"
-  working_directory project_directory
-  standard_output "append:#{node[:nominatim][:logdir]}/gunicorn.log"
-  standard_error "inherit"
-  exec_start "#{python_directory}/bin/gunicorn --max-requests 500000 -b unix:/run/gunicorn-nominatim.openstreetmap.org.sock -w #{node[:nominatim][:api_workers]} --worker-class asgi --worker-connections 1000 --protocol uwsgi 'nominatim_api.server.falcon.server:run_wsgi()'"
-  exec_reload "/bin/kill -s HUP $MAINPID"
-  kill_mode "mixed"
-  timeout_stop_sec 5
-  private_tmp true
-  requires "nominatim.socket"
-  after "network.target"
-end
-
-systemd_socket "nominatim" do
-  description "Gunicorn socket for Nominatim"
-  listen_stream "/run/gunicorn-nominatim.openstreetmap.org.sock"
-  socket_user "www-data"
+%w[nominatim nominatim_fastlane].each do |name|
+  systemd_service name do
+    description "Nominatim running as a gunicorn application (#{name})"
+    user "www-data"
+    group "www-data"
+    working_directory project_directory
+    exec_start "#{python_directory}/bin/gunicorn --max-requests 500000 -b unix:/run/gunicorn-#{name}.openstreetmap.org.sock -w #{node[:nominatim][:api_workers][name]} --worker-class asgi --worker-connections 1000 --protocol uwsgi 'nominatim_api.server.falcon.server:run_wsgi()'"
+    exec_reload "/bin/kill -s HUP $MAINPID"
+    kill_mode "mixed"
+    timeout_stop_sec 5
+    private_tmp true
+    requires "#{name}.socket"
+    after "network.target"
+  end
+
+  systemd_socket name do
+    description "Gunicorn socket for Nominatim (#{name})"
+    listen_stream "/run/gunicorn-#{name}.openstreetmap.org.sock"
+    socket_user "www-data"
+  end
 end
 
 ssl_certificate node[:fqdn] do

diff --git a/cookbooks/nominatim/templates/default/nginx.erb b/cookbooks/nominatim/templates/default/nginx.erb
index 4645cc13269f6a73f2d4fbb6ced5f85adbfcb643..afd63b5469c17e074820804d2c06eed216113282 100644 (file)
--- a/cookbooks/nominatim/templates/default/nginx.erb
+++ b/cookbooks/nominatim/templates/default/nginx.erb
@@ -1,6 +1,8 @@
-upstream nominatim_service {
-  server unix:/run/gunicorn-nominatim.openstreetmap.org.sock fail_timeout=0;
+<% %w(nominatim nominatim_fastlane).each do |sname| -%>
+upstream <%= sname %>_service {
+  server unix:/run/gunicorn-<%= sname %>.openstreetmap.org.sock fail_timeout=0;
 }
+<% end -%>
 
 map $uri $nominatim_script_name {
     ~^/*(.+?)\.php        $1;
@@ -68,6 +70,15 @@ geo $whitelisted {
     35.153.15.118 1; # gnome - https://github.com/openstreetmap/operations/issues/1160
 }
 
+geo $proxyname {
+    default nominatim_service;
+<% @frontends.each do |frontend| -%>
+<% frontend.ipaddresses(:role => :external).sort.each do |address| -%>
+    <%= address %> nominatim_fastlane_service;
+<% end -%>
+<% end -%>
+}
+
 map $server_protocol$http_user_agent $cleaned_user_agent {
     default $http_user_agent;
     "~^HTTP/1..Mozilla/" Script$http_user_agent;
@@ -223,7 +234,7 @@ server {
           return 204;
         }
 
-        uwsgi_pass nominatim_service;
+        uwsgi_pass $proxyname;
         include uwsgi_params;
 
         add_header Vary "accept-language";

diff --git a/roles/dulcy.rb b/roles/dulcy.rb
index bc21860eb38088d0b469dba97d79ca8a8edf7b96..1025522aea92010b6be138ffa7270fb42f905111 100644 (file)
--- a/roles/dulcy.rb
+++ b/roles/dulcy.rb
@@ -81,7 +81,9 @@ default_attributes(
     :dbcluster => "17/main",
     :flatnode_file => "/srv/nominatim.openstreetmap.org/planet-project/nodes.store",
     :enable_qa_tiles => false,
-    :api_workers => 12,
+    :api_workers => {
+      "nominatim" => 11
+    },
     :api_pool_size => 8
   }
 )

diff --git a/roles/longma.rb b/roles/longma.rb
index c6226ac7f82ae81d316ef87f627946aa8467386c..888d262ef70bf2e405ddbb6b733f78cee5553f19 100644 (file)
--- a/roles/longma.rb
+++ b/roles/longma.rb
@@ -44,7 +44,9 @@ default_attributes(
     :dbcluster => "17/main",
     :enable_qa_tiles => true,
     :flatnode_file => "/srv/nominatim.openstreetmap.org/planet-project/nodes.store",
-    :api_workers => 24,
+    :api_workers => {
+      "nominatim" => 22
+    },
     :api_pool_size => 8
   }
 )

diff --git a/roles/stormfly-04.rb b/roles/stormfly-04.rb
index ed65f18e8790b6d690f7b8dd13d29dd5b51dcbe9..137956d787d81ffb641df435f55cfe8a002b2656 100644 (file)
--- a/roles/stormfly-04.rb
+++ b/roles/stormfly-04.rb
@@ -95,7 +95,9 @@ default_attributes(
   :nominatim => {
     :dbcluster => "17/main",
     :flatnode_file => "/srv/nominatim.openstreetmap.org/planet-project/nodes.store",
-    :api_workers => 19,
+    :api_workers => {
+      "nominatim" => 17
+    },
     :api_pool_size => 5
   }
 )

diff --git a/roles/vhagar.rb b/roles/vhagar.rb
index c8cd4a92fbf07dbe2c2e37aa2b83927114bc70e5..c2381517a3c19cf121ddae9e1604bc56ece5a206 100644 (file)
--- a/roles/vhagar.rb
+++ b/roles/vhagar.rb
@@ -34,7 +34,9 @@ default_attributes(
     :dbcluster => "17/main",
     :flatnode_file => "/srv/nominatim.openstreetmap.org/planet-project/nodes.store",
     :api_flavour => "python",
-    :api_workers => 24,
+    :api_workers => {
+      "nominatim" => 22
+    },
     :api_pool_size => 8
   }
 )