X-Git-Url: https://git.openstreetmap.org/chef.git/blobdiff_plain/14e316cd3e1ab5f25a54d0765ebad61ac665ccb4..9ea8869c986548bd40b63d4e98cd6ab294f01c83:/cookbooks/nominatim/recipes/default.rb

diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb
index 7b218d951..cb0748457 100644
--- a/cookbooks/nominatim/recipes/default.rb
+++ b/cookbooks/nominatim/recipes/default.rb
@@ -19,9 +19,12 @@
 
 include_recipe "accounts"
 include_recipe "munin"
-include_recipe "php::fpm"
 include_recipe "prometheus"
 
+if node[:nominatim][:api_flavour] == "php"
+  include_recipe "php::fpm"
+end
+
 basedir = data_bag_item("accounts", "nominatim")["home"]
 email_errors = data_bag_item("accounts", "lonvia")["email"]
 
@@ -118,6 +121,9 @@ end
 ## Nominatim backend
 
 include_recipe "git"
+include_recipe "python"
+
+python_directory = "#{basedir}/venv"
 
 package %w[
   build-essential
@@ -131,6 +137,11 @@ package %w[
   libbz2-dev
   libpq-dev
   libproj-dev
+  liblua5.3-dev
+  libluajit-5.1-dev
+  libicu-dev
+  nlohmann-json3-dev
+  lua5.3
   python3-pyosmium
   python3-psycopg2
   python3-dotenv
@@ -139,14 +150,79 @@ package %w[
   python3-icu
   python3-datrie
   python3-yaml
-  php-pgsql
-  php-intl
+  python3-sqlalchemy-ext
+  python3-geoalchemy2
+  python3-asyncpg
+  python3-dev
+  pkg-config
   ruby
   ruby-file-tail
   ruby-pg
   ruby-webrick
 ]
 
+if node[:nominatim][:api_flavour] == "php"
+  package %w[
+    php-pgsql
+    php-intl
+  ]
+elsif node[:nominatim][:api_flavour] == "python"
+
+  python_virtualenv python_directory do
+    interpreter "/usr/bin/python3"
+  end
+
+  python_package "SQLAlchemy" do
+    python_virtualenv python_directory
+    version "2.0.20"
+  end
+
+  python_package "PyICU" do
+    python_virtualenv python_directory
+    version "2.10.2"
+  end
+
+  python_package "psycopg[binary]" do
+    python_virtualenv python_directory
+    version "3.1.10"
+  end
+
+  python_package "psycopg2-binary" do
+    python_virtualenv python_directory
+    version "2.9.7"
+  end
+
+  python_package "python-dotenv" do
+    python_virtualenv python_directory
+    version "0.21.0"
+  end
+
+  python_package "pygments" do
+    python_virtualenv python_directory
+    version "2.16.1"
+  end
+
+  python_package "PyYAML" do
+    python_virtualenv python_directory
+    version "6.0.1"
+  end
+
+  python_package "falcon" do
+    python_virtualenv python_directory
+    version "3.1.1"
+  end
+
+  python_package "uvicorn" do
+    python_virtualenv python_directory
+    version "0.23.2"
+  end
+
+  python_package "gunicorn" do
+    python_virtualenv python_directory
+    version "21.0.1"
+  end
+end
+
 source_directory = "#{basedir}/src/nominatim"
 build_directory = "#{basedir}/src/build"
 project_directory = "#{basedir}/planet-project"
@@ -177,6 +253,17 @@ if node[:nominatim][:flatnode_file]
   end
 end
 
+remote_directory "#{project_directory}/static-website" do
+  source "website"
+  owner "nominatim"
+  group "nominatim"
+  mode "755"
+  files_owner "nominatim"
+  files_group "nominatim"
+  files_mode "644"
+  purge false
+end
+
 # Normally syncing via chef is a bad idea because syncing might involve
 # an update of database functions which should not be done while an update
 # is ongoing. Therefore we sync in between update cycles. There is an
@@ -195,7 +282,7 @@ end
 
 remote_file "#{source_directory}/data/country_osm_grid.sql.gz" do
   action :create_if_missing
-  source "https://www.nominatim.org/data/country_grid.sql.gz"
+  source "https://nominatim.org/data/country_grid.sql.gz"
   owner "nominatim"
   group "nominatim"
   mode "644"
@@ -205,7 +292,7 @@ execute "compile_nominatim" do
   action :nothing
   user "nominatim"
   cwd build_directory
-  command "cmake #{source_directory} && make"
+  command "cmake -D WITH_LUAJIT=ON #{source_directory} && make"
   notifies :run, "execute[install_nominatim]"
 end
 
@@ -226,13 +313,25 @@ template "#{project_directory}/.env" do
             :dbname => node[:nominatim][:dbname],
             :flatnode_file => node[:nominatim][:flatnode_file],
             :log_file => "#{node[:nominatim][:logdir]}/query.log",
-            :tokenizer => node[:nominatim][:config][:tokenizer]
+            :tokenizer => node[:nominatim][:config][:tokenizer],
+            :forward_dependencies => node[:nominatim][:config][:forward_dependencies],
+            :pool_size => node[:nominatim][:api_pool_size],
+            :query_timeout => node[:nominatim][:api_query_timeout],
+            :request_timeout => node[:nominatim][:api_request_timeout]
+end
+
+remote_file "#{project_directory}/wikimedia-importance.sql.gz" do
+  action :create_if_missing
+  source "https://nominatim.org/data/wikimedia-importance.sql.gz"
+  owner "nominatim"
+  group "nominatim"
+  mode "644"
 end
 
-%w[wikimedia-importance.sql.gz gb_postcodes.csv.gz us_postcodes.csv.gz].each do |fname|
+%w[gb_postcodes.csv.gz us_postcodes.csv.gz].each do |fname|
   remote_file "#{project_directory}/#{fname}" do
     action :create
-    source "https://www.nominatim.org/data/#{fname}"
+    source "https://nominatim.org/data/#{fname}"
     owner "nominatim"
     group "nominatim"
     mode "644"
@@ -250,16 +349,41 @@ end
   end
 end
 
-node[:nominatim][:fpm_pools].each do |name, data|
-  php_fpm name do
-    port data[:port]
-    pm data[:pm]
-    pm_max_children data[:max_children]
-    pm_start_servers 20
-    pm_min_spare_servers 10
-    pm_max_spare_servers 20
-    pm_max_requests 10000
-    prometheus_port data[:prometheus_port]
+if node[:nominatim][:api_flavour] == "php"
+  node[:nominatim][:fpm_pools].each do |name, data|
+    php_fpm name do
+      port data[:port]
+      pm data[:pm]
+      pm_max_children data[:max_children]
+      pm_start_servers 20
+      pm_min_spare_servers 10
+      pm_max_spare_servers 20
+      pm_max_requests 10000
+      prometheus_port data[:prometheus_port]
+    end
+  end
+elsif node[:nominatim][:api_flavour] == "python"
+  systemd_service "nominatim" do
+    description "Nominatim running as a gunicorn application"
+    user "www-data"
+    group "www-data"
+    working_directory project_directory
+    standard_output "append:#{node[:nominatim][:logdir]}/gunicorn.log"
+    standard_error "inherit"
+    exec_start "#{python_directory}/bin/gunicorn --max-requests 200000 -b unix:/run/gunicorn-nominatim.openstreetmap.org.sock -w #{node[:nominatim][:api_workers]} -k uvicorn.workers.UvicornWorker nominatim.server.falcon.server:run_wsgi"
+    exec_reload "/bin/kill -s HUP $MAINPID"
+    environment :PYTHONPATH => "/usr/local/lib/nominatim/lib-python/"
+    kill_mode "mixed"
+    timeout_stop_sec 5
+    private_tmp true
+    requires "nominatim.socket"
+    after "network.target"
+  end
+
+  systemd_socket "nominatim" do
+    description "Gunicorn socket for Nominatim"
+    listen_stream "/run/gunicorn-nominatim.openstreetmap.org.sock"
+    socket_user "www-data"
   end
 end
 
@@ -509,6 +633,7 @@ end
 prometheus_exporter "nominatim" do
   port 8082
   user "www-data"
+  restrict_address_families "AF_UNIX"
   options [
     "--nominatim.query-log=#{node[:nominatim][:logdir]}/query.log",
     "--nominatim.database-name=#{node[:nominatim][:dbname]}"