]> git.openstreetmap.org Git - chef.git/commitdiff
nominatim: move to nginx for webserving
authorSarah Hoffmann <lonvia@denofr.de>
Fri, 24 Apr 2020 19:43:19 +0000 (21:43 +0200)
committerSarah Hoffmann <lonvia@denofr.de>
Fri, 24 Apr 2020 20:05:01 +0000 (22:05 +0200)
cookbooks/nominatim/attributes/default.rb
cookbooks/nominatim/metadata.rb
cookbooks/nominatim/recipes/default.rb
cookbooks/nominatim/templates/default/apache.erb [deleted file]
cookbooks/nominatim/templates/default/logrotate.nginx.erb [moved from cookbooks/nominatim/templates/default/logrotate.apache.erb with 63% similarity]
cookbooks/nominatim/templates/default/logrotate.nominatim.erb
cookbooks/nominatim/templates/default/nginx.erb [new file with mode: 0644]
cookbooks/nominatim/templates/default/nominatim.cron.erb

index 93b7daf074641b7fa957a53a041c7722ff5fe59e..d096f93941079d5136bb68fd71b1080384c2e442 100644 (file)
@@ -13,11 +13,6 @@ default[:nominatim][:fpm_pools] = {
     :port => "8000",
     :pm => "dynamic",
     :max_children => "60"
-  },
-  :bulk => {
-    :port => "8001",
-    :pm => "static",
-    :max_children => "10"
   }
 }
 
index 71c3ee93c74ba633ea48fa4536d8592afae8d9c5..57b19a03adaaf65f95baadd14673805ad0f21093 100644 (file)
@@ -6,9 +6,11 @@ description       "Installs and configures nominatim servers"
 
 version           "1.0.0"
 supports          "ubuntu"
+depends           "systemd"
 depends           "apache"
 depends           "fail2ban"
 depends           "git"
 depends           "munin"
+depends           "nginx"
 depends           "postgresql"
 depends           "python"
index 76f7cdcedb2c98c92ef22586b3ce3d4011a9e131..463b6e602d9c8298d5ecf83938f3b1e6412a118b 100644 (file)
@@ -166,6 +166,10 @@ package %w[
   python3-pyosmium
   pyosmium
   python3-psycopg2
+  php
+  php-fpm
+  php-pgsql
+  php-intl
 ]
 
 source_directory = "#{basedir}/nominatim"
@@ -302,41 +306,26 @@ end
 
 ## webserver frontend
 
-template "#{build_directory}/settings/ip_blocks.conf" do
-  action :create_if_missing
-  source "ipblocks.erb"
+directory "#{basedir}/etc" do
   owner "nominatim"
-  group "nominatim"
-  mode 0o664
+  group "adm"
+  mode 0o775
 end
 
-file "#{build_directory}/settings/apache_blocks.conf" do
+file "#{basedir}/etc/nginx_blocked_user_agent.conf" do
   action :create_if_missing
   owner "nominatim"
-  group "nominatim"
+  group "adm"
   mode 0o664
 end
 
-file "#{build_directory}/settings/ip_blocks.map" do
+file "#{basedir}/etc/nginx_blocked_referrer.conf" do
   action :create_if_missing
   owner "nominatim"
-  group "nominatim"
+  group "adm"
   mode 0o664
 end
 
-include_recipe "apache"
-
-package "php"
-package "php-fpm"
-package "php-pgsql"
-package "php-intl"
-
-apache_module "rewrite"
-apache_module "proxy"
-apache_module "proxy_fcgi"
-apache_module "proxy_http"
-apache_module "headers"
-
 service "php7.2-fpm" do
   action [:enable, :start]
   supports :status => true, :restart => true, :reload => true
@@ -353,13 +342,6 @@ node[:nominatim][:fpm_pools].each do |name, data|
   end
 end
 
-systemd_service "apache-nominatim" do
-  service "apache2"
-  dropin "nominatim"
-  tasks_max 12000
-  notifies :restart, "service[apache2]"
-end
-
 ssl_certificate node[:fqdn] do
   domains [node[:fqdn],
            "nominatim.openstreetmap.org",
@@ -368,29 +350,33 @@ ssl_certificate node[:fqdn] do
            "nominatim.openstreetmap.net",
            "nominatim.openstreetmaps.org",
            "nominatim.openmaps.org"]
-  notifies :reload, "service[apache2]"
+  notifies :reload, "service[nginx]"
 end
 
-apache_site "nominatim.openstreetmap.org" do
-  template "apache.erb"
-  directory build_directory
-  variables :pools => node[:nominatim][:fpm_pools]
-  only_if { node[:nominatim][:state] != "off" }
+package "apache2" do
+  action :remove
 end
 
-apache_site "default" do
-  action [:disable]
+include_recipe "nginx"
+
+nginx_site "default" do
+  action [:delete]
 end
 
-template "/etc/logrotate.d/apache2" do
-  source "logrotate.apache.erb"
+nginx_site "nominatim" do
+  template "nginx.erb"
+  directory build_directory
+  variables :pools => node[:nominatim][:fpm_pools],
+            :confdir => "#{basedir}/etc"
+end
+
+template "/etc/logrotate.d/nginx" do
+  source "logrotate.nginx.erb"
   owner "root"
   group "root"
   mode 0o644
 end
 
-include_recipe "fail2ban"
-
 munin_plugin_conf "nominatim" do
   template "munin.erb"
   variables :db => node[:nominatim][:dbname],
diff --git a/cookbooks/nominatim/templates/default/apache.erb b/cookbooks/nominatim/templates/default/apache.erb
deleted file mode 100644 (file)
index 46aa2d8..0000000
+++ /dev/null
@@ -1,87 +0,0 @@
-# DO NOT EDIT - This file is being maintained by Chef
-
-<VirtualHost *:443>
-    ServerName <%= node[:fqdn] %>
-    ServerAlias nominatim.openstreetmap.org
-    ServerAlias nominatim.osm.org
-    ServerAlias nominatim.openstreetmap.com
-    ServerAlias nominatim.openstreetmap.net
-    ServerAlias nominatim.openstreetmaps.org
-    ServerAlias nominatim.openmaps.org
-    ServerAdmin webmaster@openstreetmap.org
-
-    # Enable SSL
-    SSLEngine on
-    SSLProxyEngine on
-    SSLCertificateFile /etc/ssl/certs/<%= node[:fqdn] %>.pem
-    SSLCertificateKeyFile /etc/ssl/private/<%= node[:fqdn] %>.key
-
-    # Remove Proxy request header to mitigate https://httpoxy.org/
-    RequestHeader unset Proxy early
-
-    RequestReadTimeout header=15-30,MinRate=500 body=15-30,MinRate=500
-
-    CustomLog <%= node[:nominatim][:logdir] %>/nominatim.openstreetmap.org-access.log combined
-    ErrorLog /var/log/apache2/nominatim.openstreetmap.org-error.log
-
-    DocumentRoot <%= @directory %>/website
-    <Directory "<%= @directory %>/website/">
-        DirectoryIndex search.php
-        Options FollowSymLinks
-        Require all granted
-    </Directory>
-
-    ProxyPassMatch ^/([^/]*\.php(/.*)?)$ fcgi://127.0.0.1:<%= @pools[:www][:port ]%><%= @directory %>/website/$1
-
-    <% @pools.each do |name,details| -%>
-    Alias /pool-<%= name %>/ "<%= @directory %>/website/"
-    <% node[:nominatim][:redirects].each do |url,host| -%>
-    ProxyPassMatch ^/pool-<%= name %>/(<%= url %>\.php(/.*)?) http<% if port == 443 -%>s<% end -%>://<%= host %>/pool-<%= name %>/$1
-    <% end -%>
-    ProxyPassMatch ^/pool-<%= name %>/(.*\.php(/.*)?) fcgi://127.0.0.1:<%= details[:port ]%><%= @directory %>/website/$1
-    <% end -%>
-
-    Redirect 429 /pool-block/
-    ErrorDocument 429 /509.html
-    <Location /pool-block>
-        ErrorDocument 429 /509.html
-    </Location>
-    Redirect 403 /pool-ban/
-    <Location /pool-ban>
-        ErrorDocument 403 /403.html
-    </Location>
-    ErrorDocument 403 /403.html
-
-    RewriteEngine On
-
-    # manual blocks
-    Include <%= @directory %>/settings/apache_blocks.conf
-
-    # regular requests and autoblocks
-    RewriteMap bulklist txt:<%= @directory %>/settings/ip_blocks.map
-    RewriteRule ^/(search|reverse|lookup|details)(\.php)?(/.*)? /pool-${bulklist:%{REMOTE_ADDR}|www}/$1.php$3 [PT]
-</VirtualHost>
-
-<VirtualHost *:80>
-    ServerName <%= node[:fqdn] %>
-    ServerAlias nominatim.openstreetmap.org
-    ServerAlias nominatim.osm.org
-    ServerAlias nominatim.openstreetmap.com
-    ServerAlias nominatim.openstreetmap.net
-    ServerAlias nominatim.openstreetmaps.org
-    ServerAlias nominatim.openmaps.org
-    ServerAdmin webmaster@openstreetmap.org
-
-    CustomLog <%= node[:nominatim][:logdir] %>/nominatim.openstreetmap.org-access.log combined
-    ErrorLog /var/log/apache2/nominatim.openstreetmap.org-error.log
-
-    RewriteEngine On
-
-    # manual blocks
-    Include <%= @directory %>/settings/apache_blocks.conf
-
-    RewriteRule ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 [R=permanent,L]
-
-    RewriteCond %{REQUEST_URI} !^/server-status$
-    RewriteRule ^(.*)$ https://nominatim.openstreetmap.org$1 [L,NE,R=permanent]
-</VirtualHost>
similarity index 63%
rename from cookbooks/nominatim/templates/default/logrotate.apache.erb
rename to cookbooks/nominatim/templates/default/logrotate.nginx.erb
index 8835397f3081a2a0e2611d3f9c1d67acf31ccb8d..5bbd02effc2c6803445cdbb78d57b88a9ed54cd9 100644 (file)
@@ -1,30 +1,29 @@
 # DO NOT EDIT - This file is being maintained by Chef
 
-/var/log/apache2/*.log {
+/var/log/nginx/*.log {
   daily
   missingok
-  rotate 42
+  rotate 7
   compress
   delaycompress
   notifempty
-  create 640 root adm
+  create 640 nginx adm
   sharedscripts
   postrotate
-    /bin/systemctl reload apache2
+    [ -f /var/run/nginx.pid ] && kill -USR1 `cat /var/run/nginx.pid`
   endscript
 }
 
-
 <%= node[:nominatim][:logdir] %>/nominatim.openstreetmap.org-access.log {
   daily
   missingok
-  rotate 42
+  rotate 28
   compress
   delaycompress
   notifempty
   create 640 root adm
   sharedscripts
   postrotate
-    /bin/systemctl reload apache2
+    [ -f /var/run/nginx.pid ] && kill -USR1 `cat /var/run/nginx.pid`
   endscript
 }
index e1fc714162514a83c357b44f8789fc13bc521bcb..ba7a5bc3f590a2b99b84efc52f776c2383998ccc 100644 (file)
   notifempty
   create 640 www-data adm
 }
-
-<%= node[:nominatim][:logdir] %>/restricted_ips.log {
-  weekly
-  missingok
-  rotate 5
-  compress
-  delaycompress
-  notifempty
-  create 660 nominatim adm
-}
diff --git a/cookbooks/nominatim/templates/default/nginx.erb b/cookbooks/nominatim/templates/default/nginx.erb
new file mode 100644 (file)
index 0000000..1a5fa0c
--- /dev/null
@@ -0,0 +1,131 @@
+map $uri $nominatim_script_name {
+    ~^(.+?\.php)         $1;
+    ~^/([^/]+)           $1.php;
+    ^$                   search.php;
+}
+
+map $uri $nominatim_path_info {
+    ~^/([^/]+)(.*)$      $2;
+}
+
+map $query_string $email_id {
+    ~(^|&)email=([^&]+)  $2;
+}
+
+upstream nominatim_service {
+  server 127.0.0.1:<%= @pools[:www][:port ]%>;
+}
+
+# Whitelisted IPs
+geo $limit {
+    default 1;
+    2001:978:2:2c::172:6 0;
+    2001:978:2:2c::172:7 0;
+    2001:978:2:2c::172:8 0;
+    2001:978:2:2c::172:b 0;
+    2001:978:2:2c::172:c 0;
+    2001:978:2:2c::172:d 0;
+    130.117.76.6 0;
+    130.117.76.7 0;
+    130.117.76.8 0;
+    89.16.162.21 0;
+    89.16.162.22 0;
+    46.235.224.148 0;
+    209.132.180.180 0;
+    209.132.180.168 0;
+    8.43.85.23 0; # gnome
+}
+
+map $http_user_agent $blocked_user_agent {
+   default 0;
+   include <%= @confdir %>/nginx_blocked_user_agent.conf;
+}
+
+map $http_referer $blocked_referrer {
+   default 0;
+   include <%= @confdir %>/nginx_blocked_referrer.conf;
+}
+
+map $limit $limit_key {
+    0 "";
+    1 $binary_remote_addr;
+}
+
+map $blocked_user_agent $limit_tarpit {
+    0 "";
+    1 $binary_remote_addr;
+    2 $binary_remote_addr;
+}
+
+limit_req_zone $limit zone=www:50m rate=2r/s;
+limit_req_zone $limit_tarpit zone=tarpit:10m rate=1r/s;
+limit_req_zone $binary_remote_addr zone=blocked:10m rate=20r/m;
+
+server {
+    # IPv4
+    listen       80 deferred backlog=16384 reuseport fastopen=2048 default_server;
+    listen       443 ssl deferred backlog=16384 reuseport fastopen=2048 http2 default_server;
+    # IPv6
+    listen       [::]:80 deferred backlog=16384 reuseport fastopen=2048 default_server;
+    listen       [::]:443 ssl deferred backlog=16384 reuseport fastopen=2048 http2 default_server;
+    server_name  localhost;
+
+    ssl_certificate /etc/ssl/certs/<%= node[:fqdn] %>.pem;
+    ssl_certificate_key /etc/ssl/private/<%= node[:fqdn] %>.key;
+
+    root <%= @directory %>/website;
+    index search.php;
+
+    access_log <%= node[:nominatim][:logdir] %>/nominatim.openstreetmap.org-access.log combined;
+    error_log <%= node[:nominatim][:logdir] %>/nominatim.openstreetmap.org-error.log;
+
+    location /nginx_status {
+        stub_status on;
+        access_log   off;
+        allow 127.0.0.1;
+        allow ::1;
+        deny all;
+    }
+
+    error_page 403 /403.html;
+    location /403.html {
+        limit_req zone=blocked burst=5;
+    }
+
+    error_page 429 /509.html;
+    location /509.html {
+        limit_req zone=blocked burst=5;
+    }
+
+    location / {
+        set anyid = $http_referer$http_user_agent$email_id;
+        if (anyid = "")
+        { return 403; }
+        if ($blocked_user_agent ~ ^2$)
+        { return 403; }
+        if ($blocked_referrer)
+        { return 403; }
+
+        try_files $uri $uri/ @php;
+    }
+
+    location @php {
+        limit_req zone=www burst=10;
+        limit_req zone=tarpit burst=2;
+        limit_req_status 429;
+        fastcgi_pass nominatim_service;
+        include fastcgi_params;
+        fastcgi_param QUERY_STRING    $args;
+        fastcgi_param PATH_INFO       "$nominatim_path_info";
+        fastcgi_param SCRIPT_FILENAME  "$document_root/$nominatim_script_name";
+    }
+
+    location ~* \.php$ {
+        limit_req zone=www burst=10;
+        limit_req zone=tarpit burst=2;
+        limit_req_status 429;
+        fastcgi_pass    nominatim_service;
+        include         fastcgi_params;
+        fastcgi_param   SCRIPT_FILENAME    $document_root$fastcgi_script_name;
+    }
+}
index 5ae8f3ad6fcc94a2824d38e151713b878de343d9..0ef3cbb88dda57f284548bbb2eceed017fb2b71e 100644 (file)
@@ -2,7 +2,6 @@
 
 MAILTO=<%= @mailto %>
 
-* * * * * lonvia <%= @bin_directory %>/cron_ipanalyse.py <%= node[:nominatim][:logdir] %>/nominatim.openstreetmap.org-access.log
 <% if node[:nominatim][:enable_backup] -%>
 00 3 1 * * nominatim /usr/local/bin/backup-nominatim
 <% end -%>