From 611017bc534a5e2191dbf939b4be4899bbe9669c Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 24 Apr 2020 21:43:19 +0200 Subject: [PATCH] nominatim: move to nginx for webserving --- cookbooks/nominatim/attributes/default.rb | 5 - cookbooks/nominatim/metadata.rb | 2 + cookbooks/nominatim/recipes/default.rb | 68 ++++----- .../nominatim/templates/default/apache.erb | 87 ------------ ...grotate.apache.erb => logrotate.nginx.erb} | 13 +- .../templates/default/logrotate.nominatim.erb | 10 -- .../nominatim/templates/default/nginx.erb | 131 ++++++++++++++++++ .../templates/default/nominatim.cron.erb | 1 - 8 files changed, 166 insertions(+), 151 deletions(-) delete mode 100644 cookbooks/nominatim/templates/default/apache.erb rename cookbooks/nominatim/templates/default/{logrotate.apache.erb => logrotate.nginx.erb} (63%) create mode 100644 cookbooks/nominatim/templates/default/nginx.erb diff --git a/cookbooks/nominatim/attributes/default.rb b/cookbooks/nominatim/attributes/default.rb index 93b7daf07..d096f9394 100644 --- a/cookbooks/nominatim/attributes/default.rb +++ b/cookbooks/nominatim/attributes/default.rb @@ -13,11 +13,6 @@ default[:nominatim][:fpm_pools] = { :port => "8000", :pm => "dynamic", :max_children => "60" - }, - :bulk => { - :port => "8001", - :pm => "static", - :max_children => "10" } } diff --git a/cookbooks/nominatim/metadata.rb b/cookbooks/nominatim/metadata.rb index 71c3ee93c..57b19a03a 100644 --- a/cookbooks/nominatim/metadata.rb +++ b/cookbooks/nominatim/metadata.rb @@ -6,9 +6,11 @@ description "Installs and configures nominatim servers" version "1.0.0" supports "ubuntu" +depends "systemd" depends "apache" depends "fail2ban" depends "git" depends "munin" +depends "nginx" depends "postgresql" depends "python" diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb index 76f7cdced..463b6e602 100644 --- a/cookbooks/nominatim/recipes/default.rb +++ b/cookbooks/nominatim/recipes/default.rb @@ -166,6 +166,10 @@ package %w[ python3-pyosmium pyosmium python3-psycopg2 + php + php-fpm + php-pgsql + php-intl ] source_directory = "#{basedir}/nominatim" @@ -302,41 +306,26 @@ end ## webserver frontend -template "#{build_directory}/settings/ip_blocks.conf" do - action :create_if_missing - source "ipblocks.erb" +directory "#{basedir}/etc" do owner "nominatim" - group "nominatim" - mode 0o664 + group "adm" + mode 0o775 end -file "#{build_directory}/settings/apache_blocks.conf" do +file "#{basedir}/etc/nginx_blocked_user_agent.conf" do action :create_if_missing owner "nominatim" - group "nominatim" + group "adm" mode 0o664 end -file "#{build_directory}/settings/ip_blocks.map" do +file "#{basedir}/etc/nginx_blocked_referrer.conf" do action :create_if_missing owner "nominatim" - group "nominatim" + group "adm" mode 0o664 end -include_recipe "apache" - -package "php" -package "php-fpm" -package "php-pgsql" -package "php-intl" - -apache_module "rewrite" -apache_module "proxy" -apache_module "proxy_fcgi" -apache_module "proxy_http" -apache_module "headers" - service "php7.2-fpm" do action [:enable, :start] supports :status => true, :restart => true, :reload => true @@ -353,13 +342,6 @@ node[:nominatim][:fpm_pools].each do |name, data| end end -systemd_service "apache-nominatim" do - service "apache2" - dropin "nominatim" - tasks_max 12000 - notifies :restart, "service[apache2]" -end - ssl_certificate node[:fqdn] do domains [node[:fqdn], "nominatim.openstreetmap.org", @@ -368,29 +350,33 @@ ssl_certificate node[:fqdn] do "nominatim.openstreetmap.net", "nominatim.openstreetmaps.org", "nominatim.openmaps.org"] - notifies :reload, "service[apache2]" + notifies :reload, "service[nginx]" end -apache_site "nominatim.openstreetmap.org" do - template "apache.erb" - directory build_directory - variables :pools => node[:nominatim][:fpm_pools] - only_if { node[:nominatim][:state] != "off" } +package "apache2" do + action :remove end -apache_site "default" do - action [:disable] +include_recipe "nginx" + +nginx_site "default" do + action [:delete] end -template "/etc/logrotate.d/apache2" do - source "logrotate.apache.erb" +nginx_site "nominatim" do + template "nginx.erb" + directory build_directory + variables :pools => node[:nominatim][:fpm_pools], + :confdir => "#{basedir}/etc" +end + +template "/etc/logrotate.d/nginx" do + source "logrotate.nginx.erb" owner "root" group "root" mode 0o644 end -include_recipe "fail2ban" - munin_plugin_conf "nominatim" do template "munin.erb" variables :db => node[:nominatim][:dbname], diff --git a/cookbooks/nominatim/templates/default/apache.erb b/cookbooks/nominatim/templates/default/apache.erb deleted file mode 100644 index 46aa2d85e..000000000 --- a/cookbooks/nominatim/templates/default/apache.erb +++ /dev/null @@ -1,87 +0,0 @@ -# DO NOT EDIT - This file is being maintained by Chef - - - ServerName <%= node[:fqdn] %> - ServerAlias nominatim.openstreetmap.org - ServerAlias nominatim.osm.org - ServerAlias nominatim.openstreetmap.com - ServerAlias nominatim.openstreetmap.net - ServerAlias nominatim.openstreetmaps.org - ServerAlias nominatim.openmaps.org - ServerAdmin webmaster@openstreetmap.org - - # Enable SSL - SSLEngine on - SSLProxyEngine on - SSLCertificateFile /etc/ssl/certs/<%= node[:fqdn] %>.pem - SSLCertificateKeyFile /etc/ssl/private/<%= node[:fqdn] %>.key - - # Remove Proxy request header to mitigate https://httpoxy.org/ - RequestHeader unset Proxy early - - RequestReadTimeout header=15-30,MinRate=500 body=15-30,MinRate=500 - - CustomLog <%= node[:nominatim][:logdir] %>/nominatim.openstreetmap.org-access.log combined - ErrorLog /var/log/apache2/nominatim.openstreetmap.org-error.log - - DocumentRoot <%= @directory %>/website - /website/"> - DirectoryIndex search.php - Options FollowSymLinks - Require all granted - - - ProxyPassMatch ^/([^/]*\.php(/.*)?)$ fcgi://127.0.0.1:<%= @pools[:www][:port ]%><%= @directory %>/website/$1 - - <% @pools.each do |name,details| -%> - Alias /pool-<%= name %>/ "<%= @directory %>/website/" - <% node[:nominatim][:redirects].each do |url,host| -%> - ProxyPassMatch ^/pool-<%= name %>/(<%= url %>\.php(/.*)?) http<% if port == 443 -%>s<% end -%>://<%= host %>/pool-<%= name %>/$1 - <% end -%> - ProxyPassMatch ^/pool-<%= name %>/(.*\.php(/.*)?) fcgi://127.0.0.1:<%= details[:port ]%><%= @directory %>/website/$1 - <% end -%> - - Redirect 429 /pool-block/ - ErrorDocument 429 /509.html - - ErrorDocument 429 /509.html - - Redirect 403 /pool-ban/ - - ErrorDocument 403 /403.html - - ErrorDocument 403 /403.html - - RewriteEngine On - - # manual blocks - Include <%= @directory %>/settings/apache_blocks.conf - - # regular requests and autoblocks - RewriteMap bulklist txt:<%= @directory %>/settings/ip_blocks.map - RewriteRule ^/(search|reverse|lookup|details)(\.php)?(/.*)? /pool-${bulklist:%{REMOTE_ADDR}|www}/$1.php$3 [PT] - - - - ServerName <%= node[:fqdn] %> - ServerAlias nominatim.openstreetmap.org - ServerAlias nominatim.osm.org - ServerAlias nominatim.openstreetmap.com - ServerAlias nominatim.openstreetmap.net - ServerAlias nominatim.openstreetmaps.org - ServerAlias nominatim.openmaps.org - ServerAdmin webmaster@openstreetmap.org - - CustomLog <%= node[:nominatim][:logdir] %>/nominatim.openstreetmap.org-access.log combined - ErrorLog /var/log/apache2/nominatim.openstreetmap.org-error.log - - RewriteEngine On - - # manual blocks - Include <%= @directory %>/settings/apache_blocks.conf - - RewriteRule ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 [R=permanent,L] - - RewriteCond %{REQUEST_URI} !^/server-status$ - RewriteRule ^(.*)$ https://nominatim.openstreetmap.org$1 [L,NE,R=permanent] - diff --git a/cookbooks/nominatim/templates/default/logrotate.apache.erb b/cookbooks/nominatim/templates/default/logrotate.nginx.erb similarity index 63% rename from cookbooks/nominatim/templates/default/logrotate.apache.erb rename to cookbooks/nominatim/templates/default/logrotate.nginx.erb index 8835397f3..5bbd02eff 100644 --- a/cookbooks/nominatim/templates/default/logrotate.apache.erb +++ b/cookbooks/nominatim/templates/default/logrotate.nginx.erb @@ -1,30 +1,29 @@ # DO NOT EDIT - This file is being maintained by Chef -/var/log/apache2/*.log { +/var/log/nginx/*.log { daily missingok - rotate 42 + rotate 7 compress delaycompress notifempty - create 640 root adm + create 640 nginx adm sharedscripts postrotate - /bin/systemctl reload apache2 + [ -f /var/run/nginx.pid ] && kill -USR1 `cat /var/run/nginx.pid` endscript } - <%= node[:nominatim][:logdir] %>/nominatim.openstreetmap.org-access.log { daily missingok - rotate 42 + rotate 28 compress delaycompress notifempty create 640 root adm sharedscripts postrotate - /bin/systemctl reload apache2 + [ -f /var/run/nginx.pid ] && kill -USR1 `cat /var/run/nginx.pid` endscript } diff --git a/cookbooks/nominatim/templates/default/logrotate.nominatim.erb b/cookbooks/nominatim/templates/default/logrotate.nominatim.erb index e1fc71416..ba7a5bc3f 100644 --- a/cookbooks/nominatim/templates/default/logrotate.nominatim.erb +++ b/cookbooks/nominatim/templates/default/logrotate.nominatim.erb @@ -19,13 +19,3 @@ notifempty create 640 www-data adm } - -<%= node[:nominatim][:logdir] %>/restricted_ips.log { - weekly - missingok - rotate 5 - compress - delaycompress - notifempty - create 660 nominatim adm -} diff --git a/cookbooks/nominatim/templates/default/nginx.erb b/cookbooks/nominatim/templates/default/nginx.erb new file mode 100644 index 000000000..1a5fa0c98 --- /dev/null +++ b/cookbooks/nominatim/templates/default/nginx.erb @@ -0,0 +1,131 @@ +map $uri $nominatim_script_name { + ~^(.+?\.php) $1; + ~^/([^/]+) $1.php; + ^$ search.php; +} + +map $uri $nominatim_path_info { + ~^/([^/]+)(.*)$ $2; +} + +map $query_string $email_id { + ~(^|&)email=([^&]+) $2; +} + +upstream nominatim_service { + server 127.0.0.1:<%= @pools[:www][:port ]%>; +} + +# Whitelisted IPs +geo $limit { + default 1; + 2001:978:2:2c::172:6 0; + 2001:978:2:2c::172:7 0; + 2001:978:2:2c::172:8 0; + 2001:978:2:2c::172:b 0; + 2001:978:2:2c::172:c 0; + 2001:978:2:2c::172:d 0; + 130.117.76.6 0; + 130.117.76.7 0; + 130.117.76.8 0; + 89.16.162.21 0; + 89.16.162.22 0; + 46.235.224.148 0; + 209.132.180.180 0; + 209.132.180.168 0; + 8.43.85.23 0; # gnome +} + +map $http_user_agent $blocked_user_agent { + default 0; + include <%= @confdir %>/nginx_blocked_user_agent.conf; +} + +map $http_referer $blocked_referrer { + default 0; + include <%= @confdir %>/nginx_blocked_referrer.conf; +} + +map $limit $limit_key { + 0 ""; + 1 $binary_remote_addr; +} + +map $blocked_user_agent $limit_tarpit { + 0 ""; + 1 $binary_remote_addr; + 2 $binary_remote_addr; +} + +limit_req_zone $limit zone=www:50m rate=2r/s; +limit_req_zone $limit_tarpit zone=tarpit:10m rate=1r/s; +limit_req_zone $binary_remote_addr zone=blocked:10m rate=20r/m; + +server { + # IPv4 + listen 80 deferred backlog=16384 reuseport fastopen=2048 default_server; + listen 443 ssl deferred backlog=16384 reuseport fastopen=2048 http2 default_server; + # IPv6 + listen [::]:80 deferred backlog=16384 reuseport fastopen=2048 default_server; + listen [::]:443 ssl deferred backlog=16384 reuseport fastopen=2048 http2 default_server; + server_name localhost; + + ssl_certificate /etc/ssl/certs/<%= node[:fqdn] %>.pem; + ssl_certificate_key /etc/ssl/private/<%= node[:fqdn] %>.key; + + root <%= @directory %>/website; + index search.php; + + access_log <%= node[:nominatim][:logdir] %>/nominatim.openstreetmap.org-access.log combined; + error_log <%= node[:nominatim][:logdir] %>/nominatim.openstreetmap.org-error.log; + + location /nginx_status { + stub_status on; + access_log off; + allow 127.0.0.1; + allow ::1; + deny all; + } + + error_page 403 /403.html; + location /403.html { + limit_req zone=blocked burst=5; + } + + error_page 429 /509.html; + location /509.html { + limit_req zone=blocked burst=5; + } + + location / { + set anyid = $http_referer$http_user_agent$email_id; + if (anyid = "") + { return 403; } + if ($blocked_user_agent ~ ^2$) + { return 403; } + if ($blocked_referrer) + { return 403; } + + try_files $uri $uri/ @php; + } + + location @php { + limit_req zone=www burst=10; + limit_req zone=tarpit burst=2; + limit_req_status 429; + fastcgi_pass nominatim_service; + include fastcgi_params; + fastcgi_param QUERY_STRING $args; + fastcgi_param PATH_INFO "$nominatim_path_info"; + fastcgi_param SCRIPT_FILENAME "$document_root/$nominatim_script_name"; + } + + location ~* \.php$ { + limit_req zone=www burst=10; + limit_req zone=tarpit burst=2; + limit_req_status 429; + fastcgi_pass nominatim_service; + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + } +} diff --git a/cookbooks/nominatim/templates/default/nominatim.cron.erb b/cookbooks/nominatim/templates/default/nominatim.cron.erb index 5ae8f3ad6..0ef3cbb88 100644 --- a/cookbooks/nominatim/templates/default/nominatim.cron.erb +++ b/cookbooks/nominatim/templates/default/nominatim.cron.erb @@ -2,7 +2,6 @@ MAILTO=<%= @mailto %> -* * * * * lonvia <%= @bin_directory %>/cron_ipanalyse.py <%= node[:nominatim][:logdir] %>/nominatim.openstreetmap.org-access.log <% if node[:nominatim][:enable_backup] -%> 00 3 1 * * nominatim /usr/local/bin/backup-nominatim <% end -%> -- 2.45.1