From 87567b1b6fd1876f7c0258a1d2737c84808b6916 Mon Sep 17 00:00:00 2001 From: Grant Slater Date: Wed, 27 Mar 2024 16:17:30 +0000 Subject: [PATCH] imagery: add za_ngi_aerial using tiler --- cookbooks/imagery/recipes/za_ngi_aerial.rb | 39 +++++++++++++++++++ cookbooks/imagery/resources/layer.rb | 4 +- cookbooks/imagery/resources/site.rb | 7 ++++ .../templates/default/nginx_imagery.conf.erb | 8 ++++ .../nginx_imagery_layer_fragment.conf.erb | 25 ++++++++++++ .../templates/default/nginx_titiler.conf.erb | 33 +--------------- roles/ironbelly.rb | 10 ++++- 7 files changed, 92 insertions(+), 34 deletions(-) create mode 100644 cookbooks/imagery/recipes/za_ngi_aerial.rb diff --git a/cookbooks/imagery/recipes/za_ngi_aerial.rb b/cookbooks/imagery/recipes/za_ngi_aerial.rb new file mode 100644 index 000000000..26527c2a9 --- /dev/null +++ b/cookbooks/imagery/recipes/za_ngi_aerial.rb @@ -0,0 +1,39 @@ +# +# Cookbook:: imagery +# Recipe:: za_ngi_aerial +# +# Copyright:: 2024, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery::tiler" + +imagery_site "aerial.openstreetmap.org.za" do + title "OpenStreetMap - NGI - Aerial Imagery" + aliases ["aerial.osm.org.za"] + bbox [[-35.12, 16.23], [-22.1, 33.18]] + uses_tiler true +end + +imagery_layer "ngi-aerial" do + site "aerial.openstreetmap.org.za" + uses_tiler true + title "NGI Aerial 25cm/50cm" + source "https://tiler.openstreetmap.org/za-25cm/mosaic-tiler-https.json" + copyright 'State Copyright © 2024 Chief Directorate: National Geo-spatial Information' + max_zoom 20 + extension jpg + default_layer true + url_aliases ["/ngi-aerial"] +end diff --git a/cookbooks/imagery/resources/layer.rb b/cookbooks/imagery/resources/layer.rb index 7a376fdd1..62b266465 100644 --- a/cookbooks/imagery/resources/layer.rb +++ b/cookbooks/imagery/resources/layer.rb @@ -41,6 +41,7 @@ property :url_aliases, [String, Array], :default => [] property :revision, Integer, :default => 0 property :overlay, [true, false], :default => false property :default_layer, [true, false], :default => false +property :uses_tiler, [true, false], :default => false action :create do file "/srv/imagery/layers/#{new_resource.site}/#{new_resource.layer}.yml" do @@ -49,7 +50,7 @@ action :create do mode "644" content YAML.dump(:name => new_resource.layer, :title => new_resource.title || new_resource.layer, - :url => "//#{new_resource.site}/layer/#{new_resource.layer}/{z}/{x}/{y}.png", + :url => "//#{new_resource.site}/layer/#{new_resource.layer}/{z}/{x}/{y}.#{new_resource.extension}", :attribution => new_resource.copyright, :default => new_resource.default_layer, :maxZoom => new_resource.max_zoom, @@ -63,6 +64,7 @@ action :create do group "root" mode "644" variables new_resource.to_hash + not_if { new_resource.uses_tiler } end directory "/srv/imagery/nginx/#{new_resource.site}" do diff --git a/cookbooks/imagery/resources/site.rb b/cookbooks/imagery/resources/site.rb index 4fca7bb9a..a91f9794c 100644 --- a/cookbooks/imagery/resources/site.rb +++ b/cookbooks/imagery/resources/site.rb @@ -27,6 +27,7 @@ property :site, String, :name_property => true property :title, String, :required => [:create] property :aliases, [String, Array], :default => [] property :bbox, Array, :required => [:create] +property :uses_tiler, [true, false], :default => false action :create do directory "/srv/#{new_resource.site}" do @@ -117,6 +118,7 @@ action :create do restrict_address_families "AF_UNIX" # Terminate service after 30mins. Service is socket activated runtime_max_sec 1800 + not_if { new_resource.uses_tiler } end systemd_socket "mapserv-fcgi-#{new_resource.site}" do @@ -124,6 +126,7 @@ action :create do socket_user "imagery" socket_group "imagery" listen_stream "/run/mapserver-fastcgi/layer-#{new_resource.site}.socket" + not_if { new_resource.uses_tiler } end # Ensure service is stopped because otherwise the socket cannot reload @@ -132,11 +135,13 @@ action :create do action :nothing subscribes :stop, "systemd_service[mapserv-fcgi-#{new_resource.site}]" subscribes :stop, "systemd_socket[mapserv-fcgi-#{new_resource.site}]" + not_if { new_resource.uses_tiler } end systemd_unit "mapserv-fcgi-#{new_resource.site}.socket" do action [:enable, :start] subscribes :restart, "systemd_socket[mapserv-fcgi-#{new_resource.site}]" + not_if { new_resource.uses_tiler } end ssl_certificate new_resource.site do @@ -154,10 +159,12 @@ action :delete do service "mapserv-fcgi-#{new_resource.site}" do provider Chef::Provider::Service::Systemd action [:stop, :disable] + not_if { new_resource.uses_tiler } end systemd_service "mapserv-fcgi-#{new_resource.site}" do action :delete + not_if { new_resource.uses_tiler } end nginx_site new_resource.site do diff --git a/cookbooks/imagery/templates/default/nginx_imagery.conf.erb b/cookbooks/imagery/templates/default/nginx_imagery.conf.erb index e1d33c7b6..497cc1f53 100644 --- a/cookbooks/imagery/templates/default/nginx_imagery.conf.erb +++ b/cookbooks/imagery/templates/default/nginx_imagery.conf.erb @@ -7,12 +7,20 @@ server { return 301 https://$host$request_uri; } +<% if @uses_tiler -%> +upstream tiler_backend { + server 127.0.0.1:8080; + + keepalive 32; +} +<% else -%> upstream <%= @name %>_fastcgi { server "unix:/var/run/mapserver-fastcgi/layer-<%= @name %>.socket" max_fails=0; # Use default round-robin to distribute requests, rather than pick "fast" but maybe faulty. # Do not use keepalive } +<% end -%> server { listen 443 ssl http2; diff --git a/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb b/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb index f28629237..c4271690d 100644 --- a/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb +++ b/cookbooks/imagery/templates/default/nginx_imagery_layer_fragment.conf.erb @@ -1,5 +1,29 @@ +<% require 'uri' %> # DO NOT EDIT - This file is being maintained by Chef location ~* "^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(png|jpg|jpeg)$" { +<% if @uses_tiler -%> + set $args ""; + rewrite ^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.jpg /mosaicjson/tiles/WebMercatorQuad/$1/$2/$3@1x?url=<%= URI.encode_www_form_component(@source) %>&pixel_selection=first&tile_format=jpeg break; + rewrite ^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.jpeg /mosaicjson/tiles/WebMercatorQuad/$1/$2/$3@1x?url=<%= URI.encode_www_form_component(@source) %>&pixel_selection=first&tile_format=jpeg break; + rewrite ^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.png /mosaicjson/tiles/WebMercatorQuad/$1/$2/$3@1x?url=<%= URI.encode_www_form_component(@source) %>&pixel_selection=first&tile_format=png break; + proxy_pass http://tiler_backend; + proxy_set_header Host $host; + proxy_set_header Referer $http_referer; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-SSL on; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Cache-Control ""; + proxy_set_header Pragma ""; + proxy_redirect off; + proxy_cache_key "<%= @layer %><%= @revision %> $request_method $1 $2 $3"; + proxy_cache proxy_cache_zone; + proxy_cache_valid 200 204 180d; + proxy_cache_use_stale error timeout updating http_502 http_503 http_504; + proxy_cache_background_update on; + +<% else -%> # Override QUERY_STRING to force mapserver query parameters fastcgi_param QUERY_STRING "map=/srv/imagery/mapserver/layer-<%= @layer %>.map&mode=tile&layers=<%= @layer %>&tilemode=gmap&tile=$2+$3+$1"; fastcgi_pass "<%= @site %>_fastcgi"; @@ -30,6 +54,7 @@ location ~* "^/layer/<%= @layer %>/(\d+)/(\d+)/(\d+)\.(png|jpg|jpeg)$" { fastcgi_next_upstream error timeout invalid_header http_500 http_503; fastcgi_next_upstream_tries 8; +<% end -%> # Do not GZIP tiles gzip off; diff --git a/cookbooks/imagery/templates/default/nginx_titiler.conf.erb b/cookbooks/imagery/templates/default/nginx_titiler.conf.erb index e84948441..6e03a25b9 100644 --- a/cookbooks/imagery/templates/default/nginx_titiler.conf.erb +++ b/cookbooks/imagery/templates/default/nginx_titiler.conf.erb @@ -15,14 +15,6 @@ server { } } -upstream tiler_backend { - server 127.0.0.1:8080; - - keepalive 32; -} - -proxy_cache_path /var/cache/nginx-cache levels=1:2 keys_zone=ngi-aerial:64m; - server { listen 443 ssl http2; listen [::]:443 ssl http2; @@ -63,30 +55,7 @@ server { proxy_set_header X-Forwarded-Proto https; proxy_set_header X-Forwarded-SSL on; proxy_http_version 1.1; - proxy_set_header "Connection" ""; + proxy_set_header Connection ""; proxy_redirect off; } - - location /ngi-aerial { - set $args ""; - rewrite ^/ngi-aerial/(\d+)/(\d+)/(\d+)\.jpg /mosaicjson/tiles/WebMercatorQuad/$1/$2/$3@1x?url=https%3A%2F%2Ftiler.openstreetmap.org%2Fza-25cm%2Fmosaic-tiler-https.json&pixel_selection=first&tile_format=jpeg break; - proxy_pass http://tiler_backend; - proxy_set_header Host $host; - proxy_set_header Referer $http_referer; - proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Forwarded-SSL on; - proxy_http_version 1.1; - proxy_set_header "Connection" ""; - proxy_redirect off; - proxy_cache_key "$scheme$proxy_host$uri"; - proxy_cache ngi-aerial; - proxy_cache_valid 200 204 180d; - proxy_cache_use_stale error timeout updating http_502 http_503 http_504; - proxy_cache_background_update on; - proxy_ignore_headers Cache-Control; - expires max; - add_header X-Proxy-Cache $upstream_cache_status; - - } } diff --git a/roles/ironbelly.rb b/roles/ironbelly.rb index 78a9ac5ce..6363bdfd6 100644 --- a/roles/ironbelly.rb +++ b/roles/ironbelly.rb @@ -66,6 +66,14 @@ default_attributes( } } }, + :nginx => { + :proxy => { + :enable => true, + :keys_zone => "proxy_cache_zone:256M", + :inactive => "180d", + :max_size => "51200M" + } + }, :rsyncd => { :modules => { :logs => { @@ -101,5 +109,5 @@ run_list( "role[gateway]", "recipe[rsyncd]", "recipe[dhcpd]", - "recipe[imagery::tiler]" + "recipe[imagery::za_ngi_aerial]" ) -- 2.45.1