From 50c50c0943823b6885d2199816d1a795975baf68 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Thu, 26 Jun 2025 09:30:22 -0700 Subject: [PATCH] vectortile: Move ocean updates to timer This allows updates of ocean tile-layers when the shapefiles are updated --- cookbooks/vectortile/attributes/default.rb | 8 +-- cookbooks/vectortile/recipes/default.rb | 71 +++++++++---------- .../templates/default/ocean-update-notile.erb | 11 +++ .../templates/default/ocean-update-tile.erb | 24 +++++++ .../templates/default/render-ocean.erb | 11 --- .../vectortile/inspec/updates_spec.rb | 19 +++++ 6 files changed, 91 insertions(+), 53 deletions(-) create mode 100644 cookbooks/vectortile/templates/default/ocean-update-notile.erb create mode 100644 cookbooks/vectortile/templates/default/ocean-update-tile.erb delete mode 100644 cookbooks/vectortile/templates/default/render-ocean.erb create mode 100644 test/integration/vectortile/inspec/updates_spec.rb diff --git a/cookbooks/vectortile/attributes/default.rb b/cookbooks/vectortile/attributes/default.rb index 110b9e1e8..38abf55ce 100644 --- a/cookbooks/vectortile/attributes/default.rb +++ b/cookbooks/vectortile/attributes/default.rb @@ -10,12 +10,12 @@ default[:vectortile][:replication][:threads] = [0.5 * node.cpu_cores, 2].max.cei default[:vectortile][:rerender][:lowzoom][:enabled] = true default[:vectortile][:rerender][:lowzoom][:threads] = [0.5 * node.cpu_cores, 2].max.ceil default[:vectortile][:rerender][:lowzoom][:maxzoom] = 9 -default[:vectortile][:rerender][:ocean][:enabled] = true -default[:vectortile][:rerender][:ocean][:threads] = [0.5 * node.cpu_cores, 2].max.ceil -default[:vectortile][:rerender][:ocean][:minzoom] = node[:vectortile][:rerender][:lowzoom][:maxzoom] + 1 +default[:vectortile][:ocean][:enabled] = true +default[:vectortile][:ocean][:threads] = [0.5 * node.cpu_cores, 2].max.ceil +default[:vectortile][:ocean][:tileupdate] = true default[:vectortile][:tilekiln][:version] = "0.8.1" -default[:vectortile][:spirit][:version] = "6cdedcda65fbb8fec83342571c74a33208943b32" +default[:vectortile][:spirit][:version] = "d357384be76460e49f4a5f2eb33f94a1973c2628" default[:vectortile][:themepark][:version] = "beb454cc56e88533fb398ab293489c4e91f4d42b" default[:postgresql][:versions] |= [node[:vectortile][:database][:cluster].split("/").first] diff --git a/cookbooks/vectortile/recipes/default.rb b/cookbooks/vectortile/recipes/default.rb index b47815f16..257b76777 100644 --- a/cookbooks/vectortile/recipes/default.rb +++ b/cookbooks/vectortile/recipes/default.rb @@ -245,12 +245,40 @@ service "tilekiln" do action [:enable, :start] end -execute "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py" do - command "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py -R tilekiln" - cwd "/srv/vector.openstreetmap.org/spirit" +template "/usr/local/bin/ocean-update" do + source node[:vectortile][:ocean][:tileupdate] ? "ocean-update-tile.erb" : "ocean-update-notile.erb" + owner "root" + group "root" + mode "755" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :style_directory => style_directory, :config_path => shortbread_config +end + +systemd_service "ocean-update" do + description "Update ocean data" user "tileupdate" - group "tileupdate" - ignore_failure true + after "postgresql.service" + wants "postgresql.service" + sandbox :enable_network => true + restrict_address_families "AF_UNIX" + read_write_paths ["/srv/vector.openstreetmap.org/spirit/data/"] + exec_start "/usr/local/bin/ocean-update" +end + +systemd_timer "ocean-update" do + description "Update ocean data" + on_boot_sec 300 + on_unit_active_sec 3600 + accuracy_sec 60 +end + +if node[:vectortile][:ocean][:enabled] + service "ocean-update.timer" do + action [:enable, :start] + end +else + service "ocean-update.timer" do + action [:stop, :disable] + end end template "/usr/local/bin/vector-update" do @@ -332,39 +360,6 @@ else end end -template "/usr/local/bin/render-ocean" do - source "render-ocean.erb" - owner "root" - group "root" - mode "755" - variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => shortbread_config, :min_zoom => node[:vectortile][:rerender][:ocean][:minzoom], :max_zoom => 14 -end - -systemd_service "render-ocean" do - description "Render ocean tiles" - user "tileupdate" - after "postgresql.service" - wants "postgresql.service" - restrict_address_families "AF_UNIX" - sandbox true - exec_start "/usr/local/bin/render-ocean" -end - -systemd_timer "render-ocean" do - description "Render ocean tiles" - on_calendar "Sat 01:00 #{node[:timezone]}" -end - -if node[:vectortile][:rerender][:ocean][:enabled] - service "render-ocean.timer" do - action [:enable, :start] - end -else - service "render-ocean.timer" do - action [:stop, :disable] - end -end - package %w[ ruby-pg ruby-webrick diff --git a/cookbooks/vectortile/templates/default/ocean-update-notile.erb b/cookbooks/vectortile/templates/default/ocean-update-notile.erb new file mode 100644 index 000000000..67bf0bedc --- /dev/null +++ b/cookbooks/vectortile/templates/default/ocean-update-notile.erb @@ -0,0 +1,11 @@ +#!/bin/sh + +# Usage +# sudo -u tileupdate ocean-update + +set -eu + + +cd "<%= @style_directory %>" + +scripts/get-external-data.py -R tilekiln diff --git a/cookbooks/vectortile/templates/default/ocean-update-tile.erb b/cookbooks/vectortile/templates/default/ocean-update-tile.erb new file mode 100644 index 000000000..e28030cc4 --- /dev/null +++ b/cookbooks/vectortile/templates/default/ocean-update-tile.erb @@ -0,0 +1,24 @@ +#!/bin/sh + +# Usage +# sudo -u tileupdate ocean-update + +set -eu + +cd "<%= @style_directory %>" +mkdir -p "data" +rm -rf "data/ocean-tiles.txt" +scripts/get-external-data.py -R tilekiln -e "data/ocean-tiles.txt" + +if [ -f "data/ocean-tiles.txt" ] +then + wc -l "data/ocean-tiles.txt" + <%= @tilekiln_bin %> generate layers \ + --source-dbname "<%= @source_database %>" \ + --storage-dbname "<%= @storage_database %>" \ + --num-threads "<%= node[:vectortile][:replication][:threads] %>" \ + --config <%= @config_path %> \ + < "data/ocean-tiles.txt" +else + echo "No tile updates required" +fi diff --git a/cookbooks/vectortile/templates/default/render-ocean.erb b/cookbooks/vectortile/templates/default/render-ocean.erb deleted file mode 100644 index 56b7a1662..000000000 --- a/cookbooks/vectortile/templates/default/render-ocean.erb +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -set -eu -export PGAPPNAME='render-ocean' - -<%= @tilekiln_bin %> generate zooms \ ---source-dbname "<%= @source_database %>" \ ---storage-dbname "<%= @storage_database %>" \ ---num-threads "<%= node[:vectortile][:rerender][:lowzoom][:threads] %>" \ ---config <%= @config_path %> \ ---min-zoom <%= @min_zoom %> \ ---max-zoom <%= @max_zoom %> diff --git a/test/integration/vectortile/inspec/updates_spec.rb b/test/integration/vectortile/inspec/updates_spec.rb new file mode 100644 index 000000000..54e438090 --- /dev/null +++ b/test/integration/vectortile/inspec/updates_spec.rb @@ -0,0 +1,19 @@ +describe file("/usr/local/bin/ocean-update") do + it { should be_executable.by_user("tileupdate") } +end + +describe service("ocean-update") do + it { should be_installed } +end + +describe file("/usr/local/bin/vector-update") do + it { should be_executable.by_user("tileupdate") } +end + +describe file("/usr/local/bin/tiles-rerender") do + it { should be_executable.by_user("tileupdate") } +end + +describe service("replicate") do + it { should be_installed } +end -- 2.39.5