From: Tom Hughes <tom@compton.nu>
Date: Tue, 7 Aug 2018 20:07:04 +0000 (+0100)
Subject: Add osm2psql 0.96.0 compatible tile expiry script
X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/b2d0ed3e9e4f2fc899db7265a23156d5dd5cf139

Add osm2psql 0.96.0 compatible tile expiry script
---

diff --git a/cookbooks/tile/files/default/bin/expire-tiles-single b/cookbooks/tile/files/default/bin/expire-tiles-single
new file mode 100644
index 000000000..fa45be9ee
--- /dev/null
+++ b/cookbooks/tile/files/default/bin/expire-tiles-single
@@ -0,0 +1,141 @@
+#!/usr/bin/python
+"""
+Expire meta tiles from a OSM change file by resetting their modified time.
+"""
+
+import argparse
+import os.path as ospath
+import osmium as o
+import pyproj
+
+EXPIRY_TIME = 946681200 # 2000-01-01 00:00:00
+# width/height of the spherical mercator projection
+SIZE = 40075016.6855784
+
+proj_wsg84 = pyproj.Proj(init='epsg:4326')
+proj_merc = pyproj.Proj(init='epsg:3857')
+
+class TileCollector(o.SimpleHandler):
+
+    def __init__(self, node_cache, zoom):
+        super(TileCollector, self).__init__()
+        self.node_cache = o.index.create_map("dense_file_array," + node_cache)
+        self.done_nodes = set()
+        self.tile_set = set()
+        self.zoom = zoom
+
+    def add_tile_from_node(self, location):
+        if not location.valid():
+            return
+
+        lat = max(-85, min(85.0, location.lat))
+        x, y = pyproj.transform(proj_wsg84, proj_merc, location.lon, lat)
+
+        # renormalise into unit space [0,1]
+        x = 0.5 + x / SIZE
+        y = 0.5 - y / SIZE
+        # transform into tile space
+        x = x * 2**self.zoom
+        y = y * 2**self.zoom
+        # chop of the fractional parts
+        self.tile_set.add((int(x), int(y), self.zoom))
+
+    def node(self, node):
+        # we put all the nodes into the hash, as it doesn't matter whether the node was
+        # added, deleted or modified - the tile will need updating anyway.
+        self.done_nodes.add(node.id)
+        self.add_tile_from_node(node.location)
+
+    def way(self, way):
+        for n in way.nodes:
+            if not n.ref in self.done_nodes:
+                self.done_nodes.add(n.ref)
+                try:
+                    self.add_tile_from_node(self.node_cache.get(n.ref))
+                except o.NotFoundError:
+                    pass # no coordinate
+
+
+def xyz_to_meta(x, y, z, meta_size):
+    """ Return the file name of a meta tile.
+        This must match the definition of xyz to meta in mod_tile.
+    """
+    # mask off the final few bits
+    x = x & ~(meta_size - 1)
+    y = y & ~(meta_size - 1)
+
+    # generate the path
+    path = None
+    for i in range(0, 5):
+        part = str(((x & 0x0f) << 4) | (y & 0x0f))
+        x = x >> 4
+        y = y >> 4
+        if path is None:
+            path = (part + ".meta")
+        else:
+            path = ospath.join(part, path)
+
+    return ospath.join(str(z), path)
+
+
+def expire_meta(meta):
+    """Expire the meta tile by setting the modified time back.
+    """
+    if ospath.exists(meta):
+        print("Expiring " + meta)
+        os.utime(meta, (EXPIRY_TIME, EXPIRY_TIME))
+
+
+def expire_meta_tiles(options):
+    proc = TileCollector(options.node_cache, options.max_zoom)
+    proc.apply_file(options.inputfile)
+
+    tile_set = proc.tile_set
+
+    # turn all the tiles into expires, putting them in the set
+    # so that we don't expire things multiple times
+    for z in range(options.min_zoom, options.max_zoom + 1):
+        meta_set = set()
+        new_set = set()
+        for xy in tile_set:
+            meta = xyz_to_meta(xy[0], xy[1], xy[2], options.meta_size)
+
+            for tile_dir in options.tile_dir:
+                meta_set.add(ospath.join(tile_dir, meta))
+
+            # add the parent into the set for the next round
+            new_set.add((int(xy[0]/2), int(xy[1]/2), xy[2] - 1))
+
+        # expire all meta tiles
+        for meta in meta_set:
+            expire_meta(meta)
+
+        # continue with parent tiles
+        tile_set = new_set
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter,
+                                     usage='%(prog)s [options] <inputfile>')
+    parser.add_argument('--min', action='store', dest='min_zoom', default=13,
+                        type=int,
+                        help='Minimum zoom for expiry.')
+    parser.add_argument('--max', action='store', dest='max_zoom', default=20,
+                        type=int,
+                        help='Maximum zoom for expiry.')
+    parser.add_argument('-t', action='append', dest='tile_dir', default=None,
+                        required=True,
+                        help='Tile directory (repeat for multiple directories).')
+    parser.add_argument('--meta-tile-size', action='store', dest='meta_size',
+                        default=8, type=int,
+                        help='The size of the meta tile blocks.')
+    parser.add_argument('--node-cache', action='store', dest='node_cache',
+                        default='/store/database/nodes',
+                        help='osm2pgsql flatnode file.')
+    parser.add_argument('inputfile',
+                        help='OSC input file.')
+
+    options = parser.parse_args()
+
+    expire_meta_tiles(options)
diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb
index fe3b6c862..d979d69aa 100644
--- a/cookbooks/tile/recipes/default.rb
+++ b/cookbooks/tile/recipes/default.rb
@@ -436,34 +436,55 @@ package %w[
   osmosis
 ]
 
-package %w[
-  ruby
-  ruby-dev
-]
-
-package %w[
-  libproj-dev
-  libxml2-dev
-]
-
-gem_package "proj4rb"
-gem_package "libxml-ruby"
+if node[:lsb][:release].to_f >= 18.04
+  package %w[
+    pyosmium
+    python-pyproj
+  ]
+
+  remote_directory "/usr/local/bin" do
+    source "bin"
+    owner "root"
+    group "root"
+    mode 0o755
+    files_owner "root"
+    files_group "root"
+    files_mode 0o755
+  end
 
-remote_directory "/usr/local/lib/site_ruby" do
-  source "ruby"
-  owner "root"
-  group "root"
-  mode 0o755
-  files_owner "root"
-  files_group "root"
-  files_mode 0o644
-end
+  template "/usr/local/bin/expire-tiles" do
+    source "expire-tiles.bionic.erb"
+    owner "root"
+    group "root"
+    mode 0o755
+  end
+else
+  package %w[
+    ruby
+    ruby-dev
+    libproj-dev
+    libxml2-dev
+  ]
+
+  gem_package "proj4rb"
+  gem_package "libxml-ruby"
+
+  remote_directory "/usr/local/lib/site_ruby" do
+    source "ruby"
+    owner "root"
+    group "root"
+    mode 0o755
+    files_owner "root"
+    files_group "root"
+    files_mode 0o644
+  end
 
-template "/usr/local/bin/expire-tiles" do
-  source "expire-tiles.erb"
-  owner "root"
-  group "root"
-  mode 0o755
+  template "/usr/local/bin/expire-tiles" do
+    source "expire-tiles.xenial.erb"
+    owner "root"
+    group "root"
+    mode 0o755
+  end
 end
 
 directory "/var/lib/replicate" do
diff --git a/cookbooks/tile/templates/default/expire-tiles.bionic.erb b/cookbooks/tile/templates/default/expire-tiles.bionic.erb
new file mode 100644
index 000000000..2f6a97d77
--- /dev/null
+++ b/cookbooks/tile/templates/default/expire-tiles.bionic.erb
@@ -0,0 +1,15 @@
+#!/usr/bin/ruby
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+args = [
+<% node[:tile][:styles].each do |name,details| -%>
+  "-t", "/srv/tile.openstreetmap.org/tiles/<%= name %>",
+<% end -%>
+  "--min", "13",
+  "--max", "<%= node[:tile][:styles].collect { |n,d| d[:max_zoom] }.max %>"
+]
+
+Dir.glob("/var/lib/replicate/expire-queue/changes-*.gz").each do |f|
+   system("/usr/local/bin/expire-tiles-single", *args, f) && File::unlink(f)
+end
diff --git a/cookbooks/tile/templates/default/expire-tiles.erb b/cookbooks/tile/templates/default/expire-tiles.xenial.erb
similarity index 100%
rename from cookbooks/tile/templates/default/expire-tiles.erb
rename to cookbooks/tile/templates/default/expire-tiles.xenial.erb