Integrated OSM views into importance computation

[nominatim.git] / nominatim / tools / refresh.py

diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py
index 9c5b7b085e50582202a117528d87bc0ca7ff117a..531de14db8d4911270cf603a4bc6fc0c27f29337 100644 (file)
--- a/nominatim/tools/refresh.py
+++ b/nominatim/tools/refresh.py
@@ -9,13 +9,14 @@ Functions for bringing auxiliary data in the database up-to-date.
 """
 from typing import MutableSequence, Tuple, Any, Type, Mapping, Sequence, List, cast
 import logging
 """
 from typing import MutableSequence, Tuple, Any, Type, Mapping, Sequence, List, cast
 import logging

+import subprocess

 from textwrap import dedent
 from pathlib import Path
 
 from psycopg2 import sql as pysql
 
 from nominatim.config import Configuration
 from textwrap import dedent
 from pathlib import Path
 
 from psycopg2 import sql as pysql
 
 from nominatim.config import Configuration

-from nominatim.db.connection import Connection
+from nominatim.db.connection import Connection, connect

 from nominatim.db.utils import execute_file
 from nominatim.db.sql_preprocessor import SQLPreprocessor
 from nominatim.version import version_str
 from nominatim.db.utils import execute_file
 from nominatim.db.sql_preprocessor import SQLPreprocessor
 from nominatim.version import version_str


@@ -126,7 +127,7 @@ PHP_CONST_DEFS = (
 def import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = False) -> int:
     """ Replaces the wikipedia importance tables with new data.
         The import is run in a single transaction so that the new data
 def import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = False) -> int:
     """ Replaces the wikipedia importance tables with new data.
         The import is run in a single transaction so that the new data

-        is replace seemlessly.
+        is replace seamlessly.

 
         Returns 0 if all was well and 1 if the importance file could not
         be found. Throws an exception if there was an error reading the file.
 
         Returns 0 if all was well and 1 if the importance file could not
         be found. Throws an exception if there was an error reading the file.


@@ -146,6 +147,50 @@ def import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = F
 
     return 0
 
 
     return 0
 

+def import_osm_views_geotiff(dsn: str, data_path: Path) -> int:
+    """ Replaces the OSM views table with new data.
+
+        Returns 0 if all was well and 1 if the OSM views GeoTIFF file could not
+        be found. Throws an exception if there was an error reading the file.
+    """
+    datafile = data_path / 'osmviews.tiff'
+    if not datafile.exists():
+        return 1
+    with connect(dsn) as conn:
+
+        postgis_version = conn.postgis_version_tuple()
+        if postgis_version[0] < 3:
+            return 2
+
+        with conn.cursor() as cur:
+            cur.drop_table("osm_views")
+            cur.drop_table("osm_views_stat")
+
+            # -ovr: 6 -> zoom 12, 5 -> zoom 13, 4 -> zoom 14, 3 -> zoom 15
+            reproject_geotiff = f"gdalwarp -q -multi -ovr 3 -overwrite \
+                -co COMPRESS=LZW -tr 0.01 0.01 -t_srs EPSG:4326 {datafile} raster2import.tiff"
+            subprocess.run(["/bin/bash", "-c" , reproject_geotiff], check=True)
+
+            tile_size = 256
+            import_geotiff = f"raster2pgsql -I -C -Y -t {tile_size}x{tile_size} raster2import.tiff \
+                public.osm_views | psql {dsn} > /dev/null"
+            subprocess.run(["/bin/bash", "-c" , import_geotiff], check=True)
+
+            cleanup = "rm raster2import.tiff"
+            subprocess.run(["/bin/bash", "-c" , cleanup], check=True)
+
+            # To normalize osm views data, the max view value is needed
+            cur.execute(f"""
+            CREATE TABLE osm_views_stat AS (
+                SELECT MAX(ST_Value(osm_views.rast, 1, x, y)) AS max_views_count
+                FROM osm_views CROSS JOIN
+                generate_series(1, {tile_size}) As x
+                CROSS JOIN generate_series(1, {tile_size}) As y
+                WHERE x <= ST_Width(rast) AND y <= ST_Height(rast));
+            """)
+            conn.commit()
+
+    return 0

 
 def recompute_importance(conn: Connection) -> None:
     """ Recompute wikipedia links and importance for all entries in placex.
 
 def recompute_importance(conn: Connection) -> None:
     """ Recompute wikipedia links and importance for all entries in placex.


@@ -157,7 +202,7 @@ def recompute_importance(conn: Connection) -> None:
         cur.execute("""
             UPDATE placex SET (wikipedia, importance) =
                (SELECT wikipedia, importance
         cur.execute("""
             UPDATE placex SET (wikipedia, importance) =
                (SELECT wikipedia, importance

-                FROM compute_importance(extratags, country_code, osm_type, osm_id))
+                FROM compute_importance(extratags, country_code, osm_type, osm_id, centroid))

             """)
         cur.execute("""
             UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance
             """)
         cur.execute("""
             UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance