From: Sarah Hoffmann Date: Tue, 27 Sep 2022 19:08:15 +0000 (+0200) Subject: load views as a SQL file and rename to 'secondary importance' X-Git-Tag: v4.2.0~25^2~5 X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/3185fad9182b9f6b2ec609aacf9344b4922a0d35 load views as a SQL file and rename to 'secondary importance' The only requirement for secondary importance is that a raster table comes out of it. The generic name leaves open where the data comes from. --- diff --git a/nominatim/clicmd/args.py b/nominatim/clicmd/args.py index 6edfda7b..2f8273d6 100644 --- a/nominatim/clicmd/args.py +++ b/nominatim/clicmd/args.py @@ -115,7 +115,7 @@ class NominatimArgs: address_levels: bool functions: bool wiki_data: bool - osm_views: bool + secondary_importance: bool importance: bool website: bool diffs: bool diff --git a/nominatim/clicmd/refresh.py b/nominatim/clicmd/refresh.py index d03d682f..ea605ea0 100644 --- a/nominatim/clicmd/refresh.py +++ b/nominatim/clicmd/refresh.py @@ -63,8 +63,8 @@ class UpdateRefresh: help='Update the PL/pgSQL functions in the database') group.add_argument('--wiki-data', action='store_true', help='Update Wikipedia/data importance numbers') - group.add_argument('--osm-views', action='store_true', - help='Update OSM views/data importance numbers') + group.add_argument('--secondary-importance', action='store_true', + help='Update secondary importance raster data') group.add_argument('--importance', action='store_true', help='Recompute place importances (expensive!)') group.add_argument('--website', action='store_true', @@ -117,6 +117,20 @@ class UpdateRefresh: with connect(args.config.get_libpq_dsn()) as conn: refresh.load_address_levels_from_config(conn, args.config) + # Attention: must come BEFORE functions + if args.secondary_importance: + with connect(args.config.get_libpq_dsn()) as conn: + # If the table did not exist before, then the importance code + # needs to be enabled. + if not conn.table_exists('secondary_importance'): + args.functions = True + + LOG.warning('Import secondary importance raster data from %s', args.project_dir) + if refresh.import_secondary_importance(args.config.get_libpq_dsn(), + args.project_dir) > 0: + LOG.fatal('FATAL: Cannot update sendary importance raster data') + return 1 + if args.functions: LOG.warning('Create functions') with connect(args.config.get_libpq_dsn()) as conn: @@ -133,17 +147,6 @@ class UpdateRefresh: LOG.fatal('FATAL: Wikipedia importance dump file not found') return 1 - if args.osm_views: - data_path = Path(args.project_dir) - LOG.warning('Import OSM views GeoTIFF data from %s', data_path) - num = refresh.import_osm_views_geotiff(args.config.get_libpq_dsn(), data_path) - if num == 1: - LOG.fatal('FATAL: OSM views GeoTIFF file not found') - return 1 - if num == 2: - LOG.fatal('FATAL: PostGIS version number is less than 3') - return 1 - # Attention: importance MUST come after wiki data import. if args.importance: LOG.warning('Update importance values for database') diff --git a/nominatim/clicmd/setup.py b/nominatim/clicmd/setup.py index c870fb17..46eee19f 100644 --- a/nominatim/clicmd/setup.py +++ b/nominatim/clicmd/setup.py @@ -58,8 +58,6 @@ class SetupAll: group2.add_argument('--no-updates', action='store_true', help="Do not keep tables that are only needed for " "updating the database later") - group2.add_argument('--osm-views', action='store_true', - help='Import OSM views GeoTIFF') group2.add_argument('--offline', action='store_true', help="Do not attempt to load any additional data from the internet") group3 = parser.add_argument_group('Expert options') @@ -108,15 +106,11 @@ class SetupAll: 'Calculating importance values of locations will not ' 'use Wikipedia importance data.') - LOG.warning('Importing OSM views GeoTIFF data') - data_path = Path(args.project_dir) - num = refresh.import_osm_views_geotiff(args.config.get_libpq_dsn(), data_path) - if num == 1: - LOG.error('OSM views GeoTIFF file not found. ' - 'Calculating importance values of locations will not use OSM views data.') - elif num == 2: - LOG.error('PostGIS version number is less than 3. ' - 'Calculating importance values of locations will not use OSM views data.') + LOG.warning('Importing secondary importance raster data') + if refresh.import_secondary_importance(args.config.get_libpq_dsn(), + args.project_dir) != 0: + LOG.error('Secondary importance file not imported. ' + 'Falling back to default ranking.') if args.continue_at is None or args.continue_at == 'load-data': LOG.warning('Initialise tables') diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py index 531de14d..c50493cc 100644 --- a/nominatim/tools/refresh.py +++ b/nominatim/tools/refresh.py @@ -9,7 +9,6 @@ Functions for bringing auxiliary data in the database up-to-date. """ from typing import MutableSequence, Tuple, Any, Type, Mapping, Sequence, List, cast import logging -import subprocess from textwrap import dedent from pathlib import Path @@ -147,48 +146,23 @@ def import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = F return 0 -def import_osm_views_geotiff(dsn: str, data_path: Path) -> int: - """ Replaces the OSM views table with new data. +def import_secondary_importance(dsn: str, data_path: Path, ignore_errors: bool = False) -> int: + """ Replaces the secondary importance raster data table with new data. - Returns 0 if all was well and 1 if the OSM views GeoTIFF file could not + Returns 0 if all was well and 1 if the raster SQL file could not be found. Throws an exception if there was an error reading the file. """ - datafile = data_path / 'osmviews.tiff' + datafile = data_path / 'secondary_importance.sql.gz' if not datafile.exists(): return 1 - with connect(dsn) as conn: + with connect(dsn) as conn: postgis_version = conn.postgis_version_tuple() if postgis_version[0] < 3: + LOG.error('PostGIS version is too old for using OSM raster data.') return 2 - with conn.cursor() as cur: - cur.drop_table("osm_views") - cur.drop_table("osm_views_stat") - - # -ovr: 6 -> zoom 12, 5 -> zoom 13, 4 -> zoom 14, 3 -> zoom 15 - reproject_geotiff = f"gdalwarp -q -multi -ovr 3 -overwrite \ - -co COMPRESS=LZW -tr 0.01 0.01 -t_srs EPSG:4326 {datafile} raster2import.tiff" - subprocess.run(["/bin/bash", "-c" , reproject_geotiff], check=True) - - tile_size = 256 - import_geotiff = f"raster2pgsql -I -C -Y -t {tile_size}x{tile_size} raster2import.tiff \ - public.osm_views | psql {dsn} > /dev/null" - subprocess.run(["/bin/bash", "-c" , import_geotiff], check=True) - - cleanup = "rm raster2import.tiff" - subprocess.run(["/bin/bash", "-c" , cleanup], check=True) - - # To normalize osm views data, the max view value is needed - cur.execute(f""" - CREATE TABLE osm_views_stat AS ( - SELECT MAX(ST_Value(osm_views.rast, 1, x, y)) AS max_views_count - FROM osm_views CROSS JOIN - generate_series(1, {tile_size}) As x - CROSS JOIN generate_series(1, {tile_size}) As y - WHERE x <= ST_Width(rast) AND y <= ST_Height(rast)); - """) - conn.commit() + execute_file(dsn, datafile, ignore_errors=ignore_errors) return 0