From c85b74497b385a633ed3f487c77cc0736d5f7328 Mon Sep 17 00:00:00 2001 From: Tareq Al-Ahdal Date: Wed, 6 Jul 2022 08:16:41 +0200 Subject: [PATCH] Initial implementation of GeoTIFF import functionality --- nominatim/clicmd/refresh.py | 11 +++++++++++ nominatim/clicmd/setup.py | 14 ++++++++++++-- nominatim/tools/database_import.py | 6 ++++++ nominatim/tools/refresh.py | 19 +++++++++++++++++++ settings/env.defaults | 4 ++++ test/python/cli/test_cmd_import.py | 1 + test/python/cli/test_cmd_refresh.py | 11 +++++++++-- test/python/tools/test_freeze.py | 1 + test/python/tools/test_refresh.py | 15 +++++++++++++++ 9 files changed, 78 insertions(+), 4 deletions(-) diff --git a/nominatim/clicmd/refresh.py b/nominatim/clicmd/refresh.py index dce28d98..6f307a65 100644 --- a/nominatim/clicmd/refresh.py +++ b/nominatim/clicmd/refresh.py @@ -63,6 +63,8 @@ class UpdateRefresh: help='Update the PL/pgSQL functions in the database') group.add_argument('--wiki-data', action='store_true', help='Update Wikipedia/data importance numbers') + group.add_argument('--osm-views', action='store_true', + help='Update OSM views/data importance numbers') group.add_argument('--importance', action='store_true', help='Recompute place importances (expensive!)') group.add_argument('--website', action='store_true', @@ -130,6 +132,15 @@ class UpdateRefresh: data_path) > 0: LOG.fatal('FATAL: Wikipedia importance dump file not found') return 1 + + if args.osm_views: + data_path = Path(args.config.OSM_VIEWS_DATA_PATH + or args.project_dir) + LOG.warning('Import OSM views GeoTIFF data from %s', data_path) + if refresh.import_osm_views_geotiff(args.config.get_libpq_dsn(), + data_path) > 0: + LOG.fatal('FATAL: OSM views GeoTIFF file not found') + return 1 # Attention: importance MUST come after wiki data import. if args.importance: diff --git a/nominatim/clicmd/setup.py b/nominatim/clicmd/setup.py index 29724433..c7366c3a 100644 --- a/nominatim/clicmd/setup.py +++ b/nominatim/clicmd/setup.py @@ -58,8 +58,10 @@ class SetupAll: group2.add_argument('--no-updates', action='store_true', help="Do not keep tables that are only needed for " "updating the database later") + group2.add_argument('--osm-views', action='store_true', + help='Import OSM views GeoTIFF') group2.add_argument('--offline', action='store_true', - help="Do not attempt to load any additional data from the internet") + help="Do not attempt to load any additional data from the internet") group3 = parser.add_argument_group('Expert options') group3.add_argument('--ignore-errors', action='store_true', help='Continue import even when errors in SQL are present') @@ -103,7 +105,15 @@ class SetupAll: if refresh.import_wikipedia_articles(args.config.get_libpq_dsn(), data_path) > 0: LOG.error('Wikipedia importance dump file not found. ' - 'Will be using default importances.') + 'Calculating importance values of locations will not use Wikipedia importance data.') + + LOG.warning('Importing OSM views GeoTIFF data') + database_import.import_osm_views_geotiff() + data_path = Path(args.config.OSM_VIEWS_DATA_PATH or args.project_dir) + if refresh.import_osm_views_geotiff(args.config.get_libpq_dsn(), + data_path) > 0: + LOG.error('OSM views GeoTIFF file not found. ' + 'Calculating importance values of locations will not use OSM views data.') if args.continue_at is None or args.continue_at == 'load-data': LOG.warning('Initialise tables') diff --git a/nominatim/tools/database_import.py b/nominatim/tools/database_import.py index f6ebe90d..20883b96 100644 --- a/nominatim/tools/database_import.py +++ b/nominatim/tools/database_import.py @@ -75,6 +75,7 @@ def setup_database_skeleton(dsn: str, rouser: Optional[str] = None) -> None: with conn.cursor() as cur: cur.execute('CREATE EXTENSION IF NOT EXISTS hstore') cur.execute('CREATE EXTENSION IF NOT EXISTS postgis') + cur.execute('CREATE EXTENSION IF NOT EXISTS postgis_raster') conn.commit() _require_version('PostGIS', @@ -246,3 +247,8 @@ def create_search_indices(conn: Connection, config: Configuration, sql.run_parallel_sql_file(config.get_libpq_dsn(), 'indices.sql', min(8, threads), drop=drop) + + +def import_osm_views_geotiff(): + """Import OSM views GeoTIFF file""" + subprocess.run("raster2pgsql -s 4326 -I -C -t 100x100 -e osmviews.tiff public.osmviews | psql nominatim", shell=True, check=True) diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py index 8c1e9d9b..a3b6c4f0 100644 --- a/nominatim/tools/refresh.py +++ b/nominatim/tools/refresh.py @@ -146,6 +146,25 @@ def import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = F return 0 +def import_osm_views_geotiff(dsn, data_path, ignore_errors=False): + """ Replaces the OSM views table with new data. + + Returns 0 if all was well and 1 if the GeoTIFF file could not + be found. Throws an exception if there was an error reading the file. + """ + datafile = data_path / 'osmviews.tiff' + + if not datafile.exists(): + return 1 + + pre_code = """BEGIN; + DROP TABLE IF EXISTS "osmviews"; + """ + post_code = "COMMIT" + execute_file(dsn, datafile, ignore_errors=ignore_errors, + pre_code=pre_code, post_code=post_code) + + return 0 def recompute_importance(conn: Connection) -> None: """ Recompute wikipedia links and importance for all entries in placex. diff --git a/settings/env.defaults b/settings/env.defaults index 3115f438..c9759262 100644 --- a/settings/env.defaults +++ b/settings/env.defaults @@ -86,6 +86,10 @@ NOMINATIM_TIGER_DATA_PATH= # When unset, the data is expected to be located in the project directory. NOMINATIM_WIKIPEDIA_DATA_PATH= +# Directory where to find OSM views GeoTIFF file. +# When unset, the data is expected to be located in the project directory. +NOMINATIM_OSM_VIEWS_DATA_PATH= + # Configuration file for special phrase import. # OBSOLETE: use `nominatim special-phrases --config ` or simply put # a custom phrase-settings.json into your project directory. diff --git a/test/python/cli/test_cmd_import.py b/test/python/cli/test_cmd_import.py index 737c4e5c..b6a8721f 100644 --- a/test/python/cli/test_cmd_import.py +++ b/test/python/cli/test_cmd_import.py @@ -40,6 +40,7 @@ class TestCliImportWithDb: mock_func_factory(nominatim.data.country_info, 'setup_country_tables'), mock_func_factory(nominatim.tools.database_import, 'import_osm_data'), mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'), + mock_func_factory(nominatim.tools.refresh, 'import_osm_views_geotiff'), mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'), mock_func_factory(nominatim.tools.database_import, 'load_data'), mock_func_factory(nominatim.tools.database_import, 'create_tables'), diff --git a/test/python/cli/test_cmd_refresh.py b/test/python/cli/test_cmd_refresh.py index 7f44765b..ed3a68ba 100644 --- a/test/python/cli/test_cmd_refresh.py +++ b/test/python/cli/test_cmd_refresh.py @@ -24,6 +24,7 @@ class TestRefresh: @pytest.mark.parametrize("command,func", [ ('address-levels', 'load_address_levels_from_config'), ('wiki-data', 'import_wikipedia_articles'), + ('osm-views', 'import_osm_views_geotiff') ('importance', 'recompute_importance'), ('website', 'setup_website'), ]) @@ -71,15 +72,21 @@ class TestRefresh: assert self.call_nominatim('refresh', '--wiki-data') == 1 + def test_refresh_osm_views_geotiff_file_not_found(self, monkeypatch): + monkeypatch.setenv('NOMINATIM_OSM_VIEWS_DATA_PATH', 'gjoiergjeroi345Q') - def test_refresh_importance_computed_after_wiki_import(self, monkeypatch): + assert self.call_nominatim('refresh', '--osm-views') == 1 + + def test_refresh_importance_computed_after_wiki_and_osm_views_import(self, monkeypatch): calls = [] monkeypatch.setattr(nominatim.tools.refresh, 'import_wikipedia_articles', lambda *args, **kwargs: calls.append('import') or 0) + monkeypatch.setattr(nominatim.tools.refresh, 'import_osm_views_geotiff', + lambda *args, **kwargs: calls.append('import') or 0) monkeypatch.setattr(nominatim.tools.refresh, 'recompute_importance', lambda *args, **kwargs: calls.append('update')) - assert self.call_nominatim('refresh', '--importance', '--wiki-data') == 0 + assert self.call_nominatim('refresh', '--importance', '--wiki-data', '--osm-views') == 0 assert calls == ['import', 'update'] diff --git a/test/python/tools/test_freeze.py b/test/python/tools/test_freeze.py index 3ebb1730..6e852550 100644 --- a/test/python/tools/test_freeze.py +++ b/test/python/tools/test_freeze.py @@ -21,6 +21,7 @@ NOMINATIM_DROP_TABLES = [ 'address_levels', 'location_area', 'location_area_country', 'location_area_large_100', 'location_road_1', + 'osmviews' 'place', 'planet_osm_nodes', 'planet_osm_rels', 'planet_osm_ways', 'search_name_111', 'wikipedia_article', 'wikipedia_redirect' diff --git a/test/python/tools/test_refresh.py b/test/python/tools/test_refresh.py index ac52aa36..c8ebdab8 100644 --- a/test/python/tools/test_refresh.py +++ b/test/python/tools/test_refresh.py @@ -17,6 +17,10 @@ def test_refresh_import_wikipedia_not_existing(dsn): assert refresh.import_wikipedia_articles(dsn, Path('.')) == 1 +def test_refresh_import_osm_views_geotiff_not_existing(dsn): + assert refresh.import_osm_views_geotiff(dsn, Path('.')) == 1 + + @pytest.mark.parametrize("replace", (True, False)) def test_refresh_import_wikipedia(dsn, src_dir, table_factory, temp_db_cursor, replace): if replace: @@ -30,6 +34,17 @@ def test_refresh_import_wikipedia(dsn, src_dir, table_factory, temp_db_cursor, r assert temp_db_cursor.table_rows('wikipedia_redirect') > 0 +@pytest.mark.parametrize("replace", (True, False)) +def test_refresh_import_osm_views_geotiff(dsn, src_dir, table_factory, temp_db_cursor, replace): + if replace: + table_factory('osmviews') + + # use the small osm views GeoTIFF file for the API testdb + assert refresh.import_osm_views_geotiff(dsn, src_dir / 'test' / 'testdb') == 0 + + assert temp_db_cursor.table_rows('osmviews') > 0 + + def test_recompute_importance(placex_table, table_factory, temp_db_conn, temp_db_cursor): temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION compute_importance(extratags HSTORE, country_code varchar(2), -- 2.45.1