From: Sarah Hoffmann Date: Fri, 17 Feb 2023 14:52:31 +0000 (+0100) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~77 X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/96e1ef3ff846324bbfef1f209bfe30d88125b5bd?hp=381c9395747a04672acba395e0bd14332d1a9db9 Merge remote-tracking branch 'upstream/master' --- diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 3fe0b9d8..7dfb3f1d 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -74,6 +74,8 @@ jobs: php-version: ${{ matrix.php }} tools: phpunit:9, phpcs, composer ini-values: opcache.jit=disable + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: actions/setup-python@v4 with: @@ -138,7 +140,7 @@ jobs: working-directory: Nominatim/test/bdd - name: Install mypy and typechecking info - run: pip3 install -U mypy osmium uvicorn types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests types-ujson typing-extensions + run: pip3 install -U mypy osmium uvicorn types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests types-ujson types-Pygments typing-extensions if: matrix.flavour != 'oldstuff' - name: Python static typechecking diff --git a/docs/develop/Development-Environment.md b/docs/develop/Development-Environment.md index 7284e99e..3234b8cb 100644 --- a/docs/develop/Development-Environment.md +++ b/docs/develop/Development-Environment.md @@ -65,7 +65,7 @@ sudo apt install php-cgi phpunit php-codesniffer \ pip3 install --user behave mkdocs mkdocstrings pytest pytest-asyncio pylint \ mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil \ - types-ujson types-requests typing-extensions\ + types-ujson types-requests types-Pygments typing-extensions\ sanic-testing httpx asgi-lifespan ``` diff --git a/lib-php/ReverseGeocode.php b/lib-php/ReverseGeocode.php index d12e4da3..712f1480 100644 --- a/lib-php/ReverseGeocode.php +++ b/lib-php/ReverseGeocode.php @@ -122,12 +122,13 @@ class ReverseGeocode $sSQL .= ' FROM placex'; $sSQL .= ' WHERE osm_type = \'N\''; $sSQL .= ' AND country_code = \''.$sCountryCode.'\''; - $sSQL .= ' AND rank_search < 26 '; // needed to select right index + $sSQL .= ' AND rank_address between 4 and 25'; // needed to select right index $sSQL .= ' AND rank_search between 5 and ' .min(25, $iMaxRank); - $sSQL .= ' AND class = \'place\' AND type != \'postcode\''; + $sSQL .= ' AND type != \'postcode\''; $sSQL .= ' AND name IS NOT NULL '; $sSQL .= ' and indexed_status = 0 and linked_place_id is null'; - $sSQL .= ' AND ST_DWithin('.$sPointSQL.', geometry, 1.8)) p '; + $sSQL .= ' AND ST_Buffer(geometry, reverse_place_diameter(rank_search)) && '.$sPointSQL; + $sSQL .= ') as a '; $sSQL .= 'WHERE distance <= reverse_place_diameter(rank_search)'; $sSQL .= ' ORDER BY rank_search DESC, distance ASC'; $sSQL .= ' LIMIT 1'; @@ -216,23 +217,18 @@ class ReverseGeocode $sSQL .= ' ST_distance('.$sPointSQL.', geometry) as distance'; $sSQL .= ' FROM placex'; $sSQL .= ' WHERE osm_type = \'N\''; - // using rank_search because of a better differentiation - // for place nodes at rank_address 16 $sSQL .= ' AND rank_search > '.$iRankSearch; $sSQL .= ' AND rank_search <= '.$iMaxRank; - $sSQL .= ' AND rank_search < 26 '; // needed to select right index - $sSQL .= ' AND rank_address > 0'; - $sSQL .= ' AND class = \'place\''; + $sSQL .= ' AND rank_address between 4 and 25'; // needed to select right index $sSQL .= ' AND type != \'postcode\''; $sSQL .= ' AND name IS NOT NULL '; $sSQL .= ' AND indexed_status = 0 AND linked_place_id is null'; - $sSQL .= ' AND ST_DWithin('.$sPointSQL.', geometry, reverse_place_diameter('.$iRankSearch.'::smallint))'; - $sSQL .= ' ORDER BY distance ASC,'; - $sSQL .= ' rank_address DESC'; - $sSQL .= ' limit 500) as a'; - $sSQL .= ' WHERE ST_CONTAINS((SELECT geometry FROM placex WHERE place_id = '.$iPlaceID.'), geometry )'; + $sSQL .= ' AND ST_Buffer(geometry, reverse_place_diameter(rank_search)) && '.$sPointSQL; + $sSQL .= ' ORDER BY rank_search DESC, distance ASC'; + $sSQL .= ' limit 100) as a'; + $sSQL .= ' WHERE ST_Contains((SELECT geometry FROM placex WHERE place_id = '.$iPlaceID.'), geometry )'; $sSQL .= ' AND distance <= reverse_place_diameter(rank_search)'; - $sSQL .= ' ORDER BY distance ASC, rank_search DESC'; + $sSQL .= ' ORDER BY rank_search DESC, distance ASC'; $sSQL .= ' LIMIT 1'; Debug::printSQL($sSQL); diff --git a/lib-sql/indices.sql b/lib-sql/indices.sql index 9130fb52..ed078895 100644 --- a/lib-sql/indices.sql +++ b/lib-sql/indices.sql @@ -30,6 +30,13 @@ CREATE INDEX IF NOT EXISTS idx_placex_geometry_reverse_lookupPolygon AND rank_address between 4 and 25 AND type != 'postcode' AND name is not null AND indexed_status = 0 AND linked_place_id is null; --- +-- used in reverse large area lookup +CREATE INDEX IF NOT EXISTS idx_placex_geometry_reverse_lookupPlaceNode + ON placex USING gist (ST_Buffer(geometry, reverse_place_diameter(rank_search))) + {{db.tablespace.search_index}} + WHERE rank_address between 4 and 25 AND type != 'postcode' + AND name is not null AND linked_place_id is null AND osm_type = 'N'; +--- CREATE INDEX IF NOT EXISTS idx_osmline_parent_place_id ON location_property_osmline USING BTREE (parent_place_id) {{db.tablespace.search_index}} WHERE parent_place_id is not null; diff --git a/lib-sql/tables.sql b/lib-sql/tables.sql index d576485e..17216b50 100644 --- a/lib-sql/tables.sql +++ b/lib-sql/tables.sql @@ -190,7 +190,6 @@ CREATE INDEX idx_placex_geometry_buildings ON placex -- Usage: - linking of similar named places to boundaries -- - linking of place nodes with same type to boundaries --- - lookupPolygon() CREATE INDEX idx_placex_geometry_placenode ON placex USING {{postgres.spgist_geom}} (geometry) {{db.tablespace.address_index}} WHERE osm_type = 'N' and rank_search < 26 diff --git a/nominatim/api/connection.py b/nominatim/api/connection.py index 79a5e347..efa4490e 100644 --- a/nominatim/api/connection.py +++ b/nominatim/api/connection.py @@ -13,6 +13,7 @@ import sqlalchemy as sa from sqlalchemy.ext.asyncio import AsyncConnection from nominatim.db.sqlalchemy_schema import SearchTables +from nominatim.api.logging import log class SearchConnection: """ An extended SQLAlchemy connection class, that also contains @@ -34,14 +35,16 @@ class SearchConnection: ) -> Any: """ Execute a 'scalar()' query on the connection. """ + log().sql(self.connection, sql) return await self.connection.scalar(sql, params) - async def execute(self, sql: sa.sql.base.Executable, + async def execute(self, sql: 'sa.Executable', params: Union[Mapping[str, Any], Sequence[Mapping[str, Any]], None] = None - ) -> 'sa.engine.Result[Any]': + ) -> 'sa.Result[Any]': """ Execute a 'execute()' query on the connection. """ + log().sql(self.connection, sql) return await self.connection.execute(sql, params) diff --git a/nominatim/api/logging.py b/nominatim/api/logging.py new file mode 100644 index 00000000..e9c88470 --- /dev/null +++ b/nominatim/api/logging.py @@ -0,0 +1,240 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2023 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Functions for specialised logging with HTML output. +""" +from typing import Any, cast +from contextvars import ContextVar +import textwrap +import io + +import sqlalchemy as sa +from sqlalchemy.ext.asyncio import AsyncConnection + +try: + from pygments import highlight + from pygments.lexers import PythonLexer, PostgresLexer + from pygments.formatters import HtmlFormatter + CODE_HIGHLIGHT = True +except ModuleNotFoundError: + CODE_HIGHLIGHT = False + + +class BaseLogger: + """ Interface for logging function. + + The base implementation does nothing. Overwrite the functions + in derived classes which implement logging functionality. + """ + def get_buffer(self) -> str: + """ Return the current content of the log buffer. + """ + return '' + + def function(self, func: str, **kwargs: Any) -> None: + """ Start a new debug chapter for the given function and its parameters. + """ + + + def section(self, heading: str) -> None: + """ Start a new section with the given title. + """ + + + def comment(self, text: str) -> None: + """ Add a simple comment to the debug output. + """ + + + def var_dump(self, heading: str, var: Any) -> None: + """ Print the content of the variable to the debug output prefixed by + the given heading. + """ + + + def sql(self, conn: AsyncConnection, statement: 'sa.Executable') -> None: + """ Print the SQL for the given statement. + """ + + +class HTMLLogger(BaseLogger): + """ Logger that formats messages in HTML. + """ + def __init__(self) -> None: + self.buffer = io.StringIO() + + + def get_buffer(self) -> str: + return HTML_HEADER + self.buffer.getvalue() + HTML_FOOTER + + + def function(self, func: str, **kwargs: Any) -> None: + self._write(f"

Debug output for {func}()

\n

Parameters:

") + for name, value in kwargs.items(): + self._write(f'
{name}
{self._python_var(value)}
') + self._write('

') + + + def section(self, heading: str) -> None: + self._write(f"

{heading}

") + + + def comment(self, text: str) -> None: + self._write(f"

{text}

") + + + def var_dump(self, heading: str, var: Any) -> None: + self._write(f'
{heading}
{self._python_var(var)}') + + + def sql(self, conn: AsyncConnection, statement: 'sa.Executable') -> None: + sqlstr = str(cast('sa.ClauseElement', statement) + .compile(conn.sync_engine, compile_kwargs={"literal_binds": True})) + if CODE_HIGHLIGHT: + sqlstr = highlight(sqlstr, PostgresLexer(), + HtmlFormatter(nowrap=True, lineseparator='
')) + self._write(f'
{sqlstr}
') + else: + self._write(f'{sqlstr}') + + + def _python_var(self, var: Any) -> str: + if CODE_HIGHLIGHT: + fmt = highlight(repr(var), PythonLexer(), HtmlFormatter(nowrap=True)) + return f'
{fmt}
' + + return f'{str(var)}' + + + def _write(self, text: str) -> None: + """ Add the raw text to the debug output. + """ + self.buffer.write(text) + + +class TextLogger(BaseLogger): + """ Logger creating output suitable for the console. + """ + def __init__(self) -> None: + self.buffer = io.StringIO() + + + def get_buffer(self) -> str: + return self.buffer.getvalue() + + + def function(self, func: str, **kwargs: Any) -> None: + self._write(f"#### Debug output for {func}()\n\nParameters:\n") + for name, value in kwargs.items(): + self._write(f' {name}: {self._python_var(value)}\n') + self._write('\n') + + + def section(self, heading: str) -> None: + self._write(f"\n# {heading}\n\n") + + + def comment(self, text: str) -> None: + self._write(f"{text}\n") + + + def var_dump(self, heading: str, var: Any) -> None: + self._write(f'{heading}:\n {self._python_var(var)}\n\n') + + + def sql(self, conn: AsyncConnection, statement: 'sa.Executable') -> None: + sqlstr = str(cast('sa.ClauseElement', statement) + .compile(conn.sync_engine, compile_kwargs={"literal_binds": True})) + sqlstr = '\n| '.join(textwrap.wrap(sqlstr, width=78)) + self._write(f"| {sqlstr}\n\n") + + + def _python_var(self, var: Any) -> str: + return str(var) + + + def _write(self, text: str) -> None: + self.buffer.write(text) + + +logger: ContextVar[BaseLogger] = ContextVar('logger', default=BaseLogger()) + + +def set_log_output(fmt: str) -> None: + """ Enable collecting debug information. + """ + if fmt == 'html': + logger.set(HTMLLogger()) + elif fmt == 'text': + logger.set(TextLogger()) + else: + logger.set(BaseLogger()) + + +def log() -> BaseLogger: + """ Return the logger for the current context. + """ + return logger.get() + + +def get_and_disable() -> str: + """ Return the current content of the debug buffer and disable logging. + """ + buf = logger.get().get_buffer() + logger.set(BaseLogger()) + return buf + + +HTML_HEADER: str = """ + + + Nominatim - Debug + + + +""" + +HTML_FOOTER: str = "" diff --git a/nominatim/api/lookup.py b/nominatim/api/lookup.py index 56a41310..c42bf0c2 100644 --- a/nominatim/api/lookup.py +++ b/nominatim/api/lookup.py @@ -15,6 +15,7 @@ from nominatim.typing import SaColumn, SaLabel, SaRow from nominatim.api.connection import SearchConnection import nominatim.api.types as ntyp import nominatim.api.results as nres +from nominatim.api.logging import log def _select_column_geometry(column: SaColumn, geometry_output: ntyp.GeometryFormat) -> SaLabel: @@ -36,6 +37,7 @@ async def find_in_placex(conn: SearchConnection, place: ntyp.PlaceRef, """ Search for the given place in the placex table and return the base information. """ + log().section("Find in placex table") t = conn.t.placex sql = sa.select(t.c.place_id, t.c.osm_type, t.c.osm_id, t.c.name, t.c.class_, t.c.type, t.c.admin_level, @@ -44,8 +46,7 @@ async def find_in_placex(conn: SearchConnection, place: ntyp.PlaceRef, t.c.importance, t.c.wikipedia, t.c.indexed_date, t.c.parent_place_id, t.c.rank_address, t.c.rank_search, t.c.linked_place_id, - sa.func.ST_X(t.c.centroid).label('x'), - sa.func.ST_Y(t.c.centroid).label('y'), + t.c.centroid, _select_column_geometry(t.c.geometry, details.geometry_output)) if isinstance(place, ntyp.PlaceID): @@ -69,12 +70,12 @@ async def find_in_osmline(conn: SearchConnection, place: ntyp.PlaceRef, """ Search for the given place in the osmline table and return the base information. """ + log().section("Find in interpolation table") t = conn.t.osmline sql = sa.select(t.c.place_id, t.c.osm_id, t.c.parent_place_id, t.c.indexed_date, t.c.startnumber, t.c.endnumber, t.c.step, t.c.address, t.c.postcode, t.c.country_code, - sa.func.ST_X(sa.func.ST_Centroid(t.c.linegeo)).label('x'), - sa.func.ST_Y(sa.func.ST_Centroid(t.c.linegeo)).label('y'), + t.c.linegeo.ST_Centroid().label('centroid'), _select_column_geometry(t.c.linegeo, details.geometry_output)) if isinstance(place, ntyp.PlaceID): @@ -98,12 +99,12 @@ async def find_in_tiger(conn: SearchConnection, place: ntyp.PlaceRef, """ Search for the given place in the table of Tiger addresses and return the base information. Only lookup by place ID is supported. """ + log().section("Find in TIGER table") t = conn.t.tiger sql = sa.select(t.c.place_id, t.c.parent_place_id, t.c.startnumber, t.c.endnumber, t.c.step, t.c.postcode, - sa.func.ST_X(sa.func.ST_Centroid(t.c.linegeo)).label('x'), - sa.func.ST_Y(sa.func.ST_Centroid(t.c.linegeo)).label('y'), + t.c.linegeo.ST_Centroid().label('centroid'), _select_column_geometry(t.c.linegeo, details.geometry_output)) if isinstance(place, ntyp.PlaceID): @@ -119,12 +120,12 @@ async def find_in_postcode(conn: SearchConnection, place: ntyp.PlaceRef, """ Search for the given place in the postcode table and return the base information. Only lookup by place ID is supported. """ + log().section("Find in postcode table") t = conn.t.postcode sql = sa.select(t.c.place_id, t.c.parent_place_id, t.c.rank_search, t.c.rank_address, t.c.indexed_date, t.c.postcode, t.c.country_code, - sa.func.ST_X(t.c.geometry).label('x'), - sa.func.ST_Y(t.c.geometry).label('y'), + t.c.geometry.label('centroid'), _select_column_geometry(t.c.geometry, details.geometry_output)) if isinstance(place, ntyp.PlaceID): @@ -139,30 +140,36 @@ async def get_place_by_id(conn: SearchConnection, place: ntyp.PlaceRef, details: ntyp.LookupDetails) -> Optional[nres.SearchResult]: """ Retrieve a place with additional details from the database. """ + log().function('get_place_by_id', place=place, details=details) + if details.geometry_output and details.geometry_output != ntyp.GeometryFormat.GEOJSON: raise ValueError("lookup only supports geojosn polygon output.") row = await find_in_placex(conn, place, details) if row is not None: result = nres.create_from_placex_row(row) + log().var_dump('Result', result) await nres.add_result_details(conn, result, details) return result row = await find_in_osmline(conn, place, details) if row is not None: result = nres.create_from_osmline_row(row) + log().var_dump('Result', result) await nres.add_result_details(conn, result, details) return result row = await find_in_postcode(conn, place, details) if row is not None: result = nres.create_from_postcode_row(row) + log().var_dump('Result', result) await nres.add_result_details(conn, result, details) return result row = await find_in_tiger(conn, place, details) if row is not None: result = nres.create_from_tiger_row(row) + log().var_dump('Result', result) await nres.add_result_details(conn, result, details) return result diff --git a/nominatim/api/results.py b/nominatim/api/results.py index 63c9cf12..10f03393 100644 --- a/nominatim/api/results.py +++ b/nominatim/api/results.py @@ -21,6 +21,7 @@ import sqlalchemy as sa from nominatim.typing import SaSelect, SaRow from nominatim.api.types import Point, LookupDetails from nominatim.api.connection import SearchConnection +from nominatim.api.logging import log # This file defines complex result data classes. # pylint: disable=too-many-instance-attributes @@ -131,13 +132,6 @@ class SearchResult: return self.importance or (0.7500001 - (self.rank_search/40.0)) - # pylint: disable=consider-using-f-string - def centroid_as_geojson(self) -> str: - """ Get the centroid in GeoJSON format. - """ - return '{"type": "Point","coordinates": [%f, %f]}' % self.centroid - - def _filter_geometries(row: SaRow) -> Dict[str, str]: return {k[9:]: v for k, v in row._mapping.items() # pylint: disable=W0212 if k.startswith('geometry_')} @@ -165,7 +159,7 @@ def create_from_placex_row(row: SaRow) -> SearchResult: importance=row.importance, country_code=row.country_code, indexed_date=getattr(row, 'indexed_date'), - centroid=Point(row.x, row.y), + centroid=Point.from_wkb(row.centroid.data), geometry=_filter_geometries(row)) @@ -185,7 +179,7 @@ def create_from_osmline_row(row: SaRow) -> SearchResult: 'step': str(row.step)}, country_code=row.country_code, indexed_date=getattr(row, 'indexed_date'), - centroid=Point(row.x, row.y), + centroid=Point.from_wkb(row.centroid.data), geometry=_filter_geometries(row)) @@ -202,7 +196,7 @@ def create_from_tiger_row(row: SaRow) -> SearchResult: 'endnumber': str(row.endnumber), 'step': str(row.step)}, country_code='us', - centroid=Point(row.x, row.y), + centroid=Point.from_wkb(row.centroid.data), geometry=_filter_geometries(row)) @@ -218,7 +212,7 @@ def create_from_postcode_row(row: SaRow) -> SearchResult: rank_search=row.rank_search, rank_address=row.rank_address, country_code=row.country_code, - centroid=Point(row.x, row.y), + centroid=Point.from_wkb(row.centroid.data), indexed_date=row.indexed_date, geometry=_filter_geometries(row)) @@ -228,13 +222,18 @@ async def add_result_details(conn: SearchConnection, result: SearchResult, """ Retrieve more details from the database according to the parameters specified in 'details'. """ + log().section('Query details for result') if details.address_details: + log().comment('Query address details') await complete_address_details(conn, result) if details.linked_places: + log().comment('Query linked places') await complete_linked_places(conn, result) if details.parented_places: + log().comment('Query parent places') await complete_parented_places(conn, result) if details.keywords: + log().comment('Query keywords') await complete_keywords(conn, result) diff --git a/nominatim/api/types.py b/nominatim/api/types.py index 89b81111..9dc3ff2e 100644 --- a/nominatim/api/types.py +++ b/nominatim/api/types.py @@ -10,6 +10,7 @@ Complex datatypes used by the Nominatim API. from typing import Optional, Union, NamedTuple import dataclasses import enum +from struct import unpack @dataclasses.dataclass class PlaceID: @@ -55,6 +56,33 @@ class Point(NamedTuple): return self.x + def to_geojson(self) -> str: + """ Return the point in GeoJSON format. + """ + return f'{{"type": "Point","coordinates": [{self.x}, {self.y}]}}' + + + @staticmethod + def from_wkb(wkb: bytes) -> 'Point': + """ Create a point from EWKB as returned from the database. + """ + if len(wkb) != 25: + raise ValueError("Point wkb has unexpected length") + if wkb[0] == 0: + gtype, srid, x, y = unpack('>iidd', wkb[1:]) + elif wkb[0] == 1: + gtype, srid, x, y = unpack(' str: locales = options.get('locales', napi.Locales()) geom = result.geometry.get('geojson') - centroid = result.centroid_as_geojson() + centroid = result.centroid.to_geojson() out = JsonWriter() out.start_object()\ diff --git a/nominatim/api/v1/server_glue.py b/nominatim/api/v1/server_glue.py index 8aa28cfe..35028526 100644 --- a/nominatim/api/v1/server_glue.py +++ b/nominatim/api/v1/server_glue.py @@ -13,12 +13,14 @@ import abc from nominatim.config import Configuration import nominatim.api as napi +import nominatim.api.logging as loglib from nominatim.api.v1.format import dispatch as formatting CONTENT_TYPE = { 'text': 'text/plain; charset=utf-8', 'xml': 'text/xml; charset=utf-8', - 'jsonp': 'application/javascript' + 'jsonp': 'application/javascript', + 'debug': 'text/html; charset=utf-8' } @@ -131,6 +133,18 @@ class ASGIAdaptor(abc.ABC): or self.config().DEFAULT_LANGUAGE + def setup_debugging(self) -> bool: + """ Set up collection of debug information if requested. + + Return True when debugging was requested. + """ + if self.get_bool('debug', False): + loglib.set_log_output('html') + return True + + return False + + def parse_format(params: ASGIAdaptor, result_type: Type[Any], default: str) -> str: """ Get and check the 'format' parameter and prepare the formatter. `fmtter` is a formatter and `default` the @@ -175,6 +189,8 @@ async def details_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> raise params.error("Missing ID parameter 'place_id' or 'osmtype'.") place = napi.OsmID(osmtype, params.get_int('osmid'), params.get('class')) + debug = params.setup_debugging() + details = napi.LookupDetails(address_details=params.get_bool('addressdetails', False), linked_places=params.get_bool('linkedplaces', False), parented_places=params.get_bool('hierarchy', False), @@ -184,10 +200,12 @@ async def details_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> details.geometry_output = napi.GeometryFormat.GEOJSON locales = napi.Locales.from_accept_languages(params.get_accepted_languages()) - print(locales.languages) result = await api.lookup(place, details) + if debug: + return params.build_response(loglib.get_and_disable(), 'debug') + if result is None: raise params.error('No place with that OSM ID found.', status=404) diff --git a/nominatim/tools/migration.py b/nominatim/tools/migration.py index 7d117a8c..0c88493b 100644 --- a/nominatim/tools/migration.py +++ b/nominatim/tools/migration.py @@ -48,7 +48,8 @@ def migrate(config: Configuration, paths: Any) -> int: has_run_migration = False for version, func in _MIGRATION_FUNCTIONS: - if db_version <= version: + if db_version < version or \ + (db_version == (3, 5, 0, 99) and version == (3, 5, 0, 99)): title = func.__doc__ or '' LOG.warning("Running: %s (%s)", title.split('\n', 1)[0], version) kwargs = dict(conn=conn, config=config, paths=paths) @@ -371,3 +372,16 @@ def enable_forward_dependencies(conn: Connection, **_: Any) -> None: ON planet_osm_rels USING gin (parts) WITH (fastupdate=off)""") cur.execute("ANALYZE planet_osm_ways") + + +@_migration(4, 2, 99, 1) +def add_improved_geometry_reverse_placenode_index(conn: Connection, **_: Any) -> None: + """ Create improved index for reverse lookup of place nodes. + """ + with conn.cursor() as cur: + cur.execute("""CREATE INDEX IF NOT EXISTS idx_placex_geometry_reverse_lookupPlaceNode + ON placex + USING gist (ST_Buffer(geometry, reverse_place_diameter(rank_search))) + WHERE rank_address between 4 and 25 AND type != 'postcode' + AND name is not null AND linked_place_id is null AND osm_type = 'N' + """) diff --git a/nominatim/version.py b/nominatim/version.py index 40e3bda4..346af5eb 100644 --- a/nominatim/version.py +++ b/nominatim/version.py @@ -34,7 +34,7 @@ class NominatimVersion(NamedTuple): return f"{self.major}.{self.minor}.{self.patch_level}-{self.db_patch_level}" -NOMINATIM_VERSION = NominatimVersion(4, 2, 99, 0) +NOMINATIM_VERSION = NominatimVersion(4, 2, 99, 1) POSTGRESQL_REQUIRED_VERSION = (9, 6) POSTGIS_REQUIRED_VERSION = (2, 2) diff --git a/test/python/api/conftest.py b/test/python/api/conftest.py index 29424016..0275e275 100644 --- a/test/python/api/conftest.py +++ b/test/python/api/conftest.py @@ -14,6 +14,7 @@ import datetime as dt import nominatim.api as napi from nominatim.db.sql_preprocessor import SQLPreprocessor +import nominatim.api.logging as loglib class APITester: @@ -138,6 +139,8 @@ def apiobj(temp_db_with_extensions, temp_db_conn, monkeypatch): SQLPreprocessor(temp_db_conn, testapi.api.config)\ .run_sql_file(temp_db_conn, 'functions/address_lookup.sql') + loglib.set_log_output('text') yield testapi + print(loglib.get_and_disable()) testapi.api.close() diff --git a/test/python/cli/test_cmd_api.py b/test/python/cli/test_cmd_api.py index 966059c4..0b5dccfb 100644 --- a/test/python/cli/test_cmd_api.py +++ b/test/python/cli/test_cmd_api.py @@ -80,7 +80,7 @@ class TestCliDetailsCall: @pytest.fixture(autouse=True) def setup_status_mock(self, monkeypatch): result = napi.SearchResult(napi.SourceTable.PLACEX, ('place', 'thing'), - (1.0, -3.0)) + napi.Point(1.0, -3.0)) monkeypatch.setattr(napi.NominatimAPI, 'lookup', lambda *args: result) @@ -90,7 +90,7 @@ class TestCliDetailsCall: ('--relation', '1'), ('--place_id', '10001')]) - def test_status_json_format(self, cli_call, tmp_path, capsys, params): + def test_details_json_format(self, cli_call, tmp_path, capsys, params): result = cli_call('details', '--project-dir', str(tmp_path), *params) assert result == 0