X-Git-Url: https://git.openstreetmap.org/nominatim.git/blobdiff_plain/c314a3092c5b51c7782015f6fa9ac093b46fa174..refs/heads/master:/src/nominatim_api/core.py
diff --git a/src/nominatim_api/core.py b/src/nominatim_api/core.py
index 632c97a7..b6e0cb68 100644
--- a/src/nominatim_api/core.py
+++ b/src/nominatim_api/core.py
@@ -2,17 +2,23 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Implementation of classes for API access via libraries.
"""
-from typing import Mapping, Optional, Any, AsyncIterator, Dict, Sequence, List, Tuple
+from typing import Mapping, Optional, Any, AsyncIterator, Dict, Sequence, List, \
+ Union, Tuple, cast
import asyncio
import sys
import contextlib
from pathlib import Path
+if sys.version_info >= (3, 11):
+ from asyncio import timeout_at
+else:
+ from async_timeout import timeout_at
+
import sqlalchemy as sa
import sqlalchemy.ext.asyncio as sa_asyncio
@@ -20,17 +26,18 @@ from .errors import UsageError
from .sql.sqlalchemy_schema import SearchTables
from .sql.async_core_library import PGCORE_LIB, PGCORE_ERROR
from .config import Configuration
-from .sql import sqlite_functions, sqlalchemy_functions #pylint: disable=unused-import
+from .sql import sqlite_functions, sqlalchemy_functions # noqa
from .connection import SearchConnection
from .status import get_status, StatusResult
-from .lookup import get_detailed_place, get_simple_place
+from .lookup import get_places, get_detailed_place
from .reverse import ReverseGeocoder
-from .search import ForwardGeocoder, Phrase, PhraseType, make_query_analyzer
+from .timeout import Timeout
+from . import search as nsearch
from . import types as ntyp
from .results import DetailedResult, ReverseResult, SearchResults
-class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
+class NominatimAPIAsync:
""" The main frontend to the Nominatim database implements the
functions for lookup, forward and reverse geocoding using
asynchronous functions.
@@ -38,8 +45,10 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
This class shares most of the functions with its synchronous
version. There are some additional functions or parameters,
which are documented below.
+
+ This class should usually be used as a context manager in 'with' context.
"""
- def __init__(self, project_dir: Path,
+ def __init__(self, project_dir: Optional[Union[str, Path]] = None,
environ: Optional[Mapping[str, str]] = None,
loop: Optional[asyncio.AbstractEventLoop] = None) -> None:
""" Initiate a new frontend object with synchronous API functions.
@@ -58,19 +67,20 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
"""
self.config = Configuration(project_dir, environ)
self.query_timeout = self.config.get_int('QUERY_TIMEOUT') \
- if self.config.QUERY_TIMEOUT else None
+ if self.config.QUERY_TIMEOUT else None
+ self.request_timeout = self.config.get_int('REQUEST_TIMEOUT') \
+ if self.config.REQUEST_TIMEOUT else None
self.reverse_restrict_to_country_area = self.config.get_bool('SEARCH_WITHIN_COUNTRIES')
self.server_version = 0
if sys.version_info >= (3, 10):
self._engine_lock = asyncio.Lock()
else:
- self._engine_lock = asyncio.Lock(loop=loop) # pylint: disable=unexpected-keyword-arg
+ self._engine_lock = asyncio.Lock(loop=loop)
self._engine: Optional[sa_asyncio.AsyncEngine] = None
self._tables: Optional[SearchTables] = None
self._property_cache: Dict[str, Any] = {'DB:server_version': 0}
-
async def setup_database(self) -> None:
""" Set up the SQL engine and connections.
@@ -92,7 +102,6 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
extra_args['max_overflow'] = 0
extra_args['pool_size'] = self.config.get_int('API_POOL_SIZE')
-
is_sqlite = self.config.DATABASE_DSN.startswith('sqlite:')
if is_sqlite:
@@ -107,16 +116,16 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
raise UsageError(f"SQlite database '{params.get('dbname')}' does not exist.")
else:
dsn = self.config.get_database_params()
- query = {k: v for k, v in dsn.items()
+ query = {k: str(v) for k, v in dsn.items()
if k not in ('user', 'password', 'dbname', 'host', 'port')}
dburl = sa.engine.URL.create(
f'postgresql+{PGCORE_LIB}',
- database=dsn.get('dbname'),
- username=dsn.get('user'),
- password=dsn.get('password'),
- host=dsn.get('host'),
- port=int(dsn['port']) if 'port' in dsn else None,
+ database=cast(str, dsn.get('dbname')),
+ username=cast(str, dsn.get('user')),
+ password=cast(str, dsn.get('password')),
+ host=cast(str, dsn.get('host')),
+ port=int(cast(str, dsn['port'])) if 'port' in dsn else None,
query=query)
engine = sa_asyncio.create_async_engine(dburl, **extra_args)
@@ -137,26 +146,23 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
async with engine.begin() as conn:
result = await conn.scalar(sa.text('SHOW server_version_num'))
server_version = int(result)
- if server_version >= 110000:
- await conn.execute(sa.text("SET jit_above_cost TO '-1'"))
- await conn.execute(sa.text(
- "SET max_parallel_workers_per_gather TO '0'"))
+ await conn.execute(sa.text("SET jit_above_cost TO '-1'"))
+ await conn.execute(sa.text(
+ "SET max_parallel_workers_per_gather TO '0'"))
except (PGCORE_ERROR, sa.exc.OperationalError):
server_version = 0
- if server_version >= 110000:
- @sa.event.listens_for(engine.sync_engine, "connect")
- def _on_connect(dbapi_con: Any, _: Any) -> None:
- cursor = dbapi_con.cursor()
- cursor.execute("SET jit_above_cost TO '-1'")
- cursor.execute("SET max_parallel_workers_per_gather TO '0'")
+ @sa.event.listens_for(engine.sync_engine, "connect")
+ def _on_connect(dbapi_con: Any, _: Any) -> None:
+ cursor = dbapi_con.cursor()
+ cursor.execute("SET jit_above_cost TO '-1'")
+ cursor.execute("SET max_parallel_workers_per_gather TO '0'")
self._property_cache['DB:server_version'] = server_version
- self._tables = SearchTables(sa.MetaData()) # pylint: disable=no-member
+ self._tables = SearchTables(sa.MetaData())
self._engine = engine
-
async def close(self) -> None:
""" Close all active connections to the database. The NominatimAPIAsync
object remains usable after closing. If a new API functions is
@@ -165,14 +171,22 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
if self._engine is not None:
await self._engine.dispose()
+ async def __aenter__(self) -> 'NominatimAPIAsync':
+ return self
+
+ async def __aexit__(self, *_: Any) -> None:
+ await self.close()
@contextlib.asynccontextmanager
- async def begin(self) -> AsyncIterator[SearchConnection]:
+ async def begin(self, abs_timeout: Optional[float] = None) -> AsyncIterator[SearchConnection]:
""" Create a new connection with automatic transaction handling.
This function may be used to get low-level access to the database.
Refer to the documentation of SQLAlchemy for details how to use
the connection object.
+
+ You may optionally give an absolute timeout until when to wait
+ for a connection to become available.
"""
if self._engine is None:
await self.setup_database()
@@ -180,15 +194,15 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
assert self._engine is not None
assert self._tables is not None
- async with self._engine.begin() as conn:
- yield SearchConnection(conn, self._tables, self._property_cache)
-
+ async with timeout_at(abs_timeout), self._engine.begin() as conn:
+ yield SearchConnection(conn, self._tables, self._property_cache, self.config)
async def status(self) -> StatusResult:
""" Return the status of the database.
"""
+ timeout = Timeout(self.request_timeout)
try:
- async with self.begin() as conn:
+ async with self.begin(abs_timeout=timeout.abs) as conn:
conn.set_query_timeout(self.query_timeout)
status = await get_status(conn)
except (PGCORE_ERROR, sa.exc.OperationalError):
@@ -196,33 +210,35 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
return status
-
async def details(self, place: ntyp.PlaceRef, **params: Any) -> Optional[DetailedResult]:
""" Get detailed information about a place in the database.
Returns None if there is no entry under the given ID.
"""
+ timeout = Timeout(self.request_timeout)
details = ntyp.LookupDetails.from_kwargs(params)
- async with self.begin() as conn:
- conn.set_query_timeout(self.query_timeout)
- if details.keywords:
- await make_query_analyzer(conn)
- return await get_detailed_place(conn, place, details)
-
+ with details.query_stats as qs:
+ async with self.begin(abs_timeout=timeout.abs) as conn:
+ qs.log_time('start_query')
+ conn.set_query_timeout(self.query_timeout)
+ if details.keywords:
+ await nsearch.make_query_analyzer(conn)
+ return await get_detailed_place(conn, place, details)
async def lookup(self, places: Sequence[ntyp.PlaceRef], **params: Any) -> SearchResults:
""" Get simple information about a list of places.
Returns a list of place information for all IDs that were found.
"""
+ timeout = Timeout(self.request_timeout)
details = ntyp.LookupDetails.from_kwargs(params)
- async with self.begin() as conn:
- conn.set_query_timeout(self.query_timeout)
- if details.keywords:
- await make_query_analyzer(conn)
- return SearchResults(filter(None,
- [await get_simple_place(conn, p, details) for p in places]))
-
+ with details.query_stats as qs:
+ async with self.begin(abs_timeout=timeout.abs) as conn:
+ qs.log_time('start_query')
+ conn.set_query_timeout(self.query_timeout)
+ if details.keywords:
+ await nsearch.make_query_analyzer(conn)
+ return await get_places(conn, places, details)
async def reverse(self, coord: ntyp.AnyPoint, **params: Any) -> Optional[ReverseResult]:
""" Find a place by its coordinates. Also known as reverse geocoding.
@@ -235,33 +251,35 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
# There are no results to be expected outside valid coordinates.
return None
+ timeout = Timeout(self.request_timeout)
details = ntyp.ReverseDetails.from_kwargs(params)
- async with self.begin() as conn:
- conn.set_query_timeout(self.query_timeout)
- if details.keywords:
- await make_query_analyzer(conn)
- geocoder = ReverseGeocoder(conn, details,
- self.reverse_restrict_to_country_area)
- return await geocoder.lookup(coord)
-
+ with details.query_stats as qs:
+ async with self.begin(abs_timeout=timeout.abs) as conn:
+ qs.log_time('start_query')
+ conn.set_query_timeout(self.query_timeout)
+ if details.keywords:
+ await nsearch.make_query_analyzer(conn)
+ geocoder = ReverseGeocoder(conn, details,
+ self.reverse_restrict_to_country_area)
+ return await geocoder.lookup(coord)
async def search(self, query: str, **params: Any) -> SearchResults:
""" Find a place by free-text search. Also known as forward geocoding.
"""
- query = query.strip()
- if not query:
- raise UsageError('Nothing to search for.')
-
- async with self.begin() as conn:
- conn.set_query_timeout(self.query_timeout)
- geocoder = ForwardGeocoder(conn, ntyp.SearchDetails.from_kwargs(params),
- self.config.get_int('REQUEST_TIMEOUT') \
- if self.config.REQUEST_TIMEOUT else None)
- phrases = [Phrase(PhraseType.NONE, p.strip()) for p in query.split(',')]
- return await geocoder.lookup(phrases)
+ timeout = Timeout(self.request_timeout)
+ details = ntyp.SearchDetails.from_kwargs(params)
+ with details.query_stats as qs:
+ query = query.strip()
+ if not query:
+ raise UsageError('Nothing to search for.')
+ async with self.begin(abs_timeout=timeout.abs) as conn:
+ qs.log_time('start_query')
+ conn.set_query_timeout(self.query_timeout)
+ geocoder = nsearch.ForwardGeocoder(conn, details, timeout)
+ phrases = [nsearch.Phrase(nsearch.PHRASE_ANY, p.strip()) for p in query.split(',')]
+ return await geocoder.lookup(phrases)
- # pylint: disable=too-many-arguments,too-many-branches
async def search_address(self, amenity: Optional[str] = None,
street: Optional[str] = None,
city: Optional[str] = None,
@@ -272,26 +290,25 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
**params: Any) -> SearchResults:
""" Find an address using structured search.
"""
- async with self.begin() as conn:
- conn.set_query_timeout(self.query_timeout)
- details = ntyp.SearchDetails.from_kwargs(params)
-
- phrases: List[Phrase] = []
+ timeout = Timeout(self.request_timeout)
+ details = ntyp.SearchDetails.from_kwargs(params)
+ with details.query_stats as qs:
+ phrases: List[nsearch.Phrase] = []
if amenity:
- phrases.append(Phrase(PhraseType.AMENITY, amenity))
+ phrases.append(nsearch.Phrase(nsearch.PHRASE_AMENITY, amenity))
if street:
- phrases.append(Phrase(PhraseType.STREET, street))
+ phrases.append(nsearch.Phrase(nsearch.PHRASE_STREET, street))
if city:
- phrases.append(Phrase(PhraseType.CITY, city))
+ phrases.append(nsearch.Phrase(nsearch.PHRASE_CITY, city))
if county:
- phrases.append(Phrase(PhraseType.COUNTY, county))
+ phrases.append(nsearch.Phrase(nsearch.PHRASE_COUNTY, county))
if state:
- phrases.append(Phrase(PhraseType.STATE, state))
+ phrases.append(nsearch.Phrase(nsearch.PHRASE_STATE, state))
if postalcode:
- phrases.append(Phrase(PhraseType.POSTCODE, postalcode))
+ phrases.append(nsearch.Phrase(nsearch.PHRASE_POSTCODE, postalcode))
if country:
- phrases.append(Phrase(PhraseType.COUNTRY, country))
+ phrases.append(nsearch.Phrase(nsearch.PHRASE_COUNTRY, country))
if not phrases:
raise UsageError('Nothing to search for.')
@@ -309,16 +326,16 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
else:
details.restrict_min_max_rank(4, 4)
- if 'layers' not in params:
+ if details.layers is None:
details.layers = ntyp.DataLayer.ADDRESS
if amenity:
details.layers |= ntyp.DataLayer.POI
- geocoder = ForwardGeocoder(conn, details,
- self.config.get_int('REQUEST_TIMEOUT') \
- if self.config.REQUEST_TIMEOUT else None)
- return await geocoder.lookup(phrases)
-
+ async with self.begin(abs_timeout=timeout.abs) as conn:
+ qs.log_time('start_query')
+ conn.set_query_timeout(self.query_timeout)
+ geocoder = nsearch.ForwardGeocoder(conn, details, timeout)
+ return await geocoder.lookup(phrases)
async def search_category(self, categories: List[Tuple[str, str]],
near_query: Optional[str] = None,
@@ -327,33 +344,35 @@ class NominatimAPIAsync: #pylint: disable=too-many-instance-attributes
The near place may either be given as an unstructured search
query in itself or as coordinates.
"""
- if not categories:
- return SearchResults()
-
+ timeout = Timeout(self.request_timeout)
details = ntyp.SearchDetails.from_kwargs(params)
- async with self.begin() as conn:
- conn.set_query_timeout(self.query_timeout)
- if near_query:
- phrases = [Phrase(PhraseType.NONE, p) for p in near_query.split(',')]
- else:
- phrases = []
- if details.keywords:
- await make_query_analyzer(conn)
+ with details.query_stats as qs:
+ if not categories:
+ return SearchResults()
- geocoder = ForwardGeocoder(conn, details,
- self.config.get_int('REQUEST_TIMEOUT') \
- if self.config.REQUEST_TIMEOUT else None)
- return await geocoder.lookup_pois(categories, phrases)
+ async with self.begin(abs_timeout=timeout.abs) as conn:
+ qs.log_time('start_query')
+ conn.set_query_timeout(self.query_timeout)
+ if near_query:
+ phrases = [nsearch.Phrase(nsearch.PHRASE_ANY, p) for p in near_query.split(',')]
+ else:
+ phrases = []
+ if details.keywords:
+ await nsearch.make_query_analyzer(conn)
+ geocoder = nsearch.ForwardGeocoder(conn, details, timeout)
+ return await geocoder.lookup_pois(categories, phrases)
class NominatimAPI:
""" This class provides a thin synchronous wrapper around the asynchronous
Nominatim functions. It creates its own event loop and runs each
synchronous function call to completion using that loop.
+
+ This class should usually be used as a context manager in 'with' context.
"""
- def __init__(self, project_dir: Path,
+ def __init__(self, project_dir: Optional[Union[str, Path]] = None,
environ: Optional[Mapping[str, str]] = None) -> None:
""" Initiate a new frontend object with synchronous API functions.
@@ -369,20 +388,25 @@ class NominatimAPI:
self._loop = asyncio.new_event_loop()
self._async_api = NominatimAPIAsync(project_dir, environ, loop=self._loop)
-
def close(self) -> None:
""" Close all active connections to the database.
This function also closes the asynchronous worker loop making
the NominatimAPI object unusable.
"""
- self._loop.run_until_complete(self._async_api.close())
- self._loop.close()
+ if not self._loop.is_closed():
+ self._loop.run_until_complete(self._async_api.close())
+ self._loop.close()
+ def __enter__(self) -> 'NominatimAPI':
+ return self
+
+ def __exit__(self, *_: Any) -> None:
+ self.close()
@property
def config(self) -> Configuration:
- """ Provide read-only access to the [configuration](#Configuration)
+ """ Provide read-only access to the [configuration](Configuration.md)
used by the API.
"""
return self._async_api.config
@@ -405,7 +429,6 @@ class NominatimAPI:
"""
return self._loop.run_until_complete(self._async_api.status())
-
def details(self, place: ntyp.PlaceRef, **params: Any) -> Optional[DetailedResult]:
""" Get detailed information about a place in the database.
@@ -435,6 +458,8 @@ class NominatimAPI:
Only POI places can have parents. (Default: False)
keywords (bool): Add detailed information about the search terms
used for this place.
+ query_stats (QueryStatistics): When given collects statistics
+ about the query execution.
Returns:
source_table (enum): Data source of the place. See below for possible values.
@@ -488,7 +513,6 @@ class NominatimAPI:
"""
return self._loop.run_until_complete(self._async_api.details(place, **params))
-
def lookup(self, places: Sequence[ntyp.PlaceRef], **params: Any) -> SearchResults:
""" Get simple information about a list of places.
@@ -518,6 +542,8 @@ class NominatimAPI:
Only POI places can have parents. (Default: False)
keywords (bool): Add detailed information about the search terms
used for this place.
+ query_stats (QueryStatistics): When given collects statistics
+ about the query execution.
Returns:
source_table (enum): Data source of the place. See below for possible values.
@@ -565,7 +591,6 @@ class NominatimAPI:
"""
return self._loop.run_until_complete(self._async_api.lookup(places, **params))
-
def reverse(self, coord: ntyp.AnyPoint, **params: Any) -> Optional[ReverseResult]:
""" Find a place by its coordinates. Also known as reverse geocoding.
@@ -599,6 +624,8 @@ class NominatimAPI:
Only POI places can have parents. (Default: False)
keywords (bool): Add detailed information about the search terms
used for this place.
+ query_stats (QueryStatistics): When given collects statistics
+ about the query execution.
Returns:
source_table (enum): Data source of the place. See below for possible values.
@@ -647,7 +674,6 @@ class NominatimAPI:
"""
return self._loop.run_until_complete(self._async_api.reverse(coord, **params))
-
def search(self, query: str, **params: Any) -> SearchResults:
""" Find a place by free-text search. Also known as forward geocoding.
@@ -699,6 +725,8 @@ class NominatimAPI:
Only POI places can have parents. (Default: False)
keywords (bool): Add detailed information about the search terms
used for this place.
+ query_stats (QueryStatistics): When given collects statistics
+ about the query execution.
Returns:
source_table (enum): Data source of the place. See below for possible values.
@@ -747,8 +775,6 @@ class NominatimAPI:
return self._loop.run_until_complete(
self._async_api.search(query, **params))
-
- # pylint: disable=too-many-arguments
def search_address(self, amenity: Optional[str] = None,
street: Optional[str] = None,
city: Optional[str] = None,
@@ -817,6 +843,8 @@ class NominatimAPI:
Only POI places can have parents. (Default: False)
keywords (bool): Add detailed information about the search terms
used for this place.
+ query_stats (QueryStatistics): When given collects statistics
+ about the query execution.
Returns:
source_table (enum): Data source of the place. See below for possible values.
@@ -866,7 +894,6 @@ class NominatimAPI:
self._async_api.search_address(amenity, street, city, county,
state, country, postalcode, **params))
-
def search_category(self, categories: List[Tuple[str, str]],
near_query: Optional[str] = None,
**params: Any) -> SearchResults:
@@ -925,6 +952,8 @@ class NominatimAPI:
Only POI places can have parents. (Default: False)
keywords (bool): Add detailed information about the search terms
used for this place.
+ query_stats (QueryStatistics): When given collects statistics
+ about the query execution.
Returns:
source_table (enum): Data source of the place. See below for possible values.