From 177b16b89b214a74626d66174632a3bbbcf9fc65 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 10 Sep 2025 11:52:06 +0200 Subject: [PATCH] use new QueryStatistics in API server --- src/nominatim_api/server/asgi_adaptor.py | 9 +++++- src/nominatim_api/server/falcon/server.py | 34 ++++++++++++-------- src/nominatim_api/server/starlette/server.py | 30 ++++++++++------- src/nominatim_api/types.py | 13 ++++---- src/nominatim_api/v1/server_glue.py | 6 +++- test/python/api/fake_adaptor.py | 3 ++ 6 files changed, 61 insertions(+), 34 deletions(-) diff --git a/src/nominatim_api/server/asgi_adaptor.py b/src/nominatim_api/server/asgi_adaptor.py index 77c50f31..a2ded018 100644 --- a/src/nominatim_api/server/asgi_adaptor.py +++ b/src/nominatim_api/server/asgi_adaptor.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2024 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. """ Base abstraction for implementing based on different ASGI frameworks. @@ -13,6 +13,7 @@ import math from ..config import Configuration from ..core import NominatimAPIAsync +from ..types import QueryStatistics from ..result_formatting import FormatDispatcher from .content_types import CONTENT_TEXT @@ -68,6 +69,12 @@ class ASGIAdaptor(abc.ABC): """ Return the formatting object to use. """ + @abc.abstractmethod + def query_stats(self) -> Optional[QueryStatistics]: + """ Return the object for saving query statistics or None if + no statistics are required. + """ + def get_int(self, name: str, default: Optional[int] = None) -> int: """ Return an input parameter as an int. Raises an exception if the parameter is given but not in an integer format. diff --git a/src/nominatim_api/server/falcon/server.py b/src/nominatim_api/server/falcon/server.py index c16d085b..df2b3379 100644 --- a/src/nominatim_api/server/falcon/server.py +++ b/src/nominatim_api/server/falcon/server.py @@ -2,20 +2,21 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2024 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. """ Server implementation using the falcon webserver framework. """ -from typing import Optional, Mapping, Any, List +from typing import Optional, Mapping, Any, List, cast from pathlib import Path -import datetime as dt import asyncio +import datetime as dt from falcon.asgi import App, Request, Response from ...config import Configuration from ...core import NominatimAPIAsync +from ...types import QueryStatistics from ... import v1 as api_impl from ...result_formatting import FormatDispatcher, load_format_dispatcher from ... import logging as loglib @@ -95,6 +96,9 @@ class ParamWrapper(ASGIAdaptor): def formatting(self) -> FormatDispatcher: return self._formatter + def query_stats(self) -> Optional[QueryStatistics]: + return cast(Optional[QueryStatistics], getattr(self.request.context, 'query_stats', None)) + class EndpointWrapper: """ Converter for server glue endpoint functions to Falcon request handlers. @@ -124,7 +128,7 @@ class FileLoggingMiddleware: async def process_request(self, req: Request, _: Response) -> None: """ Callback before the request starts timing. """ - req.context.start = dt.datetime.now(tz=dt.timezone.utc) + req.context.query_stats = QueryStatistics() async def process_response(self, req: Request, resp: Response, resource: Optional[EndpointWrapper], @@ -132,19 +136,23 @@ class FileLoggingMiddleware: """ Callback after requests writes to the logfile. It only writes logs for successful requests for search, reverse and lookup. """ - if not req_succeeded or resource is None or resp.status != 200\ + qs = req.context.query_stats + + if not req_succeeded or 'start' not in qs\ + or resource is None or resp.status != 200\ or resource.name not in ('reverse', 'search', 'lookup', 'details'): return - finish = dt.datetime.now(tz=dt.timezone.utc) - duration = (finish - req.context.start).total_seconds() - params = req.scope['query_string'].decode('utf8') - start = req.context.start.replace(tzinfo=None)\ - .isoformat(sep=' ', timespec='milliseconds') + qs['endpoint'] = resource.name + qs['query_string'] = req.scope['query_string'].decode('utf8') + qs['results_total'] = getattr(resp.context, 'num_results', 0) + for param in ('start', 'end', 'start_query'): + if isinstance(qs.get(param), dt.datetime): + qs[param] = qs[param].replace(tzinfo=None)\ + .isoformat(sep=' ', timespec='milliseconds') - self.fd.write(f"[{start}] " - f"{duration:.4f} {getattr(resp.context, 'num_results', 0)} " - f'{resource.name} "{params}"\n') + self.fd.write(("[{start}] {total_time:.4f} {results_total} " + '{endpoint} "{query_string}"\n').format_map(qs)) class APIMiddleware: diff --git a/src/nominatim_api/server/starlette/server.py b/src/nominatim_api/server/starlette/server.py index afaf5732..15c5dd92 100644 --- a/src/nominatim_api/server/starlette/server.py +++ b/src/nominatim_api/server/starlette/server.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2024 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. """ Server implementation using the starlette webserver framework. @@ -10,9 +10,9 @@ Server implementation using the starlette webserver framework. from typing import Any, Optional, Mapping, Callable, cast, Coroutine, Dict, \ Awaitable, AsyncIterator from pathlib import Path -import datetime as dt import asyncio import contextlib +import datetime as dt from starlette.applications import Starlette from starlette.routing import Route @@ -25,6 +25,7 @@ from starlette.middleware.cors import CORSMiddleware from ...config import Configuration from ...core import NominatimAPIAsync +from ...types import QueryStatistics from ... import v1 as api_impl from ...result_formatting import FormatDispatcher, load_format_dispatcher from ..asgi_adaptor import ASGIAdaptor, EndpointFunc @@ -70,6 +71,9 @@ class ParamWrapper(ASGIAdaptor): def formatting(self) -> FormatDispatcher: return cast(FormatDispatcher, self.request.app.state.formatter) + def query_stats(self) -> Optional[QueryStatistics]: + return cast(Optional[QueryStatistics], getattr(self.request.state, 'query_stats', None)) + def _wrap_endpoint(func: EndpointFunc)\ -> Callable[[Request], Coroutine[Any, Any, Response]]: @@ -89,27 +93,29 @@ class FileLoggingMiddleware(BaseHTTPMiddleware): async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response: - start = dt.datetime.now(tz=dt.timezone.utc) + qs = QueryStatistics() + request.state.query_stats = qs response = await call_next(request) - if response.status_code != 200: + if response.status_code != 200 or 'start' not in qs: return response - finish = dt.datetime.now(tz=dt.timezone.utc) - for endpoint in ('reverse', 'search', 'lookup', 'details'): if request.url.path.startswith('/' + endpoint): - qtype = endpoint + qs['endpoint'] = endpoint break else: return response - duration = (finish - start).total_seconds() - params = request.scope['query_string'].decode('utf8') + qs['query_string'] = request.scope['query_string'].decode('utf8') + qs['results_total'] = getattr(request.state, 'num_results', 0) + for param in ('start', 'end', 'start_query'): + if isinstance(qs.get(param), dt.datetime): + qs[param] = qs[param].replace(tzinfo=None)\ + .isoformat(sep=' ', timespec='milliseconds') - self.fd.write(f"[{start.replace(tzinfo=None).isoformat(sep=' ', timespec='milliseconds')}] " - f"{duration:.4f} {getattr(request.state, 'num_results', 0)} " - f'{qtype} "{params}"\n') + self.fd.write(("[{start}] {total_time:.4f} {results_total} " + '{endpoint} "{query_string}"\n').format_map(qs)) return response diff --git a/src/nominatim_api/types.py b/src/nominatim_api/types.py index 98ec571a..f2e4c69e 100644 --- a/src/nominatim_api/types.py +++ b/src/nominatim_api/types.py @@ -340,18 +340,17 @@ class QueryStatistics(dict[str, Any]): """ def __enter__(self) -> 'QueryStatistics': - self.log_time('start_function') + self.log_time('start') return self def __exit__(self, *_: Any) -> None: - self.log_time('end_function') - self['total_time'] = (self['end_function'] - self['start_function']) \ - / dt.timedelta(microseconds=1) + self.log_time('end') + self['total_time'] = (self['end'] - self['start']).total_seconds() if 'start_query' in self: - self['wait_time'] = (self['start_query'] - self['start_function']) \ - / dt.timedelta(microseconds=1) + self['wait_time'] = (self['start_query'] - self['start']).total_seconds() else: - self['wait_time'] = 0 + self['wait_time'] = self['total_time'] + self['start_query'] = self['end'] self['query_time'] = self['total_time'] - self['wait_time'] def __missing__(self, key: str) -> str: diff --git a/src/nominatim_api/v1/server_glue.py b/src/nominatim_api/v1/server_glue.py index a3a29199..871358ef 100644 --- a/src/nominatim_api/v1/server_glue.py +++ b/src/nominatim_api/v1/server_glue.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2024 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. """ Generic part of the server implementation of the v1 API. @@ -165,6 +165,7 @@ async def details_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any: geometry_output=(GeometryFormat.GEOJSON if params.get_bool('polygon_geojson', False) else GeometryFormat.NONE), + query_stats=params.query_stats() ) if debug: @@ -197,6 +198,7 @@ async def reverse_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any: details = parse_geometry_details(params, fmt) details['max_rank'] = helpers.zoom_to_rank(params.get_int('zoom', 18)) details['layers'] = get_layers(params) + details['query_stats'] = params.query_stats() result = await api.reverse(coord, **details) @@ -234,6 +236,7 @@ async def lookup_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any: fmt = parse_format(params, SearchResults, 'xml') debug = setup_debugging(params) details = parse_geometry_details(params, fmt) + details['query_stats'] = params.query_stats() places = [] for oid in (params.get('osm_ids') or '').split(','): @@ -302,6 +305,7 @@ async def search_endpoint(api: NominatimAPIAsync, params: ASGIAdaptor) -> Any: debug = setup_debugging(params) details = parse_geometry_details(params, fmt) + details['query_stats'] = params.query_stats() details['countries'] = params.get('countrycodes', None) details['entrances'] = params.get_bool('entrances', False) details['excluded'] = params.get('exclude_place_ids', None) diff --git a/test/python/api/fake_adaptor.py b/test/python/api/fake_adaptor.py index a3a3bcf9..01050037 100644 --- a/test/python/api/fake_adaptor.py +++ b/test/python/api/fake_adaptor.py @@ -54,3 +54,6 @@ class FakeAdaptor(glue.ASGIAdaptor): def formatting(self): return formatting + + def query_stats(self): + return None -- 2.39.5