From 474d4230b88033c48ff284d58223cbc632213b85 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 6 Jan 2024 22:31:38 +0100 Subject: [PATCH] fix timezone handling for timestamps from the database SQLite is not timezone-aware, so make sure to convert to UTC before inserting any data. --- nominatim/api/status.py | 5 ++++- nominatim/tools/convert_sqlite.py | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/nominatim/api/status.py b/nominatim/api/status.py index adccf7a5..8ac92f35 100644 --- a/nominatim/api/status.py +++ b/nominatim/api/status.py @@ -37,7 +37,10 @@ async def get_status(conn: SearchConnection) -> StatusResult: status.data_updated = await conn.scalar(sql) if status.data_updated is not None: - status.data_updated = status.data_updated.replace(tzinfo=dt.timezone.utc) + if status.data_updated.tzinfo is None: + status.data_updated = status.data_updated.replace(tzinfo=dt.timezone.utc) + else: + status.data_updated = status.data_updated.astimezone(dt.timezone.utc) # Database version try: diff --git a/nominatim/tools/convert_sqlite.py b/nominatim/tools/convert_sqlite.py index 3e584710..1e7beae5 100644 --- a/nominatim/tools/convert_sqlite.py +++ b/nominatim/tools/convert_sqlite.py @@ -7,13 +7,14 @@ """ Exporting a Nominatim database to SQlite. """ -from typing import Set +from typing import Set, Any +import datetime as dt import logging from pathlib import Path import sqlalchemy as sa -from nominatim.typing import SaSelect +from nominatim.typing import SaSelect, SaRow from nominatim.db.sqlalchemy_types import Geometry, IntArray from nominatim.api.search.query_analyzer_factory import make_query_analyzer import nominatim.api as napi @@ -124,12 +125,20 @@ class SqliteWriter: async def copy_data(self) -> None: """ Copy data for all registered tables. """ + def _getfield(row: SaRow, key: str) -> Any: + value = getattr(row, key) + if isinstance(value, dt.datetime): + if value.tzinfo is not None: + value = value.astimezone(dt.timezone.utc) + return value + for table in self.dest.t.meta.sorted_tables: LOG.warning("Copying '%s'", table.name) async_result = await self.src.connection.stream(self.select_from(table.name)) async for partition in async_result.partitions(10000): - data = [{('class_' if k == 'class' else k): getattr(r, k) for k in r._fields} + data = [{('class_' if k == 'class' else k): _getfield(r, k) + for k in r._fields} for r in partition] await self.dest.execute(table.insert(), data) -- 2.45.2