From: Sarah Hoffmann <lonvia@denofr.de>
Date: Sun, 7 Jan 2024 14:24:00 +0000 (+0100)
Subject: Merge pull request #3238 from mtmail/check-database-for-version-match
X-Git-Tag: v4.4.0~29
X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/dc52d0954e65d3ed0d0bdca05ff43d8892f704f3?hp=1d7e078a2c9a12c99625a31887f23a23028124d3

Merge pull request #3238 from mtmail/check-database-for-version-match

admin --check-database also checks database vs nominatim version
---

diff --git a/nominatim/api/search/db_searches.py b/nominatim/api/search/db_searches.py
index cc352134..254d2ca6 100644
--- a/nominatim/api/search/db_searches.py
+++ b/nominatim/api/search/db_searches.py
@@ -663,7 +663,7 @@ class PlaceSearch(AbstractSearch):
                 sql = sql.where(tsearch.c.centroid
                                          .intersects(VIEWBOX_PARAM,
                                                      use_index=details.viewbox.area < 0.2))
-            elif self.expected_count >= 10000:
+            elif not self.postcodes and not self.housenumbers and self.expected_count >= 10000:
                 sql = sql.where(tsearch.c.centroid
                                          .intersects(VIEWBOX2_PARAM,
                                                      use_index=details.viewbox.area < 0.5))
diff --git a/nominatim/api/search/icu_tokenizer.py b/nominatim/api/search/icu_tokenizer.py
index eabd329d..72e0f547 100644
--- a/nominatim/api/search/icu_tokenizer.py
+++ b/nominatim/api/search/icu_tokenizer.py
@@ -8,7 +8,6 @@
 Implementation of query analysis for the ICU tokenizer.
 """
 from typing import Tuple, Dict, List, Optional, NamedTuple, Iterator, Any, cast
-from copy import copy
 from collections import defaultdict
 import dataclasses
 import difflib
@@ -188,10 +187,6 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer):
                             query.add_token(trange, qmod.TokenType.NEAR_ITEM, token)
                     else:
                         query.add_token(trange, qmod.TokenType.QUALIFIER, token)
-                        if trange.start == 0 or trange.end == query.num_token_slots():
-                            token = copy(token)
-                            token.penalty += 0.1 * (query.num_token_slots())
-                            query.add_token(trange, qmod.TokenType.NEAR_ITEM, token)
                 else:
                     query.add_token(trange, DB_TO_TOKEN_TYPE[row.type], token)
 
diff --git a/nominatim/api/status.py b/nominatim/api/status.py
index adccf7a5..8ac92f35 100644
--- a/nominatim/api/status.py
+++ b/nominatim/api/status.py
@@ -37,7 +37,10 @@ async def get_status(conn: SearchConnection) -> StatusResult:
     status.data_updated = await conn.scalar(sql)
 
     if status.data_updated is not None:
-        status.data_updated = status.data_updated.replace(tzinfo=dt.timezone.utc)
+        if status.data_updated.tzinfo is None:
+            status.data_updated = status.data_updated.replace(tzinfo=dt.timezone.utc)
+        else:
+            status.data_updated = status.data_updated.astimezone(dt.timezone.utc)
 
     # Database version
     try:
diff --git a/nominatim/tools/convert_sqlite.py b/nominatim/tools/convert_sqlite.py
index 3e584710..1e7beae5 100644
--- a/nominatim/tools/convert_sqlite.py
+++ b/nominatim/tools/convert_sqlite.py
@@ -7,13 +7,14 @@
 """
 Exporting a Nominatim database to SQlite.
 """
-from typing import Set
+from typing import Set, Any
+import datetime as dt
 import logging
 from pathlib import Path
 
 import sqlalchemy as sa
 
-from nominatim.typing import SaSelect
+from nominatim.typing import SaSelect, SaRow
 from nominatim.db.sqlalchemy_types import Geometry, IntArray
 from nominatim.api.search.query_analyzer_factory import make_query_analyzer
 import nominatim.api as napi
@@ -124,12 +125,20 @@ class SqliteWriter:
     async def copy_data(self) -> None:
         """ Copy data for all registered tables.
         """
+        def _getfield(row: SaRow, key: str) -> Any:
+            value = getattr(row, key)
+            if isinstance(value, dt.datetime):
+                if value.tzinfo is not None:
+                    value = value.astimezone(dt.timezone.utc)
+            return value
+
         for table in self.dest.t.meta.sorted_tables:
             LOG.warning("Copying '%s'", table.name)
             async_result = await self.src.connection.stream(self.select_from(table.name))
 
             async for partition in async_result.partitions(10000):
-                data = [{('class_' if k == 'class' else k): getattr(r, k) for k in r._fields}
+                data = [{('class_' if k == 'class' else k): _getfield(r, k)
+                         for k in r._fields}
                         for r in partition]
                 await self.dest.execute(table.insert(), data)
 
diff --git a/test/python/api/search/test_icu_query_analyzer.py b/test/python/api/search/test_icu_query_analyzer.py
index a88ca8b8..6a17e32a 100644
--- a/test/python/api/search/test_icu_query_analyzer.py
+++ b/test/python/api/search/test_icu_query_analyzer.py
@@ -148,9 +148,9 @@ async def test_qualifier_words(conn):
     query = await ana.analyze_query(make_phrase('foo BAR foo BAR foo'))
 
     assert query.num_token_slots() == 5
-    assert set(t.ttype for t in query.nodes[0].starting) == {TokenType.NEAR_ITEM, TokenType.QUALIFIER}
+    assert set(t.ttype for t in query.nodes[0].starting) == {TokenType.QUALIFIER}
     assert set(t.ttype for t in query.nodes[2].starting) == {TokenType.QUALIFIER}
-    assert set(t.ttype for t in query.nodes[4].starting) == {TokenType.NEAR_ITEM, TokenType.QUALIFIER}
+    assert set(t.ttype for t in query.nodes[4].starting) == {TokenType.QUALIFIER}
 
 
 @pytest.mark.asyncio