sql = sql.where(tsearch.c.centroid
.intersects(VIEWBOX_PARAM,
use_index=details.viewbox.area < 0.2))
- elif self.expected_count >= 10000:
+ elif not self.postcodes and not self.housenumbers and self.expected_count >= 10000:
sql = sql.where(tsearch.c.centroid
.intersects(VIEWBOX2_PARAM,
use_index=details.viewbox.area < 0.5))
Implementation of query analysis for the ICU tokenizer.
"""
from typing import Tuple, Dict, List, Optional, NamedTuple, Iterator, Any, cast
-from copy import copy
from collections import defaultdict
import dataclasses
import difflib
query.add_token(trange, qmod.TokenType.NEAR_ITEM, token)
else:
query.add_token(trange, qmod.TokenType.QUALIFIER, token)
- if trange.start == 0 or trange.end == query.num_token_slots():
- token = copy(token)
- token.penalty += 0.1 * (query.num_token_slots())
- query.add_token(trange, qmod.TokenType.NEAR_ITEM, token)
else:
query.add_token(trange, DB_TO_TOKEN_TYPE[row.type], token)
status.data_updated = await conn.scalar(sql)
if status.data_updated is not None:
- status.data_updated = status.data_updated.replace(tzinfo=dt.timezone.utc)
+ if status.data_updated.tzinfo is None:
+ status.data_updated = status.data_updated.replace(tzinfo=dt.timezone.utc)
+ else:
+ status.data_updated = status.data_updated.astimezone(dt.timezone.utc)
# Database version
try:
from nominatim.config import Configuration
from nominatim.db.connection import connect, Connection
+from nominatim.db import properties
from nominatim.errors import UsageError
from nominatim.tokenizer import factory as tokenizer_factory
from nominatim.tools import freeze
+from nominatim.version import NOMINATIM_VERSION, parse_version
CHECKLIST = []
return CheckState.OK
+@_check(hint="""\
+ Database version ({db_version}) doesn't match Nominatim version ({nom_version})
+
+ Hints:
+ * Are you connecting to the correct database?
+
+ {instruction}
+
+ Check the Migration chapter of the Administration Guide.
+
+ Project directory: {config.project_dir}
+ Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
+ """)
+def check_database_version(conn: Connection, config: Configuration) -> CheckResult:
+ """ Checking database_version matches Nominatim software version
+ """
+
+ if conn.table_exists('nominatim_properties'):
+ db_version_str = properties.get_property(conn, 'database_version')
+ else:
+ db_version_str = None
+
+ if db_version_str is not None:
+ db_version = parse_version(db_version_str)
+
+ if db_version == NOMINATIM_VERSION:
+ return CheckState.OK
+
+ instruction = (
+ 'Run migrations: nominatim admin --migrate'
+ if db_version < NOMINATIM_VERSION
+ else 'You need to upgrade the Nominatim software.'
+ )
+ else:
+ instruction = ''
+
+ return CheckState.FATAL, dict(db_version=db_version_str,
+ nom_version=NOMINATIM_VERSION,
+ instruction=instruction,
+ config=config)
+
@_check(hint="""\
placex table not found
Hints:
- * Are you connecting to the right database?
+ * Are you connecting to the correct database?
* Did the import process finish without errors?
Project directory: {config.project_dir}
"""
Exporting a Nominatim database to SQlite.
"""
-from typing import Set
+from typing import Set, Any
+import datetime as dt
import logging
from pathlib import Path
import sqlalchemy as sa
-from nominatim.typing import SaSelect
+from nominatim.typing import SaSelect, SaRow
from nominatim.db.sqlalchemy_types import Geometry, IntArray
from nominatim.api.search.query_analyzer_factory import make_query_analyzer
import nominatim.api as napi
async def copy_data(self) -> None:
""" Copy data for all registered tables.
"""
+ def _getfield(row: SaRow, key: str) -> Any:
+ value = getattr(row, key)
+ if isinstance(value, dt.datetime):
+ if value.tzinfo is not None:
+ value = value.astimezone(dt.timezone.utc)
+ return value
+
for table in self.dest.t.meta.sorted_tables:
LOG.warning("Copying '%s'", table.name)
async_result = await self.src.connection.stream(self.select_from(table.name))
async for partition in async_result.partitions(10000):
- data = [{('class_' if k == 'class' else k): getattr(r, k) for k in r._fields}
+ data = [{('class_' if k == 'class' else k): _getfield(r, k)
+ for k in r._fields}
for r in partition]
await self.dest.execute(table.insert(), data)
query = await ana.analyze_query(make_phrase('foo BAR foo BAR foo'))
assert query.num_token_slots() == 5
- assert set(t.ttype for t in query.nodes[0].starting) == {TokenType.NEAR_ITEM, TokenType.QUALIFIER}
+ assert set(t.ttype for t in query.nodes[0].starting) == {TokenType.QUALIFIER}
assert set(t.ttype for t in query.nodes[2].starting) == {TokenType.QUALIFIER}
- assert set(t.ttype for t in query.nodes[4].starting) == {TokenType.NEAR_ITEM, TokenType.QUALIFIER}
+ assert set(t.ttype for t in query.nodes[4].starting) == {TokenType.QUALIFIER}
@pytest.mark.asyncio
import pytest
from nominatim.tools import check_database as chkdb
+import nominatim.version
def test_check_database_unknown_db(def_config, monkeypatch):
monkeypatch.setenv('NOMINATIM_DATABASE_DSN', 'pgsql:dbname=fjgkhughwgh2423gsags')
assert chkdb.check_database(def_config) == 1
-def test_check_conection_good(temp_db_conn, def_config):
+def test_check_connection_good(temp_db_conn, def_config):
assert chkdb.check_connection(temp_db_conn, def_config) == chkdb.CheckState.OK
-def test_check_conection_bad(def_config):
+def test_check_connection_bad(def_config):
badconn = chkdb._BadConnection('Error')
assert chkdb.check_connection(badconn, def_config) == chkdb.CheckState.FATAL
+def test_check_database_version_good(property_table, temp_db_conn, def_config):
+ property_table.set('database_version',
+ '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(nominatim.version.NOMINATIM_VERSION))
+ assert chkdb.check_database_version(temp_db_conn, def_config) == chkdb.CheckState.OK
+
+def test_check_database_version_bad(property_table, temp_db_conn, def_config):
+ property_table.set('database_version', '3.9.9-9')
+ assert chkdb.check_database_version(temp_db_conn, def_config) == chkdb.CheckState.FATAL
+
+
def test_check_placex_table_good(table_factory, temp_db_conn, def_config):
table_factory('placex')
assert chkdb.check_placex_table(temp_db_conn, def_config) == chkdb.CheckState.OK
projects / nominatim.git / commitdiff