+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
"""
-Abstract class defintions for tokenizers. These base classes are here
+Abstract class definitions for tokenizers. These base classes are here
mainly for documentation purposes.
"""
from abc import ABC, abstractmethod
-from typing import List, Tuple, Dict, Any
+from typing import List, Tuple, Dict, Any, Optional, Iterable
+from pathlib import Path
from nominatim.config import Configuration
-
-# pylint: disable=unnecessary-pass
+from nominatim.db.connection import Connection
+from nominatim.data.place_info import PlaceInfo
+from nominatim.typing import Protocol
class AbstractAnalyzer(ABC):
""" The analyzer provides the functions for analysing names and building
return self
- def __exit__(self, exc_type, exc_value, traceback) -> None:
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
self.close()
def close(self) -> None:
""" Free all resources used by the analyzer.
"""
- pass
@abstractmethod
Returns:
The function returns the list of all tuples that could be
- found for the given words. Each list entry is a tuple of
- (original word, word token, word id).
+ found for the given words. Each list entry is a tuple of
+ (original word, word token, word id).
"""
- pass
@abstractmethod
Returns:
The given postcode after normalization.
"""
- pass
@abstractmethod
""" Update the tokenizer's postcode tokens from the current content
of the `location_postcode` table.
"""
- pass
@abstractmethod
- def update_special_phrases(self, phrases: List[Tuple[str, str, str, str]],
+ def update_special_phrases(self,
+ phrases: Iterable[Tuple[str, str, str, str]],
should_replace: bool) -> None:
""" Update the tokenizer's special phrase tokens from the given
list of special phrases.
When false, just add the given phrases to the
ones that already exist.
"""
- pass
@abstractmethod
- def add_country_names(self, country_code: str, names: Dict[str, str]):
+ def add_country_names(self, country_code: str, names: Dict[str, str]) -> None:
""" Add the given names to the tokenizer's list of country tokens.
Arguments:
refer to.
names: Dictionary of name type to name.
"""
- pass
@abstractmethod
- def process_place(self, place: Dict) -> Any:
+ def process_place(self, place: PlaceInfo) -> Any:
""" Extract tokens for the given place and compute the
information to be handed to the PL/pgSQL processor for building
the search index.
Arguments:
- place: Dictionary with the information about the place. Currently
- the following fields may be present:
-
- - *name* is a dictionary of names for the place together
- with the designation of the name.
- - *address* is a dictionary of address terms.
- - *country_feature* is set to a country code when the
- place describes a country.
+ place: Place information retrieved from the database.
Returns:
A JSON-serialisable structure that will be handed into
- the database via the `token_info` field.
+ the database via the `token_info` field.
"""
the tokenizer remains stable over updates.
Arguments:
- config: Read-only object with configuration obtions.
+ config: Read-only object with configuration options.
init_db: When set to False, then initialisation of database
tables should be skipped. This option is only required for
- migration purposes and can be savely ignored by custom
+ migration purposes and can be safely ignored by custom
tokenizers.
-
- TODO: can we move the init_db parameter somewhere else?
"""
- pass
@abstractmethod
- def init_from_project(self) -> None:
+ def init_from_project(self, config: Configuration) -> None:
""" Initialise the tokenizer from an existing database setup.
The function should load all previously saved configuration from
the project directory and/or the property table.
+
+ Arguments:
+ config: Read-only object with configuration options.
"""
- pass
@abstractmethod
during query time.
Arguments:
- config: Read-only object with configuration obtions.
+ config: Read-only object with configuration options.
"""
- pass
@abstractmethod
data structures or data itself must not be changed by this function.
Arguments:
- config: Read-only object with configuration obtions.
+ config: Read-only object with configuration options.
"""
- pass
@abstractmethod
- def check_database(self) -> str:
+ def check_database(self, config: Configuration) -> Optional[str]:
""" Check that the database is set up correctly and ready for being
queried.
+ Arguments:
+ config: Read-only object with configuration options.
+
Returns:
If an issue was found, return an error message with the
- description of the issue as well as hints for the user on
- how to resolve the issue.
+ description of the issue as well as hints for the user on
+ how to resolve the issue. If everything is okay, return `None`.
+ """
+
- Return `None`, if no issue was found.
+ @abstractmethod
+ def update_statistics(self, config: Configuration) -> None:
+ """ Recompute any tokenizer statistics necessary for efficient lookup.
+ This function is meant to be called from time to time by the user
+ to improve performance. However, the tokenizer must not depend on
+ it to be called in order to work.
+ """
+
+
+ @abstractmethod
+ def update_word_tokens(self) -> None:
+ """ Do house-keeping on the tokenizers internal data structures.
+ Remove unused word tokens, resort data etc.
"""
- pass
@abstractmethod
When used outside the with construct, the caller must ensure to
call the close() function before destructing the analyzer.
"""
- pass
+
+
+ @abstractmethod
+ def most_frequent_words(self, conn: Connection, num: int) -> List[str]:
+ """ Return a list of the most frequent full words in the database.
+
+ Arguments:
+ conn: Open connection to the database which may be used to
+ retrieve the words.
+ num: Maximum number of words to return.
+ """
+
+
+class TokenizerModule(Protocol):
+ """ Interface that must be exported by modules that implement their
+ own tokenizer.
+ """
+
+ def create(self, dsn: str, data_dir: Path) -> AbstractTokenizer:
+ """ Factory for new tokenizers.
+ """
projects / nominatim.git / blobdiff