add type annotations to ICU tokenizer helper modules

[nominatim.git] / nominatim / tokenizer / base.py

diff --git a/nominatim/tokenizer/base.py b/nominatim/tokenizer/base.py
index 5a3d3b1276aa7c6bf3bb36787967d519fe70aa79..6484ff6a06a798b206c9c8e54bf947cf2da4b1d4 100644 (file)
--- a/nominatim/tokenizer/base.py
+++ b/nominatim/tokenizer/base.py
@@ -10,12 +10,13 @@ mainly for documentation purposes.
 """
 from abc import ABC, abstractmethod
 from typing import List, Tuple, Dict, Any
 """
 from abc import ABC, abstractmethod
 from typing import List, Tuple, Dict, Any

+from pathlib import Path
+
+from typing_extensions import Protocol

 
 from nominatim.config import Configuration
 from nominatim.data.place_info import PlaceInfo
 
 
 from nominatim.config import Configuration
 from nominatim.data.place_info import PlaceInfo
 

-# pylint: disable=unnecessary-pass
-

 class AbstractAnalyzer(ABC):
     """ The analyzer provides the functions for analysing names and building
         the token database.
 class AbstractAnalyzer(ABC):
     """ The analyzer provides the functions for analysing names and building
         the token database.


@@ -230,3 +231,13 @@ class AbstractTokenizer(ABC):
             When used outside the with construct, the caller must ensure to
             call the close() function before destructing the analyzer.
         """
             When used outside the with construct, the caller must ensure to
             call the close() function before destructing the analyzer.
         """

+
+
+class TokenizerModule(Protocol):
+    """ Interface that must be exported by modules that implement their
+        own tokenizer.
+    """
+
+    def create(self, dsn: str, data_dir: Path) -> AbstractTokenizer:
+        """ Factory for new tokenizers.
+        """