complete tests for icu tokenizer

[nominatim.git] / nominatim / tokenizer / icu_name_processor.py

diff --git a/nominatim/tokenizer/icu_name_processor.py b/nominatim/tokenizer/icu_name_processor.py
index 0e71799507767e5e5526a37bd623d5e7641b345f..ed0a20d9539920919e5de4cfa370fab616a68a3e 100644 (file)
--- a/nominatim/tokenizer/icu_name_processor.py
+++ b/nominatim/tokenizer/icu_name_processor.py
@@ -58,6 +58,9 @@ class ICUNameProcessorRules:
 
 
 class ICUNameProcessor:
+    """ Collects the different transformation rules for normalisation of names
+        and provides the functions to aply the transformations.
+    """
 
     def __init__(self, rules):
         self.normalizer = Transliterator.createFromRules("icu_normalization",
@@ -76,7 +79,7 @@ class ICUNameProcessor:
         """ Normalize the given name, i.e. remove all elements not relevant
             for search.
         """
-        return self.normalizer.transliterate(name)
+        return self.normalizer.transliterate(name).strip()
 
     def get_variants_ascii(self, norm_name):
         """ Compute the spelling variants for the given normalized name
@@ -98,14 +101,23 @@ class ICUNameProcessor:
             else:
                 pos += 1
 
+        results = []
+
         if startpos == 0:
-            return [self.to_ascii.transliterate(norm_name)]
+            trans_name = self.to_ascii.transliterate(norm_name).strip()
+            if trans_name:
+                results.append(trans_name)
+        else:
+            for variant in variants:
+                trans_name = self.to_ascii.transliterate(variant + baseform[startpos:pos]).strip()
+                if trans_name:
+                    results.append(trans_name)
 
-        return [self.to_ascii.transliterate(v + baseform[startpos:pos]).strip() for v in variants]
+        return results
 
 
     def get_search_normalized(self, name):
         """ Return the normalized version of the name (including transliteration)
             to be applied at search time.
         """
-        return self.search.transliterate(name)
+        return self.search.transliterate(' ' + name + ' ').strip()