]> git.openstreetmap.org Git - nominatim.git/blob - nominatim/tokenizer/icu_name_processor.py
complete tests for icu tokenizer
[nominatim.git] / nominatim / tokenizer / icu_name_processor.py
1 """
2 Processor for names that are imported into the database based on the
3 ICU library.
4 """
5 import json
6 import itertools
7
8 from icu import Transliterator
9 import datrie
10
11 from nominatim.db.properties import set_property, get_property
12
13 DBCFG_IMPORT_NORM_RULES = "tokenizer_import_normalisation"
14 DBCFG_IMPORT_TRANS_RULES = "tokenizer_import_transliteration"
15 DBCFG_IMPORT_REPLACEMENTS = "tokenizer_import_replacements"
16 DBCFG_SEARCH_STD_RULES = "tokenizer_search_standardization"
17
18
19 class ICUNameProcessorRules:
20     """ Data object that saves the rules needed for the name processor.
21
22         The rules can either be initialised through an ICURuleLoader or
23         be loaded from a database when a connection is given.
24     """
25     def __init__(self, loader=None, conn=None):
26         if loader is not None:
27             self.norm_rules = loader.get_normalization_rules()
28             self.trans_rules = loader.get_transliteration_rules()
29             self.replacements = loader.get_replacement_pairs()
30             self.search_rules = loader.get_search_rules()
31         elif conn is not None:
32             self.norm_rules = get_property(conn, DBCFG_IMPORT_NORM_RULES)
33             self.trans_rules = get_property(conn, DBCFG_IMPORT_TRANS_RULES)
34             self.replacements = json.loads(get_property(conn, DBCFG_IMPORT_REPLACEMENTS))
35             self.search_rules = get_property(conn, DBCFG_SEARCH_STD_RULES)
36         else:
37             assert False, "Parameter loader or conn required."
38
39         # Compute the set of characters used in the replacement list.
40         # We need this later when computing the tree.
41         chars = set()
42         for full, repl in self.replacements:
43             chars.update(full)
44             for word in repl:
45                 chars.update(word)
46         self.replacement_charset = ''.join(chars)
47
48
49     def save_rules(self, conn):
50         """ Save the rules in the property table of the given database.
51             the rules can be loaded again by handing in a connection into
52             the constructor of the class.
53         """
54         set_property(conn, DBCFG_IMPORT_NORM_RULES, self.norm_rules)
55         set_property(conn, DBCFG_IMPORT_TRANS_RULES, self.trans_rules)
56         set_property(conn, DBCFG_IMPORT_REPLACEMENTS, json.dumps(self.replacements))
57         set_property(conn, DBCFG_SEARCH_STD_RULES, self.search_rules)
58
59
60 class ICUNameProcessor:
61     """ Collects the different transformation rules for normalisation of names
62         and provides the functions to aply the transformations.
63     """
64
65     def __init__(self, rules):
66         self.normalizer = Transliterator.createFromRules("icu_normalization",
67                                                          rules.norm_rules)
68         self.to_ascii = Transliterator.createFromRules("icu_to_ascii",
69                                                        rules.trans_rules)
70         self.search = Transliterator.createFromRules("icu_search",
71                                                      rules.search_rules)
72
73         self.replacements = datrie.Trie(rules.replacement_charset)
74         for full, repl in rules.replacements:
75             self.replacements[full] = repl
76
77
78     def get_normalized(self, name):
79         """ Normalize the given name, i.e. remove all elements not relevant
80             for search.
81         """
82         return self.normalizer.transliterate(name).strip()
83
84     def get_variants_ascii(self, norm_name):
85         """ Compute the spelling variants for the given normalized name
86             and transliterate the result.
87         """
88         baseform = ' ' + norm_name + ' '
89         variants = ['']
90
91         startpos = 0
92         pos = 0
93         while pos < len(baseform):
94             full, repl = self.replacements.longest_prefix_item(baseform[pos:],
95                                                                (None, None))
96             if full is not None:
97                 done = baseform[startpos:pos]
98                 variants = [v + done + r for v, r in itertools.product(variants, repl)]
99                 startpos = pos + len(full)
100                 pos = startpos
101             else:
102                 pos += 1
103
104         results = []
105
106         if startpos == 0:
107             trans_name = self.to_ascii.transliterate(norm_name).strip()
108             if trans_name:
109                 results.append(trans_name)
110         else:
111             for variant in variants:
112                 trans_name = self.to_ascii.transliterate(variant + baseform[startpos:pos]).strip()
113                 if trans_name:
114                     results.append(trans_name)
115
116         return results
117
118
119     def get_search_normalized(self, name):
120         """ Return the normalized version of the name (including transliteration)
121             to be applied at search time.
122         """
123         return self.search.transliterate(' ' + name + ' ').strip()