From 4fe7e3f0a6fd7612cee507a3d25ad76e2298bbef Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 26 May 2026 15:53:10 +0200 Subject: [PATCH] add new suffix-ignore parameter to language tagging sanatizer Allows to specify suffixes which are not used as language markers. --- .../sanitizers/tag_analyzer_by_language.py | 6 ++- .../test_tag_analyzer_by_language.py | 37 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/nominatim_db/tokenizer/sanitizers/tag_analyzer_by_language.py b/src/nominatim_db/tokenizer/sanitizers/tag_analyzer_by_language.py index da1f51b5..4add6906 100644 --- a/src/nominatim_db/tokenizer/sanitizers/tag_analyzer_by_language.py +++ b/src/nominatim_db/tokenizer/sanitizers/tag_analyzer_by_language.py @@ -17,6 +17,9 @@ Arguments: whitelist: Restrict the set of languages that should be tagged. Expects a list of acceptable suffixes. When unset, all 2- and 3-letter lower-case codes are accepted. + suffix-ignore: List of suffixes that are not language-related. Names + with a suffix from that list will be handled like a name + without suffix. (default: empty) use-defaults: Configure what happens when the name has no suffix. When set to 'all', a variant is created for each of the default languages in the country @@ -44,6 +47,7 @@ class _AnalyzerByLanguage: self.filter_kind = config.get_filter('filter-kind') self.replace = config.get('mode', 'replace') != 'append' self.whitelist = config.get('whitelist') + self.suffix_ignore = set(config.get_string_list('suffix-ignore')) self._compute_default_languages(config.get('use-defaults', 'no')) @@ -72,7 +76,7 @@ class _AnalyzerByLanguage: for name in (n for n in obj.names if not n.has_attr('analyzer') and self.filter_kind(n.kind)): - if name.suffix: + if name.suffix and name.suffix not in self.suffix_ignore: langs = [name.suffix] if self._suffix_matches(name.suffix) else None else: langs = self.deflangs.get(obj.place.country_code) diff --git a/test/python/tokenizer/sanitizers/test_tag_analyzer_by_language.py b/test/python/tokenizer/sanitizers/test_tag_analyzer_by_language.py index 8f1d0540..79d6f072 100644 --- a/test/python/tokenizer/sanitizers/test_tag_analyzer_by_language.py +++ b/test/python/tokenizer/sanitizers/test_tag_analyzer_by_language.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Tests for the sanitizer that enables language-dependent analyzers. @@ -258,3 +258,38 @@ class TestWhiteList: def test_empty_whitelist(self): assert self.run_sanitizer_on([], ref_yy='123') == [('123', '')] + + +class TestSuffixIgnore: + + @pytest.fixture(autouse=True) + def setup_country(self, def_config): + self.config = def_config + setup_country_config(def_config) + + def run_sanitizer_on(self, suffix_ignore, **kwargs): + place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, + 'country_code': 'de'}) + name, _ = PlaceSanitizer([{'step': 'tag-analyzer-by-language', + 'mode': 'replace', + 'use-defaults': 'mono', + 'whitelist': ['de', 'en'], + 'suffix-ignore': suffix_ignore}], + self.config).process_names(place) + + assert all(isinstance(p.attr, dict) for p in name) + assert all(len(p.attr) <= 1 for p in name) + assert all(not p.attr or ('analyzer' in p.attr and p.attr['analyzer']) + for p in name) + + return sorted([(p.name, p.attr.get('analyzer', '')) for p in name]) + + def test_ignored_suffix(self): + assert self.run_sanitizer_on(['left'], name_left='foo') == [('foo', 'de')] + + def test_not_ignored_suffix(self): + assert self.run_sanitizer_on(['left'], name_en='foo') == [('foo', 'en')] + assert self.run_sanitizer_on(['left'], name_fr='foo') == [('foo', '')] + + def test_ignored_suffix_and_whitelisted(self): + assert self.run_sanitizer_on(['de'], name_de='foo') == [('foo', 'de')] -- 2.47.3