From 28ebfe67654ecb9df452288dca4ac9bc4ca2fa17 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 28 May 2026 19:01:32 +0200 Subject: [PATCH] move sanitized names into PlaceInfo --- settings/icu_tokenizer.yaml | 29 - src/nominatim_db/data/place_info.py | 22 +- src/nominatim_db/tokenizer/icu_tokenizer.py | 17 +- src/nominatim_db/tokenizer/place_sanitizer.py | 9 +- .../sanitizers/test_affix_expansion.py | 7 +- .../sanitizers/test_clean_housenumbers.py | 16 +- .../sanitizers/test_clean_postcodes.py | 7 +- .../sanitizers/test_clean_tiger_tags.py | 7 +- .../tokenizer/sanitizers/test_delete_names.py | 39 +- .../tokenizer/sanitizers/test_delete_tags.py | 610 +++++++++--------- .../tokenizer/sanitizers/test_derive_names.py | 7 +- .../sanitizers/test_split_name_list.py | 19 +- .../sanitizers/test_strip_brace_terms.py | 12 +- .../test_tag_analyzer_by_language.py | 119 ++-- .../tokenizer/sanitizers/test_tag_japanese.py | 6 +- test/python/tokenizer/test_place_sanitizer.py | 25 +- 16 files changed, 466 insertions(+), 485 deletions(-) diff --git a/settings/icu_tokenizer.yaml b/settings/icu_tokenizer.yaml index a56685d3..261a7a74 100644 --- a/settings/icu_tokenizer.yaml +++ b/settings/icu_tokenizer.yaml @@ -29,35 +29,6 @@ transliteration: - "[^a-z0-9[:Space:]] >" - ":: NFC ()" - "[:Space:]+ > ' '" -sanitizers: - - step: clean-housenumbers - filter-kind: - - housenumber - - conscriptionnumber - - streetnumber - convert-to-name: - - (\A|.*,)[^\d,]{3,}(,.*|\Z) - - step: clean-postcodes - convert-to-address: yes - default-pattern: "[A-Z0-9- ]{3,12}" - - step: clean-tiger-tags - - step: split-name-list - delimiters: ; - - step: affix-expansion - mode: add-expanded - - step: delete-names - filter-kind: ref - filter-name: - - ".*,.*" - - ".{41,}" - - step: strip-brace-terms - - step: tag-analyzer-by-language - filter-kind: [".*name.*"] - suffix-ignore: [left,right] - whitelist: [bg,ca,cs,da,de,el,en,es,et,eu,fi,fr,gl,hu,it,ja,mg,ms,nl,"no",pl,pt,ro,ru,sk,sl,sv,tr,uk,vi] - use-defaults: all - mode: append - - step: tag-japanese token-analysis: - analyzer: generic - id: "@housenumber" diff --git a/src/nominatim_db/data/place_info.py b/src/nominatim_db/data/place_info.py index 3b87a9e4..41004530 100644 --- a/src/nominatim_db/data/place_info.py +++ b/src/nominatim_db/data/place_info.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Wrapper around place information the indexer gets from the database and hands to @@ -10,6 +10,8 @@ the tokenizer. """ from typing import Optional, Mapping, Any, Tuple, cast +from .place_name import PlaceName + class PlaceInfo: """ This data class contains all information the tokenizer can access @@ -18,6 +20,8 @@ class PlaceInfo: def __init__(self, info: Mapping[str, Any]) -> None: self._info = info + self._sanitized_names: list[PlaceName] = [] + self._sanitized_address: list[PlaceName] = [] @property def name(self) -> Optional[Mapping[str, str]]: @@ -42,6 +46,18 @@ class PlaceInfo: """ return self._info.get('address') + @property + def sanitized_names(self) -> list[PlaceName]: + """ List of place names after sanitization. + """ + return self._sanitized_names + + @property + def sanitized_address(self) -> list[PlaceName]: + """ List of address terms after sanitization. + """ + return self._sanitized_address + @property def country_code(self) -> Optional[str]: """ The country code of the country the place is in. Guaranteed @@ -83,3 +99,7 @@ class PlaceInfo: """ Return True when the place is a street object. """ return 26 <= self.rank_address <= 27 + + def set_sanitized(self, names: list[PlaceName], address: list[PlaceName]) -> None: + self._sanitized_names = names + self._sanitized_address = address diff --git a/src/nominatim_db/tokenizer/icu_tokenizer.py b/src/nominatim_db/tokenizer/icu_tokenizer.py index 445ce0e4..d0f49a55 100644 --- a/src/nominatim_db/tokenizer/icu_tokenizer.py +++ b/src/nominatim_db/tokenizer/icu_tokenizer.py @@ -455,9 +455,8 @@ class ICUNameAnalyzer(AbstractAnalyzer): info = PlaceInfo({'name': names, 'country_code': country_code, 'rank_address': 4, 'class': 'boundary', 'type': 'administrative'}) - self._add_country_full_names(country_code, - self.sanitizer.process_names(info)[0], - internal=True) + self.sanitizer.process_names(info) + self._add_country_full_names(country_code, info.sanitized_names, internal=True) def _add_country_full_names(self, country_code: str, names: Sequence[PlaceName], internal: bool = False) -> None: @@ -523,17 +522,17 @@ class ICUNameAnalyzer(AbstractAnalyzer): """ token_info = _TokenInfo() - names, address = self.sanitizer.process_names(place) + self.sanitizer.process_names(place) - if names: - token_info.set_names(self._compute_name_tokens(names)) + if place.sanitized_names: + token_info.set_names(self._compute_name_tokens(place.sanitized_names)) if place.is_country(): assert place.country_code is not None - self._add_country_full_names(place.country_code, names) + self._add_country_full_names(place.country_code, place.sanitized_names) - if address: - self._process_place_address(token_info, address) + if place.sanitized_address: + self._process_place_address(token_info, place.sanitized_address) return token_info.to_dict() diff --git a/src/nominatim_db/tokenizer/place_sanitizer.py b/src/nominatim_db/tokenizer/place_sanitizer.py index a947b876..7b4f6d3d 100644 --- a/src/nominatim_db/tokenizer/place_sanitizer.py +++ b/src/nominatim_db/tokenizer/place_sanitizer.py @@ -8,13 +8,12 @@ Handler for cleaning name and address tags in place information before it is handed to the token analysis. """ -from typing import Optional, List, Mapping, Sequence, Callable, Any, Tuple +from typing import Optional, Mapping, Sequence, Callable, Any from ..errors import UsageError from ..config import Configuration from .sanitizers.config import SanitizerConfig from .sanitizers.base import SanitizerHandler, ProcessInfo -from ..data.place_name import PlaceName from ..data.place_info import PlaceInfo @@ -25,7 +24,7 @@ class PlaceSanitizer: def __init__(self, rules: Optional[Sequence[Mapping[str, Any]]], config: Configuration) -> None: - self.handlers: List[Callable[[ProcessInfo], None]] = [] + self.handlers: list[Callable[[ProcessInfo], None]] = [] if rules: for func in rules: @@ -43,7 +42,7 @@ class PlaceSanitizer: self.handlers.append(module.create(SanitizerConfig(func))) - def process_names(self, place: PlaceInfo) -> Tuple[List[PlaceName], List[PlaceName]]: + def process_names(self, place: PlaceInfo) -> None: """ Extract a sanitized list of names and address parts from the given place. The function returns a tuple (list of names, list of address names) @@ -53,7 +52,7 @@ class PlaceSanitizer: for func in self.handlers: func(obj) - return obj.names, obj.address + place.set_sanitized(obj.names, obj.address) def load_sanitizers(config: Configuration) -> PlaceSanitizer: diff --git a/test/python/tokenizer/sanitizers/test_affix_expansion.py b/test/python/tokenizer/sanitizers/test_affix_expansion.py index 3b8f0522..781258e4 100644 --- a/test/python/tokenizer/sanitizers/test_affix_expansion.py +++ b/test/python/tokenizer/sanitizers/test_affix_expansion.py @@ -17,10 +17,9 @@ from nominatim_db.tokenizer.place_sanitizer import PlaceSanitizer def run_sanitizer(def_config): def _f(place, **kwargs): args = {k.replace('_', '-'): v for k, v in kwargs.items()} - san = PlaceSanitizer([args | {'step': 'affix-expansion'}], def_config) - names, _ = san.process_names(place) - nameset = {(p.name, p.kind, p.suffix) for p in names} - assert len(names) == len(nameset) + PlaceSanitizer([args | {'step': 'affix-expansion'}], def_config).process_names(place) + nameset = {(p.name, p.kind, p.suffix) for p in place.sanitized_names} + assert len(place.sanitized_names) == len(nameset) return nameset return _f diff --git a/test/python/tokenizer/sanitizers/test_clean_housenumbers.py b/test/python/tokenizer/sanitizers/test_clean_housenumbers.py index 2e9f4016..d2a93601 100644 --- a/test/python/tokenizer/sanitizers/test_clean_housenumbers.py +++ b/test/python/tokenizer/sanitizers/test_clean_housenumbers.py @@ -21,9 +21,9 @@ def sanitize(request, def_config): def _run(**kwargs): place = PlaceInfo({'address': kwargs}) - _, address = PlaceSanitizer([sanitizer_args], def_config).process_names(place) + PlaceSanitizer([sanitizer_args], def_config).process_names(place) - return sorted([(p.kind, p.name) for p in address]) + return sorted([(p.kind, p.name) for p in place.sanitized_address]) return _run @@ -51,10 +51,10 @@ def test_convert_to_name_converted(def_config, number): 'convert-to-name': (r'\d+', 'n/a')} place = PlaceInfo({'address': {'housenumber': number}}) - names, address = PlaceSanitizer([sanitizer_args], def_config).process_names(place) + PlaceSanitizer([sanitizer_args], def_config).process_names(place) - assert ('housenumber', number) in set((p.kind, p.name) for p in names) - assert 'housenumber' not in set(p.kind for p in address) + assert ('housenumber', number) in set((p.kind, p.name) for p in place.sanitized_names) + assert 'housenumber' not in set(p.kind for p in place.sanitized_address) @pytest.mark.parametrize('number', ('a54', 'n.a', 'bow')) @@ -63,10 +63,10 @@ def test_convert_to_name_unconverted(def_config, number): 'convert-to-name': (r'\d+', 'n/a')} place = PlaceInfo({'address': {'housenumber': number}}) - names, address = PlaceSanitizer([sanitizer_args], def_config).process_names(place) + PlaceSanitizer([sanitizer_args], def_config).process_names(place) - assert 'housenumber' not in set(p.kind for p in names) - assert ('housenumber', number) in set((p.kind, p.name) for p in address) + assert 'housenumber' not in set(p.kind for p in place.sanitized_names) + assert ('housenumber', number) in set((p.kind, p.name) for p in place.sanitized_address) @pytest.mark.parametrize('hnr,itype,out', [ diff --git a/test/python/tokenizer/sanitizers/test_clean_postcodes.py b/test/python/tokenizer/sanitizers/test_clean_postcodes.py index 433ae2b9..a0d53aaa 100644 --- a/test/python/tokenizer/sanitizers/test_clean_postcodes.py +++ b/test/python/tokenizer/sanitizers/test_clean_postcodes.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Tests for the sanitizer that normalizes postcodes. @@ -26,9 +26,10 @@ def sanitize(def_config, request): if country is not None: pi['country_code'] = country - _, address = PlaceSanitizer([sanitizer_args], def_config).process_names(PlaceInfo(pi)) + place = PlaceInfo(pi) + PlaceSanitizer([sanitizer_args], def_config).process_names(place) - return sorted([(p.kind, p.name) for p in address]) + return sorted([(p.kind, p.name) for p in place.sanitized_address]) return _run diff --git a/test/python/tokenizer/sanitizers/test_clean_tiger_tags.py b/test/python/tokenizer/sanitizers/test_clean_tiger_tags.py index d245f4df..5577389b 100644 --- a/test/python/tokenizer/sanitizers/test_clean_tiger_tags.py +++ b/test/python/tokenizer/sanitizers/test_clean_tiger_tags.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Tests for sanitizer that clean up TIGER tags. @@ -21,10 +21,9 @@ class TestCleanTigerTags: def run_sanitizer_on(self, addr): place = PlaceInfo({'address': addr}) - _, outaddr = PlaceSanitizer([{'step': 'clean-tiger-tags'}], - self.config).process_names(place) + PlaceSanitizer([{'step': 'clean-tiger-tags'}], self.config).process_names(place) - return sorted([(p.name, p.kind, p.suffix) for p in outaddr]) + return sorted([(p.name, p.kind, p.suffix) for p in place.sanitized_address]) @pytest.mark.parametrize('inname,outname', [('Hamilton, AL', 'Hamilton'), ('Little, Borough, CA', 'Little, Borough')]) diff --git a/test/python/tokenizer/sanitizers/test_delete_names.py b/test/python/tokenizer/sanitizers/test_delete_names.py index 6f6fc72c..21cd82eb 100644 --- a/test/python/tokenizer/sanitizers/test_delete_names.py +++ b/test/python/tokenizer/sanitizers/test_delete_names.py @@ -26,11 +26,12 @@ class TestWithDefault: sanitizer_args = {'step': 'delete-names'} - name, address = PlaceSanitizer([sanitizer_args], - self.config).process_names(place) + PlaceSanitizer([sanitizer_args], self.config).process_names(place) - return {'name': sorted([(p.name, p.kind, p.suffix or '') for p in name]), - 'address': sorted([(p.name, p.kind, p.suffix or '') for p in address])} + return {'name': sorted([(p.name, p.kind, p.suffix or '') + for p in place.sanitized_names]), + 'address': sorted([(p.name, p.kind, p.suffix or '') + for p in place.sanitized_address])} def test_on_name(self): res = self.run_sanitizer_on('name', name='foo', ref='bar', ref_abc='baz') @@ -58,10 +59,9 @@ class TestTypeField: sanitizer_args = {'step': 'delete-names', 'type': type} - name, _ = PlaceSanitizer([sanitizer_args], - self.config).process_names(place) + PlaceSanitizer([sanitizer_args], self.config).process_names(place) - return sorted([(p.name, p.kind, p.suffix or '') for p in name]) + return sorted([(p.name, p.kind, p.suffix or '') for p in place.sanitized_names]) def test_name_type(self): res = self.run_sanitizer_on('name', name='foo', ref='bar', ref_abc='baz') @@ -89,10 +89,9 @@ class TestFilterKind: sanitizer_args = {'step': 'delete-names', 'filter-kind': filt} - name, _ = PlaceSanitizer([sanitizer_args], - self.config).process_names(place) + PlaceSanitizer([sanitizer_args], self.config).process_names(place) - return sorted([(p.name, p.kind, p.suffix or '') for p in name]) + return sorted([(p.name, p.kind, p.suffix or '') for p in place.sanitized_names]) def test_single_exact_name(self): res = self.run_sanitizer_on(['name'], ref='foo', name='foo', @@ -129,10 +128,9 @@ class TestRankAddress: sanitizer_args = {'step': 'delete-names', 'filter-rank': rank_addr} - name, _ = PlaceSanitizer([sanitizer_args], - self.config).process_names(place) + PlaceSanitizer([sanitizer_args], self.config).process_names(place) - return sorted([(p.name, p.kind, p.suffix or '') for p in name]) + return sorted([(p.name, p.kind, p.suffix or '') for p in place.sanitized_names]) def test_single_rank(self): res = self.run_sanitizer_on('30', name='foo', ref='bar') @@ -179,10 +177,9 @@ class TestSuffix: sanitizer_args = {'step': 'delete-names', 'filter-suffix': suffix} - name, _ = PlaceSanitizer([sanitizer_args], - self.config).process_names(place) + PlaceSanitizer([sanitizer_args], self.config).process_names(place) - return sorted([(p.name, p.kind, p.suffix or '') for p in name]) + return sorted([(p.name, p.kind, p.suffix or '') for p in place.sanitized_names]) def test_single_suffix(self): res = self.run_sanitizer_on('abc', name='foo', name_abc='foo', @@ -211,10 +208,9 @@ class TestCountryCodes: sanitizer_args = {'step': 'delete-names', 'filter-country': country_code} - name, _ = PlaceSanitizer([sanitizer_args], - self.config).process_names(place) + PlaceSanitizer([sanitizer_args], self.config).process_names(place) - return sorted([(p.name, p.kind) for p in name]) + return sorted([(p.name, p.kind) for p in place.sanitized_names]) def test_single_country_code_pass(self): res = self.run_sanitizer_on('de', name='foo', ref='bar') @@ -263,10 +259,9 @@ class TestAllParameters: 'filter-name': r'[\s\S]*', } - name, _ = PlaceSanitizer([sanitizer_args], - self.config).process_names(place) + PlaceSanitizer([sanitizer_args], self.config).process_names(place) - return sorted([(p.name, p.kind, p.suffix or '') for p in name]) + return sorted([(p.name, p.kind, p.suffix or '') for p in place.sanitized_names]) def test_string_arguments_pass(self): res = self.run_sanitizer_on('de', '25-30', r'[\s\S]*', diff --git a/test/python/tokenizer/sanitizers/test_delete_tags.py b/test/python/tokenizer/sanitizers/test_delete_tags.py index 9c8f2a90..e17e87df 100644 --- a/test/python/tokenizer/sanitizers/test_delete_tags.py +++ b/test/python/tokenizer/sanitizers/test_delete_tags.py @@ -1,308 +1,302 @@ -# SPDX-License-Identifier: GPL-3.0-or-later -# -# This file is part of Nominatim. (https://nominatim.org) -# -# Copyright (C) 2025 by the Nominatim developer community. -# For a full list of authors see the git log. -""" -Tests for the sanitizer that normalizes housenumbers. -""" -import pytest - -from nominatim_db.data.place_info import PlaceInfo -from nominatim_db.tokenizer.place_sanitizer import PlaceSanitizer - - -class TestWithDefault: - - @pytest.fixture(autouse=True) - def setup_country(self, def_config): - self.config = def_config - - def run_sanitizer_on(self, type, **kwargs): - - place = PlaceInfo({type: {k.replace('_', ':'): v for k, v in kwargs.items()}, - 'country_code': 'de', 'rank_address': 30}) - - sanitizer_args = {'step': 'delete-tags'} - - name, address = PlaceSanitizer([sanitizer_args], - self.config).process_names(place) - - return {'name': sorted([(p.name, p.kind, p.suffix or '') for p in name]), - 'address': sorted([(p.name, p.kind, p.suffix or '') for p in address])} - - def test_on_name(self): - res = self.run_sanitizer_on('name', name='foo', ref='bar', ref_abc='baz') - - assert res.get('name') == [] - - def test_on_address(self): - res = self.run_sanitizer_on('address', name='foo', ref='bar', ref_abc='baz') - - assert res.get('address') == [('bar', 'ref', ''), ('baz', 'ref', 'abc'), - ('foo', 'name', '')] - - -class TestTypeField: - - @pytest.fixture(autouse=True) - def setup_country(self, def_config): - self.config = def_config - - def run_sanitizer_on(self, type, **kwargs): - - place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, - 'country_code': 'de', 'rank_address': 30}) - - sanitizer_args = {'step': 'delete-tags', - 'type': type} - - name, _ = PlaceSanitizer([sanitizer_args], - self.config).process_names(place) - - return sorted([(p.name, p.kind, p.suffix or '') for p in name]) - - def test_name_type(self): - res = self.run_sanitizer_on('name', name='foo', ref='bar', ref_abc='baz') - - assert res == [] - - def test_address_type(self): - res = self.run_sanitizer_on('address', name='foo', ref='bar', ref_abc='baz') - - assert res == [('bar', 'ref', ''), ('baz', 'ref', 'abc'), - ('foo', 'name', '')] - - -class TestFilterKind: - - @pytest.fixture(autouse=True) - def setup_country(self, def_config): - self.config = def_config - - def run_sanitizer_on(self, filt, **kwargs): - - place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, - 'country_code': 'de', 'rank_address': 30}) - - sanitizer_args = {'step': 'delete-tags', - 'filter-kind': filt} - - name, _ = PlaceSanitizer([sanitizer_args], - self.config).process_names(place) - - return sorted([(p.name, p.kind, p.suffix or '') for p in name]) - - def test_single_exact_name(self): - res = self.run_sanitizer_on(['name'], ref='foo', name='foo', - name_abc='bar', ref_abc='bar') - - assert res == [('bar', 'ref', 'abc'), ('foo', 'ref', '')] - - def test_single_pattern(self): - res = self.run_sanitizer_on(['.*name'], - name_fr='foo', ref_fr='foo', namexx_fr='bar', - shortname_fr='bar', name='bar') - - assert res == [('bar', 'namexx', 'fr'), ('foo', 'ref', 'fr')] - - def test_multiple_patterns(self): - res = self.run_sanitizer_on(['.*name', 'ref'], - name_fr='foo', ref_fr='foo', oldref_fr='foo', - namexx_fr='bar', shortname_fr='baz', name='baz') - - assert res == [('bar', 'namexx', 'fr'), ('foo', 'oldref', 'fr')] - - -class TestRankAddress: - - @pytest.fixture(autouse=True) - def setup_country(self, def_config): - self.config = def_config - - def run_sanitizer_on(self, rank_addr, **kwargs): - - place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, - 'country_code': 'de', 'rank_address': 30}) - - sanitizer_args = {'step': 'delete-tags', - 'rank_address': rank_addr} - - name, _ = PlaceSanitizer([sanitizer_args], - self.config).process_names(place) - - return sorted([(p.name, p.kind, p.suffix or '') for p in name]) - - def test_single_rank(self): - res = self.run_sanitizer_on('30', name='foo', ref='bar') - - assert res == [] - - def test_single_rank_fail(self): - res = self.run_sanitizer_on('28', name='foo', ref='bar') - - assert res == [('bar', 'ref', ''), ('foo', 'name', '')] - - def test_ranged_rank_pass(self): - res = self.run_sanitizer_on('26-30', name='foo', ref='bar') - - assert res == [] - - def test_ranged_rank_fail(self): - res = self.run_sanitizer_on('26-29', name='foo', ref='bar') - - assert res == [('bar', 'ref', ''), ('foo', 'name', '')] - - def test_mixed_rank_pass(self): - res = self.run_sanitizer_on(['4', '20-28', '30', '10-12'], name='foo', ref='bar') - - assert res == [] - - def test_mixed_rank_fail(self): - res = self.run_sanitizer_on(['4-8', '10', '26-29', '18'], name='foo', ref='bar') - - assert res == [('bar', 'ref', ''), ('foo', 'name', '')] - - -class TestSuffix: - - @pytest.fixture(autouse=True) - def setup_country(self, def_config): - self.config = def_config - - def run_sanitizer_on(self, suffix, **kwargs): - - place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, - 'country_code': 'de', 'rank_address': 30}) - - sanitizer_args = {'step': 'delete-tags', - 'suffix': suffix} - - name, _ = PlaceSanitizer([sanitizer_args], - self.config).process_names(place) - - return sorted([(p.name, p.kind, p.suffix or '') for p in name]) - - def test_single_suffix(self): - res = self.run_sanitizer_on('abc', name='foo', name_abc='foo', - name_pqr='bar', ref='bar', ref_abc='baz') - - assert res == [('bar', 'name', 'pqr'), ('bar', 'ref', ''), ('foo', 'name', '')] - - def test_multiple_suffix(self): - res = self.run_sanitizer_on(['abc.*', 'pqr'], name='foo', name_abcxx='foo', - ref_pqr='bar', name_pqrxx='baz') - - assert res == [('baz', 'name', 'pqrxx'), ('foo', 'name', '')] - - -class TestCountryCodes: - - @pytest.fixture(autouse=True) - def setup_country(self, def_config): - self.config = def_config - - def run_sanitizer_on(self, country_code, **kwargs): - - place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, - 'country_code': 'de', 'rank_address': 30}) - - sanitizer_args = {'step': 'delete-tags', - 'country_code': country_code} - - name, _ = PlaceSanitizer([sanitizer_args], - self.config).process_names(place) - - return sorted([(p.name, p.kind) for p in name]) - - def test_single_country_code_pass(self): - res = self.run_sanitizer_on('de', name='foo', ref='bar') - - assert res == [] - - def test_single_country_code_fail(self): - res = self.run_sanitizer_on('in', name='foo', ref='bar') - - assert res == [('bar', 'ref'), ('foo', 'name')] - - def test_empty_country_code_list(self): - res = self.run_sanitizer_on([], name='foo', ref='bar') - - assert res == [('bar', 'ref'), ('foo', 'name')] - - def test_multiple_country_code_pass(self): - res = self.run_sanitizer_on(['in', 'de', 'fr'], name='foo', ref='bar') - - assert res == [] - - def test_multiple_country_code_fail(self): - res = self.run_sanitizer_on(['in', 'au', 'fr'], name='foo', ref='bar') - - assert res == [('bar', 'ref'), ('foo', 'name')] - - -class TestAllParameters: - - @pytest.fixture(autouse=True) - def setup_country(self, def_config): - self.config = def_config - - def run_sanitizer_on(self, country_code, rank_addr, suffix, **kwargs): - - place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, - 'country_code': 'de', 'rank_address': 30}) - - sanitizer_args = { - 'step': 'delete-tags', - 'type': 'name', - 'filter-kind': ['name', 'ref'], - 'country_code': country_code, - 'rank_address': rank_addr, - 'suffix': suffix, - 'name': r'[\s\S]*', - } - - name, _ = PlaceSanitizer([sanitizer_args], - self.config).process_names(place) - - return sorted([(p.name, p.kind, p.suffix or '') for p in name]) - - def test_string_arguments_pass(self): - res = self.run_sanitizer_on('de', '25-30', r'[\s\S]*', - name='foo', ref='foo', name_abc='bar', ref_abc='baz') - - assert res == [] - - def test_string_arguments_fail(self): - res = self.run_sanitizer_on('in', '25-30', r'[\s\S]*', - name='foo', ref='foo', name_abc='bar', ref_abc='baz') - - assert res == [('bar', 'name', 'abc'), ('baz', 'ref', 'abc'), - ('foo', 'name', ''), ('foo', 'ref', '')] - - def test_list_arguments_pass(self): - res = self.run_sanitizer_on(['de', 'in'], ['20-28', '30'], [r'abc.*', r'[\s\S]*'], - name='foo', ref='foo', name_abcxx='bar', ref_pqr='baz') - - assert res == [] - - def test_list_arguments_fail(self): - res = self.run_sanitizer_on(['de', 'in'], ['14', '20-29'], [r'abc.*', r'pqr'], - name='foo', ref_abc='foo', name_abcxx='bar', ref_pqr='baz') - - assert res == [('bar', 'name', 'abcxx'), ('baz', 'ref', 'pqr'), - ('foo', 'name', ''), ('foo', 'ref', 'abc')] - - def test_mix_arguments_pass(self): - res = self.run_sanitizer_on('de', ['10', '20-28', '30'], r'[\s\S]*', - name_abc='foo', ref_abc='foo', name_abcxx='bar', ref_pqr='baz') - - assert res == [] - - def test_mix_arguments_fail(self): - res = self.run_sanitizer_on(['de', 'in'], ['10', '20-28', '30'], r'abc.*', - name='foo', ref='foo', name_pqr='bar', ref_pqr='baz') - - assert res == [('bar', 'name', 'pqr'), ('baz', 'ref', 'pqr'), - ('foo', 'name', ''), ('foo', 'ref', '')] +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2026 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Tests for the sanitizer that normalizes housenumbers. +""" +import pytest + +from nominatim_db.data.place_info import PlaceInfo +from nominatim_db.tokenizer.place_sanitizer import PlaceSanitizer + + +class TestWithDefault: + + @pytest.fixture(autouse=True) + def setup_country(self, def_config): + self.config = def_config + + def run_sanitizer_on(self, type, **kwargs): + + place = PlaceInfo({type: {k.replace('_', ':'): v for k, v in kwargs.items()}, + 'country_code': 'de', 'rank_address': 30}) + + sanitizer_args = {'step': 'delete-tags'} + + PlaceSanitizer([sanitizer_args], self.config).process_names(place) + + return {'name': sorted([(p.name, p.kind, p.suffix or '') + for p in place.sanitized_names]), + 'address': sorted([(p.name, p.kind, p.suffix or '') + for p in place.sanitized_address])} + + def test_on_name(self): + res = self.run_sanitizer_on('name', name='foo', ref='bar', ref_abc='baz') + + assert res.get('name') == [] + + def test_on_address(self): + res = self.run_sanitizer_on('address', name='foo', ref='bar', ref_abc='baz') + + assert res.get('address') == [('bar', 'ref', ''), ('baz', 'ref', 'abc'), + ('foo', 'name', '')] + + +class TestTypeField: + + @pytest.fixture(autouse=True) + def setup_country(self, def_config): + self.config = def_config + + def run_sanitizer_on(self, type, **kwargs): + + place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, + 'country_code': 'de', 'rank_address': 30}) + + sanitizer_args = {'step': 'delete-tags', + 'type': type} + + PlaceSanitizer([sanitizer_args], self.config).process_names(place) + + return sorted([(p.name, p.kind, p.suffix or '') for p in place.sanitized_names]) + + def test_name_type(self): + res = self.run_sanitizer_on('name', name='foo', ref='bar', ref_abc='baz') + + assert res == [] + + def test_address_type(self): + res = self.run_sanitizer_on('address', name='foo', ref='bar', ref_abc='baz') + + assert res == [('bar', 'ref', ''), ('baz', 'ref', 'abc'), + ('foo', 'name', '')] + + +class TestFilterKind: + + @pytest.fixture(autouse=True) + def setup_country(self, def_config): + self.config = def_config + + def run_sanitizer_on(self, filt, **kwargs): + + place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, + 'country_code': 'de', 'rank_address': 30}) + + sanitizer_args = {'step': 'delete-tags', + 'filter-kind': filt} + + PlaceSanitizer([sanitizer_args], self.config).process_names(place) + + return sorted([(p.name, p.kind, p.suffix or '') for p in place.sanitized_names]) + + def test_single_exact_name(self): + res = self.run_sanitizer_on(['name'], ref='foo', name='foo', + name_abc='bar', ref_abc='bar') + + assert res == [('bar', 'ref', 'abc'), ('foo', 'ref', '')] + + def test_single_pattern(self): + res = self.run_sanitizer_on(['.*name'], + name_fr='foo', ref_fr='foo', namexx_fr='bar', + shortname_fr='bar', name='bar') + + assert res == [('bar', 'namexx', 'fr'), ('foo', 'ref', 'fr')] + + def test_multiple_patterns(self): + res = self.run_sanitizer_on(['.*name', 'ref'], + name_fr='foo', ref_fr='foo', oldref_fr='foo', + namexx_fr='bar', shortname_fr='baz', name='baz') + + assert res == [('bar', 'namexx', 'fr'), ('foo', 'oldref', 'fr')] + + +class TestRankAddress: + + @pytest.fixture(autouse=True) + def setup_country(self, def_config): + self.config = def_config + + def run_sanitizer_on(self, rank_addr, **kwargs): + + place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, + 'country_code': 'de', 'rank_address': 30}) + + sanitizer_args = {'step': 'delete-tags', + 'rank_address': rank_addr} + + PlaceSanitizer([sanitizer_args], self.config).process_names(place) + + return sorted([(p.name, p.kind, p.suffix or '') for p in place.sanitized_names]) + + def test_single_rank(self): + res = self.run_sanitizer_on('30', name='foo', ref='bar') + + assert res == [] + + def test_single_rank_fail(self): + res = self.run_sanitizer_on('28', name='foo', ref='bar') + + assert res == [('bar', 'ref', ''), ('foo', 'name', '')] + + def test_ranged_rank_pass(self): + res = self.run_sanitizer_on('26-30', name='foo', ref='bar') + + assert res == [] + + def test_ranged_rank_fail(self): + res = self.run_sanitizer_on('26-29', name='foo', ref='bar') + + assert res == [('bar', 'ref', ''), ('foo', 'name', '')] + + def test_mixed_rank_pass(self): + res = self.run_sanitizer_on(['4', '20-28', '30', '10-12'], name='foo', ref='bar') + + assert res == [] + + def test_mixed_rank_fail(self): + res = self.run_sanitizer_on(['4-8', '10', '26-29', '18'], name='foo', ref='bar') + + assert res == [('bar', 'ref', ''), ('foo', 'name', '')] + + +class TestSuffix: + + @pytest.fixture(autouse=True) + def setup_country(self, def_config): + self.config = def_config + + def run_sanitizer_on(self, suffix, **kwargs): + + place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, + 'country_code': 'de', 'rank_address': 30}) + + sanitizer_args = {'step': 'delete-tags', + 'suffix': suffix} + + PlaceSanitizer([sanitizer_args], self.config).process_names(place) + + return sorted([(p.name, p.kind, p.suffix or '') for p in place.sanitized_names]) + + def test_single_suffix(self): + res = self.run_sanitizer_on('abc', name='foo', name_abc='foo', + name_pqr='bar', ref='bar', ref_abc='baz') + + assert res == [('bar', 'name', 'pqr'), ('bar', 'ref', ''), ('foo', 'name', '')] + + def test_multiple_suffix(self): + res = self.run_sanitizer_on(['abc.*', 'pqr'], name='foo', name_abcxx='foo', + ref_pqr='bar', name_pqrxx='baz') + + assert res == [('baz', 'name', 'pqrxx'), ('foo', 'name', '')] + + +class TestCountryCodes: + + @pytest.fixture(autouse=True) + def setup_country(self, def_config): + self.config = def_config + + def run_sanitizer_on(self, country_code, **kwargs): + + place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, + 'country_code': 'de', 'rank_address': 30}) + + sanitizer_args = {'step': 'delete-tags', + 'country_code': country_code} + + PlaceSanitizer([sanitizer_args], self.config).process_names(place) + + return sorted([(p.name, p.kind) for p in place.sanitized_names]) + + def test_single_country_code_pass(self): + res = self.run_sanitizer_on('de', name='foo', ref='bar') + + assert res == [] + + def test_single_country_code_fail(self): + res = self.run_sanitizer_on('in', name='foo', ref='bar') + + assert res == [('bar', 'ref'), ('foo', 'name')] + + def test_empty_country_code_list(self): + res = self.run_sanitizer_on([], name='foo', ref='bar') + + assert res == [('bar', 'ref'), ('foo', 'name')] + + def test_multiple_country_code_pass(self): + res = self.run_sanitizer_on(['in', 'de', 'fr'], name='foo', ref='bar') + + assert res == [] + + def test_multiple_country_code_fail(self): + res = self.run_sanitizer_on(['in', 'au', 'fr'], name='foo', ref='bar') + + assert res == [('bar', 'ref'), ('foo', 'name')] + + +class TestAllParameters: + + @pytest.fixture(autouse=True) + def setup_country(self, def_config): + self.config = def_config + + def run_sanitizer_on(self, country_code, rank_addr, suffix, **kwargs): + + place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, + 'country_code': 'de', 'rank_address': 30}) + + sanitizer_args = {'step': 'delete-tags', + 'type': 'name', + 'filter-kind': ['name', 'ref'], + 'country_code': country_code, + 'rank_address': rank_addr, + 'suffix': suffix, + 'name': r'[\s\S]*', + } + + PlaceSanitizer([sanitizer_args], self.config).process_names(place) + + return sorted([(p.name, p.kind, p.suffix or '') for p in place.sanitized_names]) + + def test_string_arguments_pass(self): + res = self.run_sanitizer_on('de', '25-30', r'[\s\S]*', + name='foo', ref='foo', name_abc='bar', ref_abc='baz') + + assert res == [] + + def test_string_arguments_fail(self): + res = self.run_sanitizer_on('in', '25-30', r'[\s\S]*', + name='foo', ref='foo', name_abc='bar', ref_abc='baz') + + assert res == [('bar', 'name', 'abc'), ('baz', 'ref', 'abc'), + ('foo', 'name', ''), ('foo', 'ref', '')] + + def test_list_arguments_pass(self): + res = self.run_sanitizer_on(['de', 'in'], ['20-28', '30'], [r'abc.*', r'[\s\S]*'], + name='foo', ref='foo', name_abcxx='bar', ref_pqr='baz') + + assert res == [] + + def test_list_arguments_fail(self): + res = self.run_sanitizer_on(['de', 'in'], ['14', '20-29'], [r'abc.*', r'pqr'], + name='foo', ref_abc='foo', name_abcxx='bar', ref_pqr='baz') + + assert res == [('bar', 'name', 'abcxx'), ('baz', 'ref', 'pqr'), + ('foo', 'name', ''), ('foo', 'ref', 'abc')] + + def test_mix_arguments_pass(self): + res = self.run_sanitizer_on('de', ['10', '20-28', '30'], r'[\s\S]*', + name_abc='foo', ref_abc='foo', name_abcxx='bar', ref_pqr='baz') + + assert res == [] + + def test_mix_arguments_fail(self): + res = self.run_sanitizer_on(['de', 'in'], ['10', '20-28', '30'], r'abc.*', + name='foo', ref='foo', name_pqr='bar', ref_pqr='baz') + + assert res == [('bar', 'name', 'pqr'), ('baz', 'ref', 'pqr'), + ('foo', 'name', ''), ('foo', 'ref', '')] diff --git a/test/python/tokenizer/sanitizers/test_derive_names.py b/test/python/tokenizer/sanitizers/test_derive_names.py index 99d6ff0e..4d8b2af8 100644 --- a/test/python/tokenizer/sanitizers/test_derive_names.py +++ b/test/python/tokenizer/sanitizers/test_derive_names.py @@ -37,7 +37,8 @@ def test_name_deletion(mk_sanitizer, prim, out, keep): place = PlaceInfo({'name': names, 'address': names, 'country_code': 'de', 'rank_address': 30}) - res = san.process_names(place) + san.process_names(place) + res = [place.sanitized_names, place.sanitized_address] assert len(res[(out + 1) % 2]) == 3 if keep: @@ -53,9 +54,9 @@ def simple_replace(mk_sanitizer): san = mk_sanitizer(name_pattern=pattern, type='name', keep_original=keep, variants=variants) place = PlaceInfo({'name': {'name': name}, 'country_code': 'de', 'rank_address': 30}) - out, _ = san.process_names(place) + san.process_names(place) - return {p.name for p in out} + return {p.name for p in place.sanitized_names} return _impl diff --git a/test/python/tokenizer/sanitizers/test_split_name_list.py b/test/python/tokenizer/sanitizers/test_split_name_list.py index ec4869b3..c19d5f87 100644 --- a/test/python/tokenizer/sanitizers/test_split_name_list.py +++ b/test/python/tokenizer/sanitizers/test_split_name_list.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Tests for the sanitizer that splits multivalue lists. @@ -23,17 +23,16 @@ class TestSplitName: def run_sanitizer_on(self, **kwargs): place = PlaceInfo({'name': kwargs}) - name, _ = PlaceSanitizer([{'step': 'split-name-list'}], self.config).process_names(place) + PlaceSanitizer([{'step': 'split-name-list'}], self.config).process_names(place) - return sorted([(p.name, p.kind, p.suffix) for p in name]) + return sorted([(p.name, p.kind, p.suffix) for p in place.sanitized_names]) def sanitize_with_delimiter(self, delimiter, name): place = PlaceInfo({'name': {'name': name}}) - san = PlaceSanitizer([{'step': 'split-name-list', 'delimiters': delimiter}], - self.config) - name, _ = san.process_names(place) + PlaceSanitizer([{'step': 'split-name-list', 'delimiters': delimiter}], + self.config).process_names(place) - return sorted([p.name for p in name]) + return sorted([p.name for p in place.sanitized_names]) def test_simple(self): assert self.run_sanitizer_on(name='ABC') == [('ABC', 'name', None)] @@ -67,7 +66,7 @@ class TestSplitName: def test_no_name_list(def_config): place = PlaceInfo({'address': {'housenumber': '3'}}) - name, address = PlaceSanitizer([{'step': 'split-name-list'}], def_config).process_names(place) + PlaceSanitizer([{'step': 'split-name-list'}], def_config).process_names(place) - assert not name - assert len(address) == 1 + assert not place.sanitized_names + assert len(place.sanitized_address) == 1 diff --git a/test/python/tokenizer/sanitizers/test_strip_brace_terms.py b/test/python/tokenizer/sanitizers/test_strip_brace_terms.py index 8aece57f..ff872153 100644 --- a/test/python/tokenizer/sanitizers/test_strip_brace_terms.py +++ b/test/python/tokenizer/sanitizers/test_strip_brace_terms.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Tests for the sanitizer that handles braced suffixes. @@ -21,9 +21,9 @@ class TestStripBrace: def run_sanitizer_on(self, **kwargs): place = PlaceInfo({'name': kwargs}) - name, _ = PlaceSanitizer([{'step': 'strip-brace-terms'}], self.config).process_names(place) + PlaceSanitizer([{'step': 'strip-brace-terms'}], self.config).process_names(place) - return sorted([(p.name, p.kind, p.suffix) for p in name]) + return sorted([(p.name, p.kind, p.suffix) for p in place.sanitized_names]) def test_no_braces(self): assert self.run_sanitizer_on(name='foo', ref='23') == [('23', 'ref', None), @@ -49,7 +49,7 @@ class TestStripBrace: def test_no_names(def_config): place = PlaceInfo({'address': {'housenumber': '3'}}) - name, address = PlaceSanitizer([{'step': 'strip-brace-terms'}], def_config).process_names(place) + PlaceSanitizer([{'step': 'strip-brace-terms'}], def_config).process_names(place) - assert not name - assert len(address) == 1 + assert not place.sanitized_names + assert len(place.sanitized_address) == 1 diff --git a/test/python/tokenizer/sanitizers/test_tag_analyzer_by_language.py b/test/python/tokenizer/sanitizers/test_tag_analyzer_by_language.py index 79d6f072..dbc10ce0 100644 --- a/test/python/tokenizer/sanitizers/test_tag_analyzer_by_language.py +++ b/test/python/tokenizer/sanitizers/test_tag_analyzer_by_language.py @@ -23,10 +23,9 @@ class TestWithDefaults: def run_sanitizer_on(self, country, **kwargs): place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, 'country_code': country}) - name, _ = PlaceSanitizer([{'step': 'tag-analyzer-by-language'}], - self.config).process_names(place) + PlaceSanitizer([{'step': 'tag-analyzer-by-language'}], self.config).process_names(place) - return sorted([(p.name, p.kind, p.suffix, p.attr) for p in name]) + return sorted([(p.name, p.kind, p.suffix, p.attr) for p in place.sanitized_names]) def test_no_names(self): assert self.run_sanitizer_on('de') == [] @@ -53,11 +52,11 @@ class TestFilterKind: def run_sanitizer_on(self, filt, **kwargs): place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, 'country_code': 'de'}) - name, _ = PlaceSanitizer([{'step': 'tag-analyzer-by-language', - 'filter-kind': filt}], - self.config).process_names(place) + PlaceSanitizer([{'step': 'tag-analyzer-by-language', + 'filter-kind': filt}], + self.config).process_names(place) - return sorted([(p.name, p.kind, p.suffix, p.attr) for p in name]) + return sorted([(p.name, p.kind, p.suffix, p.attr) for p in place.sanitized_names]) def test_single_exact_name(self): res = self.run_sanitizer_on(['name'], name_fr='A', ref_fr='12', @@ -102,44 +101,44 @@ class TestDefaultCountry: def run_sanitizer_append(self, mode, country, **kwargs): place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, 'country_code': country}) - name, _ = PlaceSanitizer([{'step': 'tag-analyzer-by-language', - 'use-defaults': mode, - 'mode': 'append'}], - self.config).process_names(place) + PlaceSanitizer([{'step': 'tag-analyzer-by-language', + 'use-defaults': mode, + 'mode': 'append'}], + self.config).process_names(place) - assert all(isinstance(p.attr, dict) for p in name) - assert all(len(p.attr) <= 1 for p in name) + assert all(isinstance(p.attr, dict) for p in place.sanitized_names) + assert all(len(p.attr) <= 1 for p in place.sanitized_names) assert all(not p.attr or ('analyzer' in p.attr and p.attr['analyzer']) - for p in name) + for p in place.sanitized_names) - return sorted([(p.name, p.attr.get('analyzer', '')) for p in name]) + return sorted([(p.name, p.attr.get('analyzer', '')) for p in place.sanitized_names]) def run_sanitizer_replace(self, mode, country, **kwargs): place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, 'country_code': country}) - name, _ = PlaceSanitizer([{'step': 'tag-analyzer-by-language', - 'use-defaults': mode, - 'mode': 'replace'}], - self.config).process_names(place) + PlaceSanitizer([{'step': 'tag-analyzer-by-language', + 'use-defaults': mode, + 'mode': 'replace'}], + self.config).process_names(place) - assert all(isinstance(p.attr, dict) for p in name) - assert all(len(p.attr) <= 1 for p in name) + assert all(isinstance(p.attr, dict) for p in place.sanitized_names) + assert all(len(p.attr) <= 1 for p in place.sanitized_names) assert all(not p.attr or ('analyzer' in p.attr and p.attr['analyzer']) - for p in name) + for p in place.sanitized_names) - return sorted([(p.name, p.attr.get('analyzer', '')) for p in name]) + return sorted([(p.name, p.attr.get('analyzer', '')) for p in place.sanitized_names]) def test_missing_country(self): place = PlaceInfo({'name': {'name': 'something'}}) - name, _ = PlaceSanitizer([{'step': 'tag-analyzer-by-language', - 'use-defaults': 'all', - 'mode': 'replace'}], - self.config).process_names(place) + PlaceSanitizer([{'step': 'tag-analyzer-by-language', + 'use-defaults': 'all', + 'mode': 'replace'}], + self.config).process_names(place) - assert len(name) == 1 - assert name[0].name == 'something' - assert name[0].suffix is None - assert 'analyzer' not in name[0].attr + assert len(place.sanitized_names) == 1 + assert place.sanitized_names[0].name == 'something' + assert place.sanitized_names[0].suffix is None + assert 'analyzer' not in place.sanitized_names[0].attr def test_mono_unknown_country(self): expect = [('XX', '')] @@ -200,18 +199,18 @@ class TestCountryWithWhitelist: def run_sanitizer_on(self, mode, country, **kwargs): place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, 'country_code': country}) - name, _ = PlaceSanitizer([{'step': 'tag-analyzer-by-language', - 'use-defaults': mode, - 'mode': 'replace', - 'whitelist': ['de', 'fr', 'ru']}], - self.config).process_names(place) - - assert all(isinstance(p.attr, dict) for p in name) - assert all(len(p.attr) <= 1 for p in name) + PlaceSanitizer([{'step': 'tag-analyzer-by-language', + 'use-defaults': mode, + 'mode': 'replace', + 'whitelist': ['de', 'fr', 'ru']}], + self.config).process_names(place) + + assert all(isinstance(p.attr, dict) for p in place.sanitized_names) + assert all(len(p.attr) <= 1 for p in place.sanitized_names) assert all(not p.attr or ('analyzer' in p.attr and p.attr['analyzer']) - for p in name) + for p in place.sanitized_names) - return sorted([(p.name, p.attr.get('analyzer', '')) for p in name]) + return sorted([(p.name, p.attr.get('analyzer', '')) for p in place.sanitized_names]) def test_mono_monoling(self): assert self.run_sanitizer_on('mono', 'de', name='Foo') == [('Foo', 'de')] @@ -238,17 +237,17 @@ class TestWhiteList: def run_sanitizer_on(self, whitelist, **kwargs): place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}}) - name, _ = PlaceSanitizer([{'step': 'tag-analyzer-by-language', - 'mode': 'replace', - 'whitelist': whitelist}], - self.config).process_names(place) + PlaceSanitizer([{'step': 'tag-analyzer-by-language', + 'mode': 'replace', + 'whitelist': whitelist}], + self.config).process_names(place) - assert all(isinstance(p.attr, dict) for p in name) - assert all(len(p.attr) <= 1 for p in name) + assert all(isinstance(p.attr, dict) for p in place.sanitized_names) + assert all(len(p.attr) <= 1 for p in place.sanitized_names) assert all(not p.attr or ('analyzer' in p.attr and p.attr['analyzer']) - for p in name) + for p in place.sanitized_names) - return sorted([(p.name, p.attr.get('analyzer', '')) for p in name]) + return sorted([(p.name, p.attr.get('analyzer', '')) for p in place.sanitized_names]) def test_in_whitelist(self): assert self.run_sanitizer_on(['de', 'xx'], ref_xx='123') == [('123', 'xx')] @@ -270,19 +269,19 @@ class TestSuffixIgnore: def run_sanitizer_on(self, suffix_ignore, **kwargs): place = PlaceInfo({'name': {k.replace('_', ':'): v for k, v in kwargs.items()}, 'country_code': 'de'}) - name, _ = PlaceSanitizer([{'step': 'tag-analyzer-by-language', - 'mode': 'replace', - 'use-defaults': 'mono', - 'whitelist': ['de', 'en'], - 'suffix-ignore': suffix_ignore}], - self.config).process_names(place) - - assert all(isinstance(p.attr, dict) for p in name) - assert all(len(p.attr) <= 1 for p in name) + PlaceSanitizer([{'step': 'tag-analyzer-by-language', + 'mode': 'replace', + 'use-defaults': 'mono', + 'whitelist': ['de', 'en'], + 'suffix-ignore': suffix_ignore}], + self.config).process_names(place) + + assert all(isinstance(p.attr, dict) for p in place.sanitized_names) + assert all(len(p.attr) <= 1 for p in place.sanitized_names) assert all(not p.attr or ('analyzer' in p.attr and p.attr['analyzer']) - for p in name) + for p in place.sanitized_names) - return sorted([(p.name, p.attr.get('analyzer', '')) for p in name]) + return sorted([(p.name, p.attr.get('analyzer', '')) for p in place.sanitized_names]) def test_ignored_suffix(self): assert self.run_sanitizer_on(['left'], name_left='foo') == [('foo', 'de')] diff --git a/test/python/tokenizer/sanitizers/test_tag_japanese.py b/test/python/tokenizer/sanitizers/test_tag_japanese.py index 6db7a3c3..f66b28ee 100644 --- a/test/python/tokenizer/sanitizers/test_tag_japanese.py +++ b/test/python/tokenizer/sanitizers/test_tag_japanese.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. import pytest @@ -21,8 +21,8 @@ class TestTagJapanese: 'country_code': 'jp' }) sanitizer_args = {'step': 'tag-japanese'} - _, address = PlaceSanitizer([sanitizer_args], self.config).process_names(place) - tmp_list = [(p.name, p.kind) for p in address] + PlaceSanitizer([sanitizer_args], self.config).process_names(place) + tmp_list = [(p.name, p.kind) for p in place.sanitized_address] return sorted(tmp_list) def test_on_address(self): diff --git a/test/python/tokenizer/test_place_sanitizer.py b/test/python/tokenizer/test_place_sanitizer.py index 3eea6400..b4826fca 100644 --- a/test/python/tokenizer/test_place_sanitizer.py +++ b/test/python/tokenizer/test_place_sanitizer.py @@ -12,10 +12,11 @@ import pytest from nominatim_db.errors import UsageError import nominatim_db.tokenizer.place_sanitizer as sanitizer from nominatim_db.data.place_info import PlaceInfo +from nominatim_db.data.place_name import PlaceName def test_placeinfo_clone_new_name(): - place = sanitizer.PlaceName('foo', 'ki', 'su') + place = PlaceName('foo', 'ki', 'su') newplace = place.clone(name='bar') @@ -26,7 +27,7 @@ def test_placeinfo_clone_new_name(): def test_placeinfo_clone_merge_attr(): - place = sanitizer.PlaceName('foo', 'ki', 'su') + place = PlaceName('foo', 'ki', 'su') place.set_attr('a1', 'v1') place.set_attr('a2', 'v2') @@ -40,7 +41,7 @@ def test_placeinfo_clone_merge_attr(): def test_placeinfo_has_attr(): - place = sanitizer.PlaceName('foo', 'ki', 'su') + place = PlaceName('foo', 'ki', 'su') place.set_attr('a1', 'v1') assert place.has_attr('a1') @@ -50,26 +51,30 @@ def test_placeinfo_has_attr(): def test_sanitizer_default(def_config): san = sanitizer.PlaceSanitizer([{'step': 'split-name-list'}], def_config) - name, address = san.process_names(PlaceInfo({'name': {'name:de:de': '1;2;3'}, - 'address': {'street': 'Bald'}})) + place = PlaceInfo({'name': {'name:de:de': '1;2;3'}, + 'address': {'street': 'Bald'}}) + san.process_names(place) + name = place.sanitized_names + address = place.sanitized_address assert len(name) == 3 - assert all(isinstance(n, sanitizer.PlaceName) for n in name) + assert all(isinstance(n, PlaceName) for n in name) assert all(n.kind == 'name' for n in name) assert all(n.suffix == 'de:de' for n in name) assert len(address) == 1 - assert all(isinstance(n, sanitizer.PlaceName) for n in address) + assert all(isinstance(n, PlaceName) for n in address) @pytest.mark.parametrize('rules', [None, []]) def test_sanitizer_empty_list(def_config, rules): san = sanitizer.PlaceSanitizer(rules, def_config) - name, address = san.process_names(PlaceInfo({'name': {'name:de:de': '1;2;3'}})) + place = PlaceInfo({'name': {'name:de:de': '1;2;3'}}) + san.process_names(place) - assert len(name) == 1 - assert all(isinstance(n, sanitizer.PlaceName) for n in name) + assert len(place.sanitized_names) == 1 + assert all(isinstance(n, PlaceName) for n in place.sanitized_names) def test_sanitizer_missing_step_definition(def_config): -- 2.47.3