also switch legacy tokenizer to new street/place choice behaviour

diff --git a/lib-sql/tokenizer/legacy_tokenizer.sql b/lib-sql/tokenizer/legacy_tokenizer.sql
index 3b82619ff591ee969204509e82597818111d7ebb..8c8f56e1c264e937bf8617ab56b5e6e9d83325e2 100644 (file)
--- a/lib-sql/tokenizer/legacy_tokenizer.sql
+++ b/lib-sql/tokenizer/legacy_tokenizer.sql
@@ -44,14 +44,14 @@ $$ LANGUAGE SQL IMMUTABLE STRICT;
 CREATE OR REPLACE FUNCTION token_is_street_address(info JSONB)
   RETURNS BOOLEAN
 AS $$
-  SELECT info->>'street' is not null or info->>'place' is null;
+  SELECT info->>'street' is not null or info->>'place_search' is null;
 $$ LANGUAGE SQL IMMUTABLE;
 
 
 CREATE OR REPLACE FUNCTION token_has_addr_street(info JSONB)
   RETURNS BOOLEAN
 AS $$
-  SELECT info->>'street' is not null;
+  SELECT info->>'street' is not null and info->>'street' != '{}';
 $$ LANGUAGE SQL IMMUTABLE;
 
 

diff --git a/nominatim/tokenizer/legacy_tokenizer.py b/nominatim/tokenizer/legacy_tokenizer.py
index a50dedb2da7f2ffb51ad836445d2282bd0408280..e09700d9ddb8856a8d52fc6f7de1e9b748b9bbd7 100644 (file)
--- a/nominatim/tokenizer/legacy_tokenizer.py
+++ b/nominatim/tokenizer/legacy_tokenizer.py
@@ -564,14 +564,13 @@ class _TokenInfo:
     def add_street(self, conn: Connection, street: str) -> None:
         """ Add addr:street match terms.
         """
-        def _get_street(name: str) -> List[int]:
+        def _get_street(name: str) -> Optional[str]:
             with conn.cursor() as cur:
-                return cast(List[int],
+                return cast(Optional[str],
                             cur.scalar("SELECT word_ids_from_name(%s)::text", (name, )))
 
         tokens = self.cache.streets.get(street, _get_street)
-        if tokens:
-            self.data['street'] = tokens
+        self.data['street'] = tokens or '{}'
 
 
     def add_place(self, conn: Connection, place: str) -> None:

diff --git a/test/python/tokenizer/test_legacy.py b/test/python/tokenizer/test_legacy.py
index 57a82b8a0f4aa5af2f1dc6c2377db0fca8d83959..d63ee8e14624d5db3e652cfd677511f755b6a487 100644 (file)
--- a/test/python/tokenizer/test_legacy.py
+++ b/test/python/tokenizer/test_legacy.py
@@ -549,7 +549,7 @@ class TestPlaceAddress:
     def test_process_place_street_empty(self):
         info = self.process_address(street='🜵')
 
-        assert 'street' not in info
+        assert info['street'] == '{}'
 
 
     def test_process_place_place(self):