From 28916e121b9deac8790edd8b2a8998f9ebebf0d2 Mon Sep 17 00:00:00 2001 From: Itz-Agasta Date: Wed, 12 Aug 2026 13:40:06 +0530 Subject: [PATCH] Filter search results by category Applies to every search returning placex rows. Sources without categories return nothing when an include filter is given. --- .../search/db_searches/address_search.py | 15 +++- src/nominatim_api/search/db_searches/base.py | 35 +++++++++- .../search/db_searches/country_search.py | 6 ++ .../search/db_searches/near_search.py | 1 + .../search/db_searches/place_search.py | 1 + .../search/db_searches/poi_search.py | 2 + .../search/db_searches/postcode_search.py | 5 ++ src/nominatim_api/sql/sqlalchemy_functions.py | 40 +++++++++++ test/python/api/search/test_search_poi.py | 68 +++++++++++++++++++ 9 files changed, 170 insertions(+), 3 deletions(-) diff --git a/src/nominatim_api/search/db_searches/address_search.py b/src/nominatim_api/search/db_searches/address_search.py index 76999c89..8fa69399 100644 --- a/src/nominatim_api/search/db_searches/address_search.py +++ b/src/nominatim_api/search/db_searches/address_search.py @@ -74,6 +74,8 @@ async def _get_placex_housenumbers(conn: SearchConnection, sql = base.select_placex(t).add_columns(t.c.importance)\ .where(t.c.place_id.in_(place_ids)) + sql = base.filter_by_category(sql, t, details) + if details.geometry_output: sql = base.add_geometry_columns(sql, t.c.geometry, details) @@ -210,6 +212,14 @@ class AddressSearch(base.AbstractSearch): sql = base.select_placex(t).join(tsearch, t.c.place_id == tsearch.c.place_id) + # The category filter must not restrict the query itself: the rows + # here are the parents used to find the housenumbers and only become + # a result of their own when no housenumber was found. Carry the + # match along as a column and apply it when collecting the results. + catfilter = base.category_restriction(t, details) + if catfilter is not None: + sql = sql.add_columns(catfilter.label('cat_match')) + if details.geometry_output: sql = base.add_geometry_columns(sql, t.c.geometry, details) @@ -269,7 +279,9 @@ class AddressSearch(base.AbstractSearch): if n.isdecimal() and len(n) < 8] interpol_sql: SaColumn tiger_sql: SaColumn - if numerals and \ + # Interpolations and Tiger data carry no categories, so they cannot + # satisfy an include filter. + if numerals and not details.include and \ (not self.qualifiers or ('place', 'house') in self.qualifiers.values): # Housenumbers from interpolations interpol_sql = _make_interpolation_subquery(conn.t.osmline, inner, @@ -339,6 +351,7 @@ class AddressSearch(base.AbstractSearch): # filter conditions. if (not details.excluded or result.place_id not in details.excluded_place_ids)\ and (not self.qualifiers or result.category in self.qualifiers.values)\ + and (catfilter is None or row.cat_match)\ and result.rank_address >= details.min_rank: result.accuracy += 1.0 # penalty for missing housenumber results.append(result) diff --git a/src/nominatim_api/search/db_searches/base.py b/src/nominatim_api/search/db_searches/base.py index b4884e8f..fd67dde7 100644 --- a/src/nominatim_api/search/db_searches/base.py +++ b/src/nominatim_api/search/db_searches/base.py @@ -7,7 +7,7 @@ """ Interface for classes implementing a database search. """ -from typing import Callable, List +from typing import Callable, List, Optional import abc import re @@ -15,7 +15,7 @@ import sqlalchemy as sa from ...typing import SaFromClause, SaSelect, SaColumn, SaExpression, SaLambdaSelect from ...sql.sqlalchemy_types import Geometry -from ...sql.sqlalchemy_functions import CategoryMatch +from ...sql.sqlalchemy_functions import CategoryMatch, CategoryContains from ...connection import SearchConnection from ...types import SearchDetails, DataLayer, GeometryFormat from ...results import SearchResults @@ -58,6 +58,37 @@ def category_filter(table: SaFromClause, cls: str, typ: str) -> SaExpression: return CategoryMatch(table, category_ltree(cls, typ), cls, typ) +def category_restriction(table: SaFromClause, + details: SearchDetails) -> Optional[SaExpression]: + """ Build a boolean expression for the include/exclude category filters. + + A row must match at least one category of every include group and + must not match all categories of any of the exclude groups. Returns + None when no category filters are requested. + """ + terms: list[SaExpression] = [] + + for group in details.include: + terms.append(sa.or_(*(CategoryContains(table, cat) for cat in group))) + + for group in details.exclude: + terms.append(sa.not_(sa.and_(*(CategoryContains(table, cat) for cat in group)))) + + if not terms: + return None + + return sa.and_(*terms) + + +def filter_by_category(sql: SaSelect, t: SaFromClause, + details: SearchDetails) -> SaSelect: + """ Apply the include/exclude category filters, if applicable. + """ + restriction = category_restriction(t, details) + + return sql if restriction is None else sql.where(restriction) + + class AbstractSearch(abc.ABC): """ Encapuslation of a single lookup in the database. """ diff --git a/src/nominatim_api/search/db_searches/country_search.py b/src/nominatim_api/search/db_searches/country_search.py index e53465b2..77603c7d 100644 --- a/src/nominatim_api/search/db_searches/country_search.py +++ b/src/nominatim_api/search/db_searches/country_search.py @@ -44,6 +44,7 @@ class CountrySearch(base.AbstractSearch): if details.excluded: sql = sql.where(base.exclude_places(t)) + sql = base.filter_by_category(sql, t, details) sql = base.filter_by_area(sql, t, details) bind_params = { @@ -79,6 +80,11 @@ class CountrySearch(base.AbstractSearch): if details.excluded: return nres.SearchResults() + # The fallback tables carry no categories, so there is no way to tell + # if the country would satisfy the include filter. + if details.include: + return nres.SearchResults() + t = conn.t.country_name tgrid = conn.t.country_grid diff --git a/src/nominatim_api/search/db_searches/near_search.py b/src/nominatim_api/search/db_searches/near_search.py index 10adb5ec..f595c0a7 100644 --- a/src/nominatim_api/search/db_searches/near_search.py +++ b/src/nominatim_api/search/db_searches/near_search.py @@ -104,6 +104,7 @@ class NearSearch(base.AbstractSearch): .order_by(inner.c.dist) sql = sql.where(base.no_index(t.c.rank_address).between(MIN_RANK_PARAM, MAX_RANK_PARAM)) + sql = base.filter_by_category(sql, t, details) if details.countries: sql = sql.where(t.c.country_code.in_(COUNTRIES_PARAM)) if details.excluded: diff --git a/src/nominatim_api/search/db_searches/place_search.py b/src/nominatim_api/search/db_searches/place_search.py index e50e3fd9..4bddb1b0 100644 --- a/src/nominatim_api/search/db_searches/place_search.py +++ b/src/nominatim_api/search/db_searches/place_search.py @@ -183,6 +183,7 @@ class PlaceSearch(base.AbstractSearch): .where(t.c.indexed_status == 0) if self.qualifiers: sql = sql.where(self.qualifiers.sql_restrict(t)) + sql = base.filter_by_category(sql, t, details) if details.layers is not None: sql = sql.where(base.filter_by_layer(t, details.layers)) diff --git a/src/nominatim_api/search/db_searches/poi_search.py b/src/nominatim_api/search/db_searches/poi_search.py index 5e82bd0a..f4b062fc 100644 --- a/src/nominatim_api/search/db_searches/poi_search.py +++ b/src/nominatim_api/search/db_searches/poi_search.py @@ -68,6 +68,8 @@ class PoiSearch(base.AbstractSearch): if self.countries: sql = sql.where(t.c.country_code.in_(self.countries.values)) + sql = base.filter_by_category(sql, t, details) + if details.excluded: sql = sql.where(base.exclude_places(t)) diff --git a/src/nominatim_api/search/db_searches/postcode_search.py b/src/nominatim_api/search/db_searches/postcode_search.py index 3caa1b4f..e5177e92 100644 --- a/src/nominatim_api/search/db_searches/postcode_search.py +++ b/src/nominatim_api/search/db_searches/postcode_search.py @@ -39,6 +39,11 @@ class PostcodeSearch(base.AbstractSearch): details: SearchDetails) -> nres.SearchResults: """ Find results for the search in the database. """ + # Postcodes carry no categories, so they can never satisfy an + # include filter. + if details.include: + return nres.SearchResults() + t = conn.t.postcode pcs = self.postcodes.values diff --git a/src/nominatim_api/sql/sqlalchemy_functions.py b/src/nominatim_api/sql/sqlalchemy_functions.py index f05dbae6..b07f5381 100644 --- a/src/nominatim_api/sql/sqlalchemy_functions.py +++ b/src/nominatim_api/sql/sqlalchemy_functions.py @@ -254,3 +254,43 @@ def _sqlite_category_match(element: CategoryMatch, compiler.process(cls_lit, **kw), compiler.process(typ_col, **kw), compiler.process(typ_lit, **kw)) + + +class CategoryContains(sa.sql.functions.GenericFunction[Any]): + """ Match a placex row against a category or any of its descendants. + + On PostgreSQL this is the ltree containment operator + (``categories <@ ''``). On SQLite, where the categories + are stored as a comma-separated text, the same is emulated with a + substring search. + """ + name = 'CategoryContains' + inherit_cache = True + + def __init__(self, table: SaFromClause, category: str) -> None: + # The needles for the SQLite variant are precomputed here because + # SQLAlchemy 1.4 binds a parameter only once per statement, even + # when it is rendered more than once. + super().__init__(table.c.categories, sa.literal(category), + sa.literal(f',{category},'), sa.literal(f',{category}.')) + + +@compiles(CategoryContains) +def _default_category_contains(element: CategoryContains, + compiler: 'sa.Compiled', **kw: Any) -> str: + cats, category, _, _ = list(element.clauses) + return "(%s <@ (%s)::ltree)" % (compiler.process(cats, **kw), + compiler.process(category, **kw)) + + +@compiles(CategoryContains, 'sqlite') +def _sqlite_category_contains(element: CategoryContains, + compiler: 'sa.Compiled', **kw: Any) -> str: + cats, _, exact, descendants = list(element.clauses) + # The categories are padded with the separator on both sides, so that + # the needles match only on a full label boundary: ',,' is an + # exact hit, ',.' one of its descendants. + haystack = "(',' || %s || ',')" % compiler.process(cats, **kw) + return "(instr(%s, %s) > 0 OR instr(%s, %s) > 0)" \ + % (haystack, compiler.process(exact, **kw), + haystack, compiler.process(descendants, **kw)) diff --git a/test/python/api/search/test_search_poi.py b/test/python/api/search/test_search_poi.py index c92ae6f8..41a1b1d0 100644 --- a/test/python/api/search/test_search_poi.py +++ b/test/python/api/search/test_search_poi.py @@ -118,3 +118,71 @@ class TestPoiSearchWithRestrictions: details=SearchDetails.from_kwargs(args)) assert [r.place_id for r in results] == [2] + + +class TestCategoryFilters: + + @pytest.fixture(autouse=True) + def fill_database(self, apiobj): + # A restaurant that is also a hotel. + apiobj.add_placex(place_id=1, class_='amenity', type='restaurant', + categories=['osm.amenity.restaurant', 'osm.tourism.hotel'], + centroid=(10.0, 10.0)) + # A plain restaurant. + apiobj.add_placex(place_id=2, class_='amenity', type='restaurant', + categories=['osm.amenity.restaurant'], + centroid=(10.0, 10.0)) + # A restaurant that is also a fast food place. + apiobj.add_placex(place_id=3, class_='amenity', type='restaurant', + categories=['osm.amenity.restaurant', 'osm.amenity.fast_food'], + centroid=(10.0, 10.0)) + + def run(self, apiobj, frontend, **kwargs): + results = run_search(apiobj, frontend, 0.1, [('amenity', 'restaurant')], + details=SearchDetails.from_kwargs(kwargs)) + return sorted(r.place_id for r in results) + + def test_no_filter(self, apiobj, frontend): + assert self.run(apiobj, frontend) == [1, 2, 3] + + def test_include_exact(self, apiobj, frontend): + assert self.run(apiobj, frontend, include=['osm.tourism.hotel']) == [1] + + @pytest.mark.parametrize('category,expected', [('osm.amenity', [1, 2, 3]), + ('osm.tourism', [1])]) + def test_include_matches_descendants(self, apiobj, frontend, category, expected): + assert self.run(apiobj, frontend, include=[category]) == expected + + @pytest.mark.parametrize('category', ['osm.amenity.fast', 'osm.amen.restaurant']) + def test_include_does_not_match_partial_label(self, apiobj, frontend, category): + assert self.run(apiobj, frontend, include=[category]) == [] + + def test_include_comma_is_or(self, apiobj, frontend): + assert self.run(apiobj, frontend, + include=['osm.tourism.hotel,osm.amenity.fast_food']) == [1, 3] + + @pytest.mark.parametrize('categories,expected', [ + (['osm.tourism.hotel', 'osm.amenity.fast_food'], []), + (['osm.amenity.restaurant', 'osm.tourism.hotel'], [1])]) + def test_include_repeated_is_and(self, apiobj, frontend, categories, expected): + assert self.run(apiobj, frontend, include=categories) == expected + + def test_exclude_exact(self, apiobj, frontend): + assert self.run(apiobj, frontend, exclude=['osm.tourism.hotel']) == [2, 3] + + def test_exclude_matches_descendants(self, apiobj, frontend): + assert self.run(apiobj, frontend, exclude=['osm.amenity']) == [] + + @pytest.mark.parametrize('group,expected', [ + ('osm.tourism.hotel,osm.amenity.fast_food', [1, 2, 3]), + ('osm.tourism.hotel,osm.amenity.restaurant', [2, 3])]) + def test_exclude_comma_needs_all(self, apiobj, frontend, group, expected): + assert self.run(apiobj, frontend, exclude=[group]) == expected + + def test_exclude_repeated_is_or(self, apiobj, frontend): + assert self.run(apiobj, frontend, + exclude=['osm.tourism.hotel', 'osm.amenity.fast_food']) == [2] + + def test_include_and_exclude(self, apiobj, frontend): + assert self.run(apiobj, frontend, include=['osm.amenity'], + exclude=['osm.tourism.hotel']) == [2, 3] -- 2.47.3