From: mtmail Date: Thu, 22 Jun 2023 10:14:55 +0000 (+0200) Subject: Merge branch 'osm-search:master' into check-database-on-frozen-database X-Git-Tag: v4.3.0~61^2 X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/15a66e7b7d79d273f1674f673161cb440458ca7d?hp=2337cc653b679c9873344cc7278410a714a94a23 Merge branch 'osm-search:master' into check-database-on-frozen-database --- diff --git a/.github/actions/build-nominatim/action.yml b/.github/actions/build-nominatim/action.yml index 5d06267c..eaab5ae0 100644 --- a/.github/actions/build-nominatim/action.yml +++ b/.github/actions/build-nominatim/action.yml @@ -25,7 +25,7 @@ runs: shell: bash - name: Install${{ matrix.flavour }} prerequisites run: | - sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION} + sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION} lua-dkjson if [ "$FLAVOUR" == "oldstuff" ]; then pip3 install MarkupSafe==2.0.1 python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4 GeoAlchemy2==0.10.0 datrie asyncpg else diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 7dfb3f1d..a22a89c0 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -113,7 +113,7 @@ jobs: if: matrix.flavour == 'oldstuff' - name: Install Python webservers - run: pip3 install falcon sanic sanic-testing sanic-cors starlette + run: pip3 install falcon starlette - name: Install latest pylint run: pip3 install -U pylint asgi_lifespan @@ -250,6 +250,9 @@ jobs: - name: Prepare import environment run: | mv Nominatim/test/testdb/apidb-test-data.pbf test.pbf + mv Nominatim/settings/flex-base.lua flex-base.lua + mv Nominatim/settings/import-extratags.lua import-extratags.lua + mv Nominatim/settings/taginfo.lua taginfo.lua rm -rf Nominatim mkdir data-env-reverse working-directory: /home/nominatim @@ -258,6 +261,10 @@ jobs: run: nominatim --version working-directory: /home/nominatim/nominatim-project + - name: Print taginfo + run: lua taginfo.lua + working-directory: /home/nominatim + - name: Collect host OS information run: nominatim admin --collect-os-info working-directory: /home/nominatim/nominatim-project diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index e55e4b37..61fc1bce 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -67,9 +67,8 @@ For running the experimental Python frontend: * one of the following web frameworks: * [falcon](https://falconframework.org/) (3.0+) - * [sanic](https://sanic.dev) and (optionally) [sanic-cors](https://github.com/ashleysommer/sanic-cors) * [starlette](https://www.starlette.io/) - * [uvicorn](https://www.uvicorn.org/) (only with falcon and starlette framworks) + * [uvicorn](https://www.uvicorn.org/) For dependencies for running tests and building documentation, see the [Development section](../develop/Development-Environment.md). diff --git a/docs/develop/Development-Environment.md b/docs/develop/Development-Environment.md index 3234b8cb..d0369ea1 100644 --- a/docs/develop/Development-Environment.md +++ b/docs/develop/Development-Environment.md @@ -41,7 +41,6 @@ It has the following additional requirements: For testing the Python search frontend, you need to install extra dependencies depending on your choice of webserver framework: -* [sanic-testing](https://sanic.dev/en/plugins/sanic-testing/getting-started.html) (sanic only) * [httpx](https://www.python-httpx.org/) (starlette only) * [asgi-lifespan](https://github.com/florimondmanca/asgi-lifespan) (starlette only) @@ -66,7 +65,7 @@ sudo apt install php-cgi phpunit php-codesniffer \ pip3 install --user behave mkdocs mkdocstrings pytest pytest-asyncio pylint \ mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil \ types-ujson types-requests types-Pygments typing-extensions\ - sanic-testing httpx asgi-lifespan + httpx asgi-lifespan ``` The `mkdocs` executable will be located in `.local/bin`. You may have to add diff --git a/nominatim/api/logging.py b/nominatim/api/logging.py index 6bf3ed38..6c8b1b38 100644 --- a/nominatim/api/logging.py +++ b/nominatim/api/logging.py @@ -178,7 +178,7 @@ class HTMLLogger(BaseLogger): self._write(f"rank={res.rank_address}, ") self._write(f"osm={format_osm(res.osm_object)}, ") self._write(f'cc={res.country_code}, ') - self._write(f'importance={res.importance or -1:.5f})') + self._write(f'importance={res.importance or float("nan"):.5f})') total += 1 self._write(f'TOTAL: {total}

') @@ -196,7 +196,7 @@ class HTMLLogger(BaseLogger): def _python_var(self, var: Any) -> str: if CODE_HIGHLIGHT: - fmt = highlight(repr(var), PythonLexer(), HtmlFormatter(nowrap=True)) + fmt = highlight(str(var), PythonLexer(), HtmlFormatter(nowrap=True)) return f'
{fmt}
' return f'{str(var)}' diff --git a/nominatim/api/search/db_search_builder.py b/nominatim/api/search/db_search_builder.py index b6ba211c..9ff8c03c 100644 --- a/nominatim/api/search/db_search_builder.py +++ b/nominatim/api/search/db_search_builder.py @@ -141,12 +141,14 @@ class SearchBuilder: yield dbs.CountrySearch(sdata) if sdata.postcodes and (is_category or self.configured_for_postcode): + penalty = 0.0 if sdata.countries else 0.1 if address: sdata.lookups = [dbf.FieldLookup('nameaddress_vector', [t.token for r in address for t in self.query.get_partials_list(r)], 'restrict')] - yield dbs.PostcodeSearch(0.4, sdata) + penalty += 0.2 + yield dbs.PostcodeSearch(penalty, sdata) def build_housenumber_search(self, sdata: dbf.SearchData, hnrs: List[Token], diff --git a/nominatim/api/search/db_searches.py b/nominatim/api/search/db_searches.py index db35f726..76ff368f 100644 --- a/nominatim/api/search/db_searches.py +++ b/nominatim/api/search/db_searches.py @@ -403,6 +403,12 @@ class CountrySearch(AbstractSearch): details: SearchDetails) -> nres.SearchResults: """ Look up the country in the fallback country tables. """ + # Avoid the fallback search when this is a more search. Country results + # usually are in the first batch of results and it is not possible + # to exclude these fallbacks. + if details.excluded: + return nres.SearchResults() + t = conn.t.country_name tgrid = conn.t.country_grid @@ -562,6 +568,8 @@ class PlaceSearch(AbstractSearch): sql = sql.where(tsearch.c.country_code.in_(self.countries.values)) if self.postcodes: + # if a postcode is given, don't search for state or country level objects + sql = sql.where(tsearch.c.address_rank > 9) tpc = conn.t.postcode if self.expected_count > 1000: # Many results expected. Restrict by postcode. diff --git a/nominatim/api/search/geocoder.py b/nominatim/api/search/geocoder.py index 5e90d408..0ef649d9 100644 --- a/nominatim/api/search/geocoder.py +++ b/nominatim/api/search/geocoder.py @@ -180,7 +180,7 @@ def _dump_searches(searches: List[AbstractSearch], query: QueryStruct, return f'{c[0]}^{c[1]}' for search in searches[start:]: - fields = ('name_lookups', 'name_ranking', 'countries', 'housenumbers', + fields = ('lookups', 'rankings', 'countries', 'housenumbers', 'postcodes', 'qualifier') iters = itertools.zip_longest([f"{search.penalty:.3g}"], *(getattr(search, attr, []) for attr in fields), diff --git a/nominatim/api/search/icu_tokenizer.py b/nominatim/api/search/icu_tokenizer.py index 17e67905..f259995d 100644 --- a/nominatim/api/search/icu_tokenizer.py +++ b/nominatim/api/search/icu_tokenizer.py @@ -153,7 +153,7 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer): """ log().section('Analyze query (using ICU tokenizer)') normalized = list(filter(lambda p: p.text, - (qmod.Phrase(p.ptype, self.normalizer.transliterate(p.text)) + (qmod.Phrase(p.ptype, self.normalize_text(p.text)) for p in phrases))) query = qmod.QueryStruct(normalized) log().var_dump('Normalized query', query.source) @@ -187,6 +187,14 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer): return query + def normalize_text(self, text: str) -> str: + """ Bring the given text into a normalized form. That is the + standardized form search will work with. All information removed + at this stage is inevitably lost. + """ + return cast(str, self.normalizer.transliterate(text)) + + def split_query(self, query: qmod.QueryStruct) -> Tuple[QueryParts, WordDict]: """ Transliterate the phrases and split them into tokens. @@ -248,12 +256,11 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer): and (repl.ttype != qmod.TokenType.HOUSENUMBER or len(tlist.tokens[0].lookup_word) > 4): repl.add_penalty(0.39) - elif tlist.ttype == qmod.TokenType.HOUSENUMBER: + elif tlist.ttype == qmod.TokenType.HOUSENUMBER \ + and len(tlist.tokens[0].lookup_word) <= 3: if any(c.isdigit() for c in tlist.tokens[0].lookup_word): for repl in node.starting: - if repl.end == tlist.end and repl.ttype != qmod.TokenType.HOUSENUMBER \ - and (repl.ttype != qmod.TokenType.HOUSENUMBER - or len(tlist.tokens[0].lookup_word) <= 3): + if repl.end == tlist.end and repl.ttype != qmod.TokenType.HOUSENUMBER: repl.add_penalty(0.5 - tlist.tokens[0].penalty) elif tlist.ttype not in (qmod.TokenType.COUNTRY, qmod.TokenType.PARTIAL): norm = parts[i].normalized diff --git a/nominatim/api/search/legacy_tokenizer.py b/nominatim/api/search/legacy_tokenizer.py index 96975704..3346584c 100644 --- a/nominatim/api/search/legacy_tokenizer.py +++ b/nominatim/api/search/legacy_tokenizer.py @@ -233,12 +233,11 @@ class LegacyQueryAnalyzer(AbstractQueryAnalyzer): and (repl.ttype != qmod.TokenType.HOUSENUMBER or len(tlist.tokens[0].lookup_word) > 4): repl.add_penalty(0.39) - elif tlist.ttype == qmod.TokenType.HOUSENUMBER: + elif tlist.ttype == qmod.TokenType.HOUSENUMBER \ + and len(tlist.tokens[0].lookup_word) <= 3: if any(c.isdigit() for c in tlist.tokens[0].lookup_word): for repl in node.starting: - if repl.end == tlist.end and repl.ttype != qmod.TokenType.HOUSENUMBER \ - and (repl.ttype != qmod.TokenType.HOUSENUMBER - or len(tlist.tokens[0].lookup_word) <= 3): + if repl.end == tlist.end and repl.ttype != qmod.TokenType.HOUSENUMBER: repl.add_penalty(0.5 - tlist.tokens[0].penalty) diff --git a/nominatim/api/search/token_assignment.py b/nominatim/api/search/token_assignment.py index 747fea6c..11da2359 100644 --- a/nominatim/api/search/token_assignment.py +++ b/nominatim/api/search/token_assignment.py @@ -270,7 +270,12 @@ class _TokenSequence: if (base.postcode.start == 0 and self.direction != -1)\ or (base.postcode.end == query.num_token_slots() and self.direction != 1): log().comment('postcode search') - yield dataclasses.replace(base, penalty=self.penalty) + #
, should give preference to address search + if base.postcode.start == 0: + penalty = self.penalty + else: + penalty = self.penalty + 0.1 + yield dataclasses.replace(base, penalty=penalty) # Postcode or country-only search if not base.address: @@ -278,6 +283,9 @@ class _TokenSequence: log().comment('postcode/country search') yield dataclasses.replace(base, penalty=self.penalty) else: + # ,
should give preference to postcode search + if base.postcode and base.postcode.start == 0: + self.penalty += 0.1 # Use entire first word as name if self.direction != -1: log().comment('first word = name') diff --git a/nominatim/api/types.py b/nominatim/api/types.py index c7e15843..87568a09 100644 --- a/nominatim/api/types.py +++ b/nominatim/api/types.py @@ -302,10 +302,11 @@ def format_excluded(ids: Any) -> List[int]: else: raise UsageError("Parameter 'excluded' needs to be a comma-separated list " "or a Python list of numbers.") - if not all(isinstance(i, int) or (isinstance(i, str) and i.isdigit()) for i in plist): + if not all(isinstance(i, int) or + (isinstance(i, str) and (not i or i.isdigit())) for i in plist): raise UsageError("Parameter 'excluded' only takes place IDs.") - return [int(id) for id in plist if id] + return [int(id) for id in plist if id] or [0] def format_categories(categories: List[Tuple[str, str]]) -> List[Tuple[str, str]]: diff --git a/nominatim/api/v1/helpers.py b/nominatim/api/v1/helpers.py index 62e5e943..ea7c125d 100644 --- a/nominatim/api/v1/helpers.py +++ b/nominatim/api/v1/helpers.py @@ -62,13 +62,13 @@ def extend_query_parts(queryparts: Dict[str, Any], details: Dict[str, Any], """ parsed = SearchDetails.from_kwargs(details) if parsed.geometry_output != GeometryFormat.NONE: - if parsed.geometry_output & GeometryFormat.GEOJSON: + if GeometryFormat.GEOJSON in parsed.geometry_output: queryparts['polygon_geojson'] = '1' - if parsed.geometry_output & GeometryFormat.KML: + if GeometryFormat.KML in parsed.geometry_output: queryparts['polygon_kml'] = '1' - if parsed.geometry_output & GeometryFormat.SVG: + if GeometryFormat.SVG in parsed.geometry_output: queryparts['polygon_svg'] = '1' - if parsed.geometry_output & GeometryFormat.TEXT: + if GeometryFormat.TEXT in parsed.geometry_output: queryparts['polygon_text'] = '1' if parsed.address_details: queryparts['addressdetails'] = '1' diff --git a/nominatim/api/v1/server_glue.py b/nominatim/api/v1/server_glue.py index 43cc6e56..865e1331 100644 --- a/nominatim/api/v1/server_glue.py +++ b/nominatim/api/v1/server_glue.py @@ -185,7 +185,7 @@ class ASGIAdaptor(abc.ABC): """ Return the accepted languages. """ return self.get('accept-language')\ - or self.get_header('http_accept_language')\ + or self.get_header('accept-language')\ or self.config().DEFAULT_LANGUAGE diff --git a/nominatim/cli.py b/nominatim/cli.py index 6a89a8de..836f9037 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -215,7 +215,7 @@ class AdminServe: group.add_argument('--server', default='127.0.0.1:8088', help='The address the server will listen to.') group.add_argument('--engine', default='php', - choices=('php', 'sanic', 'falcon', 'starlette'), + choices=('php', 'falcon', 'starlette'), help='Webserver framework to run. (default: php)') @@ -223,6 +223,7 @@ class AdminServe: if args.engine == 'php': run_php_server(args.server, args.project_dir / 'website') else: + import uvicorn # pylint: disable=import-outside-toplevel server_info = args.server.split(':', 1) host = server_info[0] if len(server_info) > 1: @@ -232,21 +233,10 @@ class AdminServe: else: port = 8088 - if args.engine == 'sanic': - server_module = importlib.import_module('nominatim.server.sanic.server') + server_module = importlib.import_module(f'nominatim.server.{args.engine}.server') - app = server_module.get_application(args.project_dir) - app.run(host=host, port=port, debug=True, single_process=True) - else: - import uvicorn # pylint: disable=import-outside-toplevel - - if args.engine == 'falcon': - server_module = importlib.import_module('nominatim.server.falcon.server') - elif args.engine == 'starlette': - server_module = importlib.import_module('nominatim.server.starlette.server') - - app = server_module.get_application(args.project_dir) - uvicorn.run(app, host=host, port=port) + app = server_module.get_application(args.project_dir) + uvicorn.run(app, host=host, port=port) return 0 diff --git a/nominatim/server/sanic/__init__.py b/nominatim/server/sanic/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/nominatim/server/sanic/server.py b/nominatim/server/sanic/server.py deleted file mode 100644 index 15887eef..00000000 --- a/nominatim/server/sanic/server.py +++ /dev/null @@ -1,78 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# This file is part of Nominatim. (https://nominatim.org) -# -# Copyright (C) 2023 by the Nominatim developer community. -# For a full list of authors see the git log. -""" -Server implementation using the sanic webserver framework. -""" -from typing import Any, Optional, Mapping, Callable, cast, Coroutine -from pathlib import Path - -from sanic import Request, HTTPResponse, Sanic -from sanic.exceptions import SanicException -from sanic.response import text as TextResponse - -from nominatim.api import NominatimAPIAsync -import nominatim.api.v1 as api_impl -from nominatim.config import Configuration - -class ParamWrapper(api_impl.ASGIAdaptor): - """ Adaptor class for server glue to Sanic framework. - """ - - def __init__(self, request: Request) -> None: - self.request = request - - - def get(self, name: str, default: Optional[str] = None) -> Optional[str]: - return cast(Optional[str], self.request.args.get(name, default)) - - - def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]: - return cast(Optional[str], self.request.headers.get(name, default)) - - - def error(self, msg: str, status: int = 400) -> SanicException: - exception = SanicException(msg, status_code=status) - - return exception - - - def create_response(self, status: int, output: str) -> HTTPResponse: - return TextResponse(output, status=status, content_type=self.content_type) - - - def config(self) -> Configuration: - return cast(Configuration, self.request.app.ctx.api.config) - - -def _wrap_endpoint(func: api_impl.EndpointFunc)\ - -> Callable[[Request], Coroutine[Any, Any, HTTPResponse]]: - async def _callback(request: Request) -> HTTPResponse: - return cast(HTTPResponse, await func(request.app.ctx.api, ParamWrapper(request))) - - return _callback - - -def get_application(project_dir: Path, - environ: Optional[Mapping[str, str]] = None) -> Sanic: - """ Create a Nominatim sanic ASGI application. - """ - app = Sanic("NominatimInstance") - - app.ctx.api = NominatimAPIAsync(project_dir, environ) - - if app.ctx.api.config.get_bool('CORS_NOACCESSCONTROL'): - from sanic_cors import CORS # pylint: disable=import-outside-toplevel - CORS(app) - - legacy_urls = app.ctx.api.config.get_bool('SERVE_LEGACY_URLS') - for name, func in api_impl.ROUTES: - endpoint = _wrap_endpoint(func) - app.add_route(endpoint, f"/{name}", name=f"v1_{name}_simple") - if legacy_urls: - app.add_route(endpoint, f"/{name}.php", name=f"v1_{name}_legacy") - - return app diff --git a/settings/flex-base.lua b/settings/flex-base.lua index 58d60228..fbfb4d54 100644 --- a/settings/flex-base.lua +++ b/settings/flex-base.lua @@ -11,6 +11,11 @@ local ADDRESS_TAGS = nil local SAVE_EXTRA_MAINS = false local POSTCODE_FALLBACK = true +-- tables required for taginfo +module.TAGINFO_MAIN = {keys = {}, delete_tags = {}} +module.TAGINFO_NAME_KEYS = {} +module.TAGINFO_ADDRESS_KEYS = {} + -- The single place table. local place_table = osm2pgsql.define_table{ @@ -372,6 +377,17 @@ function module.tag_group(data) end end +-- Returns prefix part of the keys, and reject suffix matching keys +local function process_key(key) + if key:sub(1, 1) == '*' then + return nil + end + if key:sub(#key, #key) == '*' then + return key:sub(1, #key - 2) + end + return key +end + -- Process functions for all data types function module.process_node(object) @@ -465,14 +481,29 @@ function module.set_prefilters(data) PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} PRE_EXTRAS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags} + module.TAGINFO_MAIN.delete_tags = data.delete_tags end function module.set_main_tags(data) MAIN_KEYS = data + local keys = {} + for k, _ in pairs(data) do + table.insert(keys, k) + end + module.TAGINFO_MAIN.keys = keys end function module.set_name_tags(data) NAMES = module.tag_group(data) + + for _, lst in pairs(data) do + for _, k in ipairs(lst) do + local key = process_key(k) + if key ~= nil then + module.TAGINFO_NAME_KEYS[key] = true + end + end + end end function module.set_address_tags(data) @@ -480,8 +511,18 @@ function module.set_address_tags(data) POSTCODE_FALLBACK = data.postcode_fallback data.postcode_fallback = nil end - ADDRESS_TAGS = module.tag_group(data) + + for _, lst in pairs(data) do + if lst ~= nil then + for _, k in ipairs(lst) do + local key = process_key(k) + if key ~= nil then + module.TAGINFO_ADDRESS_KEYS[key] = true + end + end + end + end end function module.set_unused_handling(data) diff --git a/settings/import-extratags.lua b/settings/import-extratags.lua index fd9a15f3..204bd1c8 100644 --- a/settings/import-extratags.lua +++ b/settings/import-extratags.lua @@ -7,7 +7,6 @@ flex.set_main_tags{ historic = 'always', military = 'always', natural = 'named', - landuse = 'named', highway = {'always', street_lamp = 'named', traffic_signals = 'named', diff --git a/settings/import-full.lua b/settings/import-full.lua index 1b64124d..1dc317a9 100644 --- a/settings/import-full.lua +++ b/settings/import-full.lua @@ -7,7 +7,6 @@ flex.set_main_tags{ historic = 'always', military = 'always', natural = 'named', - landuse = 'named', highway = {'always', street_lamp = 'named', traffic_signals = 'named', diff --git a/settings/taginfo.lua b/settings/taginfo.lua new file mode 100644 index 00000000..dba395e9 --- /dev/null +++ b/settings/taginfo.lua @@ -0,0 +1,74 @@ +-- Prints taginfo project description in the standard output +-- + +-- create fake "osm2pgsql" table for flex-base, originally created by the main C++ program +osm2pgsql = {} +function osm2pgsql.define_table(...) end + +-- provide path to flex-style lua file +flex = require('import-extratags') +local json = require ('dkjson') + + +------------ helper functions --------------------- + +function get_key_description(key, description) + local desc = {} + desc.key = key + desc.description = description + set_keyorder(desc, {'key', 'description'}) + return desc +end + +-- Sets the key order for the resulting JSON table +function set_keyorder(table, order) + setmetatable(table, { + __jsonorder = order + }) +end + + +-- Prints the collected tags in the required format in JSON +function print_taginfo() + local tags = {} + + for _, k in ipairs(flex.TAGINFO_MAIN.keys) do + local desc = get_key_description(k, 'POI/feature in the search database') + if flex.TAGINFO_MAIN.delete_tags[k] ~= nil then + desc.description = string.format('%s(except for values: %s).', desc.description, + table.concat(flex.TAGINFO_MAIN.delete_tags[k], ', ')) + end + table.insert(tags, desc) + end + + for k, _ in pairs(flex.TAGINFO_NAME_KEYS) do + local desc = get_key_description(k, 'Searchable name of the place.') + table.insert(tags, desc) + end + for k, _ in pairs(flex.TAGINFO_ADDRESS_KEYS) do + local desc = get_key_description(k, 'Used to determine the address of a place.') + table.insert(tags, desc) + end + + local format = { + data_format = 1, + data_url = 'https://nominatim.openstreetmap.org/taginfo.json', + project = { + name = 'Nominatim', + description = 'OSM search engine.', + project_url = 'https://nominatim.openstreetmap.org', + doc_url = 'https://nominatim.org/release-docs/develop/', + contact_name = 'Sarah Hoffmann', + contact_email = 'lonvia@denofr.de' + } + } + format.tags = tags + + set_keyorder(format, {'data_format', 'data_url', 'project', 'tags'}) + set_keyorder(format.project, {'name', 'description', 'project_url', 'doc_url', + 'contact_name', 'contact_email'}) + + print(json.encode(format)) +end + +print_taginfo() diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index 64b62aba..572c571a 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -350,20 +350,6 @@ class NominatimEnvironment: return _request - def create_api_request_func_sanic(self): - import nominatim.server.sanic.server - - async def _request(endpoint, params, project_dir, environ, http_headers): - app = nominatim.server.sanic.server.get_application(project_dir, environ) - - _, response = await app.asgi_client.get(f"/{endpoint}", params=params, - headers=http_headers) - - return response.text, response.status_code - - return _request - - def create_api_request_func_falcon(self): import nominatim.server.falcon.server import falcon.testing diff --git a/test/python/api/search/test_icu_query_analyzer.py b/test/python/api/search/test_icu_query_analyzer.py index 78cd2c4d..faf81375 100644 --- a/test/python/api/search/test_icu_query_analyzer.py +++ b/test/python/api/search/test_icu_query_analyzer.py @@ -118,10 +118,10 @@ async def test_penalty_postcodes_and_housenumbers(conn, term, order): assert query.num_token_slots() == 1 - torder = [(tl.tokens[0].penalty, tl.ttype) for tl in query.nodes[0].starting] + torder = [(tl.tokens[0].penalty, tl.ttype.name) for tl in query.nodes[0].starting] torder.sort() - assert [t[1] for t in torder] == [TokenType[o] for o in order] + assert [t[1] for t in torder] == order @pytest.mark.asyncio async def test_category_words_only_at_beginning(conn): diff --git a/test/python/api/search/test_legacy_query_analyzer.py b/test/python/api/search/test_legacy_query_analyzer.py index c2115853..cdea6ede 100644 --- a/test/python/api/search/test_legacy_query_analyzer.py +++ b/test/python/api/search/test_legacy_query_analyzer.py @@ -195,11 +195,10 @@ async def test_penalty_postcodes_and_housenumbers(conn, term, order): assert query.num_token_slots() == 1 - torder = [(tl.tokens[0].penalty, tl.ttype) for tl in query.nodes[0].starting] - print(query.nodes[0].starting) + torder = [(tl.tokens[0].penalty, tl.ttype.name) for tl in query.nodes[0].starting] torder.sort() - assert [t[1] for t in torder] == [TokenType[o] for o in order] + assert [t[1] for t in torder] == order @pytest.mark.asyncio diff --git a/test/python/api/search/test_token_assignment.py b/test/python/api/search/test_token_assignment.py index f78d5430..dc123403 100644 --- a/test/python/api/search/test_token_assignment.py +++ b/test/python/api/search/test_token_assignment.py @@ -253,7 +253,7 @@ def test_postcode_with_designation(): (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)])) check_assignments(yield_token_assignments(q), - TokenAssignment(name=TokenRange(1, 2), + TokenAssignment(penalty=0.1, name=TokenRange(1, 2), postcode=TokenRange(0, 1)), TokenAssignment(postcode=TokenRange(0, 1), address=[TokenRange(1, 2)])) @@ -266,7 +266,7 @@ def test_postcode_with_designation_backwards(): check_assignments(yield_token_assignments(q), TokenAssignment(name=TokenRange(0, 1), postcode=TokenRange(1, 2)), - TokenAssignment(postcode=TokenRange(1, 2), + TokenAssignment(penalty=0.1, postcode=TokenRange(1, 2), address=[TokenRange(0, 1)])) diff --git a/test/python/api/test_server_glue_v1.py b/test/python/api/test_server_glue_v1.py index 538d91f1..a731e720 100644 --- a/test/python/api/test_server_glue_v1.py +++ b/test/python/api/test_server_glue_v1.py @@ -123,7 +123,7 @@ def test_accepted_languages_from_param(): def test_accepted_languages_from_header(): - a = FakeAdaptor(headers={'http_accept_language': 'de'}) + a = FakeAdaptor(headers={'accept-language': 'de'}) assert a.get_accepted_languages() == 'de' @@ -135,13 +135,13 @@ def test_accepted_languages_from_default(monkeypatch): def test_accepted_languages_param_over_header(): a = FakeAdaptor(params={'accept-language': 'de'}, - headers={'http_accept_language': 'en'}) + headers={'accept-language': 'en'}) assert a.get_accepted_languages() == 'de' def test_accepted_languages_header_over_default(monkeypatch): monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', 'en') - a = FakeAdaptor(headers={'http_accept_language': 'de'}) + a = FakeAdaptor(headers={'accept-language': 'de'}) assert a.get_accepted_languages() == 'de' @@ -197,14 +197,14 @@ def test_raise_error_during_debug(): loglib.log().section('Ongoing') with pytest.raises(FakeError) as excinfo: - a.raise_error('bad state') + a.raise_error('badstate') content = ET.fromstring(excinfo.value.msg) assert content.tag == 'html' assert '>Ongoing<' in excinfo.value.msg - assert 'bad state' in excinfo.value.msg + assert 'badstate' in excinfo.value.msg # ASGIAdaptor.build_response diff --git a/test/python/cli/test_cli.py b/test/python/cli/test_cli.py index d0e3307e..f1bb75a9 100644 --- a/test/python/cli/test_cli.py +++ b/test/python/cli/test_cli.py @@ -68,15 +68,6 @@ def test_cli_serve_php(cli_call, mock_func_factory): assert func.called == 1 -def test_cli_serve_sanic(cli_call, mock_func_factory): - mod = pytest.importorskip("sanic") - func = mock_func_factory(mod.Sanic, "run") - - cli_call('serve', '--engine', 'sanic') == 0 - - assert func.called == 1 - - def test_cli_serve_starlette_custom_server(cli_call, mock_func_factory): pytest.importorskip("starlette") mod = pytest.importorskip("uvicorn") diff --git a/vagrant/Install-on-Ubuntu-20.sh b/vagrant/Install-on-Ubuntu-20.sh index 52967092..ef776579 100755 --- a/vagrant/Install-on-Ubuntu-20.sh +++ b/vagrant/Install-on-Ubuntu-20.sh @@ -23,7 +23,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: sudo apt install -y php-cgi sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \ libboost-filesystem-dev libexpat1-dev zlib1g-dev \ - libbz2-dev libpq-dev liblua5.3-dev lua5.3 \ + libbz2-dev libpq-dev liblua5.3-dev lua5.3 lua-dkjson \ postgresql-12-postgis-3 \ postgresql-contrib-12 postgresql-12-postgis-3-scripts \ php-cli php-pgsql php-intl libicu-dev python3-dotenv \ diff --git a/vagrant/Install-on-Ubuntu-22.sh b/vagrant/Install-on-Ubuntu-22.sh index 7a4d146f..c44cf87d 100755 --- a/vagrant/Install-on-Ubuntu-22.sh +++ b/vagrant/Install-on-Ubuntu-22.sh @@ -23,7 +23,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: sudo apt install -y php-cgi sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \ libboost-filesystem-dev libexpat1-dev zlib1g-dev \ - libbz2-dev libpq-dev liblua5.3-dev lua5.3 \ + libbz2-dev libpq-dev liblua5.3-dev lua5.3 lua-dkjson \ postgresql-server-dev-14 postgresql-14-postgis-3 \ postgresql-contrib-14 postgresql-14-postgis-3-scripts \ php-cli php-pgsql php-intl libicu-dev python3-dotenv \