From: Sarah Hoffmann Date: Mon, 2 Jun 2025 13:19:18 +0000 (+0200) Subject: release 5.1.0.post8 X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/HEAD?ds=inline;hp=0a7624039bc4189fd999eb23ea5a175a1c4b2dfb release 5.1.0.post8 --- diff --git a/.flake8 b/.flake8 index cf87715a..1aae19dc 100644 --- a/.flake8 +++ b/.flake8 @@ -7,5 +7,5 @@ extend-ignore = per-file-ignores = __init__.py: F401 test/python/utils/test_json_writer.py: E131 - test/python/conftest.py: E402 + **/conftest.py: E402 test/bdd/*: F821 diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index a8bf957f..4d555416 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -68,26 +68,34 @@ jobs: with: dependencies: ${{ matrix.dependencies }} + - uses: actions/cache@v4 + with: + path: | + /usr/local/bin/osm2pgsql + key: osm2pgsql-bin-22-1 + if: matrix.ubuntu == '22' + - name: Compile osm2pgsql run: | - sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua-dkjson nlohmann-json3-dev - mkdir osm2pgsql-build - cd osm2pgsql-build - git clone https://github.com/osm2pgsql-dev/osm2pgsql - mkdir build - cd build - cmake ../osm2pgsql - make - sudo make install - cd ../.. - rm -rf osm2pgsql-build + if [ ! -f /usr/local/bin/osm2pgsql ]; then + sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua-dkjson nlohmann-json3-dev + mkdir osm2pgsql-build + cd osm2pgsql-build + git clone https://github.com/osm2pgsql-dev/osm2pgsql + mkdir build + cd build + cmake ../osm2pgsql + make + sudo make install + cd ../.. + rm -rf osm2pgsql-build + else + sudo apt-get install -y -qq libexpat1 liblua${LUA_VERSION} + fi if: matrix.ubuntu == '22' env: LUA_VERSION: ${{ matrix.lua }} - - name: Install test prerequisites - run: ./venv/bin/pip install behave==1.2.6 - - name: Install test prerequisites (apt) run: sudo apt-get install -y -qq python3-pytest python3-pytest-asyncio uvicorn python3-falcon python3-aiosqlite python3-pyosmium if: matrix.dependencies == 'apt' @@ -96,6 +104,9 @@ jobs: run: ./venv/bin/pip install pytest-asyncio falcon starlette asgi_lifespan aiosqlite osmium uvicorn if: matrix.dependencies == 'pip' + - name: Install test prerequisites + run: ./venv/bin/pip install pytest-bdd + - name: Install latest flake8 run: ./venv/bin/pip install -U flake8 @@ -118,8 +129,8 @@ jobs: - name: BDD tests run: | - ../../../venv/bin/python -m behave -DREMOVE_TEMPLATE=1 --format=progress3 - working-directory: Nominatim/test/bdd + ../venv/bin/python -m pytest test/bdd --nominatim-purge + working-directory: Nominatim install: runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 311414fe..6c90cd3c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -113,3 +113,5 @@ Checklist for releases: * run `nominatim --version` to confirm correct version * [ ] tag new release and add a release on github.com * [ ] build pip packages and upload to pypi + * `make build` + * `twine upload dist/*` diff --git a/ChangeLog b/ChangeLog index 9ffe4038..dff198eb 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,22 @@ +5.1.0 + * replace datrie with simple internal trie implementation + * add pattern-based postcode parser for queries, + postcodes no longer need to be present in OSM to be found + * take variants into account when computing token similarity + * add extratags output to geocodejson format + * fix default layer setting used for structured queries + * update abbreviation lists for Russian and English + (thanks @shoorick, @IvanShift, @mhsrn21) + * fix variant generation for Norwegian + * fix normalization around space-like characters + * improve postcode search and handling of postcodes in queries + * reorganise internal query structure and get rid of slow enums + * enable code linting for tests + * various code moderinsations in test code (thanks @eumiro) + * remove setting osm2pgsql location via config.lib_dir + * make SQL functions parallel save as far as possible (thanks @otbutz) + * various fixes and improvements to documentation (thanks @TuringVerified) + 5.0.0 * increase required versions for PostgreSQL (12+), PostGIS (3.0+) * remove installation via cmake and debundle osm2pgsql diff --git a/Makefile b/Makefile index f35c9782..d6423add 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ lint: flake8 src test/python test/bdd bdd: - cd test/bdd; behave -DREMOVE_TEMPLATE=1 + pytest test/bdd --nominatim-purge # Documentation diff --git a/README.md b/README.md index 3b0f328a..54bdbf8e 100644 --- a/README.md +++ b/README.md @@ -27,18 +27,25 @@ can be found at nominatim.org as well. A quick summary of the necessary steps: -1. Create a Python virtualenv and install the packages: + +1. Clone this git repository and download the country grid + + git clone https://github.com/osm-search/Nominatim.git + wget -O Nominatim/data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz + +2. Create a Python virtualenv and install the packages: python3 -m venv nominatim-venv ./nominatim-venv/bin/pip install packaging/nominatim-{api,db} -2. Create a project directory, get OSM data and import: +3. Create a project directory, get OSM data and import: mkdir nominatim-project cd nominatim-project - ../nominatim-venv/bin/nominatim import --osm-file + ../nominatim-venv/bin/nominatim import --osm-file 2>&1 | tee setup.log + -3. Start the webserver: +4. Start the webserver: ./nominatim-venv/bin/pip install uvicorn falcon ../nominatim-venv/bin/nominatim serve diff --git a/SECURITY.md b/SECURITY.md index e3660bcd..98295e1f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -9,7 +9,8 @@ versions. | Version | End of support for security updates | | ------- | ----------------------------------- | -| 5.0.x | 2027-02-06 +| 5.1.x | 2027-04-01 | +| 5.0.x | 2027-02-06 | | 4.5.x | 2026-09-12 | | 4.4.x | 2026-03-07 | | 4.3.x | 2025-09-07 | diff --git a/docs/api/Search.md b/docs/api/Search.md index 1c269168..3c9a7148 100644 --- a/docs/api/Search.md +++ b/docs/api/Search.md @@ -212,7 +212,7 @@ other layers. The featureType allows to have a more fine-grained selection for places from the address layer. Results can be restricted to places that make up the 'state', 'country' or 'city' part of an address. A featureType of -settlement selects any human inhabited feature from 'state' down to +`settlement` selects any human inhabited feature from 'state' down to 'neighbourhood'. When featureType is set, then results are automatically restricted diff --git a/docs/customize/Settings.md b/docs/customize/Settings.md index 94726ca7..edf2241b 100644 --- a/docs/customize/Settings.md +++ b/docs/customize/Settings.md @@ -602,6 +602,43 @@ results gathered so far. Note that under high load you may observe that users receive different results than usual without seeing an error. This may cause some confusion. +#### NOMINATIM_OUTPUT_NAMES + +| Summary | | +| -------------- | --------------------------------------------------- | +| **Description:** | Specifies order of name tags | +| **Format:** | string: comma-separated list of tag names | +| **Default:** | name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref | + +Specifies the order in which different name tags are used. +The values in this list determine the preferred order of name variants, +including language-specific names (in OSM: the name tag with and without any language suffix). + +Comma-separated list, where :XX stands for language suffix +(e.g. name:en) and no :XX stands for general tags (e.g. name). + +See also [NOMINATIM_DEFAULT_LANGUAGE](#nominatim_default_language). + +!!! note + If NOMINATIM_OUTPUT_NAMES = `name:XX,name,short_name:XX,short_name` the search follows + + ``` + 'name', 'short_name' + ``` + + if we have no preferred language order for showing search results. + + For languages ['en', 'es'] the search follows + + ``` + 'name:en', 'name:es', + 'name', + 'short_name:en', 'short_name:es', + 'short_name' + ``` + + For those familiar with the internal implementation, the `_place_*` expansion is added, but to simplify, it is not included in this example. + ### Logging Settings #### NOMINATIM_LOG_DB diff --git a/docs/customize/Tokenizers.md b/docs/customize/Tokenizers.md index d290c148..23db34c9 100644 --- a/docs/customize/Tokenizers.md +++ b/docs/customize/Tokenizers.md @@ -67,7 +67,13 @@ Here is an example configuration file: ``` yaml query-preprocessing: - - normalize + - step: split_japanese_phrases + - step: regex_replace + replacements: + - pattern: https?://[^\s]* # Filter URLs starting with http or https + replace: '' + - step: normalize + normalization: - ":: lower ()" - "ß > 'ss'" # German szet is unambiguously equal to double ss @@ -88,8 +94,8 @@ token-analysis: replacements: ['ä', 'ae'] ``` -The configuration file contains four sections: -`normalization`, `transliteration`, `sanitizers` and `token-analysis`. +The configuration file contains five sections: +`query-preprocessing`, `normalization`, `transliteration`, `sanitizers` and `token-analysis`. #### Query preprocessing @@ -106,6 +112,19 @@ The following is a list of preprocessors that are shipped with Nominatim. heading_level: 6 docstring_section_style: spacy +##### regex-replace + +::: nominatim_api.query_preprocessing.regex_replace + options: + members: False + heading_level: 6 + docstring_section_style: spacy + description: + This option runs any given regex pattern on the input and replaces values accordingly + replacements: + - pattern: regex pattern + replace: string to replace with + #### Normalization and Transliteration diff --git a/docs/develop/Development-Environment.md b/docs/develop/Development-Environment.md index 9ade7916..5f247455 100644 --- a/docs/develop/Development-Environment.md +++ b/docs/develop/Development-Environment.md @@ -25,15 +25,15 @@ following packages should get you started: ## Prerequisites for testing and documentation -The Nominatim test suite consists of behavioural tests (using behave) and +The Nominatim test suite consists of behavioural tests (using pytest-bdd) and unit tests (using pytest). It has the following additional requirements: -* [behave test framework](https://behave.readthedocs.io) >= 1.2.6 * [flake8](https://flake8.pycqa.org/en/stable/) (CI always runs the latest version from pip) * [mypy](http://mypy-lang.org/) (plus typing information for external libs) * [Python Typing Extensions](https://github.com/python/typing_extensions) (for Python < 3.9) * [pytest](https://pytest.org) * [pytest-asyncio](https://pytest-asyncio.readthedocs.io) +* [pytest-bdd](https://pytest-bdd.readthedocs.io) For testing the Python search frontend, you need to install extra dependencies depending on your choice of webserver framework: @@ -48,9 +48,6 @@ The documentation is built with mkdocs: * [mkdocs-material](https://squidfunk.github.io/mkdocs-material/) * [mkdocs-gen-files](https://oprypin.github.io/mkdocs-gen-files/) -Please be aware that tests always run against the globally installed -osm2pgsql, so you need to have this set up. If you want to test against -the vendored version of osm2pgsql, you need to set the PATH accordingly. ### Installing prerequisites on Ubuntu/Debian @@ -69,9 +66,10 @@ To set up the virtual environment with all necessary packages run: ```sh virtualenv ~/nominatim-dev-venv ~/nominatim-dev-venv/bin/pip install\ - psutil psycopg[binary] PyICU SQLAlchemy \ - python-dotenv jinja2 pyYAML behave \ - mkdocs mkdocstrings mkdocs-gen-files pytest pytest-asyncio flake8 \ + psutil 'psycopg[binary]' PyICU SQLAlchemy \ + python-dotenv jinja2 pyYAML \ + mkdocs 'mkdocstrings[python]' mkdocs-gen-files \ + pytest pytest-asyncio pytest-bdd flake8 \ types-jinja2 types-markupsafe types-psutil types-psycopg2 \ types-pygments types-pyyaml types-requests types-ujson \ types-urllib3 typing-extensions unicorn falcon starlette \ diff --git a/docs/develop/Testing.md b/docs/develop/Testing.md index 12673d40..738fa4b8 100644 --- a/docs/develop/Testing.md +++ b/docs/develop/Testing.md @@ -43,53 +43,53 @@ The name of the pytest binary depends on your installation. ## BDD Functional Tests (`test/bdd`) Functional tests are written as BDD instructions. For more information on -the philosophy of BDD testing, see the -[Behave manual](http://pythonhosted.org/behave/philosophy.html). - -The following explanation assume that the reader is familiar with the BDD -notations of features, scenarios and steps. - -All possible steps can be found in the `steps` directory and should ideally -be documented. +the philosophy of BDD testing, read the Wikipedia article on +[Behaviour-driven development](https://en.wikipedia.org/wiki/Behavior-driven_development). ### General Usage To run the functional tests, do - cd test/bdd - behave - -The tests can be configured with a set of environment variables (`behave -D key=val`): - - * `TEMPLATE_DB` - name of template database used as a skeleton for - the test databases (db tests) - * `TEST_DB` - name of test database (db tests) - * `API_TEST_DB` - name of the database containing the API test data (api tests) - * `API_TEST_FILE` - OSM file to be imported into the API test database (api tests) - * `API_ENGINE` - webframe to use for running search queries, same values as - `nominatim serve --engine` parameter - * `DB_HOST` - (optional) hostname of database host - * `DB_PORT` - (optional) port of database on host - * `DB_USER` - (optional) username of database login - * `DB_PASS` - (optional) password for database login - * `REMOVE_TEMPLATE` - if true, the template and API database will not be reused - during the next run. Reusing the base templates speeds - up tests considerably but might lead to outdated errors - for some changes in the database layout. - * `KEEP_TEST_DB` - if true, the test database will not be dropped after a test - is finished. Should only be used if one single scenario is - run, otherwise the result is undefined. - -Logging can be defined through command line parameters of behave itself. Check -out `behave --help` for details. Also have a look at the 'work-in-progress' -feature of behave which comes in handy when writing new tests. + pytest test/bdd + +The BDD tests create databases for the tests. You can set name of the databases +through configuration variables in your `pytest.ini`: + + * `nominatim_test_db` defines the name of the temporary database created for + a single test (default: `test_nominatim`) + * `nominatim_api_test_db` defines the name of the database containing + the API test data, see also below (default: `test_api_nominatim`) + * `nominatim_template_db` defines the name of the template database used + for creating the temporary test databases. It contains some static setup + which usually doesn't change between imports of OSM data + (default: `test_template_nominatim`) + +To change other connection parameters for the PostgreSQL database, use +the [libpq enivronment variables](https://www.postgresql.org/docs/current/libpq-envars.html). +Never set a password through these variables. Use a +[password file](https://www.postgresql.org/docs/current/libpq-pgpass.html) instead. + +The API test database and the template database are only created once and then +left untouched. This is usually what you want because it speeds up subsequent +runs of BDD tests. If you do change code that has an influence on the content +of these databases, you can run pytest with the `--nominatim-purge` parameter +and the databases will be dropped and recreated from scratch. + +When running the BDD tests with make (using `make tests` or `make bdd`), then +the databases will always be purged. + +The temporary test database is usually dropped directly after the test, so +it does not take up unnecessary space. If you want to keep the database around, +for example while debugging a specific BDD test, use the parameter +`--nominatim-keep-db`. + ### API Tests (`test/bdd/api`) These tests are meant to test the different API endpoints and their parameters. They require to import several datasets into a test database. This is normally done automatically during setup of the test. The API test database is then -kept around and reused in subsequent runs of behave. Use `behave -DREMOVE_TEMPLATE` +kept around and reused in subsequent runs of behave. Use `--nominatim-purge` to force a reimport of the database. The official test dataset is saved in the file `test/testdb/apidb-test-data.pbf` @@ -109,12 +109,12 @@ test the correctness of osm2pgsql. Each test will write some data into the `plac table (and optionally the `planet_osm_*` tables if required) and then run Nominatim's processing functions on that. -These tests need to create their own test databases. By default they will be -called `test_template_nominatim` and `test_nominatim`. Names can be changed with -the environment variables `TEMPLATE_DB` and `TEST_DB`. The user running the tests -needs superuser rights for postgres. +These tests use the template database and create temporary test databases for +each test. ### Import Tests (`test/bdd/osm2pgsql`) -These tests check that data is imported correctly into the place table. They -use the same template database as the DB Creation tests, so the same remarks apply. +These tests check that data is imported correctly into the place table. + +These tests also use the template database and create temporary test databases +for each test. diff --git a/lib-lua/themes/nominatim/presets.lua b/lib-lua/themes/nominatim/presets.lua index aa51ac14..2bccc3af 100644 --- a/lib-lua/themes/nominatim/presets.lua +++ b/lib-lua/themes/nominatim/presets.lua @@ -187,7 +187,7 @@ module.MAIN_TAGS_POIS = function (group) passing_place = group, street_lamp = 'named', traffic_signals = 'named'}, - historic = {'always', + historic = {'fallback', yes = group, no = group}, information = {include_when_tag_present('tourism', 'information'), @@ -196,6 +196,7 @@ module.MAIN_TAGS_POIS = function (group) trail_blaze = 'never'}, junction = {'fallback', no = group}, + landuse = {cemetery = 'always'}, leisure = {'always', nature_reserve = 'fallback', swimming_pool = 'named', @@ -229,6 +230,7 @@ module.MAIN_TAGS_POIS = function (group) shop = {'always', no = group}, tourism = {'always', + attraction = 'fallback', no = group, yes = group, information = exclude_when_key_present('information')}, @@ -330,7 +332,7 @@ module.NAME_TAGS.core = {main = {'name', 'name:*', } module.NAME_TAGS.address = {house = {'addr:housename'}} module.NAME_TAGS.poi = group_merge({main = {'brand'}, - extra = {'iata', 'icao'}}, + extra = {'iata', 'icao', 'faa'}}, module.NAME_TAGS.core) -- Address tagging diff --git a/lib-sql/functions.sql b/lib-sql/functions.sql index 158969d9..737a3f21 100644 --- a/lib-sql/functions.sql +++ b/lib-sql/functions.sql @@ -8,7 +8,6 @@ {% include('functions/utils.sql') %} {% include('functions/ranking.sql') %} {% include('functions/importance.sql') %} -{% include('functions/address_lookup.sql') %} {% include('functions/interpolation.sql') %} {% if 'place' in db.tables %} diff --git a/lib-sql/functions/address_lookup.sql b/lib-sql/functions/address_lookup.sql deleted file mode 100644 index b59b7656..00000000 --- a/lib-sql/functions/address_lookup.sql +++ /dev/null @@ -1,334 +0,0 @@ --- SPDX-License-Identifier: GPL-2.0-only --- --- This file is part of Nominatim. (https://nominatim.org) --- --- Copyright (C) 2022 by the Nominatim developer community. --- For a full list of authors see the git log. - --- Functions for returning address information for a place. - -DROP TYPE IF EXISTS addressline CASCADE; -CREATE TYPE addressline as ( - place_id BIGINT, - osm_type CHAR(1), - osm_id BIGINT, - name HSTORE, - class TEXT, - type TEXT, - place_type TEXT, - admin_level INTEGER, - fromarea BOOLEAN, - isaddress BOOLEAN, - rank_address INTEGER, - distance FLOAT -); - - -CREATE OR REPLACE FUNCTION get_name_by_language(name hstore, languagepref TEXT[]) - RETURNS TEXT - AS $$ -DECLARE - result TEXT; -BEGIN - IF name is null THEN - RETURN null; - END IF; - - FOR j IN 1..array_upper(languagepref,1) LOOP - IF name ? languagepref[j] THEN - result := trim(name->languagepref[j]); - IF result != '' THEN - return result; - END IF; - END IF; - END LOOP; - - -- as a fallback - take the last element since it is the default name - RETURN trim((avals(name))[array_length(avals(name), 1)]); -END; -$$ -LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; - - ---housenumber only needed for tiger data -CREATE OR REPLACE FUNCTION get_address_by_language(for_place_id BIGINT, - housenumber INTEGER, - languagepref TEXT[]) - RETURNS TEXT - AS $$ -DECLARE - result TEXT[]; - currresult TEXT; - prevresult TEXT; - location RECORD; -BEGIN - - result := '{}'; - prevresult := ''; - - FOR location IN - SELECT name, - CASE WHEN place_id = for_place_id THEN 99 ELSE rank_address END as rank_address - FROM get_addressdata(for_place_id, housenumber) - WHERE isaddress order by rank_address desc - LOOP - currresult := trim(get_name_by_language(location.name, languagepref)); - IF currresult != prevresult AND currresult IS NOT NULL - AND result[(100 - location.rank_address)] IS NULL - THEN - result[(100 - location.rank_address)] := currresult; - prevresult := currresult; - END IF; - END LOOP; - - RETURN array_to_string(result,', '); -END; -$$ -LANGUAGE plpgsql STABLE PARALLEL SAFE; - -DROP TYPE IF EXISTS addressdata_place; -CREATE TYPE addressdata_place AS ( - place_id BIGINT, - country_code VARCHAR(2), - housenumber TEXT, - postcode TEXT, - class TEXT, - type TEXT, - name HSTORE, - address HSTORE, - centroid GEOMETRY -); - --- Compute the list of address parts for the given place. --- --- If in_housenumber is greator or equal 0, look for an interpolation. -CREATE OR REPLACE FUNCTION get_addressdata(in_place_id BIGINT, in_housenumber INTEGER) - RETURNS setof addressline - AS $$ -DECLARE - place addressdata_place; - location RECORD; - country RECORD; - current_rank_address INTEGER; - location_isaddress BOOLEAN; -BEGIN - -- The place in question might not have a direct entry in place_addressline. - -- Look for the parent of such places then and save it in place. - - -- first query osmline (interpolation lines) - IF in_housenumber >= 0 THEN - SELECT parent_place_id as place_id, country_code, - in_housenumber as housenumber, postcode, - 'place' as class, 'house' as type, - null as name, null as address, - ST_Centroid(linegeo) as centroid - INTO place - FROM location_property_osmline - WHERE place_id = in_place_id - AND in_housenumber between startnumber and endnumber; - END IF; - - --then query tiger data - {% if config.get_bool('USE_US_TIGER_DATA') %} - IF place IS NULL AND in_housenumber >= 0 THEN - SELECT parent_place_id as place_id, 'us' as country_code, - in_housenumber as housenumber, postcode, - 'place' as class, 'house' as type, - null as name, null as address, - ST_Centroid(linegeo) as centroid - INTO place - FROM location_property_tiger - WHERE place_id = in_place_id - AND in_housenumber between startnumber and endnumber; - END IF; - {% endif %} - - -- postcode table - IF place IS NULL THEN - SELECT parent_place_id as place_id, country_code, - null::text as housenumber, postcode, - 'place' as class, 'postcode' as type, - null as name, null as address, - null as centroid - INTO place - FROM location_postcode - WHERE place_id = in_place_id; - END IF; - - -- POI objects in the placex table - IF place IS NULL THEN - SELECT parent_place_id as place_id, country_code, - coalesce(address->'housenumber', - address->'streetnumber', - address->'conscriptionnumber')::text as housenumber, - postcode, - class, type, - name, address, - centroid - INTO place - FROM placex - WHERE place_id = in_place_id and rank_search > 27; - END IF; - - -- If place is still NULL at this point then the object has its own - -- entry in place_address line. However, still check if there is not linked - -- place we should be using instead. - IF place IS NULL THEN - select coalesce(linked_place_id, place_id) as place_id, country_code, - null::text as housenumber, postcode, - class, type, - null as name, address, - null as centroid - INTO place - FROM placex where place_id = in_place_id; - END IF; - ---RAISE WARNING '% % % %',searchcountrycode, searchhousenumber, searchpostcode; - - -- --- Return the record for the base entry. - - current_rank_address := 1000; - FOR location IN - SELECT placex.place_id, osm_type, osm_id, name, - coalesce(extratags->'linked_place', extratags->'place') as place_type, - class, type, admin_level, - CASE WHEN rank_address = 0 THEN 100 - WHEN rank_address = 11 THEN 5 - ELSE rank_address END as rank_address, - country_code - FROM placex - WHERE place_id = place.place_id - LOOP ---RAISE WARNING '%',location; - -- mix in default names for countries - IF location.rank_address = 4 and place.country_code is not NULL THEN - FOR country IN - SELECT coalesce(name, ''::hstore) as name FROM country_name - WHERE country_code = place.country_code LIMIT 1 - LOOP - place.name := country.name || place.name; - END LOOP; - END IF; - - IF location.rank_address < 4 THEN - -- no country locations for ranks higher than country - place.country_code := NULL::varchar(2); - ELSEIF place.country_code IS NULL AND location.country_code IS NOT NULL THEN - place.country_code := location.country_code; - END IF; - - RETURN NEXT ROW(location.place_id, location.osm_type, location.osm_id, - location.name, location.class, location.type, - location.place_type, - location.admin_level, true, - location.type not in ('postcode', 'postal_code'), - location.rank_address, 0)::addressline; - - current_rank_address := location.rank_address; - END LOOP; - - -- --- Return records for address parts. - - FOR location IN - SELECT placex.place_id, osm_type, osm_id, name, class, type, - coalesce(extratags->'linked_place', extratags->'place') as place_type, - admin_level, fromarea, isaddress and linked_place_id is NULL as isaddress, - CASE WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address, - distance, country_code, postcode - FROM place_addressline join placex on (address_place_id = placex.place_id) - WHERE place_addressline.place_id IN (place.place_id, in_place_id) - AND linked_place_id is null - AND (placex.country_code IS NULL OR place.country_code IS NULL - OR placex.country_code = place.country_code) - ORDER BY rank_address desc, - (place_addressline.place_id = in_place_id) desc, - (CASE WHEN coalesce((avals(name) && avals(place.address)), False) THEN 2 - WHEN isaddress THEN 0 - WHEN fromarea - and place.centroid is not null - and ST_Contains(geometry, place.centroid) THEN 1 - ELSE -1 END) desc, - fromarea desc, distance asc, rank_search desc - LOOP - -- RAISE WARNING '%',location; - location_isaddress := location.rank_address != current_rank_address; - - IF place.country_code IS NULL AND location.country_code IS NOT NULL THEN - place.country_code := location.country_code; - END IF; - IF location.type in ('postcode', 'postal_code') - AND place.postcode is not null - THEN - -- If the place had a postcode assigned, take this one only - -- into consideration when it is an area and the place does not have - -- a postcode itself. - IF location.fromarea AND location_isaddress - AND (place.address is null or not place.address ? 'postcode') - THEN - place.postcode := null; -- remove the less exact postcode - ELSE - location_isaddress := false; - END IF; - END IF; - RETURN NEXT ROW(location.place_id, location.osm_type, location.osm_id, - location.name, location.class, location.type, - location.place_type, - location.admin_level, location.fromarea, - location_isaddress, - location.rank_address, - location.distance)::addressline; - - current_rank_address := location.rank_address; - END LOOP; - - -- If no country was included yet, add the name information from country_name. - IF current_rank_address > 4 THEN - FOR location IN - SELECT name || coalesce(derived_name, ''::hstore) as name FROM country_name - WHERE country_code = place.country_code LIMIT 1 - LOOP ---RAISE WARNING '% % %',current_rank_address,searchcountrycode,countryname; - RETURN NEXT ROW(null, null, null, location.name, 'place', 'country', NULL, - null, true, true, 4, 0)::addressline; - END LOOP; - END IF; - - -- Finally add some artificial rows. - IF place.country_code IS NOT NULL THEN - location := ROW(null, null, null, hstore('ref', place.country_code), - 'place', 'country_code', null, null, true, false, 4, 0)::addressline; - RETURN NEXT location; - END IF; - - IF place.name IS NOT NULL THEN - location := ROW(in_place_id, null, null, place.name, place.class, - place.type, null, null, true, true, 29, 0)::addressline; - RETURN NEXT location; - END IF; - - IF place.housenumber IS NOT NULL THEN - location := ROW(null, null, null, hstore('ref', place.housenumber), - 'place', 'house_number', null, null, true, true, 28, 0)::addressline; - RETURN NEXT location; - END IF; - - IF place.address is not null and place.address ? '_unlisted_place' THEN - RETURN NEXT ROW(null, null, null, hstore('name', place.address->'_unlisted_place'), - 'place', 'locality', null, null, true, true, 25, 0)::addressline; - END IF; - - IF place.postcode is not null THEN - location := ROW(null, null, null, hstore('ref', place.postcode), 'place', - 'postcode', null, null, false, true, 5, 0)::addressline; - RETURN NEXT location; - ELSEIF place.address is not null and place.address ? 'postcode' - and not place.address->'postcode' SIMILAR TO '%(,|;)%' THEN - location := ROW(null, null, null, hstore('ref', place.address->'postcode'), 'place', - 'postcode', null, null, false, true, 5, 0)::addressline; - RETURN NEXT location; - END IF; - - RETURN; -END; -$$ -LANGUAGE plpgsql STABLE PARALLEL SAFE; diff --git a/lib-sql/tokenizer/icu_tokenizer.sql b/lib-sql/tokenizer/icu_tokenizer.sql index f0c30f1b..8cf13120 100644 --- a/lib-sql/tokenizer/icu_tokenizer.sql +++ b/lib-sql/tokenizer/icu_tokenizer.sql @@ -128,16 +128,14 @@ DECLARE partial_terms TEXT[] = '{}'::TEXT[]; term TEXT; term_id INTEGER; - term_count INTEGER; BEGIN SELECT min(word_id) INTO full_token FROM word WHERE word = norm_term and type = 'W'; IF full_token IS NULL THEN full_token := nextval('seq_word'); - INSERT INTO word (word_id, word_token, type, word, info) - SELECT full_token, lookup_term, 'W', norm_term, - json_build_object('count', 0) + INSERT INTO word (word_id, word_token, type, word) + SELECT full_token, lookup_term, 'W', norm_term FROM unnest(lookup_terms) as lookup_term; END IF; @@ -150,14 +148,67 @@ BEGIN partial_tokens := '{}'::INT[]; FOR term IN SELECT unnest(partial_terms) LOOP - SELECT min(word_id), max(info->>'count') INTO term_id, term_count + SELECT min(word_id) INTO term_id + FROM word WHERE word_token = term and type = 'w'; + + IF term_id IS NULL THEN + term_id := nextval('seq_word'); + INSERT INTO word (word_id, word_token, type) + VALUES (term_id, term, 'w'); + END IF; + + partial_tokens := array_merge(partial_tokens, ARRAY[term_id]); + END LOOP; +END; +$$ +LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION getorcreate_full_word(norm_term TEXT, + lookup_terms TEXT[], + lookup_norm_terms TEXT[], + OUT full_token INT, + OUT partial_tokens INT[]) + AS $$ +DECLARE + partial_terms TEXT[] = '{}'::TEXT[]; + term TEXT; + term_id INTEGER; +BEGIN + SELECT min(word_id) INTO full_token + FROM word WHERE word = norm_term and type = 'W'; + + IF full_token IS NULL THEN + full_token := nextval('seq_word'); + IF lookup_norm_terms IS NULL THEN + INSERT INTO word (word_id, word_token, type, word) + SELECT full_token, lookup_term, 'W', norm_term + FROM unnest(lookup_terms) as lookup_term; + ELSE + INSERT INTO word (word_id, word_token, type, word, info) + SELECT full_token, t.lookup, 'W', norm_term, + CASE WHEN norm_term = t.norm THEN null + ELSE json_build_object('lookup', t.norm) END + FROM unnest(lookup_terms, lookup_norm_terms) as t(lookup, norm); + END IF; + END IF; + + FOR term IN SELECT unnest(string_to_array(unnest(lookup_terms), ' ')) LOOP + term := trim(term); + IF NOT (ARRAY[term] <@ partial_terms) THEN + partial_terms := partial_terms || term; + END IF; + END LOOP; + + partial_tokens := '{}'::INT[]; + FOR term IN SELECT unnest(partial_terms) LOOP + SELECT min(word_id) INTO term_id FROM word WHERE word_token = term and type = 'w'; IF term_id IS NULL THEN term_id := nextval('seq_word'); - term_count := 0; - INSERT INTO word (word_id, word_token, type, info) - VALUES (term_id, term, 'w', json_build_object('count', term_count)); + INSERT INTO word (word_id, word_token, type) + VALUES (term_id, term, 'w'); END IF; partial_tokens := array_merge(partial_tokens, ARRAY[term_id]); diff --git a/packaging/nominatim-api/pyproject.toml b/packaging/nominatim-api/pyproject.toml index 601029ca..14b23cd2 100644 --- a/packaging/nominatim-api/pyproject.toml +++ b/packaging/nominatim-api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nominatim-api" -version = "5.0.0.post7" +version = "5.1.0.post8" description = "A tool for building a database of OpenStreetMap for geocoding and for searching the database. Search library." readme = "README.md" requires-python = ">=3.7" @@ -16,9 +16,9 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "SQLAlchemy==2.0.39", + "SQLAlchemy==2.0.41", "falcon==4.0.2", - "uvicorn==0.34.0", + "uvicorn==0.34.3", "gunicorn==23.0.0" ] diff --git a/packaging/nominatim-db/pyproject.toml b/packaging/nominatim-db/pyproject.toml index 0c9a7055..09494a10 100644 --- a/packaging/nominatim-db/pyproject.toml +++ b/packaging/nominatim-db/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nominatim-db" -version = "5.0.0.post7" +version = "5.1.0.post8" description = "A tool for building a database of OpenStreetMap for geocoding and for searching the database. Database backend." readme = "README.md" requires-python = ">=3.7" @@ -16,12 +16,12 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "psycopg[binary]==3.2.6", - "python-dotenv==1.0.1", + "psycopg[binary]==3.2.9", + "python-dotenv==1.1.0", "jinja2==3.1.6", "pyYAML==6.0.2", "psutil==7.0.0", - "PyICU==2.14", + "PyICU==2.15.2", "osmium==4.0.2", ] diff --git a/settings/country_settings.yaml b/settings/country_settings.yaml index a2ca7412..88ace911 100644 --- a/settings/country_settings.yaml +++ b/settings/country_settings.yaml @@ -944,7 +944,7 @@ kp: # South Korea (대한민국) kr: partition: 49 - languages: ko, en + languages: ko names: !include country-names/kr.yaml postcode: pattern: "ddddd" diff --git a/settings/env.defaults b/settings/env.defaults index b8c66667..3ebb288f 100644 --- a/settings/env.defaults +++ b/settings/env.defaults @@ -192,6 +192,13 @@ NOMINATIM_REQUEST_TIMEOUT=60 # to geocode" instead. NOMINATIM_SEARCH_WITHIN_COUNTRIES=False +# Specifies the order in which different name tags are used. +# The values in this list determine the preferred order of name variants, +# including language-specific names. +# Comma-separated list, where :XX stands for language-specific tags +# (e.g. name:en) and no :XX stands for general tags (e.g. name). +NOMINATIM_OUTPUT_NAMES=name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref + ### Log settings # # The following options allow to enable logging of API requests. diff --git a/settings/icu-rules/variants-en.yaml b/settings/icu-rules/variants-en.yaml index 99cd6da6..54a7b475 100644 --- a/settings/icu-rules/variants-en.yaml +++ b/settings/icu-rules/variants-en.yaml @@ -6,25 +6,19 @@ - Air Force Base -> AFB - Air National Guard Base -> ANGB - Airport -> Aprt - - Alley -> Al - - Alley -> All - - Alley -> Ally - - Alley -> Aly + - Alley -> Al,All,Ally,Aly - Alleyway -> Alwy - Amble -> Ambl - Anex -> Anx - Apartments -> Apts - - Approach -> Apch - - Approach -> App + - Approach -> Apch,App - Arcade -> Arc - Arterial -> Artl - Artery -> Arty - - Avenue -> Av - - Avenue -> Ave + - Avenue -> Av,Ave - Back -> Bk - Banan -> Ba - - Basin -> Basn - - Basin -> Bsn + - Basin -> Basn,Bsn - Bayou -> Byu - Beach -> Bch - Bend -> Bnd @@ -33,71 +27,51 @@ - Bluffs -> Blfs - Boardwalk -> Bwlk - Bottom -> Btm - - Boulevard -> Blvd - - Boulevard -> Bvd + - Boulevard -> Blvd,Bvd - Boundary -> Bdy - Bowl -> Bl - Brace -> Br - Brae -> Br - Branch -> Br - Break -> Brk - - Bridge -> Bdge - - Bridge -> Br - - Bridge -> Brdg - - Bridge -> Brg - - Bridge -> Bri - - Broadway -> Bdwy - - Broadway -> Bway - - Broadway -> Bwy + - Bridge$ -> Bdge,Br,Brdg,Brg,Bri + - Broadway -> Bdwy,Bway,Bwy - Brook -> Brk - Brooks -> Brks - Brow -> Brw - - Buildings -> Bldgs - - Buildings -> Bldngs + - Buildings -> Bldgs,Bldngs - Business -> Bus - Burg -> Bg - Burgs -> Bgs - - Bypass -> Bps - - Bypass -> Byp - - Bypass -> Bypa + - Bypass -> Bps,Byp,Bypa - Byway -> Bywy - Camp -> Cp - Canyon -> Cyn - Cape -> Cpe - Caravan -> Cvn - - Causeway -> Caus - - Causeway -> Cswy - - Causeway -> Cway - - Center -> Cen - - Center -> Ctr + - Causeway -> Caus,Cswy,Cway + - Center,Centre -> Cen,Ctr - Centers -> Ctrs - Central -> Ctrl - - Centre -> Cen - - Centre -> Ctr - Centreway -> Cnwy - Chase -> Ch - Church -> Ch - Circle -> Cir - Circles -> Cirs - - Circuit -> Cct - - Circuit -> Ci - - Circus -> Crc - - Circus -> Crcs + - Circuit -> Cct,Ci + - Circus -> Crc,Crcs - City -> Cty - Cliff -> Clf - Cliffs -> Clfs - Close -> Cl - Club -> Clb - - Common -> Cmn - - Common -> Comm + - Common -> Cmn,Comm - Commons -> Cmns - Community -> Comm - Concourse -> Cnc - Concourse -> Con - Copse -> Cps - - Corner -> Cor - - Corner -> Cnr - - Corner -> Crn + - Corner -> Cor,Cnr,Crn - Corners -> Cors - Corso -> Cso - Cottages -> Cotts @@ -105,36 +79,24 @@ - County Road -> CR - County Route -> CR - Course -> Crse - - Court -> Crt - - Court -> Ct + - Court -> Crt,Ct - Courts -> Cts - Courtyard -> Cyd - Courtyard -> Ctyd - - Cove -> Ce - - Cove -> Cov - - Cove -> Cv + - Cove$ -> Ce,Cov,Cv - Coves -> Cvs - - Creek -> Ck - - Creek -> Cr - - Creek -> Crk + - Creek$ -> Ck,Cr,Crk - Crescent -> Cr - Crescent -> Cres - - Crest -> Crst - - Crest -> Cst + - Crest -> Crst,Cst - Croft -> Cft - - Cross -> Cs - - Cross -> Crss - - Crossing -> Crsg - - Crossing -> Csg - - Crossing -> Xing - - Crossroad -> Crd - - Crossroad -> Xrd + - Cross -> Cs,Crss + - Crossing -> Crsg,Csg,Xing + - Crossroad -> Crd,Xrd - Crossroads -> Xrds - Crossway -> Cowy - - Cul-de-sac -> Cds - - Cul-de-sac -> Csac - - Curve -> Cve - - Curve -> Curv + - Cul-de-sac -> Cds,Csac + - Curve -> Cve,Curv - Cutting -> Cutt - Dale -> Dle - Dam -> Dm @@ -143,14 +105,10 @@ - Divide -> Dv - Down -> Dn - Downs -> Dn - - Drive -> Dr - - Drive -> Drv - - Drive -> Dv + - Drive -> Dr,Drv,Dv - Drives -> Drs - Drive-In => Drive-In # prevent abbreviation here - - Driveway -> Drwy - - Driveway -> Dvwy - - Driveway -> Dwy + - Driveway -> Drwy,Dvwy,Dwy - East -> E - Edge -> Edg - Elbow -> Elb @@ -158,25 +116,18 @@ - Esplanade -> Esp - Estate -> Est - Estates -> Ests - - Expressway -> Exp - - Expressway -> Expy - - Expressway -> Expwy - - Expressway -> Xway + - Expressway -> Exp,Expy,Expwy,Xway - Extension -> Ex - Extensions -> Exts - - Fairway -> Fawy - - Fairway -> Fy + - Fairway -> Fawy,Fy - Falls -> Fls - Father -> Fr - - Ferry -> Fy - - Ferry -> Fry - - Field -> Fd - - Field -> Fld + - Ferry -> Fy,Fry + - Field -> Fd,Fld - Fields -> Flds - Fire Track -> Ftrk - Firetrail -> Fit - - Flat -> Fl - - Flat -> Flt + - Flat -> Fl,Flt - Flats -> Flts - Follow -> Folw - Footway -> Ftwy @@ -191,67 +142,47 @@ - Fork -> Frk - Forks -> Frks - Fort -> Ft - - Freeway -> Frwy - - Freeway -> Fwy + - Freeway -> Frwy,Fwy - Front -> Frnt - - Frontage -> Fr - - Frontage -> Frtg + - Frontage -> Fr,Frtg - Garden -> Gdn - - Gardens -> Gdn - - Gardens -> Gdns - - Gate -> Ga - - Gate -> Gte - - Gates -> Ga - - Gates -> Gte - - Gateway -> Gwy - - Gateway -> Gtwy + - Gardens -> Gdn,Gdns + - Gate,Gates -> Ga,Gte + - Gateway -> Gwy,Gtwy - George -> Geo - - Glade -> Gl - - Glade -> Gld - - Glade -> Glde + - Glade$ -> Gl,Gld,Glde - Glen -> Gln - Glens -> Glns - Grange -> Gra - - Green -> Gn - - Green -> Grn + - Green -> Gn,Grn - Greens -> Grns - Ground -> Grnd - - Grove -> Gr - - Grove -> Gro - - Grove -> Grv + - Grove$ -> Gr,Gro,Grv - Groves -> Grvs - Grovet -> Gr - Gully -> Gly - - Harbor -> Hbr + - Harbor -> Hbr,Harbour - Harbors -> Hbrs - - Harbour -> Hbr + - Harbour -> Hbr,Harbor - Haven -> Hvn - Head -> Hd - Heads -> Hd - - Heights -> Hgts - - Heights -> Ht - - Heights -> Hts + - Heights -> Hgts,Ht,Hts - High School -> HS - - Highroad -> Hird - - Highroad -> Hrd + - Highroad -> Hird,Hrd - Highway -> Hwy - Hill -> Hl - - Hills -> Hl - - Hills -> Hls + - Hills -> Hl,Hls - Hollow -> Holw - Hospital -> Hosp - - House -> Ho - - House -> Hse + - House -> Ho,Hse - Industrial -> Ind - Inlet -> Inlt - Interchange -> Intg - International -> Intl - - Island -> I - - Island -> Is + - Island -> I,Is - Islands -> Iss - - Junction -> Jct - - Junction -> Jctn - - Junction -> Jnc + - Junction -> Jct,Jctn,Jnc - Junctions -> Jcts - Junior -> Jr - Key -> Ky @@ -260,40 +191,31 @@ - Knolls -> Knls - Lagoon -> Lgn - Lake -> Lk - - Lakes -> L - - Lakes -> Lks - - Landing -> Ldg - - Landing -> Lndg - - Lane -> La - - Lane -> Ln + - Lakes -> L,Lks + - Landing -> Ldg,Lndg + - Lane -> La,Ln - Laneway -> Lnwy - Light -> Lgt - Lights -> Lgts - Line -> Ln - Link -> Lk - - Little -> Lit - - Little -> Lt + - Little -> Lit,Lt - Loaf -> Lf - Lock -> Lck - Locks -> Lcks - Lodge -> Ldg - Lookout -> Lkt - Loop -> Lp - - Lower -> Low - - Lower -> Lr - - Lower -> Lwr + - Lower -> Low,Lr,Lwr - Mall -> Ml - Manor -> Mnr - Manors -> Mnrs - Mansions -> Mans - Market -> Mkt - Meadow -> Mdw - - Meadows -> Mdw - - Meadows -> Mdws + - Meadows -> Mdw,Mdws - Mead -> Md - - Meander -> Mdr - - Meander -> Mndr - - Meander -> Mr + - Meander -> Mdr,Mndr,Mr - Medical -> Med - Memorial -> Mem - Mews -> Mw @@ -304,12 +226,10 @@ - Mill -> Ml - Mills -> Mls - Mission -> Msn - - Motorway -> Mtwy - - Motorway -> Mwy + - Motorway -> Mtwy,Mwy - Mount -> Mt - Mountain -> Mtn - - Mountains -> Mtn - - Mountains -> Mtns + - Mountains$ -> Mtn,Mtns - Municipal -> Mun - Museum -> Mus - National Park -> NP @@ -321,50 +241,37 @@ - Northeast -> NE - Northwest -> NW - Orchard -> Orch - - Outlook -> Out - - Outlook -> Otlk + - Outlook -> Out,Otlk - Overpass -> Opas - Parade -> Pde - Paradise -> Pdse - Park -> Pk - Parklands -> Pkld - - Parkway -> Pkwy - - Parkway -> Pky - - Parkway -> Pwy + - Parkway -> Pkwy,Pky,Pwy - Parkways -> Pkwy - Pass -> Ps - Passage -> Psge - - Pathway -> Phwy - - Pathway -> Pway - - Pathway -> Pwy + - Pathway -> Phwy,Pway,Pwy - Piazza -> Piaz - Pike -> Pk - Pine -> Pne - Pines -> Pnes - Place -> Pl - - Plain -> Pl - - Plain -> Pln - - Plains -> Pl - - Plains -> Plns + - Plain -> Pl,Pln + - Plains -> Pl,Plns - Plateau -> Plat - - Plaza -> Pl - - Plaza -> Plz - - Plaza -> Plza + - Plaza -> Pl,Plz,Plza - Pocket -> Pkt - - Point -> Pnt - - Point -> Pt + - Point -> Pnt,Pt - Points -> Pts - - Port -> Prt - - Port -> Pt + - Port -> Prt,Pt - Ports -> Prts - Post Office -> PO - Prairie -> Pr - Precinct -> Pct - - Promenade -> Prm - - Promenade -> Prom + - Promenade -> Prm,Prom - Quadrangle -> Qdgl - - Quadrant -> Qdrt - - Quadrant -> Qd + - Quadrant -> Qdrt,Qd - Quay -> Qy - Quays -> Qy - Quays -> Qys @@ -372,8 +279,7 @@ - Ramble -> Ra - Ramble -> Rmbl - Ranch -> Rnch - - Range -> Rge - - Range -> Rnge + - Range -> Rge,Rnge - Rapid -> Rpd - Rapids -> Rpds - Reach -> Rch @@ -381,37 +287,31 @@ - Reserve -> Res - Reservoir -> Res - Rest -> Rst - - Retreat -> Rt - - Retreat -> Rtt + - Retreat -> Rt,Rtt - Return -> Rtn - - Ridge -> Rdg - - Ridge -> Rdge + - Ridge -> Rdg,Rdge - Ridges -> Rdgs - Ridgeway -> Rgwy - Right of Way -> Rowy - Rise -> Ri - - River -> R - - River -> Riv - - River -> Rvr + - ^River -> R,Riv,Rvr + - River$ -> R,Riv,Rvr - Riverway -> Rvwy - Riviera -> Rvra - Road -> Rd - Roads -> Rds - Roadside -> Rdsd - - Roadway -> Rdwy - - Roadway -> Rdy + - Roadway -> Rdwy,Rdy - Rocks -> Rks - Ronde -> Rnde - Rosebowl -> Rsbl - Rotary -> Rty - Round -> Rnd - - Route -> Rt - - Route -> Rte + - Route -> Rt,Rte - Saint -> St - Saints -> SS - Senior -> Sr - - Serviceway -> Swy - - Serviceway -> Svwy + - Serviceway -> Swy,Svwy - Shoal -> Shl - Shore -> Shr - Shores -> Shrs @@ -421,8 +321,7 @@ - Skyway -> Skwy - Slope -> Slpe - Sound -> Snd - - South -> S - - South -> Sth + - South -> S,Sth - Southeast -> SE - Southwest -> SW - Spring -> Spg @@ -431,13 +330,10 @@ - Square -> Sq - Squares -> Sqs - Stairway -> Strwy - - State Highway -> SH - - State Highway -> SHwy + - State Highway -> SH,SHwy - State Route -> SR - - Station -> Sta - - Station -> Stn - - Strand -> Sd - - Strand -> Stra + - Station -> Sta,Stn + - Strand -> Sd,Stra - Stravenue -> Stra - Stream -> Strm - Street -> St @@ -447,61 +343,43 @@ - Summit -> Smt - Tarn -> Tn - Terminal -> Term - - Terrace -> Tce - - Terrace -> Ter - - Terrace -> Terr - - Thoroughfare -> Thfr - - Thoroughfare -> Thor + - Terrace -> Tce,Ter,Terr + - Thoroughfare -> Thfr,Thor - Throughway -> Trwy - - Tollway -> Tlwy - - Tollway -> Twy + - Tollway -> Tlwy,Twy - Towers -> Twrs - Township -> Twp - Trace -> Trce - - Track -> Tr - - Track -> Trak - - Track -> Trk + - Track -> Tr,Trak,Trk - Trafficway -> Trfy - Trail -> Trl - Trailer -> Trlr - Triangle -> Tri - Trunkway -> Tkwy - - Tunnel -> Tun - - Tunnel -> Tunl - - Turn -> Tn - - Turn -> Trn - - Turnpike -> Tpk - - Turnpike -> Tpke - - Underpass -> Upas - - Underpass -> Ups + - Tunnel -> Tun,Tunl + - Turn -> Tn,Trn + - Turnpike -> Tpk,Tpke + - Underpass -> Upas,Ups - Union -> Un - Unions -> Uns - - University -> Uni - - University -> Univ + - University -> Uni,Univ - Upper -> Up - Upper -> Upr - Vale -> Va - Valley -> Vly - Valley -> Vy - Valleys -> Vlys - - Viaduct -> Vdct - - Viaduct -> Via - - Viaduct -> Viad + - Viaduct$ -> Vdct,Via,Viad - View -> Vw - Views -> Vws - - Village -> Vill - - Village -> Vlg + - Village -> Vill,Vlg - Villages -> Vlgs - Villas -> Vlls - Ville -> Vl - - Vista -> Vis - - Vista -> Vst - - Vista -> Vsta - - Walk -> Wk - - Walk -> Wlk + - Vista -> Vis,Vst,Vsta + - Walk -> Wk,Wlk - Walks -> Walk - - Walkway -> Wkwy - - Walkway -> Wky + - Walkway -> Wkwy,Wky - Waters -> Wtr - Way -> Wy - Well -> Wl diff --git a/src/nominatim_api/localization.py b/src/nominatim_api/localization.py index bbf9225b..3414286e 100644 --- a/src/nominatim_api/localization.py +++ b/src/nominatim_api/localization.py @@ -8,6 +8,7 @@ Helper functions for localizing names of results. """ from typing import Mapping, List, Optional +from .config import Configuration import re @@ -20,14 +21,18 @@ class Locales: """ def __init__(self, langs: Optional[List[str]] = None): + self.config = Configuration(None) self.languages = langs or [] self.name_tags: List[str] = [] - # Build the list of supported tags. It is currently hard-coded. - self._add_lang_tags('name') - self._add_tags('name', 'brand') - self._add_lang_tags('official_name', 'short_name') - self._add_tags('official_name', 'short_name', 'ref') + parts = self.config.OUTPUT_NAMES.split(',') + + for part in parts: + part = part.strip() + if part.endswith(":XX"): + self._add_lang_tags(part[:-3]) + else: + self._add_tags(part) def __bool__(self) -> bool: return len(self.languages) > 0 diff --git a/src/nominatim_api/logging.py b/src/nominatim_api/logging.py index 1a6aef9b..64d43fdc 100644 --- a/src/nominatim_api/logging.py +++ b/src/nominatim_api/logging.py @@ -342,7 +342,8 @@ HTML_HEADER: str = """ Nominatim - Debug