From: Sarah Hoffmann Date: Wed, 28 May 2025 12:13:49 +0000 (+0200) Subject: release 5.1.0.post7 X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/HEAD?hp=84bac1bb4df234d0c25d94fb1abbf9289d2d55c1 release 5.1.0.post7 --- diff --git a/.flake8 b/.flake8 index 82d77ed3..1aae19dc 100644 --- a/.flake8 +++ b/.flake8 @@ -6,3 +6,6 @@ extend-ignore = E711 per-file-ignores = __init__.py: F401 + test/python/utils/test_json_writer.py: E131 + **/conftest.py: E402 + test/bdd/*: F821 diff --git a/.github/actions/setup-postgresql/action.yml b/.github/actions/setup-postgresql/action.yml index 331d094a..7a9590c1 100644 --- a/.github/actions/setup-postgresql/action.yml +++ b/.github/actions/setup-postgresql/action.yml @@ -11,10 +11,8 @@ runs: steps: - name: Remove existing PostgreSQL run: | + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y sudo apt-get purge -yq postgresql* - sudo apt install curl ca-certificates gnupg - curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null - sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' sudo apt-get update -qq shell: bash diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 8b38d3dc..4d555416 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -37,10 +37,10 @@ jobs: needs: create-archive strategy: matrix: - flavour: ["ubuntu-20", "ubuntu-24"] + flavour: ["ubuntu-22", "ubuntu-24"] include: - - flavour: ubuntu-20 - ubuntu: 20 + - flavour: ubuntu-22 + ubuntu: 22 postgresql: 12 lua: '5.1' dependencies: pip @@ -68,26 +68,34 @@ jobs: with: dependencies: ${{ matrix.dependencies }} + - uses: actions/cache@v4 + with: + path: | + /usr/local/bin/osm2pgsql + key: osm2pgsql-bin-22-1 + if: matrix.ubuntu == '22' + - name: Compile osm2pgsql run: | - sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua-dkjson nlohmann-json3-dev - mkdir osm2pgsql-build - cd osm2pgsql-build - git clone https://github.com/osm2pgsql-dev/osm2pgsql - mkdir build - cd build - cmake ../osm2pgsql - make - sudo make install - cd ../.. - rm -rf osm2pgsql-build - if: matrix.ubuntu == '20' + if [ ! -f /usr/local/bin/osm2pgsql ]; then + sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua-dkjson nlohmann-json3-dev + mkdir osm2pgsql-build + cd osm2pgsql-build + git clone https://github.com/osm2pgsql-dev/osm2pgsql + mkdir build + cd build + cmake ../osm2pgsql + make + sudo make install + cd ../.. + rm -rf osm2pgsql-build + else + sudo apt-get install -y -qq libexpat1 liblua${LUA_VERSION} + fi + if: matrix.ubuntu == '22' env: LUA_VERSION: ${{ matrix.lua }} - - name: Install test prerequisites - run: ./venv/bin/pip install behave==1.2.6 - - name: Install test prerequisites (apt) run: sudo apt-get install -y -qq python3-pytest python3-pytest-asyncio uvicorn python3-falcon python3-aiosqlite python3-pyosmium if: matrix.dependencies == 'apt' @@ -96,11 +104,14 @@ jobs: run: ./venv/bin/pip install pytest-asyncio falcon starlette asgi_lifespan aiosqlite osmium uvicorn if: matrix.dependencies == 'pip' + - name: Install test prerequisites + run: ./venv/bin/pip install pytest-bdd + - name: Install latest flake8 run: ./venv/bin/pip install -U flake8 - name: Python linting - run: ../venv/bin/python -m flake8 src + run: ../venv/bin/python -m flake8 src test/python test/bdd working-directory: Nominatim - name: Install mypy and typechecking info @@ -118,8 +129,8 @@ jobs: - name: BDD tests run: | - ../../../venv/bin/python -m behave -DREMOVE_TEMPLATE=1 --format=progress3 - working-directory: Nominatim/test/bdd + ../venv/bin/python -m pytest test/bdd --nominatim-purge + working-directory: Nominatim install: runs-on: ubuntu-latest @@ -185,9 +196,6 @@ jobs: - name: Prepare import environment run: | mv Nominatim/test/testdb/apidb-test-data.pbf test.pbf - mv Nominatim/settings/flex-base.lua flex-base.lua - mv Nominatim/settings/import-extratags.lua import-extratags.lua - mv Nominatim/settings/taginfo.lua taginfo.lua rm -rf Nominatim mkdir data-env-reverse working-directory: /home/nominatim @@ -205,7 +213,7 @@ jobs: working-directory: /home/nominatim/nominatim-project - name: Print taginfo - run: lua taginfo.lua + run: lua ./nominatim-venv/lib/*/site-packages/nominatim_db/resources/lib-lua/taginfo.lua working-directory: /home/nominatim - name: Collect host OS information diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b9bf2920..6c90cd3c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,7 +87,6 @@ Checklist for releases: * [ ] increase versions in * `src/nominatim_api/version.py` * `src/nominatim_db/version.py` - * CMakeLists.txt * [ ] update `ChangeLog` (copy information from patch releases from release branch) * [ ] complete `docs/admin/Migration.md` * [ ] update EOL dates in `SECURITY.md` @@ -114,3 +113,5 @@ Checklist for releases: * run `nominatim --version` to confirm correct version * [ ] tag new release and add a release on github.com * [ ] build pip packages and upload to pypi + * `make build` + * `twine upload dist/*` diff --git a/ChangeLog b/ChangeLog index b7609255..dff198eb 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,47 @@ +5.1.0 + * replace datrie with simple internal trie implementation + * add pattern-based postcode parser for queries, + postcodes no longer need to be present in OSM to be found + * take variants into account when computing token similarity + * add extratags output to geocodejson format + * fix default layer setting used for structured queries + * update abbreviation lists for Russian and English + (thanks @shoorick, @IvanShift, @mhsrn21) + * fix variant generation for Norwegian + * fix normalization around space-like characters + * improve postcode search and handling of postcodes in queries + * reorganise internal query structure and get rid of slow enums + * enable code linting for tests + * various code moderinsations in test code (thanks @eumiro) + * remove setting osm2pgsql location via config.lib_dir + * make SQL functions parallel save as far as possible (thanks @otbutz) + * various fixes and improvements to documentation (thanks @TuringVerified) + +5.0.0 + * increase required versions for PostgreSQL (12+), PostGIS (3.0+) + * remove installation via cmake and debundle osm2pgsql + * remove deprecated PHP frontend + * remove deprecated legacy tokenizer + * add configurable pre-processing of queries + * add query pre-processor to split up Japanese addresses + * rewrite of osm2pgsql style implementation + (also adds support for osm2pgsql-themepark) + * reduce the number of SQL queries needed to complete a 'lookup' call + * improve computation of centroid for lines with only two points + * improve bbox output for postcode areas + * improve result order by returning the largest object when other things are + equal + * add fallback for reverse geocoding to default country tables + * exclude postcode areas from reverse geocoding + * disable search endpoint when database is reverse-only (regression) + * minor performance improvements to area split algorithm + * switch table and index creation to use autocommit mode to avoid deadlocks + * drop overly long ways during import + * restrict automatic migrations to versions 4.3+ + * switch linting from pylint to flake8 + * switch tests to use a wikimedia test file in the new CSV style + * various fixes and improvements to documentation + 4.5.0 * allow building Nominatim as a pip package * make osm2pgsql building optional diff --git a/Makefile b/Makefile index 9e914850..d6423add 100644 --- a/Makefile +++ b/Makefile @@ -24,10 +24,10 @@ pytest: pytest test/python lint: - flake8 src + flake8 src test/python test/bdd bdd: - cd test/bdd; behave -DREMOVE_TEMPLATE=1 + pytest test/bdd --nominatim-purge # Documentation diff --git a/README.md b/README.md index 3b0f328a..54bdbf8e 100644 --- a/README.md +++ b/README.md @@ -27,18 +27,25 @@ can be found at nominatim.org as well. A quick summary of the necessary steps: -1. Create a Python virtualenv and install the packages: + +1. Clone this git repository and download the country grid + + git clone https://github.com/osm-search/Nominatim.git + wget -O Nominatim/data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz + +2. Create a Python virtualenv and install the packages: python3 -m venv nominatim-venv ./nominatim-venv/bin/pip install packaging/nominatim-{api,db} -2. Create a project directory, get OSM data and import: +3. Create a project directory, get OSM data and import: mkdir nominatim-project cd nominatim-project - ../nominatim-venv/bin/nominatim import --osm-file + ../nominatim-venv/bin/nominatim import --osm-file 2>&1 | tee setup.log + -3. Start the webserver: +4. Start the webserver: ./nominatim-venv/bin/pip install uvicorn falcon ../nominatim-venv/bin/nominatim serve diff --git a/SECURITY.md b/SECURITY.md index 3ec22cbd..98295e1f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -9,10 +9,11 @@ versions. | Version | End of support for security updates | | ------- | ----------------------------------- | +| 5.1.x | 2027-04-01 | +| 5.0.x | 2027-02-06 | | 4.5.x | 2026-09-12 | | 4.4.x | 2026-03-07 | | 4.3.x | 2025-09-07 | -| 4.2.x | 2024-11-24 | ## Reporting a Vulnerability diff --git a/docs/admin/Deployment-Python.md b/docs/admin/Deployment-Python.md index 5932f4e6..463e15c0 100644 --- a/docs/admin/Deployment-Python.md +++ b/docs/admin/Deployment-Python.md @@ -37,7 +37,7 @@ cd Nominatim ``` The recommended way to deploy a Python ASGI application is to run -the ASGI runner [uvicorn](https://uvicorn.org/) +the ASGI runner [uvicorn](https://www.uvicorn.org/) together with [gunicorn](https://gunicorn.org/) HTTP server. We use Falcon here as the web framework. diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index dadbe460..2571de5d 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -26,7 +26,7 @@ For running Nominatim: * [PostgreSQL](https://www.postgresql.org) (12+ will work, 13+ strongly recommended) * [PostGIS](https://postgis.net) (3.0+ will work, 3.2+ strongly recommended) - * [osm2pgsql](https://osm2pgsql.org) (1.8+, optional when building with CMake) + * [osm2pgsql](https://osm2pgsql.org) (1.8+) * [Python 3](https://www.python.org/) (3.7+) Furthermore the following Python libraries are required: @@ -37,7 +37,6 @@ Furthermore the following Python libraries are required: * [Jinja2](https://palletsprojects.com/p/jinja/) * [PyICU](https://pypi.org/project/PyICU/) * [PyYaml](https://pyyaml.org/) (5.1+) - * [datrie](https://github.com/pytries/datrie) These will be installed automatically when using pip installation. @@ -102,6 +101,15 @@ you might consider setting: and even reduce `autovacuum_work_mem` further. This will reduce the amount of memory that autovacuum takes away from the import process. +## Installing the latest release + +Nominatim is easiest installed directly from Pypi. Make sure you have installed +osm2pgsql, PostgreSQL/PostGIS and libICU together with its header files. + +Then you can install Nominatim with: + + pip install nominatim-db nominatim-api + ## Downloading and building Nominatim ### Downloading the latest release diff --git a/docs/admin/Migration.md b/docs/admin/Migration.md index 13e6d7f5..becf2f7b 100644 --- a/docs/admin/Migration.md +++ b/docs/admin/Migration.md @@ -9,19 +9,15 @@ the following steps: * Update the frontend: `pip install -U nominatim-api` * (optionally) Restart updates -If you are still using CMake for the installation of Nominatim, then you -need to update the software in one step before migrating the database. -It is not recommended to do this while the machine is serving requests. - Below you find additional migrations and hints about other structural and breaking changes. **Please read them before running the migration.** !!! note If you are migrating from a version <4.3, you need to install 4.3 - first and migrate to 4.3 first. Then you can migrate to the current + and migrate to 4.3 first. Then you can migrate to the current version. It is strongly recommended to do a reimport instead. -## 4.5.0 -> master +## 4.5.0 -> 5.0.0 ### PHP frontend removed @@ -33,6 +29,42 @@ needed. It currently omits a warning and does otherwise nothing. It will be removed in later versions of Nominatim. So make sure you remove it from your scripts. +### CMake building removed + +Nominatim can now only be installed via pip. Please follow the installation +instructions for the current version to change to pip. + +### osm2pgsql no longer vendored in + +Nominatim no longer ships its own version of osm2pgsql. Please install a +stock version of osm2pgsql from your distribution. See the +[installation instruction for osm2pgsql](https://osm2pgsql.org/doc/install.html) +for details. A minimum version of 1.8 is required. The current stable versions +of Ubuntu and Debian already ship with an appropriate versions. For older +installation, you may have to compile a newer osm2pgsql yourself. + +### Legacy tokenizer removed + +The `legacy` tokenizer is no longer enabled. This tokenizer has been superseded +by the `ICU` tokenizer a long time ago. In the unlikely case that your database +still uses the `legacy` tokenizer, you must reimport your database. + +### osm2pgsql style overhauled + +There are some fundamental changes to how customized osm2pgsql styles should +be written. The changes are mostly backwards compatible, i.e. custom styles +should still work with the new implementation. The only exception is a +customization of the `process_tags()` function. This function is no longer +considered public and neither are the helper functions used in it. +They currently still work but will be removed at some point. If you have +been making changes to `process_tags`, please review your style and try +to switch to the new convenience functions. + +For more information on the changes, see the +[pull request](https://github.com/osm-search/Nominatim/pull/3615) +and read the new +[customization documentation](https://nominatim.org/release-docs/latest/customize/Import-Styles/). + ## 4.4.0 -> 4.5.0 ### New structure for Python packages diff --git a/docs/admin/Update.md b/docs/admin/Update.md index 5d1324d0..cdb79cae 100644 --- a/docs/admin/Update.md +++ b/docs/admin/Update.md @@ -68,10 +68,10 @@ the update interval no new data has been published yet, it will go to sleep until the next expected update and only then attempt to download the next batch. The one-time mode is particularly useful if you want to run updates continuously -but need to schedule other work in between updates. For example, the main -service at osm.org uses it, to regularly recompute postcodes -- a process that -must not be run while updates are in progress. Its update script -looks like this: +but need to schedule other work in between updates. For example, you might +want to regularly recompute postcodes -- a process that +must not be run while updates are in progress. An update script refreshing +postcodes regularly might look like this: ```sh #!/bin/bash @@ -109,17 +109,19 @@ Unit=nominatim-updates.service WantedBy=multi-user.target ``` -And then a similar service definition: `/etc/systemd/system/nominatim-updates.service`: +`OnUnitActiveSec` defines how often the individual update command is run. + +Then add a service definition for the timer in `/etc/systemd/system/nominatim-updates.service`: ``` [Unit] Description=Single updates of Nominatim [Service] -WorkingDirectory=/srv/nominatim -ExecStart=nominatim replication --once -StandardOutput=append:/var/log/nominatim-updates.log -StandardError=append:/var/log/nominatim-updates.error.log +WorkingDirectory=/srv/nominatim-project +ExecStart=/srv/nominatim-venv/bin/nominatim replication --once +StandardOutput=journald +StandardError=inherit User=nominatim Group=nominatim Type=simple @@ -128,9 +130,9 @@ Type=simple WantedBy=multi-user.target ``` -Replace the `WorkingDirectory` with your project directory. Also adapt user and -group names as required. `OnUnitActiveSec` defines how often the individual -update command is run. +Replace the `WorkingDirectory` with your project directory. `ExecStart` points +to the nominatim binary that was installed in your virtualenv earlier. +Finally, you might need to adapt user and group names as required. Now activate the service and start the updates: @@ -140,12 +142,13 @@ sudo systemctl enable nominatim-updates.timer sudo systemctl start nominatim-updates.timer ``` -You can stop future data updates, while allowing any current, in-progress +You can stop future data updates while allowing any current, in-progress update steps to finish, by running `sudo systemctl stop nominatim-updates.timer` and waiting until `nominatim-updates.service` isn't -running (`sudo systemctl is-active nominatim-updates.service`). Current output -from the update can be seen like above (`systemctl status -nominatim-updates.service`). +running (`sudo systemctl is-active nominatim-updates.service`). + +To check the output from the update process, use journalctl: `journalctl -u +nominatim-updates.service` #### Catch-up mode @@ -155,13 +158,13 @@ all changes from the server until the database is up-to-date. The catch-up mode still respects the parameter `NOMINATIM_REPLICATION_MAX_DIFF`. It downloads and applies the changes in appropriate batches until all is done. -The catch-up mode is foremost useful to bring the database up to speed after the +The catch-up mode is foremost useful to bring the database up to date after the initial import. Give that the service usually is not in production at this point, you can temporarily be a bit more generous with the batch size and number of threads you use for the updates by running catch-up like this: ``` -cd /srv/nominatim +cd /srv/nominatim-project NOMINATIM_REPLICATION_MAX_DIFF=5000 nominatim replication --catch-up --threads 15 ``` @@ -173,13 +176,13 @@ replication catch-up at whatever interval you desire. When running scheduled updates with catch-up, it is a good idea to choose a replication source with an update frequency that is an order of magnitude lower. For example, if you want to update once a day, use an hourly updated - source. This makes sure that you don't miss an entire day of updates when + source. This ensures that you don't miss an entire day of updates when the source is unexpectedly late to publish its update. If you want to use the source with the same update frequency (e.g. a daily updated source with daily updates), use the - continuous update mode. It ensures to re-request the newest update until it - is published. + once mode together with a frequently run systemd script as described above. + It ensures to re-request the newest update until they have been published. #### Continuous updates @@ -197,36 +200,3 @@ parameters: The update application keeps running forever and retrieves and applies new updates from the server as they are published. - -You can run this command as a simple systemd service. Create a service -description like that in `/etc/systemd/system/nominatim-updates.service`: - -``` -[Unit] -Description=Continuous updates of Nominatim - -[Service] -WorkingDirectory=/srv/nominatim -ExecStart=nominatim replication -StandardOutput=append:/var/log/nominatim-updates.log -StandardError=append:/var/log/nominatim-updates.error.log -User=nominatim -Group=nominatim -Type=simple - -[Install] -WantedBy=multi-user.target -``` - -Replace the `WorkingDirectory` with your project directory. Also adapt user -and group names as required. - -Now activate the service and start the updates: - -``` -sudo systemctl daemon-reload -sudo systemctl enable nominatim-updates -sudo systemctl start nominatim-updates -``` - - diff --git a/docs/api/Output.md b/docs/api/Output.md index 75220cf5..a5ec933e 100644 --- a/docs/api/Output.md +++ b/docs/api/Output.md @@ -106,8 +106,11 @@ The following feature attributes are implemented: * `name` - localised name of the place * `housenumber`, `street`, `locality`, `district`, `postcode`, `city`, `county`, `state`, `country` - - provided when it can be determined from the address + provided when it can be determined from the address (only with `addressdetails=1`) * `admin` - list of localised names of administrative boundaries (only with `addressdetails=1`) + * `extra` - dictionary with additional useful tags like `website` or `maxspeed` + (only with `extratags=1`) + Use `polygon_geojson` to output the full geometry of the object instead of the centroid. diff --git a/docs/api/Search.md b/docs/api/Search.md index 1c269168..3c9a7148 100644 --- a/docs/api/Search.md +++ b/docs/api/Search.md @@ -212,7 +212,7 @@ other layers. The featureType allows to have a more fine-grained selection for places from the address layer. Results can be restricted to places that make up the 'state', 'country' or 'city' part of an address. A featureType of -settlement selects any human inhabited feature from 'state' down to +`settlement` selects any human inhabited feature from 'state' down to 'neighbourhood'. When featureType is set, then results are automatically restricted diff --git a/docs/customize/Import-Styles.md b/docs/customize/Import-Styles.md index 003e56e3..23778f77 100644 --- a/docs/customize/Import-Styles.md +++ b/docs/customize/Import-Styles.md @@ -1,95 +1,123 @@ -## Configuring the Import +# Configuring the Import of OSM data In the very first step of a Nominatim import, OSM data is loaded into the database. Nominatim uses [osm2pgsql](https://osm2pgsql.org) for this task. It comes with a [flex style](https://osm2pgsql.org/doc/manual.html#the-flex-output) specifically tailored to filter and convert OSM data into Nominatim's -internal data representation. - -There are a number of default configurations for the flex style which -result in geocoding databases of different detail. The +internal data representation. Nominatim ships with a few preset +configurations for this import, each results in a geocoding database of +different detail. The [Import section](../admin/Import.md#filtering-imported-data) explains these default configurations in detail. -You can also create your own custom style. Put the style file into your -project directory and then set `NOMINATIM_IMPORT_STYLE` to the name of the file. -It is always recommended to start with one of the standard styles and customize -those. You find the standard styles under the name `import-.lua` -in the standard Nominatim configuration path (usually `/etc/nominatim` or -`/usr/local/etc/nominatim`). +If you want to have more control over which OSM data is added to the database, +you can also create your own custom style. Create a new lua style file, put it +into your project directory and then set `NOMINATIM_IMPORT_STYLE` to the name +of the file. Custom style files can be used to modify the existing preset +configurations or to implement your own configuration from scratch. The remainder of the page describes how the flex style works and how to customize it. -### The `flex-base.lua` module +## The `flex-base` lua module The core of Nominatim's flex import configuration is the `flex-base` module. It defines the table layout used by Nominatim and provides standard -implementations for the import callbacks that make it easy to customize +implementations for the import callbacks that help with customizing how OSM tags are used by Nominatim. -Every custom style should include this module to make sure that the correct +Every custom style must include this module to make sure that the correct tables are created. Thus start your custom style as follows: ``` lua local flex = require('flex-base') +``` + +### Using preset configurations + +If you want to start with one of the existing presets, then you can import +its settings using the `import_topic()` function: + +``` +local flex = require('flex-base') +flex.import_topic('streets') ``` -The following sections explain how the module can be customized. +The `import_topic` function takes an optional second configuration +parameter. The available options are explained in the +[themepark section](#using-osm2pgsql-themepark). +!!! note + You can also directly import the preset style files, e.g. + `local flex = require('import-street')`. It is not possible to + set extra configuration this way. -### Changing the recognized tags +### How processing works -If you just want to change which OSM tags are recognized during import, -then there are a number of convenience functions to set the tag lists used -during the processing. +When Nominatim processes an OSM object, it looks for four kinds of tags: +The _main tags_ classify what kind of place the OSM object represents. One +OSM object can have more than one main tag. In such case one database entry +is created for each main tag. _Name tags_ represent searchable names of the +place. _Address tags_ are used to compute the address hierarchy of the place. +Address tags are used for searching and for creating a display name of the place. +_Extra tags_ are any tags that are not directly related to search but +contain interesting additional information. -!!! warning - There are no built-in defaults for the tag lists, so all the functions - need to be called from your style script to fully process the data. - Make sure you start from one of the default style and only modify - the data you are interested in. You can also derive your style from an - existing style by importing the appropriate module, e.g. - `local flex = require('import-street')`. +!!! danger + Some tags in the extratags category are used by Nominatim to better + classify the place. You want to make sure these are always present + in custom styles. -Many of the following functions take _key match lists_. These lists can +Configuring the style means deciding which key and/or key/value is used +in which category. + +## Changing the recognized tags + +The flex style offers a number of functions to set the classification of +each OSM tag. Most of these functions can also take a preset string instead +of a tag description. These presets describe common configurations that +are also used in the definition of the predefined styles. This section +lists the configuration functions and the accepted presets. + +#### Key match lists + +Some of the following functions take _key match lists_. These lists can contain three kinds of strings to match against tag keys: A string that ends in an asterisk `*` is a prefix match and accordingly matches against any key that starts with the given string (minus the `*`). A suffix match can be defined similarly with a string that starts with a `*`. Any other string is matched exactly against tag keys. +### Main tags + +`set/modify_main_tags()` allow to define which tags are used as main tags. It +takes a lua table parameter which defines for keys and key/value +combinations, how they are classified. + +The following classifications are recognized: -#### `set_main_tags()` - principal tags - -If a principal or main tag is found on an OSM object, then the object -is included in Nominatim's search index. A single object may also have -multiple main tags. In that case, the object will be included multiple -times in the index, once for each main tag. - -The flex script distinguishes between four types of main tags: - -* __always__: a main tag that is used unconditionally -* __named__: consider this main tag only, if the object has a proper name - (a reference is not enough, see below). -* __named_with_key__: consider this main tag only, when the object has - a proper name with a domain prefix. For example, if the main tag is - `bridge=yes`, then it will only be added as an extra row, if there is - a tag `bridge:name[:XXX]` for the same object. If this property is set, - all other names that are not domain-specific are ignored. -* __fallback__: use this main tag only, if there is no other main tag. - Fallback always implied `named`, i.e. fallbacks are only tried for - named objects. - -The `set_main_tags()` function takes exactly one table parameter which -defines the keys and key/value combinations to include and the kind of -main tag. Each lua table key defines an OSM tag key. The value may -be a string defining the kind of main key as described above. Then the tag will -be considered a main tag for any possible value. To further restrict -which values are acceptable, give a table with the permitted values -and their kind of main tag. If the table contains a simple value without -key, then this is used as default for values that are not listed. +| classification | meaning | +| :-------------- | :------ | +| always | Unconditionally use this tag as a main tag. | +| named | Consider as main tag, when the object has a primary name (see [names](#name-tags) below) | +| named_with_key | Consider as main tag, when the object has a primary name with a domain prefix. For example, if the main tag is `bridge=yes`, then it will only be added as an extra entry, if there is a tag `bridge:name[:XXX]` for the same object. If this property is set, all names that are not domain-specific are ignored. | +| fallback | Consider as main tag only when no other main tag was found. Fallback always implies `named`, i.e. fallbacks are only tried for objects with primary names. | +| delete | Completely ignore the tag in any further processing | +| extra | Move the tag to extratags and then ignore it for further processing | +| ``| Advanced handling, see [below](#advanced-main-tag-handling) | + +Each key in the table parameter defines an OSM tag key. The value may +be directly a classification as described above. Then the tag will +be considered a main tag for any possible value that is not further defined. +To further restrict which values are acceptable, give a table with the +permitted values and their kind of main tag. If the table contains a simple +value without key, then this is used as default for values that are not listed. + +`set_main_tags()` will completely replace the current main tag configuration +with the new configuration. `modify_main_tags()` will merge the new +configuration with the existing one. Otherwise, the two functions do exactly +the same. !!! example ``` lua @@ -97,80 +125,188 @@ key, then this is used as default for values that are not listed. flex.set_main_tags{ boundary = {administrative = 'named'}, - highway = {'always', street_lamp = 'named'}, + highway = {'always', street_lamp = 'named', no = 'delete'}, landuse = 'fallback' } ``` In this example an object with a `boundary` tag will only be included when it has a value of `administrative`. Objects with `highway` tags are - always included. However when the value is `street_lamp` then the object - must have a name, too. With any other value, the object is included - independently of the name. Finally, if a `landuse` tag is present then - it will be used independely of the concrete value if neither boundary + always included with two exceptions: the troll tag `highway=no` is + deleted on the spot. And when the value is `street_lamp` then the object + must have a name, too. Finally, if a `landuse` tag is present then + it will be used independently of the concrete value when neither boundary nor highway tags were found and the object is named. +##### Presets -#### `set_prefilters()` - ignoring tags +| Name | Description | +| :----- | :---------- | +| admin | Basic tag set collecting places and administrative boundaries. This set is needed also to ensure proper address computation and should therefore always be present. You can disable selected place types like `place=locality` after adding this set, if they are not relevant for your use case. | +| all_boundaries | Extends the set of recognized boundaries and places to all available ones. | +| natural | Tags for natural features like rivers and mountain peaks. | +| street/default | Tags for streets. Major streets are always included, minor ones only when they have a name. | +| street/car | Tags for all streets that can be used by a motor vehicle. | +| street/all | Includes all highway features named and unnamed. | +| poi/delete | Adds most POI features with and without name. Some frequent but very domain-specific values are excluded by deleting them. | +| poi/extra | Like 'poi/delete' but excluded values are moved to extratags. | -Pre-filtering of tags allows to ignore them for any further processing. -Thus pre-filtering takes precedence over any other tag processing. This is -useful when some specific key/value combinations need to be excluded from -processing. When tags are filtered, they may either be deleted completely -or moved to `extratags`. Extra tags are saved with the object and returned -to the user when requested, but are not used otherwise. -`set_prefilters()` takes a table with four optional fields: +##### Advanced main tag handling -* __delete_keys__ is a _key match list_ for tags that should be deleted -* __delete_tags__ contains a table of tag keys pointing to a list of tag - values. Tags with matching key/value pairs are deleted. -* __extra_keys__ is a _key match list_ for tags which should be saved into - extratags -* __extra_tags__ contains a table of tag keys pointing to a list of tag - values. Tags with matching key/value pairs are moved to extratags. +The groups described above are in fact only a preset for a filtering function +that is used to make the final decision how a pre-selected main tag is entered +into Nominatim's internal table. To further customize handling you may also +supply your own filtering function. -Key list may contain three kinds of strings: -A string that ends in an asterisk `*` is a prefix match and accordingly matches -against any key that starts with the given string (minus the `*`). -A suffix match can be defined similarly with a string that starts with a `*`. -Any other string is matched exactly against tag keys. +The function takes up to three parameters: a Place object of the object +being processed, the key of the main tag and the value of the main tag. +The function may return one of three values: + +* `nil` or `false` causes the entry to be ignored +* the Place object causes the place to be added as is +* `Place.copy(names=..., address=..., extratags=...) causes the + place to be enter into the database but with name/address/extratags + set to the given different values. + +The Place object has some read-only values that can be used to determine +the handling: + +* **object** is the original OSM object data handed in by osm2pgsql +* **admin_level** is the content of the admin_level tag, parsed into an integer and normalized to a value between 0 and 15 +* **has_name** is a boolean indicating if the object has a primary name tag +* **names** is a table with the collected list of name tags +* **address** is a table with the collected list of address tags +* **extratags** is a table with the collected list of additional tags to save !!! example ``` lua - local flex = require('import-full') + local flex = require('flex-base') - flex.set_prefilters{ - delete_keys = {'source', 'source:*'}, - extra_tags = {amenity = {'yes', 'no'}} - } - flex.set_main_tags{ - amenity = 'always' - } + flex.add_topic('street') + + local function no_sidewalks(place, k, v) + if place.object.tags.footway == 'sidewalk' then + return false + end + + -- default behaviour is to have all footways + return place + end + + flex.modify_main_tags(highway = {'footway' = no_sidewalks} + ``` + This script adds a custom handler for `highway=footway`. It only includes + them in the database, when the object doesn't have a tag `footway=sidewalk` + indicating that it is just part of a larger street which should already + be indexed. Note that it is not necessary to check the key and value + of the main tag because the function is only used for the specific + main tag. + + +### Ignored tags + +The function `ignore_keys()` sets the `delete` classification for keys. +This function takes a _key match list_ so that it is possible to exclude +groups of keys. + +Note that full matches always take precedence over suffix matches, which +in turn take precedence over prefix matches. + +!!! example + ``` lua + local flex = require('flex-base') + + flex.add_topic('admin') + flex.ignore_keys{'old_name', 'old_name:*'} ``` - In this example any tags `source` and tags that begin with `source:` are - deleted before any other processing is done. Getting rid of frequent tags - this way can speed up the import. + This example uses the `admin` preset with the exception that names + that are no longer are in current use, are ignored. + +##### Presets - Tags with `amenity=yes` or `amenity=no` are moved to extratags. Later - all tags with an `amenity` key are made a main tag. This effectively means - that Nominatim will use all amenity tags except for those with value - yes and no. +| Name | Description | +| :----- | :---------- | +| metatags | Tags with meta information about the OSM tag like source, notes and import sources. | +| name | Non-names that actually describe properties or name parts. These names can throw off search and should always be removed. | +| address | Extra `addr:*` tags that are not useful for Nominatim. | -#### `set_name_tags()` - defining names -The flex script distinguishes between two kinds of names: +### Tags for `extratags` -* __main__: the primary names make an object fully searchable. - Main tags of type _named_ will only cause the object to be included when - such a primary name is present. Primary names are usually those found - in the `name` tag and its variants. -* __extra__: extra names are still added to the search index but they are - alone not sufficient to make an object named. +The function `add_for_extratags()` sets the `extra` classification for keys. +This function takes a +_key match list_ so that it is possible to move groups of keys to extratags. -`set_name_tags()` takes a table with two optional fields `main` and `extra`. -They take _key match lists_ for main and extra names respectively. +Note that full matches always take precedence over suffix matches, which +in turn take precedence over prefix matches. + +!!! example + ``` lua + local flex = require('flex-base') + + flex.add_topic('street') + flex.add_for_extratags{'surface', 'access', 'vehicle', 'maxspeed'} + ``` + + This example uses the `street` preset but adds a couple of tags that + are of interest about the condition of the street. + +##### Presets + +| Name | Description | +| :----- | :---------- | +| required | Tags that Nominatim will use for various computations when present in extratags. Always include these. | + +In addition, all [presets from ignored tags](#presets_1) are accepted. + +### General pre-filtering + +_(deprecated)_ `set_prefilters()` allows to set the `delete` and `extra` +classification for main tags. + +This function removes all previously set main tags with `delete` and `extra` +classification and then adds the newly defined tags. + +`set_prefilters()` takes a table with four optional fields: + +* __delete_keys__ is a _key match list_ for tags that should be deleted +* __delete_tags__ contains a table of tag keys pointing to a list of tag + values. Tags with matching key/value pairs are deleted. +* __extra_keys__ is a _key match list_ for tags which should be saved into + extratags +* __extra_tags__ contains a table of tag keys pointing to a list of tag + values. Tags with matching key/value pairs are moved to extratags. + +!!! danger "Deprecation warning" + Use of this function should be replaced with `modify_main_tags()` to + set the data from `delete_tags` and `extra_tags`, with `ignore_keys()` + for the `delete_keys` parameter and with `add_for_extratags()` for the + `extra_keys` parameter. + +### Name tags + +`set/modify_name_tags()` allow to define the tags used for naming places. Name tags +can only be selected by their keys. The import script distinguishes +between primary and auxiliary names. A primary name is the given name of +a place. Having a primary name makes a place _named_. This is important +for main tags that are only included when a name is present. Auxiliary names +are identifiers like references. They may be searched for but should not +be included on their own. + +The functions take a table with two optional fields `main` and `extra`. +They take _key match lists_ for primary and auxiliary names respectively. +A third field `house` can contain tags for names that appear in place of +house numbers in addresses. This field can only contain complete key names. +'house tags' are special in that they cause the OSM object to be added to +the database independently of the presence of other main tags. + +`set_name_tags()` overwrites the current configuration, while +`modify_name_tags()` replaces the fields that are given. (Be aware that +the fields are replaced as a whole. `main = {'foo_name'}` will cause +`foo_name` to become the only recognized primary name. Any previously +defined primary names are forgotten.) !!! example ``` lua @@ -186,29 +322,33 @@ They take _key match lists_ for main and extra names respectively. only include those that have a common name and not those which just have some reference ID from the city. -#### `set_address_tags()` - defining address parts +##### Presets -Address tags will be used to build up the address of an object. +| Name | Description | +| :----- | :---------- | +| core | Basic set of recognized names for all places. | +| address | Additional names useful when indexing full addresses. | +| poi | Extended set of recognized names for pois. Use on top of the core set. | -`set_address_tags()` takes a table with arbitrary fields pointing to -_key match lists_. Two fields have a special meaning: +### Address tags -* __main__: defines -the tags that make a full address object out of the OSM object. This -is usually the housenumber or variants thereof. If a main address tag -appears, then the object will always be included, if necessary with a -fallback of `place=house`. If the key has a prefix of `addr:` or `is_in:` -this will be stripped. +`set/modify_address_tags()` defines the tags that will be used to build +up the address of an object. Address tags can only be chosen by their key. -* __extra__: defines all supplementary tags for addresses, tags like `addr:street`, `addr:city` etc. If the key has a prefix of `addr:` or `is_in:` this will be stripped. +The functions take a table with arbitrary fields, each defining +a key list or _key match list_. Some fields have a special meaning: -All other fields will be handled as summary fields. If a key matches the -key match list, then its value will be added to the address tags with the -name of the field as key. If multiple tags match, then an arbitrary one -wins. +| Field | Type | Description | +| :---------| :-------- | :-----------| +| main | key list | Tags that make a full address object out of the OSM object. This is usually the house number or variants thereof. If a main address tag appears, then the object will always be included, if necessary with a fallback of `place=house`. If the key has a prefix of `addr:` or `is_in:` this will be stripped. | +| extra | key match list | Supplementary tags for addresses, tags like `addr:street`, `addr:city` etc. If the key has a prefix of `addr:` or `is_in:` this will be stripped. | +| interpolation | key list | Tags that identify address interpolation lines. | +| country | key match list | Tags that may contain the country the place is in. The first found value with a two-letter code will be accepted, all other values are discarded. | +| _other_ | key match list | Summary field. If a key matches the key match list, then its value will be added to the address tags with the name of the field as key. If multiple tags match, then an arbitrary one wins. | -Country tags are handled slightly special. Only tags with a two-letter code -are accepted, all other values are discarded. +`set_address_tags()` overwrites the current configuration, while +`modify_address_tags()` replaces the fields that are given. (Be aware that +the fields are replaced as a whole.) !!! example ``` lua @@ -232,21 +372,33 @@ are accepted, all other values are discarded. to postcodes, they will always be saved under the key `postcode` thus normalizing the multitude of keys that are used in the OSM database. +##### Presets + +| Name | Description | +| :----- | :---------- | +| core | Basic set of tags needed to recognize address relationship for any place. Always include this. | +| houses | Additional set of tags needed to recognize proper addresses | -#### `set_unused_handling()` - processing remaining tags +### Handling of unclassified tags -This function defines what to do with tags that remain after all tags +`set_unused_handling()` defines what to do with tags that remain after all tags have been classified using the functions above. There are two ways in which the function can be used: `set_unused_handling(delete_keys = ..., delete_tags = ...)` deletes all keys that match the descriptions in the parameters and moves all remaining tags into the extratags list. + `set_unused_handling(extra_keys = ..., extra_tags = ...)` moves all tags matching the parameters into the extratags list and then deletes the remaining tags. For the format of the parameters see the description in `set_prefilters()` above. +When no special handling is set, then unused tags will be discarded with one +exception: place tags are kept in extratags for administrative boundaries. +When using a custom setting, you should also make sure that the place tag +is added for extratags. + !!! example ``` lua local flex = require('import-full') @@ -263,17 +415,23 @@ above. already delete the tiger tags with `set_prefilters()` because that would remove tiger:county before the address tags are processed. -### Customizing osm2pgsql callbacks +## Customizing osm2pgsql callbacks osm2pgsql expects the flex style to implement three callbacks, one process function per OSM type. If you want to implement special handling for certain OSM types, you can override the default implementations provided by the flex-base module. -#### Changing the relation types to be handled +### Enabling additional relation types -The default scripts only allows relations of type `multipolygon`, `boundary` -and `waterway`. To add other types relations, set `RELATION_TYPES` for +OSM relations can represent very diverse +[types of real-world objects](https://wiki.openstreetmap.org/wiki/Key:type). To +be able to process them correctly, Nominatim needs to understand how to +create a geometry for each type. By default, the script knows how to +process relations of type `multipolygon`, `boundary` and `waterway`. All +other relation types are ignored. + +To add other types relations, set `RELATION_TYPES` for the type to the kind of geometry that should be created. The following kinds of geometries can be used: @@ -297,7 +455,7 @@ kinds of geometries can be used: geometry. -#### Adding additional logic to processing functions +### Adding additional logic to processing functions The default processing functions are also exported by the flex-base module as `process_node`, `process_way` and `process_relation`. These can be used @@ -322,110 +480,83 @@ logic. ### Customizing the main processing function -The main processing function of the flex style can be found in the function -`process_tags`. This function is called for all OSM object kinds and is -responsible for filtering the tags and writing out the rows into Postgresql. +!!! danger "Deprecation Warning" + The style used to allow overwriting the internal processing function + `process_tags()`. While this is currently still possible, it is no longer + encouraged and may stop working in future versions. The internal + `Place` class should now be considered read-only. + + +## Using osm2pgsql-themepark + +The Nominatim osm2pgsql style is designed so that it can also be used as +a theme for [osm2pgsql-themepark](https://osm2pgsql.org/themepark/). This +makes it easy to combine Nominatim with other projects like +[openstreetmap-carto](https://github.com/gravitystorm/openstreetmap-carto) +in the same database. + +To set up one of the preset styles, simply include a topic with the same name: + +``` +local themepark = require('themepark') +themepark:add_topic('nominatim/address') +``` + +Themepark topics offer two configuration options: + +* **street_theme** allows to choose one of the sub topics for streets: + * _default_ - include all major streets and named minor paths + * _car_ - include all streets physically usable by cars + * _all_ - include all major streets and minor paths +* **with_extratags**, when set to a truthy value, then tags that are + not specifically used for address or naming are added to the + extratags column + +The customization functions described in the +[Changing recognized tags](#changing-the-recognized-tags) section +are available from the theme. To access the theme you need to explicitly initialize it. !!! Example ``` lua - local flex = require('import-full') + local themepark = require('themepark') - local original_process_tags = flex.process_tags + themepark:add_topic('nominatim/full', {with_extratags = true}) - function flex.process_tags(o) - if o.object.tags.highway ~= nil and o.object.tags.access == 'no' then - return - end + local flex = themepark:init_theme('nominatim') + + flex.modify_main_tags{'amenity' = { + 'waste_basket' = 'delete'} + } + ``` + This example uses the full Nominatim configuration but disables + importing waste baskets. + +You may also write a new configuration from scratch. Simply omit including +a Nominatim topic and only call the required customization functions. - original_process_tags(o) +Customizing the osm2pgsql processing functions as explained +[above](#adding-additional-logic-to-processing-functions) is not possible +when running under themepark. Instead include other topics that make the +necessary modifications or add an additional processor before including +the Nominatim topic. + +!!! Example + ``` lua + local themepark = require('themepark') + + local function discard_country_boundaries(object) + if object.tags.boundary == 'administrative' and object.tags.admin_level == '2' then + return 'stop' + end end + + themepark:add_proc('relation', discard_country_boundaries) + -- Order matters here. The topic needs to be added after the custom callback. + themepark:add_topic('nominatim/full', {with_extratags = true}) ``` + Discarding country-level boundaries when running under themepark. - This example shows the most simple customization of the process_tags function. - It simply adds some additional processing before running the original code. - To do that, first save the original function and then overwrite process_tags - from the module. In this example all highways which are not accessible - by anyone will be ignored. - - -#### The `Place` class - -The `process_tags` function receives a Lua object of `Place` type which comes -with some handy functions to collect the data necessary for geocoding and -writing it into the place table. Always use this object to fill the table. - -The Place class has some attributes which you may access read-only: - -* __object__ is the original OSM object data handed in by osm2pgsql -* __admin_level__ is the content of the admin_level tag, parsed into an - integer and normalized to a value between 0 and 15 -* __has_name__ is a boolean indicating if the object has a full name -* __names__ is a table with the collected list of name tags -* __address__ is a table with the collected list of address tags -* __extratags__ is a table with the collected list of additional tags to save - -There are a number of functions to fill these fields. All functions expect -a table parameter with fields as indicated in the description. -Many of these functions expect match functions which are described in detail -further below. - -* __delete{match=...}__ removes all tags that match the match function given - in _match_. -* __grab_extratags{match=...}__ moves all tags that match the match function - given in _match_ into extratags. Returns the number of tags moved. -* __clean{delete=..., extra=...}__ deletes all tags that match _delete_ and - moves the ones that match _extra_ into extratags -* __grab_address_parts{groups=...}__ moves matching tags into the address table. - _groups_ must be a group match function. Tags of the group `main` and - `extra` are added to the address table as is but with `addr:` and `is_in:` - prefixes removed from the tag key. All other groups are added with the - group name as key and the value from the tag. Multiple values of the same - group overwrite each other. The function returns the number of tags saved - from the main group. -* __grab_main_parts{groups=...}__ moves matching tags into the name table. - _groups_ must be a group match function. If a tags of the group `main` is - present, the object will be marked as having a name. Tags of group `house` - produce a fallback to `place=house`. This fallback is return by the function - if present. - -There are two functions to write a row into the place table. Both functions -expect the main tag (key and value) for the row and then use the collected -information from the name, address, extratags etc. fields to complete the row. -They also have a boolean parameter `save_extra_mains` which defines how any -unprocessed tags are handled: when True, the tags will be saved as extratags, -when False, they will be simply discarded. - -* __write_row(key, value, save_extra_mains)__ creates a new table row from - the current state of the Place object. -* __write_place(key, value, mtype, save_extra_mains)__ creates a new row - conditionally. When value is nil, the function will attempt to look up the - value in the object tags. If value is still nil or mtype is nil, the row - is ignored. An mtype of `always` will then always write out the row, - a mtype of `named` only, when the object has a full name. When mtype - is `named_with_key`, the function checks for a domain name, i.e. a name - tag prefixed with the name of the main key. Only if at least one is found, - the row will be written. The names are replaced with the domain names found. - -#### Match functions - -The Place functions usually expect either a _match function_ or a -_group match function_ to find the tags to apply their function to. - -The __match function__ is a Lua function which takes two parameters, -key and value, and returns a boolean to indicate that a tag matches. The -flex-base module has a convenience function `tag_match()` to create such a -function. It takes a table with two optional fields: `keys` takes a key match -list (see above), `tags` takes a table with keys that point to a list of -possible values, thus defining key/value matches. - -The __group match function__ is a Lua function which also takes two parameters, -key and value, and returns a string indicating to which group or type they -belong to. The `tag_group()` can be used to create such a function. It expects -a table where the group names are the keys and the values are a key match list. - - - -### Using the gazetteer output of osm2pgsql +## osm2pgsql gazetteer output Nominatim still allows you to configure the gazetteer output to remain backwards compatible with older imports. It will be automatically used @@ -435,9 +566,9 @@ of Nominatim. Do not use the gazetteer output for new imports. There is no guarantee that new versions of Nominatim are fully compatible with the gazetteer output. -### Changing the Style of Existing Databases +## Changing the style of existing databases -There is normally no issue changing the style of a database that is already +There is usually no issue changing the style of a database that is already imported and now kept up-to-date with change files. Just be aware that any change in the style applies to updates only. If you want to change the data that is already in the database, then a reimport is necessary. diff --git a/docs/customize/Settings.md b/docs/customize/Settings.md index b00d04cf..edf2241b 100644 --- a/docs/customize/Settings.md +++ b/docs/customize/Settings.md @@ -336,7 +336,7 @@ NOMINATIM_TABLESPACE_SEARCH_INDEX NOMINATIM_TABLESPACE_OSM_DATA : Raw OSM data cache used for import and updates. -NOMINATIM_TABLESPACE_OSM_DATA +NOMINATIM_TABLESPACE_OSM_INDEX : Indexes on the raw OSM data cache. NOMINATIM_TABLESPACE_PLACE_DATA @@ -602,6 +602,43 @@ results gathered so far. Note that under high load you may observe that users receive different results than usual without seeing an error. This may cause some confusion. +#### NOMINATIM_OUTPUT_NAMES + +| Summary | | +| -------------- | --------------------------------------------------- | +| **Description:** | Specifies order of name tags | +| **Format:** | string: comma-separated list of tag names | +| **Default:** | name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref | + +Specifies the order in which different name tags are used. +The values in this list determine the preferred order of name variants, +including language-specific names (in OSM: the name tag with and without any language suffix). + +Comma-separated list, where :XX stands for language suffix +(e.g. name:en) and no :XX stands for general tags (e.g. name). + +See also [NOMINATIM_DEFAULT_LANGUAGE](#nominatim_default_language). + +!!! note + If NOMINATIM_OUTPUT_NAMES = `name:XX,name,short_name:XX,short_name` the search follows + + ``` + 'name', 'short_name' + ``` + + if we have no preferred language order for showing search results. + + For languages ['en', 'es'] the search follows + + ``` + 'name:en', 'name:es', + 'name', + 'short_name:en', 'short_name:es', + 'short_name' + ``` + + For those familiar with the internal implementation, the `_place_*` expansion is added, but to simplify, it is not included in this example. + ### Logging Settings #### NOMINATIM_LOG_DB diff --git a/docs/customize/Special-Phrases.md b/docs/customize/Special-Phrases.md index 4824512b..3ab837f2 100644 --- a/docs/customize/Special-Phrases.md +++ b/docs/customize/Special-Phrases.md @@ -17,7 +17,7 @@ columns: * **phrase**: the keyword to look for * **class**: key of the main tag of the place to find - (see [principal tags in import style](Import-Styles.md#set_main_tags-principal-tags) + (see [Import styles](Import-Styles.md#how-processing-works) * **type**: value of the main tag * **operator**: type of special phrase, may be one of: * *in*: place is within the place defined by the search term (e.g. "_Hotels in_ Berlin") diff --git a/docs/customize/Tokenizers.md b/docs/customize/Tokenizers.md index 30be170e..23db34c9 100644 --- a/docs/customize/Tokenizers.md +++ b/docs/customize/Tokenizers.md @@ -4,12 +4,11 @@ The tokenizer module in Nominatim is responsible for analysing the names given to OSM objects and the terms of an incoming query in order to make sure, they can be matched appropriately. -Nominatim offers different tokenizer modules, which behave differently and have -different configuration options. This sections describes the tokenizers and how -they can be configured. +Nominatim currently offers only one tokenizer module, the ICU tokenizer. This section +describes the tokenizer and how it can be configured. !!! important - The use of a tokenizer is tied to a database installation. You need to choose + The selection of tokenizer is tied to a database installation. You need to choose and configure the tokenizer before starting the initial import. Once the import is done, you cannot switch to another tokenizer anymore. Reconfiguring the chosen tokenizer is very limited as well. See the comments in each tokenizer @@ -43,10 +42,19 @@ On import the tokenizer processes names in the following three stages: See the [Token analysis](#token-analysis) section below for more information. -During query time, only normalization and transliteration are relevant. -An incoming query is first split into name chunks (this usually means splitting -the string at the commas) and the each part is normalised and transliterated. -The result is used to look up places in the search index. +During query time, the tokeinzer is responsible for processing incoming +queries. This happens in two stages: + +1. During **query preprocessing** the incoming text is split into name + chunks and normalised. This usually means applying the same normalisation + as during the import process but may involve other processing like, + for example, word break detection. +2. The **token analysis** step breaks down the query parts into tokens, + looks them up in the database and assigns them possible functions and + probabilities. + +Query processing can be further customized while the rest of the analysis +is hard-coded. ### Configuration @@ -58,6 +66,14 @@ have no effect. Here is an example configuration file: ``` yaml +query-preprocessing: + - step: split_japanese_phrases + - step: regex_replace + replacements: + - pattern: https?://[^\s]* # Filter URLs starting with http or https + replace: '' + - step: normalize + normalization: - ":: lower ()" - "ß > 'ss'" # German szet is unambiguously equal to double ss @@ -78,8 +94,37 @@ token-analysis: replacements: ['ä', 'ae'] ``` -The configuration file contains four sections: -`normalization`, `transliteration`, `sanitizers` and `token-analysis`. +The configuration file contains five sections: +`query-preprocessing`, `normalization`, `transliteration`, `sanitizers` and `token-analysis`. + +#### Query preprocessing + +The section for `query-preprocessing` defines an ordered list of functions +that are applied to the query before the token analysis. + +The following is a list of preprocessors that are shipped with Nominatim. + +##### normalize + +::: nominatim_api.query_preprocessing.normalize + options: + members: False + heading_level: 6 + docstring_section_style: spacy + +##### regex-replace + +::: nominatim_api.query_preprocessing.regex_replace + options: + members: False + heading_level: 6 + docstring_section_style: spacy + description: + This option runs any given regex pattern on the input and replaces values accordingly + replacements: + - pattern: regex pattern + replace: string to replace with + #### Normalization and Transliteration diff --git a/docs/develop/Development-Environment.md b/docs/develop/Development-Environment.md index a6558c7d..5f247455 100644 --- a/docs/develop/Development-Environment.md +++ b/docs/develop/Development-Environment.md @@ -25,15 +25,15 @@ following packages should get you started: ## Prerequisites for testing and documentation -The Nominatim test suite consists of behavioural tests (using behave) and +The Nominatim test suite consists of behavioural tests (using pytest-bdd) and unit tests (using pytest). It has the following additional requirements: -* [behave test framework](https://behave.readthedocs.io) >= 1.2.6 * [flake8](https://flake8.pycqa.org/en/stable/) (CI always runs the latest version from pip) * [mypy](http://mypy-lang.org/) (plus typing information for external libs) * [Python Typing Extensions](https://github.com/python/typing_extensions) (for Python < 3.9) * [pytest](https://pytest.org) * [pytest-asyncio](https://pytest-asyncio.readthedocs.io) +* [pytest-bdd](https://pytest-bdd.readthedocs.io) For testing the Python search frontend, you need to install extra dependencies depending on your choice of webserver framework: @@ -48,9 +48,6 @@ The documentation is built with mkdocs: * [mkdocs-material](https://squidfunk.github.io/mkdocs-material/) * [mkdocs-gen-files](https://oprypin.github.io/mkdocs-gen-files/) -Please be aware that tests always run against the globally installed -osm2pgsql, so you need to have this set up. If you want to test against -the vendored version of osm2pgsql, you need to set the PATH accordingly. ### Installing prerequisites on Ubuntu/Debian @@ -59,7 +56,9 @@ The easiest way, to handle these Python dependencies is to run your development from within a virtual environment. ```sh -sudo apt install libsqlite3-mod-spatialite +sudo apt install libsqlite3-mod-spatialite osm2pgsql \ + postgresql-postgis postgresql-postgis-scripts \ + pkg-config libicu-dev virtualenv ``` To set up the virtual environment with all necessary packages run: @@ -67,9 +66,10 @@ To set up the virtual environment with all necessary packages run: ```sh virtualenv ~/nominatim-dev-venv ~/nominatim-dev-venv/bin/pip install\ - psutil psycopg[binary] PyICU SQLAlchemy \ - python-dotenv jinja2 pyYAML datrie behave \ - mkdocs mkdocstrings mkdocs-gen-files pytest pytest-asyncio flake8 \ + psutil 'psycopg[binary]' PyICU SQLAlchemy \ + python-dotenv jinja2 pyYAML \ + mkdocs 'mkdocstrings[python]' mkdocs-gen-files \ + pytest pytest-asyncio pytest-bdd flake8 \ types-jinja2 types-markupsafe types-psutil types-psycopg2 \ types-pygments types-pyyaml types-requests types-ujson \ types-urllib3 typing-extensions unicorn falcon starlette \ diff --git a/docs/develop/ICU-Tokenizer-Modules.md b/docs/develop/ICU-Tokenizer-Modules.md index d189422a..30b85ac7 100644 --- a/docs/develop/ICU-Tokenizer-Modules.md +++ b/docs/develop/ICU-Tokenizer-Modules.md @@ -14,10 +14,11 @@ of sanitizers and token analysis. implemented, it is not guaranteed to be stable at the moment. -## Using non-standard sanitizers and token analyzers +## Using non-standard modules -Sanitizer names (in the `step` property) and token analysis names (in the -`analyzer`) may refer to externally supplied modules. There are two ways +Sanitizer names (in the `step` property), token analysis names (in the +`analyzer`) and query preprocessor names (in the `step` property) +may refer to externally supplied modules. There are two ways to include external modules: through a library or from the project directory. To include a module from a library, use the absolute import path as name and @@ -27,6 +28,53 @@ To use a custom module without creating a library, you can put the module somewhere in your project directory and then use the relative path to the file. Include the whole name of the file including the `.py` ending. +## Custom query preprocessors + +A query preprocessor must export a single factory function `create` with +the following signature: + +``` python +create(self, config: QueryConfig) -> Callable[[list[Phrase]], list[Phrase]] +``` + +The function receives the custom configuration for the preprocessor and +returns a callable (function or class) with the actual preprocessing +code. When a query comes in, then the callable gets a list of phrases +and needs to return the transformed list of phrases. The list and phrases +may be changed in place or a completely new list may be generated. + +The `QueryConfig` is a simple dictionary which contains all configuration +options given in the yaml configuration of the ICU tokenizer. It is up to +the function to interpret the values. + +A `nominatim_api.search.Phrase` describes a part of the query that contains one or more independent +search terms. Breaking a query into phrases helps reducing the number of +possible tokens Nominatim has to take into account. However a phrase break +is definitive: a multi-term search word cannot go over a phrase break. +A Phrase object has two fields: + + * `ptype` further refines the type of phrase (see list below) + * `text` contains the query text for the phrase + +The order of phrases matters to Nominatim when doing further processing. +Thus, while you may split or join phrases, you should not reorder them +unless you really know what you are doing. + +Phrase types can further help narrowing down how the tokens in the phrase +are interpreted. The following phrase types are known: + +| Name | Description | +|----------------|-------------| +| PHRASE_ANY | No specific designation (i.e. source is free-form query) | +| PHRASE_AMENITY | Contains name or type of a POI | +| PHRASE_STREET | Contains a street name optionally with a housenumber | +| PHRASE_CITY | Contains the postal city | +| PHRASE_COUNTY | Contains the equivalent of a county | +| PHRASE_STATE | Contains a state or province | +| PHRASE_POSTCODE| Contains a postal code | +| PHRASE_COUNTRY | Contains the country name or code | + + ## Custom sanitizer modules A sanitizer module must export a single factory function `create` with the @@ -90,21 +138,22 @@ adding extra attributes) or completely replace the list with a different one. The following sanitizer removes the directional prefixes from street names in the US: -``` python -import re - -def _filter_function(obj): - if obj.place.country_code == 'us' \ - and obj.place.rank_address >= 26 and obj.place.rank_address <= 27: - for name in obj.names: - name.name = re.sub(r'^(north|south|west|east) ', - '', - name.name, - flags=re.IGNORECASE) - -def create(config): - return _filter_function -``` +!!! example + ``` python + import re + + def _filter_function(obj): + if obj.place.country_code == 'us' \ + and obj.place.rank_address >= 26 and obj.place.rank_address <= 27: + for name in obj.names: + name.name = re.sub(r'^(north|south|west|east) ', + '', + name.name, + flags=re.IGNORECASE) + + def create(config): + return _filter_function + ``` This is the most simple form of a sanitizer module. If defines a single filter function and implements the required `create()` function by returning @@ -128,13 +177,13 @@ sanitizers: !!! warning This example is just a simplified show case on how to create a sanitizer. - It is not really read for real-world use: while the sanitizer would + It is not really meant for real-world use: while the sanitizer would correctly transform `West 5th Street` into `5th Street`. it would also shorten a simple `North Street` to `Street`. For more sanitizer examples, have a look at the sanitizers provided by Nominatim. They can be found in the directory -[`nominatim/tokenizer/sanitizers`](https://github.com/osm-search/Nominatim/tree/master/nominatim/tokenizer/sanitizers). +[`src/nominatim_db/tokenizer/sanitizers`](https://github.com/osm-search/Nominatim/tree/master/src/nominatim_db/tokenizer/sanitizers). ## Custom token analysis module diff --git a/docs/develop/Testing.md b/docs/develop/Testing.md index 12673d40..738fa4b8 100644 --- a/docs/develop/Testing.md +++ b/docs/develop/Testing.md @@ -43,53 +43,53 @@ The name of the pytest binary depends on your installation. ## BDD Functional Tests (`test/bdd`) Functional tests are written as BDD instructions. For more information on -the philosophy of BDD testing, see the -[Behave manual](http://pythonhosted.org/behave/philosophy.html). - -The following explanation assume that the reader is familiar with the BDD -notations of features, scenarios and steps. - -All possible steps can be found in the `steps` directory and should ideally -be documented. +the philosophy of BDD testing, read the Wikipedia article on +[Behaviour-driven development](https://en.wikipedia.org/wiki/Behavior-driven_development). ### General Usage To run the functional tests, do - cd test/bdd - behave - -The tests can be configured with a set of environment variables (`behave -D key=val`): - - * `TEMPLATE_DB` - name of template database used as a skeleton for - the test databases (db tests) - * `TEST_DB` - name of test database (db tests) - * `API_TEST_DB` - name of the database containing the API test data (api tests) - * `API_TEST_FILE` - OSM file to be imported into the API test database (api tests) - * `API_ENGINE` - webframe to use for running search queries, same values as - `nominatim serve --engine` parameter - * `DB_HOST` - (optional) hostname of database host - * `DB_PORT` - (optional) port of database on host - * `DB_USER` - (optional) username of database login - * `DB_PASS` - (optional) password for database login - * `REMOVE_TEMPLATE` - if true, the template and API database will not be reused - during the next run. Reusing the base templates speeds - up tests considerably but might lead to outdated errors - for some changes in the database layout. - * `KEEP_TEST_DB` - if true, the test database will not be dropped after a test - is finished. Should only be used if one single scenario is - run, otherwise the result is undefined. - -Logging can be defined through command line parameters of behave itself. Check -out `behave --help` for details. Also have a look at the 'work-in-progress' -feature of behave which comes in handy when writing new tests. + pytest test/bdd + +The BDD tests create databases for the tests. You can set name of the databases +through configuration variables in your `pytest.ini`: + + * `nominatim_test_db` defines the name of the temporary database created for + a single test (default: `test_nominatim`) + * `nominatim_api_test_db` defines the name of the database containing + the API test data, see also below (default: `test_api_nominatim`) + * `nominatim_template_db` defines the name of the template database used + for creating the temporary test databases. It contains some static setup + which usually doesn't change between imports of OSM data + (default: `test_template_nominatim`) + +To change other connection parameters for the PostgreSQL database, use +the [libpq enivronment variables](https://www.postgresql.org/docs/current/libpq-envars.html). +Never set a password through these variables. Use a +[password file](https://www.postgresql.org/docs/current/libpq-pgpass.html) instead. + +The API test database and the template database are only created once and then +left untouched. This is usually what you want because it speeds up subsequent +runs of BDD tests. If you do change code that has an influence on the content +of these databases, you can run pytest with the `--nominatim-purge` parameter +and the databases will be dropped and recreated from scratch. + +When running the BDD tests with make (using `make tests` or `make bdd`), then +the databases will always be purged. + +The temporary test database is usually dropped directly after the test, so +it does not take up unnecessary space. If you want to keep the database around, +for example while debugging a specific BDD test, use the parameter +`--nominatim-keep-db`. + ### API Tests (`test/bdd/api`) These tests are meant to test the different API endpoints and their parameters. They require to import several datasets into a test database. This is normally done automatically during setup of the test. The API test database is then -kept around and reused in subsequent runs of behave. Use `behave -DREMOVE_TEMPLATE` +kept around and reused in subsequent runs of behave. Use `--nominatim-purge` to force a reimport of the database. The official test dataset is saved in the file `test/testdb/apidb-test-data.pbf` @@ -109,12 +109,12 @@ test the correctness of osm2pgsql. Each test will write some data into the `plac table (and optionally the `planet_osm_*` tables if required) and then run Nominatim's processing functions on that. -These tests need to create their own test databases. By default they will be -called `test_template_nominatim` and `test_nominatim`. Names can be changed with -the environment variables `TEMPLATE_DB` and `TEST_DB`. The user running the tests -needs superuser rights for postgres. +These tests use the template database and create temporary test databases for +each test. ### Import Tests (`test/bdd/osm2pgsql`) -These tests check that data is imported correctly into the place table. They -use the same template database as the DB Creation tests, so the same remarks apply. +These tests check that data is imported correctly into the place table. + +These tests also use the template database and create temporary test databases +for each test. diff --git a/docs/develop/Tokenizers.md b/docs/develop/Tokenizers.md index f4a55adc..a1dae78b 100644 --- a/docs/develop/Tokenizers.md +++ b/docs/develop/Tokenizers.md @@ -91,14 +91,19 @@ for a custom tokenizer implementation. ### Directory Structure -Nominatim expects a single file `src/nominatim_db/tokenizer/_tokenizer.py` -containing the Python part of the implementation. +Nominatim expects two files containing the Python part of the implementation: + + * `src/nominatim_db/tokenizer/_tokenizer.py` contains the tokenizer + code used during import and + * `src/nominatim_api/search/_tokenizer.py` has the code used during + query time. + `` is a unique name for the tokenizer consisting of only lower-case letters, digits and underscore. A tokenizer also needs to install some SQL functions. By convention, these should be placed in `lib-sql/tokenizer`. If the tokenizer has a default configuration file, this should be saved in -the `settings/_tokenizer.`. +`settings/_tokenizer.`. ### Configuration and Persistence @@ -110,9 +115,11 @@ are tied to a database installation and must only be read during installation time. If they are needed for the runtime then they must be saved into the `nominatim_properties` table and later loaded from there. -### The Python module +### The Python modules -The Python module is expect to export a single factory function: +#### `src/nominatim_db/tokenizer/` + +The import Python module is expected to export a single factory function: ```python def create(dsn: str, data_dir: Path) -> AbstractTokenizer @@ -123,6 +130,20 @@ is a directory in the project directory that the tokenizer may use to save database-specific data. The function must return the instance of the tokenizer class as defined below. +#### `src/nominatim_api/search/` + +The query-time Python module must also export a factory function: + +``` python +def create_query_analyzer(conn: SearchConnection) -> AbstractQueryAnalyzer +``` + +The `conn` parameter contains the current search connection. See the +[library documentation](../library/Low-Level-DB-Access.md#searchconnection-class) +for details on the class. The function must return the instance of the tokenizer +class as defined below. + + ### Python Tokenizer Class All tokenizers must inherit from `nominatim_db.tokenizer.base.AbstractTokenizer` @@ -138,6 +159,13 @@ and implement the abstract functions defined there. options: heading_level: 6 + +### Python Query Analyzer Class + +::: nominatim_api.search.query_analyzer_factory.AbstractQueryAnalyzer + options: + heading_level: 6 + ### PL/pgSQL Functions The tokenizer must provide access functions for the `token_info` column diff --git a/docs/extra.css b/docs/extra.css index 1decc478..033e9903 100644 --- a/docs/extra.css +++ b/docs/extra.css @@ -2,8 +2,8 @@ display: none!important } -.wy-nav-content { - max-width: 900px!important +.md-content { + max-width: 800px } table { diff --git a/lib-lua/flex-base.lua b/lib-lua/flex-base.lua new file mode 100644 index 00000000..1173c53f --- /dev/null +++ b/lib-lua/flex-base.lua @@ -0,0 +1,14 @@ +-- This is just an alias for the Nominatim themepark theme module +local flex = require('themes/nominatim/init') + +function flex.load_topic(name, cfg) + local topic_file = debug.getinfo(1, "S").source:sub(2):match("(.*/)") .. 'themes/nominatim/topics/'.. name .. '.lua' + + if topic_file == nil then + error('Cannot find topic: ' .. name) + end + + loadfile(topic_file)(nil, flex, cfg or {}) +end + +return flex diff --git a/lib-lua/import-address.lua b/lib-lua/import-address.lua new file mode 100644 index 00000000..bec21505 --- /dev/null +++ b/lib-lua/import-address.lua @@ -0,0 +1,6 @@ +-- This is just an alias for the Nominatim themepark address topic +local flex = require('flex-base') + +flex.load_topic('address') + +return flex diff --git a/lib-lua/import-admin.lua b/lib-lua/import-admin.lua new file mode 100644 index 00000000..8d1230a1 --- /dev/null +++ b/lib-lua/import-admin.lua @@ -0,0 +1,6 @@ +-- This is just an alias for the Nominatim themepark admin topic +local flex = require('flex-base') + +flex.load_topic('admin') + +return flex diff --git a/lib-lua/import-extratags.lua b/lib-lua/import-extratags.lua new file mode 100644 index 00000000..53b1c81e --- /dev/null +++ b/lib-lua/import-extratags.lua @@ -0,0 +1,6 @@ +-- This is just an alias for the Nominatim themepark full topic +local flex = require('flex-base') + +flex.load_topic('full', {with_extratags = true}) + +return flex diff --git a/lib-lua/import-full.lua b/lib-lua/import-full.lua new file mode 100644 index 00000000..59308a67 --- /dev/null +++ b/lib-lua/import-full.lua @@ -0,0 +1,6 @@ +-- This is just an alias for the Nominatim themepark full topic +local flex = require('flex-base') + +flex.load_topic('full') + +return flex diff --git a/lib-lua/import-street.lua b/lib-lua/import-street.lua new file mode 100644 index 00000000..1a02b098 --- /dev/null +++ b/lib-lua/import-street.lua @@ -0,0 +1,6 @@ +-- This is just an alias for the Nominatim themepark street topic +local flex = require('flex-base') + +flex.load_topic('street') + +return flex diff --git a/lib-lua/taginfo.lua b/lib-lua/taginfo.lua new file mode 100644 index 00000000..402499ad --- /dev/null +++ b/lib-lua/taginfo.lua @@ -0,0 +1,118 @@ +-- Prints taginfo project description in the standard output +-- + +-- create fake "osm2pgsql" table for flex-base, originally created by the main C++ program +osm2pgsql = {} +function osm2pgsql.define_table(...) end + +-- provide path to flex-style lua file +package.path = arg[0]:match("(.*/)") .. "?.lua;" .. package.path +local flex = require('import-' .. (arg[1] or 'extratags')) +local json = require ('dkjson') + +local NAME_DESCRIPTIONS = { + 'Searchable auxiliary name of the place', + main = 'Searchable primary name of the place', + house = 'House name part of an address, searchable' +} +local ADDRESS_DESCRIPTIONS = { + 'Used to determine the address of a place', + main = 'Primary key for an address point', + postcode = 'Used to determine the postcode of a place', + country = 'Used to determine country of a place (only if written as two-letter code)', + interpolation = 'Primary key for an address interpolation line' +} + +------------ helper functions --------------------- +-- Sets the key order for the resulting JSON table +local function set_keyorder(table, order) + setmetatable(table, { + __jsonorder = order + }) +end + +local function get_key_description(key, description) + local desc = {} + desc.key = key + desc.description = description + set_keyorder(desc, {'key', 'description'}) + return desc +end + +local function get_key_value_description(key, value, description) + local desc = {key = key, value = value, description = description} + set_keyorder(desc, {'key', 'value', 'description'}) + return desc +end + +local function group_table_to_keys(tags, data, descriptions) + for group, values in pairs(data) do + local desc = descriptions[group] or descriptions[1] + for _, key in pairs(values) do + if key:sub(1, 1) ~= '*' and key:sub(#key, #key) ~= '*' then + table.insert(tags, get_key_description(key, desc)) + end + end + end +end + +-- Prints the collected tags in the required format in JSON +local function print_taginfo() + local taginfo = flex.get_taginfo() + local tags = {} + + for k, values in pairs(taginfo.main) do + if values[1] == nil or values[1] == 'delete' or values[1] == 'extra' then + for v, group in pairs(values) do + if type(v) == 'string' and group ~= 'delete' and group ~= 'extra' then + local text = 'POI/feature in the search database' + if type(group) ~= 'function' then + text = 'Fallback ' .. text + end + table.insert(tags, get_key_value_description(k, v, text)) + end + end + elseif type(values[1]) == 'function' or values[1] == 'fallback' then + local desc = 'POI/feature in the search database' + if values[1] == 'fallback' then + desc = 'Fallback ' .. desc + end + local excp = {} + for v, group in pairs(values) do + if group == 'delete' or group == 'extra' then + table.insert(excp, v) + end + end + if next(excp) ~= nil then + desc = desc .. string.format(' (except for values: %s)', + table.concat(excp, ', ')) + end + table.insert(tags, get_key_description(k, desc)) + end + end + + group_table_to_keys(tags, taginfo.name, NAME_DESCRIPTIONS) + group_table_to_keys(tags, taginfo.address, ADDRESS_DESCRIPTIONS) + + local format = { + data_format = 1, + data_url = 'https://nominatim.openstreetmap.org/taginfo.json', + project = { + name = 'Nominatim', + description = 'OSM search engine.', + project_url = 'https://nominatim.openstreetmap.org', + doc_url = 'https://nominatim.org/release-docs/develop/', + contact_name = 'Sarah Hoffmann', + contact_email = 'lonvia@denofr.de' + } + } + format.tags = tags + + set_keyorder(format, {'data_format', 'data_url', 'project', 'tags'}) + set_keyorder(format.project, {'name', 'description', 'project_url', 'doc_url', + 'contact_name', 'contact_email'}) + + print(json.encode(format)) +end + +print_taginfo() diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua new file mode 100644 index 00000000..fef86f91 --- /dev/null +++ b/lib-lua/themes/nominatim/init.lua @@ -0,0 +1,925 @@ +-- Nominatim themepark theme. +-- +-- The Nominatim theme creates a fixed set of import tables for use with +-- Nominatim. Creation and object processing are directly controlled by +-- the theme. Topics provide preset configurations. You should add exactly +-- one topic to your project. +-- +-- The theme also exports a number of functions that can be used to configure +-- its behaviour. These may be directly called in the style file after +-- importing the theme: +-- +-- local nominatim = themepark:init_theme('nominatim') +-- nominatim.set_main_tags{boundary = 'always'} +-- +-- This allows to write your own configuration from scratch. You can also +-- use it to customize topics. In that case, first add the topic, then +-- change the configuration: +-- +-- themepark:add_topic('nominatim/full') +-- local nominatim = themepark:init_theme('nominatim') +-- nominatim.ignore_tags{'amenity'} + +local module = {} + +local MAIN_KEYS = {admin_level = {'delete'}} +local PRE_FILTER = {prefix = {}, suffix = {}} +local NAMES = {} +local NAME_FILTER = nil +local ADDRESS_TAGS = {} +local ADDRESS_FILTER = nil +local EXTRATAGS_FILTER +local POSTCODE_FALLBACK = true + +-- This file can also be directly require'd instead of running it under +-- the themepark framework. In that case the first parameter is usually +-- the module name. Lets check for that, so that further down we can call +-- the low-level osm2pgsql functions instead of themepark functions. +local themepark = ... +if type(themepark) ~= 'table' then + themepark = nil +end + +-- The single place table. +local place_table_definition = { + name = "place", + ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, + columns = { + { column = 'class', type = 'text', not_null = true }, + { column = 'type', type = 'text', not_null = true }, + { column = 'admin_level', type = 'smallint' }, + { column = 'name', type = 'hstore' }, + { column = 'address', type = 'hstore' }, + { column = 'extratags', type = 'hstore' }, + { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true }, + }, + data_tablespace = os.getenv("NOMINATIM_TABLESPACE_PLACE_DATA"), + index_tablespace = os.getenv("NOMINATIM_TABLESPACE_PLACE_INDEX"), + indexes = {} +} + +local insert_row +local script_path = debug.getinfo(1, "S").source:match("@?(.*/)") +local PRESETS = loadfile(script_path .. 'presets.lua')() + +if themepark then + themepark:add_table(place_table_definition) + insert_row = function(columns) + themepark:insert('place', columns, {}, {}) + end +else + local place_table = osm2pgsql.define_table(place_table_definition) + insert_row = function(columns) + place_table:insert(columns) + end +end + +------------ Geometry functions for relations --------------------- + +function module.relation_as_multipolygon(o) + return o:as_multipolygon() +end + +function module.relation_as_multiline(o) + return o:as_multilinestring():line_merge() +end + + +module.RELATION_TYPES = { + multipolygon = module.relation_as_multipolygon, + boundary = module.relation_as_multipolygon, + waterway = module.relation_as_multiline +} + +--------- Built-in place transformation functions -------------------------- + +local PlaceTransform = {} + +-- Special transform meanings which are interpreted elsewhere +PlaceTransform.fallback = 'fallback' +PlaceTransform.delete = 'delete' +PlaceTransform.extra = 'extra' + +-- always: unconditionally use that place +function PlaceTransform.always(place) + return place +end + +-- never: unconditionally drop the place +function PlaceTransform.never() + return nil +end + +-- named: use the place if it has a fully-qualified name +function PlaceTransform.named(place) + if place.has_name then + return place + end +end + +-- named_with_key: use place if there is a name with the main key prefix +function PlaceTransform.named_with_key(place, k) + local names = {} + local prefix = k .. ':name' + for namek, namev in pairs(place.intags) do + if namek:sub(1, #prefix) == prefix + and (#namek == #prefix + or namek:sub(#prefix + 1, #prefix + 1) == ':') then + names[namek:sub(#k + 2)] = namev + end + end + + if next(names) ~= nil then + return place:clone{names=names} + end +end + +-- Special transform used with address fallbacks: ignore all names +-- except for those marked as being part of the address. +local function address_fallback(place) + if next(place.names) == nil or NAMES.house == nil then + return place + end + + local names = {} + for k, v in pairs(place.names) do + if NAME_FILTER(k, v) == 'house' then + names[k] = v + end + end + return place:clone{names=names} +end + +--------- Built-in extratags transformation functions --------------- + +local function default_extratags_filter(p, k) + -- Default handling is to copy over place tag for boundaries. + -- Nominatim needs this. + if k ~= 'boundary' or p.intags.place == nil then + return p.extratags + end + + local extra = { place = p.intags.place } + for kin, vin in pairs(p.extratags) do + extra[kin] = vin + end + + return extra +end +EXTRATAGS_FILTER = default_extratags_filter + +----------------- other helper functions ----------------------------- + +local function lookup_prefilter_classification(k, v) + -- full matches + local desc = MAIN_KEYS[k] + local fullmatch = desc and (desc[v] or desc[1]) + if fullmatch ~= nil then + return fullmatch + end + -- suffixes + for slen, slist in pairs(PRE_FILTER.suffix) do + if #k >= slen then + local group = slist[k:sub(-slen)] + if group ~= nil then + return group + end + end + end + -- prefixes + for slen, slist in pairs(PRE_FILTER.prefix) do + if #k >= slen then + local group = slist[k:sub(1, slen)] + if group ~= nil then + return group + end + end + end +end + + +local function merge_filters_into_main(group, keys, tags) + if keys ~= nil then + for _, key in pairs(keys) do + -- ignore suffix and prefix matches + if key:sub(1, 1) ~= '*' and key:sub(#key, #key) ~= '*' then + if MAIN_KEYS[key] == nil then + MAIN_KEYS[key] = {} + end + MAIN_KEYS[key][1] = group + end + end + end + + if tags ~= nil then + for key, values in pairs(tags) do + if MAIN_KEYS[key] == nil then + MAIN_KEYS[key] = {} + end + for _, v in pairs(values) do + MAIN_KEYS[key][v] = group + end + end + end +end + + +local function remove_group_from_main(group) + for key, values in pairs(MAIN_KEYS) do + for _, ttype in pairs(values) do + if ttype == group then + values[ttype] = nil + end + end + if next(values) == nil then + MAIN_KEYS[key] = nil + end + end +end + + +local function add_pre_filter(data) + for group, keys in pairs(data) do + for _, key in pairs(keys) do + local klen = #key - 1 + if key:sub(1, 1) == '*' then + if klen > 0 then + if PRE_FILTER.suffix[klen] == nil then + PRE_FILTER.suffix[klen] = {} + end + PRE_FILTER.suffix[klen][key:sub(2)] = group + end + elseif key:sub(#key, #key) == '*' then + if PRE_FILTER.prefix[klen] == nil then + PRE_FILTER.prefix[klen] = {} + end + PRE_FILTER.prefix[klen][key:sub(1, klen)] = group + end + end + end +end + +------------- Place class ------------------------------------------ + +local Place = {} +Place.__index = Place + +function Place.new(object, geom_func) + local self = setmetatable({}, Place) + self.object = object + self.geom_func = geom_func + + self.admin_level = tonumber(self.object.tags.admin_level or 15) or 15 + if self.admin_level == nil + or self.admin_level <= 0 or self.admin_level > 15 + or math.floor(self.admin_level) ~= self.admin_level then + self.admin_level = 15 + end + + self.num_entries = 0 + self.has_name = false + self.names = {} + self.address = {} + self.extratags = {} + + self.intags = {} + + local has_main_tags = false + for k, v in pairs(self.object.tags) do + local group = lookup_prefilter_classification(k, v) + if group == 'extra' then + self.extratags[k] = v + elseif group ~= 'delete' then + self.intags[k] = v + if group ~= nil then + has_main_tags = true + end + end + end + + if not has_main_tags then + -- no interesting tags, don't bother processing + self.intags = {} + end + + return self +end + +function Place:clean(data) + for k, v in pairs(self.intags) do + if data.delete ~= nil and data.delete(k, v) then + self.intags[k] = nil + elseif data.extra ~= nil and data.extra(k, v) then + self.extratags[k] = v + self.intags[k] = nil + end + end +end + +function Place:delete(data) + if data.match ~= nil then + for k, v in pairs(self.intags) do + if data.match(k, v) then + self.intags[k] = nil + end + end + end +end + +function Place:grab_extratags(data) + local count = 0 + + if data.match ~= nil then + for k, v in pairs(self.intags) do + if data.match(k, v) then + self.intags[k] = nil + self.extratags[k] = v + count = count + 1 + end + end + end + + return count +end + +local function strip_address_prefix(k) + if k:sub(1, 5) == 'addr:' then + return k:sub(6) + end + + if k:sub(1, 6) == 'is_in:' then + return k:sub(7) + end + + return k +end + + +function Place:grab_address_parts(data) + local count = 0 + + if data.groups ~= nil then + for k, v in pairs(self.intags) do + local atype = data.groups(k, v) + + if atype ~= nil then + if atype == 'main' then + self.has_name = true + self.address[strip_address_prefix(k)] = v + count = count + 1 + elseif atype == 'extra' then + self.address[strip_address_prefix(k)] = v + else + self.address[atype] = v + end + self.intags[k] = nil + end + end + end + + return count +end + + +function Place:grab_name_parts(data) + local fallback = nil + + if data.groups ~= nil then + for k, v in pairs(self.intags) do + local atype = data.groups(k, v) + + if atype ~= nil then + self.names[k] = v + self.intags[k] = nil + if atype == 'main' then + self.has_name = true + elseif atype == 'house' then + self.has_name = true + fallback = {'place', 'house', address_fallback} + end + end + end + end + + return fallback +end + + +function Place:write_place(k, v, mfunc) + v = v or self.intags[k] + if v == nil then + return 0 + end + + local place = mfunc(self, k, v) + if place then + local res = place:write_row(k, v) + self.num_entries = self.num_entries + res + return res + end + + return 0 +end + +function Place:write_row(k, v) + if self.geometry == nil then + self.geometry = self.geom_func(self.object) + end + if self.geometry == nil or self.geometry:is_null() then + return 0 + end + + local extratags = EXTRATAGS_FILTER(self, k, v) + if not (extratags and next(extratags)) then + extratags = nil + end + + insert_row{ + class = k, + type = v, + admin_level = self.admin_level, + name = next(self.names) and self.names, + address = next(self.address) and self.address, + extratags = extratags, + geometry = self.geometry + } + + return 1 +end + + +function Place:clone(data) + local cp = setmetatable({}, Place) + cp.object = self.object + cp.geometry = data.geometry or self.geometry + cp.geom_func = self.geom_func + cp.intags = data.intags or self.intags + cp.admin_level = data.admin_level or self.admin_level + cp.names = data.names or self.names + cp.address = data.address or self.address + cp.extratags = data.extratags or self.extratags + + return cp +end + + +function module.tag_match(data) + if data == nil or next(data) == nil then + return nil + end + + local fullmatches = {} + local key_prefixes = {} + local key_suffixes = {} + + if data.keys ~= nil then + for _, key in pairs(data.keys) do + if key:sub(1, 1) == '*' then + if #key > 1 then + if key_suffixes[#key - 1] == nil then + key_suffixes[#key - 1] = {} + end + key_suffixes[#key - 1][key:sub(2)] = true + end + elseif key:sub(#key, #key) == '*' then + if key_prefixes[#key - 1] == nil then + key_prefixes[#key - 1] = {} + end + key_prefixes[#key - 1][key:sub(1, #key - 1)] = true + else + fullmatches[key] = true + end + end + end + + if data.tags ~= nil then + for k, vlist in pairs(data.tags) do + if fullmatches[k] == nil then + fullmatches[k] = {} + for _, v in pairs(vlist) do + fullmatches[k][v] = true + end + end + end + end + + return function (k, v) + if fullmatches[k] ~= nil and (fullmatches[k] == true or fullmatches[k][v] ~= nil) then + return true + end + + for slen, slist in pairs(key_suffixes) do + if #k >= slen and slist[k:sub(-slen)] ~= nil then + return true + end + end + + for slen, slist in pairs(key_prefixes) do + if #k >= slen and slist[k:sub(1, slen)] ~= nil then + return true + end + end + + return false + end +end + + +function module.tag_group(data) + if data == nil or next(data) == nil then + return nil + end + + local fullmatches = {} + local key_prefixes = {} + local key_suffixes = {} + + for group, tags in pairs(data) do + for _, key in pairs(tags) do + if key:sub(1, 1) == '*' then + if #key > 1 then + if key_suffixes[#key - 1] == nil then + key_suffixes[#key - 1] = {} + end + key_suffixes[#key - 1][key:sub(2)] = group + end + elseif key:sub(#key, #key) == '*' then + if key_prefixes[#key - 1] == nil then + key_prefixes[#key - 1] = {} + end + key_prefixes[#key - 1][key:sub(1, #key - 1)] = group + else + fullmatches[key] = group + end + end + end + + return function (k) + local val = fullmatches[k] + if val ~= nil then + return val + end + + for slen, slist in pairs(key_suffixes) do + if #k >= slen then + val = slist[k:sub(-slen)] + if val ~= nil then + return val + end + end + end + + for slen, slist in pairs(key_prefixes) do + if #k >= slen then + val = slist[k:sub(1, slen)] + if val ~= nil then + return val + end + end + end + end +end + +-- Returns prefix part of the keys, and reject suffix matching keys +local function process_key(key) + if key:sub(1, 1) == '*' then + return nil + end + if key:sub(#key, #key) == '*' then + return key:sub(1, #key - 2) + end + return key +end + +-- Process functions for all data types +function module.process_node(object) + + local function geom_func(o) + return o:as_point() + end + + module.process_tags(Place.new(object, geom_func)) +end + +function module.process_way(object) + + local function geom_func(o) + local geom = o:as_polygon() + + if geom:is_null() then + geom = o:as_linestring() + if geom:is_null() or geom:length() > 30 then + return nil + end + end + + return geom + end + + module.process_tags(Place.new(object, geom_func)) +end + +function module.process_relation(object) + local geom_func = module.RELATION_TYPES[object.tags.type] + + if geom_func ~= nil then + module.process_tags(Place.new(object, geom_func)) + end +end + +-- The process functions are used by default by osm2pgsql. +if themepark then + themepark:add_proc('node', module.process_node) + themepark:add_proc('way', module.process_way) + themepark:add_proc('relation', module.process_relation) +else + osm2pgsql.process_node = module.process_node + osm2pgsql.process_way = module.process_way + osm2pgsql.process_relation = module.process_relation +end + +function module.process_tags(o) + if next(o.intags) == nil then + return -- shortcut when pre-filtering has removed all tags + end + + -- Exception for boundary/place double tagging + if o.intags.boundary == 'administrative' then + o:grab_extratags{match = function (k, v) + return k == 'place' and v:sub(1,3) ~= 'isl' + end} + end + + -- name keys + local fallback = o:grab_name_parts{groups=NAME_FILTER} + + -- address keys + if o:grab_address_parts{groups=ADDRESS_FILTER} > 0 and fallback == nil then + fallback = {'place', 'house', address_fallback} + end + if o.address.country ~= nil and #o.address.country ~= 2 then + o.address['country'] = nil + end + if POSTCODE_FALLBACK and fallback == nil and o.address.postcode ~= nil then + fallback = {'place', 'postcode', PlaceTransform.always} + end + + if o.address.interpolation ~= nil then + o:write_place('place', 'houses', PlaceTransform.always) + return + end + + -- collect main keys + for k, v in pairs(o.intags) do + local ktable = MAIN_KEYS[k] + if ktable then + local ktype = ktable[v] or ktable[1] + if type(ktype) == 'function' then + o:write_place(k, v, ktype) + elseif ktype == 'fallback' and o.has_name then + fallback = {k, v, PlaceTransform.named} + end + end + end + + if fallback ~= nil and o.num_entries == 0 then + o:write_place(fallback[1], fallback[2], fallback[3]) + end +end + +--------- Convenience functions for simple style configuration ----------------- + +function module.set_prefilters(data) + remove_group_from_main('delete') + merge_filters_into_main('delete', data.delete_keys, data.delete_tags) + + remove_group_from_main('extra') + merge_filters_into_main('extra', data.extra_keys, data.extra_tags) + + PRE_FILTER = {prefix = {}, suffix = {}} + add_pre_filter{delete = data.delete_keys, extra = data.extra_keys} +end + + +function module.ignore_keys(data) + if type(data) == 'string' then + local preset = data + data = PRESETS.IGNORE_KEYS[data] + if data == nil then + error('Unknown preset for ignored keys: ' .. preset) + end + end + merge_filters_into_main('delete', data) + add_pre_filter{delete = data} +end + + +function module.add_for_extratags(data) + if type(data) == 'string' then + local preset = data + data = PRESETS.EXTRATAGS[data] or PRESETS.IGNORE_KEYS[data] + if data == nil then + error('Unknown preset for extratags: ' .. preset) + end + end + merge_filters_into_main('extra', data) + add_pre_filter{extra = data} +end + + +function module.set_main_tags(data) + for key, values in pairs(MAIN_KEYS) do + for _, ttype in pairs(values) do + if ttype == 'fallback' or type(ttype) == 'function' then + values[ttype] = nil + end + end + if next(values) == nil then + MAIN_KEYS[key] = nil + end + end + module.modify_main_tags(data) +end + + +function module.modify_main_tags(data) + if type(data) == 'string' then + local preset = data + if data:sub(1, 7) == 'street/' then + data = PRESETS.MAIN_TAGS_STREETS[data:sub(8)] + elseif data:sub(1, 4) == 'poi/' then + data = PRESETS.MAIN_TAGS_POIS(data:sub(5)) + else + data = PRESETS.MAIN_TAGS[data] + end + if data == nil then + error('Unknown preset for main tags: ' .. preset) + end + end + + for k, v in pairs(data) do + if MAIN_KEYS[k] == nil then + MAIN_KEYS[k] = {} + end + if type(v) == 'function' then + MAIN_KEYS[k][1] = v + elseif type(v) == 'string' then + MAIN_KEYS[k][1] = PlaceTransform[v] + elseif type(v) == 'table' then + for subk, subv in pairs(v) do + if type(subv) == 'function' then + MAIN_KEYS[k][subk] = subv + else + MAIN_KEYS[k][subk] = PlaceTransform[subv] + end + end + end + end +end + + +function module.modify_name_tags(data) + if type(data) == 'string' then + local preset = data + data = PRESETS.NAME_TAGS[data] + if data == nil then + error('Unknown preset for name keys: ' .. preset) + end + end + + for k,v in pairs(data) do + if next(v) then + NAMES[k] = v + else + NAMES[k] = nil + end + end + NAME_FILTER = module.tag_group(NAMES) + remove_group_from_main('fallback:name') + if data.house ~= nil then + merge_filters_into_main('fallback:name', data.house) + end +end + + +function module.set_name_tags(data) + NAMES = {} + module.modify_name_tags(data) +end + + +function module.set_address_tags(data) + ADDRESS_TAGS = {} + module.modify_address_tags(data) +end + + +function module.modify_address_tags(data) + if type(data) == 'string' then + local preset = data + data = PRESETS.ADDRESS_TAGS[data] + if data == nil then + error('Unknown preset for address keys: ' .. preset) + end + end + + for k, v in pairs(data) do + if k == 'postcode_fallback' then + POSTCODE_FALLBACK = v + elseif next(v) == nil then + ADDRESS_TAGS[k] = nil + else + ADDRESS_TAGS[k] = v + end + end + + ADDRESS_FILTER = module.tag_group(ADDRESS_TAGS) + + remove_group_from_main('fallback:address') + merge_filters_into_main('fallback:address', data.main) + merge_filters_into_main('fallback:address', data.interpolation) + remove_group_from_main('fallback:postcode') + if POSTCODE_FALLBACK then + merge_filters_into_main('fallback:postcode', data.postcode) + end +end + + +function module.set_address_tags(data) + ADDRESS_TAGS_SOURCE = {} + module.modify_address_tags(data) +end + + +function module.set_postcode_fallback(enable) + if POSTCODE_FALLBACK ~= enable then + remove_group_from_main('fallback:postcode') + if enable then + merge_filters_into_main('fallback:postcode', ADDRESS_TAGS.postcode) + end + end + POSTCODE_FALLBACK = enable +end + + +function module.set_unused_handling(data) + if type(data) == 'function' then + EXTRATAGS_FILTER = data + elseif data == nil then + EXTRATAGS_FILTER = default_extratags_filter + elseif data.extra_keys == nil and data.extra_tags == nil then + local delfilter = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} + EXTRATAGS_FILTER = function (p, k) + local extra = {} + for kin, vin in pairs(p.intags) do + if kin ~= k and not delfilter(kin, vin) then + extra[kin] = vin + end + end + if next(extra) == nil then + return p.extratags + end + for kextra, vextra in pairs(p.extratags) do + extra[kextra] = vextra + end + return extra + end + elseif data.delete_keys == nil and data.delete_tags == nil then + local incfilter = module.tag_match{keys = data.extra_keys, tags = data.extra_tags} + EXTRATAGS_FILTER = function (p, k) + local extra = {} + for kin, vin in pairs(p.intags) do + if kin ~= k and incfilter(kin, vin) then + extra[kin] = vin + end + end + if next(extra) == nil then + return p.extratags + end + for kextra, vextra in pairs(p.extratags) do + extra[kextra] = vextra + end + return extra + end + else + error("unused handler can have only 'extra_keys' or 'delete_keys' set.") + end +end + +function module.set_relation_types(data) + module.RELATION_TYPES = {} + for k, v in data do + if v == 'multipolygon' then + module.RELATION_TYPES[k] = module.relation_as_multipolygon + elseif v == 'multiline' then + module.RELATION_TYPES[k] = module.relation_as_multiline + end + end +end + + +function module.get_taginfo() + return {main = MAIN_KEYS, name = NAMES, address = ADDRESS_TAGS} +end + +return module diff --git a/lib-lua/themes/nominatim/presets.lua b/lib-lua/themes/nominatim/presets.lua new file mode 100644 index 00000000..00ff0f52 --- /dev/null +++ b/lib-lua/themes/nominatim/presets.lua @@ -0,0 +1,382 @@ +-- Defines defaults used in the topic definitions. + +local module = {} + +-- Helper functions + +local function group_merge(group1, group2) + for name, values in pairs(group2) do + if group1[name] == nil then + group1[name] = values + else + for _, v in pairs(values) do + table.insert(group1[name], v) + end + end + end + + return group1 +end + +-- Customized main tag filter functions + +local EXCLUDED_FOOTWAYS = { sidewalk = 1, crossing = 1, link = 1, traffic_aisle } + +local function filter_footways(place) + if place.has_name then + local footway = place.object.tags.footway + if footway == nil or EXCLUDED_FOOTWAYS[footway] ~= 1 then + return place + end + end + return false +end + +local function include_when_tag_present(key, value, named) + if named then + return function(place) + if place.has_name and place.intags[key] == value then + return place + end + return false + end + else + return function(place) + if place.intags[key] == value then + return place + end + return false + end + end +end + +local function exclude_when_key_present(key, named) + if named then + return function(place) + if place.has_name and place.intags[key] == nil then + return place + end + return false + end + else + return function(place) + if place.intags[key] == nil then + return place + end + return false + end + + end +end + +local function lock_transform(place) + if place.object.tags.waterway ~= nil then + local name = place.object.tags.lock_name + if name ~= nil then + return place:clone{names={name=name, ref=place.object.tags.lock_ref}} + end + end + + return false +end + +-- Main tag definition + +module.MAIN_TAGS = {} + +module.MAIN_TAGS.admin = { + boundary = {administrative = 'named'}, + landuse = {residential = 'fallback', + farm = 'fallback', + farmyard = 'fallback', + industrial = 'fallback', + commercial = 'fallback', + allotments = 'fallback', + retail = 'fallback'}, + place = {county = 'always', + district = 'always', + municipality = 'always', + city = 'always', + town = 'always', + borough = 'always', + village = 'always', + suburb = 'always', + hamlet = 'always', + croft = 'always', + subdivision = 'always', + allotments = 'always', + neighbourhood = 'always', + quarter = 'always', + isolated_dwelling = 'always', + farm = 'always', + city_block = 'always', + locality = 'always'} +} + +module.MAIN_TAGS.all_boundaries = { + boundary = {'named', + place = 'delete', + land_area = 'delete', + postal_code = 'always'}, + landuse = 'fallback', + place = 'always' +} + +module.MAIN_TAGS.natural = { + waterway = {'named', + riverbank = 'delete'}, + natural = {'named', + yes = 'delete', + no = 'delete', + coastline = 'delete', + saddle = 'fallback', + water = exclude_when_key_present('water', true)}, + mountain_pass = {'always', + no = 'delete'}, + water = {include_when_tag_present('natural', 'water', true), + river = 'never', + stream = 'never', + canal = 'never', + ditch = 'never', + drain = 'never', + fish_pass = 'never', + yes = 'delete', + intermittent = 'delete', + tidal = 'delete' + } +} + +module.MAIN_TAGS_POIS = function (group) + group = group or 'delete' + return { + aerialway = {'always', + no = group, + pylon = group}, + aeroway = {'always', + no = group}, + amenity = {'always', + no = group, + parking_space = group, + parking_entrance = group, + waste_disposal = group, + hunting_stand = group}, + building = {'fallback', + no = group}, + bridge = {'named_with_key', + no = group}, + club = {'always', + no = group}, + craft = {'always', + no = group}, + emergency = {'always', + no = group, + yes = group, + fire_hydrant = group}, + healthcare = {'fallback', + yes = group, + no = group}, + highway = {'always', + no = group, + turning_circle = group, + mini_roundabout = group, + noexit = group, + crossing = group, + give_way = group, + stop = group, + turning_loop = group, + passing_place = group, + street_lamp = 'named', + traffic_signals = 'named'}, + historic = {'fallback', + yes = group, + no = group}, + information = {include_when_tag_present('tourism', 'information'), + yes = 'delete', + route_marker = 'never', + trail_blaze = 'never'}, + junction = {'fallback', + no = group}, + landuse = {cemetery = 'always'}, + leisure = {'always', + nature_reserve = 'fallback', + swimming_pool = 'named', + no = group}, + lock = {yes = lock_transform}, + man_made = {pier = 'always', + tower = 'always', + bridge = 'always', + works = 'named', + water_tower = 'always', + dyke = 'named', + adit = 'named', + lighthouse = 'always', + watermill = 'always', + tunnel = 'always'}, + military = {'always', + yes = group, + no = group}, + office = {'always', + no = group}, + railway = {'named', + rail = group, + no = group, + abandoned = group, + disused = group, + razed = group, + level_crossing = group, + switch = group, + signal = group, + buffer_stop = group}, + shop = {'always', + no = group}, + tourism = {'always', + attraction = 'fallback', + no = group, + yes = group, + information = exclude_when_key_present('information')}, + tunnel = {'named_with_key', + no = group} +} end + +module.MAIN_TAGS_STREETS = {} + +module.MAIN_TAGS_STREETS.default = { + place = {square = 'always'}, + highway = {motorway = 'always', + trunk = 'always', + primary = 'always', + secondary = 'always', + tertiary = 'always', + unclassified = 'always', + residential = 'always', + road = 'always', + living_street = 'always', + pedestrian = 'always', + service = 'named', + cycleway = 'named', + path = 'named', + footway = filter_footways, + steps = 'named', + bridleway = 'named', + track = 'named', + motorway_link = 'named', + trunk_link = 'named', + primary_link = 'named', + secondary_link = 'named', + tertiary_link = 'named'} +} + +module.MAIN_TAGS_STREETS.car = { + place = {square = 'always'}, + highway = {motorway = 'always', + trunk = 'always', + primary = 'always', + secondary = 'always', + tertiary = 'always', + unclassified = 'always', + residential = 'always', + road = 'always', + living_street = 'always', + service = 'always', + track = 'always', + motorway_link = 'always', + trunk_link = 'always', + primary_link = 'always', + secondary_link = 'always', + tertiary_link = 'always'} +} + +module.MAIN_TAGS_STREETS.all = { + place = {square = 'always'}, + highway = {motorway = 'always', + trunk = 'always', + primary = 'always', + secondary = 'always', + tertiary = 'always', + unclassified = 'always', + residential = 'always', + road = 'always', + living_street = 'always', + pedestrian = 'always', + service = 'always', + cycleway = 'always', + path = 'always', + footway = 'always', + steps = 'always', + bridleway = 'always', + track = 'always', + motorway_link = 'always', + trunk_link = 'always', + primary_link = 'always', + secondary_link = 'always', + tertiary_link = 'always'} +} + + +-- name tags + +module.NAME_TAGS = {} + +module.NAME_TAGS.core = {main = {'name', 'name:*', + 'int_name', 'int_name:*', + 'nat_name', 'nat_name:*', + 'reg_name', 'reg_name:*', + 'loc_name', 'loc_name:*', + 'old_name', 'old_name:*', + 'alt_name', 'alt_name:*', 'alt_name_*', + 'official_name', 'official_name:*', + 'place_name', 'place_name:*', + 'short_name', 'short_name:*'}, + extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref', + 'loc_ref', 'old_ref', 'ISO3166-2'} + } +module.NAME_TAGS.address = {house = {'addr:housename'}} +module.NAME_TAGS.poi = group_merge({main = {'brand'}, + extra = {'iata', 'icao'}}, + module.NAME_TAGS.core) + +-- Address tagging + +module.ADDRESS_TAGS = {} + +module.ADDRESS_TAGS.core = { extra = {'addr:*', 'is_in:*', 'tiger:county'}, + postcode = {'postal_code', 'postcode', 'addr:postcode', + 'tiger:zip_left', 'tiger:zip_right'}, + country = {'country_code', 'ISO3166-1', + 'addr:country_code', 'is_in:country_code', + 'addr:country', 'is_in:country'} + } + +module.ADDRESS_TAGS.houses = { main = {'addr:housenumber', + 'addr:conscriptionnumber', + 'addr:streetnumber'}, + interpolation = {'addr:interpolation'} + } + +-- Ignored tags (prefiltered away) + +module.IGNORE_KEYS = {} + +module.IGNORE_KEYS.metatags = {'note', 'note:*', 'source', 'source:*', '*source', + 'attribution', 'comment', 'fixme', 'created_by', + 'tiger:cfcc', 'tiger:reviewed', 'nysgissam:*', + 'NHD:*', 'nhd:*', 'gnis:*', 'geobase:*', 'yh:*', + 'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*', + 'lacounty:*', 'ref:linz:*', + 'ref:bygningsnr', 'ref:ruian:*', 'building:ruian:type', + 'type', + 'is_in:postcode'} +module.IGNORE_KEYS.name = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', + 'name:etymology', 'name:etymology:*', + 'name:signed', 'name:botanical'} +module.IGNORE_KEYS.address = {'addr:street:*', 'addr:city:*', 'addr:district:*', + 'addr:province:*', 'addr:subdistrict:*', 'addr:place:*', + 'addr:TW:dataset'} + +-- Extra tags (prefiltered away) + +module.EXTRATAGS = {} + +module.EXTRATAGS.required = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'} + +return module diff --git a/lib-lua/themes/nominatim/topics/address.lua b/lib-lua/themes/nominatim/topics/address.lua new file mode 100644 index 00000000..0e813673 --- /dev/null +++ b/lib-lua/themes/nominatim/topics/address.lua @@ -0,0 +1,23 @@ +local _, flex, cfg = ... + +flex.set_main_tags('admin') +flex.modify_main_tags('street/' .. (cfg.street_theme or 'default')) +flex.modify_main_tags{boundary = {postal_code = 'always'}} + +flex.set_name_tags('core') +flex.modify_name_tags('address') + +flex.set_address_tags('core') +flex.modify_address_tags('houses') + +flex.ignore_keys('metatags') +flex.add_for_extratags('required') + +if cfg.with_extratags then + flex.set_unused_handling{delete_keys = {'tiger:*'}} + flex.add_for_extratags('name') + flex.add_for_extratags('address') +else + flex.ignore_keys('name') + flex.ignore_keys('address') +end diff --git a/lib-lua/themes/nominatim/topics/admin.lua b/lib-lua/themes/nominatim/topics/admin.lua new file mode 100644 index 00000000..47f4e248 --- /dev/null +++ b/lib-lua/themes/nominatim/topics/admin.lua @@ -0,0 +1,20 @@ +local _, flex, cfg = ... + +flex.set_main_tags('admin') + +flex.set_name_tags('core') + +flex.set_address_tags('core') +flex.set_postcode_fallback(false) + +flex.ignore_keys('metatags') +flex.add_for_extratags('required') + +if cfg.with_extratags then + flex.set_unused_handling{delete_keys = {'tiger:*'}} + flex.add_for_extratags('name') + flex.add_for_extratags('address') +else + flex.ignore_keys('name') + flex.ignore_keys('address') +end diff --git a/lib-lua/themes/nominatim/topics/full.lua b/lib-lua/themes/nominatim/topics/full.lua new file mode 100644 index 00000000..a0b61b0f --- /dev/null +++ b/lib-lua/themes/nominatim/topics/full.lua @@ -0,0 +1,32 @@ +local _, flex, cfg = ... + +local group +if cfg.with_extratags then + group = 'extra' +else + group = 'delete' +end + +flex.set_main_tags('all_boundaries') +flex.modify_main_tags('natural') +flex.modify_main_tags('street/' .. (cfg.street_theme or 'default')) +flex.modify_main_tags('poi/' .. group) + +flex.set_name_tags('core') +flex.modify_name_tags('address') +flex.modify_name_tags('poi') + +flex.set_address_tags('core') +flex.modify_address_tags('houses') + +flex.ignore_keys('metatags') +flex.add_for_extratags('required') + +if cfg.with_extratags then + flex.set_unused_handling{delete_keys = {'tiger:*'}} + flex.add_for_extratags('name') + flex.add_for_extratags('address') +else + flex.ignore_keys('name') + flex.ignore_keys('address') +end diff --git a/lib-lua/themes/nominatim/topics/street.lua b/lib-lua/themes/nominatim/topics/street.lua new file mode 100644 index 00000000..89bed442 --- /dev/null +++ b/lib-lua/themes/nominatim/topics/street.lua @@ -0,0 +1,22 @@ +local _, flex, cfg = ... + +flex.set_main_tags('admin') +flex.modify_main_tags('street/' .. (cfg.street_theme or 'default')) +flex.modify_main_tags{boundary = {postal_code = 'always'}} + +flex.set_name_tags('core') + +flex.set_address_tags('core') +flex.set_postcode_fallback(false) + +flex.ignore_keys('metatags') +flex.add_for_extratags('required') + +if cfg.with_extratags then + flex.set_unused_handling{delete_keys = {'tiger:*'}} + flex.add_for_extratags('name') + flex.add_for_extratags('address') +else + flex.ignore_keys('name') + flex.ignore_keys('address') +end diff --git a/lib-sql/functions.sql b/lib-sql/functions.sql index 158969d9..737a3f21 100644 --- a/lib-sql/functions.sql +++ b/lib-sql/functions.sql @@ -8,7 +8,6 @@ {% include('functions/utils.sql') %} {% include('functions/ranking.sql') %} {% include('functions/importance.sql') %} -{% include('functions/address_lookup.sql') %} {% include('functions/interpolation.sql') %} {% if 'place' in db.tables %} diff --git a/lib-sql/functions/address_lookup.sql b/lib-sql/functions/address_lookup.sql deleted file mode 100644 index cba11dbf..00000000 --- a/lib-sql/functions/address_lookup.sql +++ /dev/null @@ -1,334 +0,0 @@ --- SPDX-License-Identifier: GPL-2.0-only --- --- This file is part of Nominatim. (https://nominatim.org) --- --- Copyright (C) 2022 by the Nominatim developer community. --- For a full list of authors see the git log. - --- Functions for returning address information for a place. - -DROP TYPE IF EXISTS addressline CASCADE; -CREATE TYPE addressline as ( - place_id BIGINT, - osm_type CHAR(1), - osm_id BIGINT, - name HSTORE, - class TEXT, - type TEXT, - place_type TEXT, - admin_level INTEGER, - fromarea BOOLEAN, - isaddress BOOLEAN, - rank_address INTEGER, - distance FLOAT -); - - -CREATE OR REPLACE FUNCTION get_name_by_language(name hstore, languagepref TEXT[]) - RETURNS TEXT - AS $$ -DECLARE - result TEXT; -BEGIN - IF name is null THEN - RETURN null; - END IF; - - FOR j IN 1..array_upper(languagepref,1) LOOP - IF name ? languagepref[j] THEN - result := trim(name->languagepref[j]); - IF result != '' THEN - return result; - END IF; - END IF; - END LOOP; - - -- as a fallback - take the last element since it is the default name - RETURN trim((avals(name))[array_length(avals(name), 1)]); -END; -$$ -LANGUAGE plpgsql IMMUTABLE; - - ---housenumber only needed for tiger data -CREATE OR REPLACE FUNCTION get_address_by_language(for_place_id BIGINT, - housenumber INTEGER, - languagepref TEXT[]) - RETURNS TEXT - AS $$ -DECLARE - result TEXT[]; - currresult TEXT; - prevresult TEXT; - location RECORD; -BEGIN - - result := '{}'; - prevresult := ''; - - FOR location IN - SELECT name, - CASE WHEN place_id = for_place_id THEN 99 ELSE rank_address END as rank_address - FROM get_addressdata(for_place_id, housenumber) - WHERE isaddress order by rank_address desc - LOOP - currresult := trim(get_name_by_language(location.name, languagepref)); - IF currresult != prevresult AND currresult IS NOT NULL - AND result[(100 - location.rank_address)] IS NULL - THEN - result[(100 - location.rank_address)] := currresult; - prevresult := currresult; - END IF; - END LOOP; - - RETURN array_to_string(result,', '); -END; -$$ -LANGUAGE plpgsql STABLE; - -DROP TYPE IF EXISTS addressdata_place; -CREATE TYPE addressdata_place AS ( - place_id BIGINT, - country_code VARCHAR(2), - housenumber TEXT, - postcode TEXT, - class TEXT, - type TEXT, - name HSTORE, - address HSTORE, - centroid GEOMETRY -); - --- Compute the list of address parts for the given place. --- --- If in_housenumber is greator or equal 0, look for an interpolation. -CREATE OR REPLACE FUNCTION get_addressdata(in_place_id BIGINT, in_housenumber INTEGER) - RETURNS setof addressline - AS $$ -DECLARE - place addressdata_place; - location RECORD; - country RECORD; - current_rank_address INTEGER; - location_isaddress BOOLEAN; -BEGIN - -- The place in question might not have a direct entry in place_addressline. - -- Look for the parent of such places then and save it in place. - - -- first query osmline (interpolation lines) - IF in_housenumber >= 0 THEN - SELECT parent_place_id as place_id, country_code, - in_housenumber as housenumber, postcode, - 'place' as class, 'house' as type, - null as name, null as address, - ST_Centroid(linegeo) as centroid - INTO place - FROM location_property_osmline - WHERE place_id = in_place_id - AND in_housenumber between startnumber and endnumber; - END IF; - - --then query tiger data - {% if config.get_bool('USE_US_TIGER_DATA') %} - IF place IS NULL AND in_housenumber >= 0 THEN - SELECT parent_place_id as place_id, 'us' as country_code, - in_housenumber as housenumber, postcode, - 'place' as class, 'house' as type, - null as name, null as address, - ST_Centroid(linegeo) as centroid - INTO place - FROM location_property_tiger - WHERE place_id = in_place_id - AND in_housenumber between startnumber and endnumber; - END IF; - {% endif %} - - -- postcode table - IF place IS NULL THEN - SELECT parent_place_id as place_id, country_code, - null::text as housenumber, postcode, - 'place' as class, 'postcode' as type, - null as name, null as address, - null as centroid - INTO place - FROM location_postcode - WHERE place_id = in_place_id; - END IF; - - -- POI objects in the placex table - IF place IS NULL THEN - SELECT parent_place_id as place_id, country_code, - coalesce(address->'housenumber', - address->'streetnumber', - address->'conscriptionnumber')::text as housenumber, - postcode, - class, type, - name, address, - centroid - INTO place - FROM placex - WHERE place_id = in_place_id and rank_search > 27; - END IF; - - -- If place is still NULL at this point then the object has its own - -- entry in place_address line. However, still check if there is not linked - -- place we should be using instead. - IF place IS NULL THEN - select coalesce(linked_place_id, place_id) as place_id, country_code, - null::text as housenumber, postcode, - class, type, - null as name, address, - null as centroid - INTO place - FROM placex where place_id = in_place_id; - END IF; - ---RAISE WARNING '% % % %',searchcountrycode, searchhousenumber, searchpostcode; - - -- --- Return the record for the base entry. - - current_rank_address := 1000; - FOR location IN - SELECT placex.place_id, osm_type, osm_id, name, - coalesce(extratags->'linked_place', extratags->'place') as place_type, - class, type, admin_level, - CASE WHEN rank_address = 0 THEN 100 - WHEN rank_address = 11 THEN 5 - ELSE rank_address END as rank_address, - country_code - FROM placex - WHERE place_id = place.place_id - LOOP ---RAISE WARNING '%',location; - -- mix in default names for countries - IF location.rank_address = 4 and place.country_code is not NULL THEN - FOR country IN - SELECT coalesce(name, ''::hstore) as name FROM country_name - WHERE country_code = place.country_code LIMIT 1 - LOOP - place.name := country.name || place.name; - END LOOP; - END IF; - - IF location.rank_address < 4 THEN - -- no country locations for ranks higher than country - place.country_code := NULL::varchar(2); - ELSEIF place.country_code IS NULL AND location.country_code IS NOT NULL THEN - place.country_code := location.country_code; - END IF; - - RETURN NEXT ROW(location.place_id, location.osm_type, location.osm_id, - location.name, location.class, location.type, - location.place_type, - location.admin_level, true, - location.type not in ('postcode', 'postal_code'), - location.rank_address, 0)::addressline; - - current_rank_address := location.rank_address; - END LOOP; - - -- --- Return records for address parts. - - FOR location IN - SELECT placex.place_id, osm_type, osm_id, name, class, type, - coalesce(extratags->'linked_place', extratags->'place') as place_type, - admin_level, fromarea, isaddress and linked_place_id is NULL as isaddress, - CASE WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address, - distance, country_code, postcode - FROM place_addressline join placex on (address_place_id = placex.place_id) - WHERE place_addressline.place_id IN (place.place_id, in_place_id) - AND linked_place_id is null - AND (placex.country_code IS NULL OR place.country_code IS NULL - OR placex.country_code = place.country_code) - ORDER BY rank_address desc, - (place_addressline.place_id = in_place_id) desc, - (CASE WHEN coalesce((avals(name) && avals(place.address)), False) THEN 2 - WHEN isaddress THEN 0 - WHEN fromarea - and place.centroid is not null - and ST_Contains(geometry, place.centroid) THEN 1 - ELSE -1 END) desc, - fromarea desc, distance asc, rank_search desc - LOOP - -- RAISE WARNING '%',location; - location_isaddress := location.rank_address != current_rank_address; - - IF place.country_code IS NULL AND location.country_code IS NOT NULL THEN - place.country_code := location.country_code; - END IF; - IF location.type in ('postcode', 'postal_code') - AND place.postcode is not null - THEN - -- If the place had a postcode assigned, take this one only - -- into consideration when it is an area and the place does not have - -- a postcode itself. - IF location.fromarea AND location_isaddress - AND (place.address is null or not place.address ? 'postcode') - THEN - place.postcode := null; -- remove the less exact postcode - ELSE - location_isaddress := false; - END IF; - END IF; - RETURN NEXT ROW(location.place_id, location.osm_type, location.osm_id, - location.name, location.class, location.type, - location.place_type, - location.admin_level, location.fromarea, - location_isaddress, - location.rank_address, - location.distance)::addressline; - - current_rank_address := location.rank_address; - END LOOP; - - -- If no country was included yet, add the name information from country_name. - IF current_rank_address > 4 THEN - FOR location IN - SELECT name || coalesce(derived_name, ''::hstore) as name FROM country_name - WHERE country_code = place.country_code LIMIT 1 - LOOP ---RAISE WARNING '% % %',current_rank_address,searchcountrycode,countryname; - RETURN NEXT ROW(null, null, null, location.name, 'place', 'country', NULL, - null, true, true, 4, 0)::addressline; - END LOOP; - END IF; - - -- Finally add some artificial rows. - IF place.country_code IS NOT NULL THEN - location := ROW(null, null, null, hstore('ref', place.country_code), - 'place', 'country_code', null, null, true, false, 4, 0)::addressline; - RETURN NEXT location; - END IF; - - IF place.name IS NOT NULL THEN - location := ROW(in_place_id, null, null, place.name, place.class, - place.type, null, null, true, true, 29, 0)::addressline; - RETURN NEXT location; - END IF; - - IF place.housenumber IS NOT NULL THEN - location := ROW(null, null, null, hstore('ref', place.housenumber), - 'place', 'house_number', null, null, true, true, 28, 0)::addressline; - RETURN NEXT location; - END IF; - - IF place.address is not null and place.address ? '_unlisted_place' THEN - RETURN NEXT ROW(null, null, null, hstore('name', place.address->'_unlisted_place'), - 'place', 'locality', null, null, true, true, 25, 0)::addressline; - END IF; - - IF place.postcode is not null THEN - location := ROW(null, null, null, hstore('ref', place.postcode), 'place', - 'postcode', null, null, false, true, 5, 0)::addressline; - RETURN NEXT location; - ELSEIF place.address is not null and place.address ? 'postcode' - and not place.address->'postcode' SIMILAR TO '%(,|;)%' THEN - location := ROW(null, null, null, hstore('ref', place.address->'postcode'), 'place', - 'postcode', null, null, false, true, 5, 0)::addressline; - RETURN NEXT location; - END IF; - - RETURN; -END; -$$ -LANGUAGE plpgsql STABLE; diff --git a/lib-sql/functions/importance.sql b/lib-sql/functions/importance.sql index 1de5899c..4993d70b 100644 --- a/lib-sql/functions/importance.sql +++ b/lib-sql/functions/importance.sql @@ -65,7 +65,7 @@ BEGIN RETURN NULL; END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; {% else %} @@ -78,7 +78,7 @@ SELECT convert_from(CAST(E'\\x' || array_to_string(ARRAY( FROM regexp_matches($1, '%[0-9a-f][0-9a-f]|.', 'gi') AS r(m) ), '') AS bytea), 'UTF8'); $$ -LANGUAGE SQL IMMUTABLE STRICT; +LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION catch_decode_url_part(p varchar) @@ -91,7 +91,7 @@ EXCEPTION WHEN others THEN return null; END; $$ -LANGUAGE plpgsql IMMUTABLE STRICT; +LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_wikipedia_match(extratags HSTORE, country_code varchar(2)) @@ -139,7 +139,7 @@ BEGIN RETURN NULL; END; $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; {% endif %} @@ -203,5 +203,5 @@ BEGIN RETURN result; END; $$ -LANGUAGE plpgsql; +LANGUAGE plpgsql PARALLEL SAFE; diff --git a/lib-sql/functions/interpolation.sql b/lib-sql/functions/interpolation.sql index 2fd21e8a..452ef7c3 100644 --- a/lib-sql/functions/interpolation.sql +++ b/lib-sql/functions/interpolation.sql @@ -34,7 +34,7 @@ BEGIN RETURN in_address; END; $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; @@ -70,7 +70,7 @@ BEGIN RETURN parent_place_id; END; $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION reinsert_interpolation(way_id BIGINT, addr HSTORE, diff --git a/lib-sql/functions/partition-functions.sql b/lib-sql/functions/partition-functions.sql index 20ec3da6..d3c83615 100644 --- a/lib-sql/functions/partition-functions.sql +++ b/lib-sql/functions/partition-functions.sql @@ -17,28 +17,6 @@ CREATE TYPE nearfeaturecentr AS ( centroid GEOMETRY ); --- feature intersects geometry --- for areas and linestrings they must touch at least along a line -CREATE OR REPLACE FUNCTION is_relevant_geometry(de9im TEXT, geom_type TEXT) -RETURNS BOOLEAN -AS $$ -BEGIN - IF substring(de9im from 1 for 2) != 'FF' THEN - RETURN TRUE; - END IF; - - IF geom_type = 'ST_Point' THEN - RETURN substring(de9im from 4 for 1) = '0'; - END IF; - - IF geom_type in ('ST_LineString', 'ST_MultiLineString') THEN - RETURN substring(de9im from 4 for 1) = '1'; - END IF; - - RETURN substring(de9im from 4 for 1) = '2'; -END -$$ LANGUAGE plpgsql IMMUTABLE; - CREATE OR REPLACE function getNearFeatures(in_partition INTEGER, feature GEOMETRY, feature_centroid GEOMETRY, maxrank INTEGER) @@ -59,7 +37,12 @@ BEGIN isguess, postcode, centroid FROM location_area_large_{{ partition }} WHERE geometry && feature - AND is_relevant_geometry(ST_Relate(geometry, feature), ST_GeometryType(feature)) + AND CASE WHEN ST_Dimension(feature) = 0 + THEN _ST_Covers(geometry, feature) + WHEN ST_Dimension(feature) = 2 + THEN ST_Relate(geometry, feature, 'T********') + ELSE ST_NPoints(ST_Intersection(geometry, feature)) > 1 + END AND rank_address < maxrank -- Postcodes currently still use rank_search to define for which -- features they are relevant. @@ -75,7 +58,7 @@ BEGIN RAISE EXCEPTION 'Unknown partition %', in_partition; END $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_address_place(in_partition SMALLINT, feature GEOMETRY, @@ -104,7 +87,7 @@ BEGIN RAISE EXCEPTION 'Unknown partition %', in_partition; END; $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; create or replace function deleteLocationArea(in_partition INTEGER, in_place_id BIGINT, in_rank_search INTEGER) RETURNS BOOLEAN AS $$ @@ -142,14 +125,16 @@ BEGIN IF in_rank_search <= 4 and not in_estimate THEN INSERT INTO location_area_country (place_id, country_code, geometry) - values (in_place_id, in_country_code, in_geometry); + (SELECT in_place_id, in_country_code, geom + FROM split_geometry(in_geometry) as geom); RETURN TRUE; END IF; {% for partition in db.partitions %} IF in_partition = {{ partition }} THEN INSERT INTO location_area_large_{{ partition }} (partition, place_id, country_code, keywords, rank_search, rank_address, isguess, postcode, centroid, geometry) - values (in_partition, in_place_id, in_country_code, in_keywords, in_rank_search, in_rank_address, in_estimate, postcode, in_centroid, in_geometry); + (SELECT in_partition, in_place_id, in_country_code, in_keywords, in_rank_search, in_rank_address, in_estimate, postcode, in_centroid, geom + FROM split_geometry(in_geometry) as geom); RETURN TRUE; END IF; {% endfor %} @@ -187,7 +172,7 @@ BEGIN RAISE EXCEPTION 'Unknown partition %', in_partition; END $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION getNearestNamedPlacePlaceId(in_partition INTEGER, point GEOMETRY, @@ -217,7 +202,7 @@ BEGIN RAISE EXCEPTION 'Unknown partition %', in_partition; END $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; create or replace function insertSearchName( in_partition INTEGER, in_place_id BIGINT, in_name_vector INTEGER[], @@ -325,7 +310,7 @@ BEGIN RAISE EXCEPTION 'Unknown partition %', in_partition; END $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION getNearestParallelRoadFeature(in_partition INTEGER, line GEOMETRY) @@ -369,4 +354,4 @@ BEGIN RAISE EXCEPTION 'Unknown partition %', in_partition; END $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; diff --git a/lib-sql/functions/placex_triggers.sql b/lib-sql/functions/placex_triggers.sql index 9d0d73b5..8524ffc3 100644 --- a/lib-sql/functions/placex_triggers.sql +++ b/lib-sql/functions/placex_triggers.sql @@ -109,7 +109,7 @@ BEGIN RETURN result; END; $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION find_associated_street(poi_osm_type CHAR(1), @@ -200,7 +200,7 @@ BEGIN RETURN result; END; $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; -- Find the parent road of a POI. @@ -286,7 +286,7 @@ BEGIN RETURN parent_place_id; END; $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; -- Try to find a linked place for the given object. CREATE OR REPLACE FUNCTION find_linked_place(bnd placex) @@ -404,7 +404,7 @@ BEGIN RETURN NULL; END; $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_poi_search_terms(obj_place_id BIGINT, diff --git a/lib-sql/functions/ranking.sql b/lib-sql/functions/ranking.sql index 97a0cde3..c16ad1db 100644 --- a/lib-sql/functions/ranking.sql +++ b/lib-sql/functions/ranking.sql @@ -29,7 +29,7 @@ BEGIN RETURN 0.02; END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; -- Return an approximate update radius according to the search rank. @@ -60,7 +60,7 @@ BEGIN RETURN 0; END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; -- Compute a base address rank from the extent of the given geometry. -- @@ -107,7 +107,7 @@ BEGIN RETURN 23; END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; -- Guess a ranking for postcodes from country and postcode format. @@ -167,7 +167,7 @@ BEGIN END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; -- Get standard search and address rank for an object. @@ -236,7 +236,7 @@ BEGIN END IF; END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_addr_tag_rank(key TEXT, country TEXT, OUT from_rank SMALLINT, @@ -283,7 +283,7 @@ BEGIN END LOOP; END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION weigh_search(search_vector INT[], @@ -304,4 +304,4 @@ BEGIN RETURN def_weight; END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; diff --git a/lib-sql/functions/utils.sql b/lib-sql/functions/utils.sql index df00f916..30f94080 100644 --- a/lib-sql/functions/utils.sql +++ b/lib-sql/functions/utils.sql @@ -24,7 +24,7 @@ BEGIN RETURN ST_PointOnSurface(place); END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION geometry_sector(partition INTEGER, place GEOMETRY) @@ -34,7 +34,7 @@ BEGIN RETURN (partition*1000000) + (500-ST_X(place)::INTEGER)*1000 + (500-ST_Y(place)::INTEGER); END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; @@ -60,7 +60,7 @@ BEGIN RETURN r; END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; -- Return the node members with a given label from a relation member list -- as a set. @@ -88,7 +88,7 @@ BEGIN RETURN; END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_rel_node_members(members JSONB, memberLabels TEXT[]) @@ -107,7 +107,7 @@ BEGIN RETURN; END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; -- Copy 'name' to or from the default language. @@ -136,7 +136,7 @@ BEGIN END IF; END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; -- Find the nearest artificial postcode for the given geometry. @@ -172,7 +172,7 @@ BEGIN RETURN outcode; END; $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_country_code(place geometry) @@ -233,7 +233,7 @@ BEGIN RETURN NULL; END; $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_country_language_code(search_country_code VARCHAR(2)) @@ -251,7 +251,7 @@ BEGIN RETURN NULL; END; $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_partition(in_country_code VARCHAR(10)) @@ -268,7 +268,7 @@ BEGIN RETURN 0; END; $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; -- Find the parent of an address with addr:street/addr:place tag. @@ -299,7 +299,7 @@ BEGIN RETURN parent_place_id; END; $$ -LANGUAGE plpgsql STABLE; +LANGUAGE plpgsql STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION delete_location(OLD_place_id BIGINT) @@ -337,7 +337,7 @@ BEGIN ST_Project(geom::geography, radius, 3.9269908)::geometry)); END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION add_location(place_id BIGINT, country_code varchar(2), @@ -348,8 +348,6 @@ CREATE OR REPLACE FUNCTION add_location(place_id BIGINT, country_code varchar(2) RETURNS BOOLEAN AS $$ DECLARE - locationid INTEGER; - secgeo GEOMETRY; postcode TEXT; BEGIN PERFORM deleteLocationArea(partition, place_id, rank_search); @@ -360,18 +358,19 @@ BEGIN postcode := upper(trim (in_postcode)); END IF; - IF ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') THEN - FOR secgeo IN select split_geometry(geometry) AS geom LOOP - PERFORM insertLocationAreaLarge(partition, place_id, country_code, keywords, rank_search, rank_address, false, postcode, centroid, secgeo); - END LOOP; - - ELSEIF ST_GeometryType(geometry) = 'ST_Point' THEN - secgeo := place_node_fuzzy_area(geometry, rank_search); - PERFORM insertLocationAreaLarge(partition, place_id, country_code, keywords, rank_search, rank_address, true, postcode, centroid, secgeo); + IF ST_Dimension(geometry) = 2 THEN + RETURN insertLocationAreaLarge(partition, place_id, country_code, keywords, + rank_search, rank_address, false, postcode, + centroid, geometry); + END IF; + IF ST_Dimension(geometry) = 0 THEN + RETURN insertLocationAreaLarge(partition, place_id, country_code, keywords, + rank_search, rank_address, true, postcode, + centroid, place_node_fuzzy_area(geometry, rank_search)); END IF; - RETURN true; + RETURN false; END; $$ LANGUAGE plpgsql; @@ -394,19 +393,21 @@ DECLARE geo RECORD; area FLOAT; remainingdepth INTEGER; - added INTEGER; BEGIN - -- RAISE WARNING 'quad_split_geometry: maxarea=%, depth=%',maxarea,maxdepth; - IF (ST_GeometryType(geometry) not in ('ST_Polygon','ST_MultiPolygon') OR NOT ST_IsValid(geometry)) THEN + IF not ST_IsValid(geometry) THEN + RETURN; + END IF; + + IF ST_Dimension(geometry) != 2 OR maxdepth <= 1 THEN RETURN NEXT geometry; RETURN; END IF; remainingdepth := maxdepth - 1; area := ST_AREA(geometry); - IF remainingdepth < 1 OR area < maxarea THEN + IF area < maxarea THEN RETURN NEXT geometry; RETURN; END IF; @@ -426,7 +427,6 @@ BEGIN xmid := (xmin+xmax)/2; ymid := (ymin+ymax)/2; - added := 0; FOR seg IN 1..4 LOOP IF seg = 1 THEN @@ -442,23 +442,20 @@ BEGIN secbox := ST_SetSRID(ST_MakeBox2D(ST_Point(xmid,ymid),ST_Point(xmax,ymax)),4326); END IF; - IF st_intersects(geometry, secbox) THEN - secgeo := st_intersection(geometry, secbox); - IF NOT ST_IsEmpty(secgeo) AND ST_GeometryType(secgeo) in ('ST_Polygon','ST_MultiPolygon') THEN - FOR geo IN select quad_split_geometry(secgeo, maxarea, remainingdepth) as geom LOOP - IF NOT ST_IsEmpty(geo.geom) AND ST_GeometryType(geo.geom) in ('ST_Polygon','ST_MultiPolygon') THEN - added := added + 1; - RETURN NEXT geo.geom; - END IF; - END LOOP; - END IF; + secgeo := st_intersection(geometry, secbox); + IF NOT ST_IsEmpty(secgeo) AND ST_Dimension(secgeo) = 2 THEN + FOR geo IN SELECT quad_split_geometry(secgeo, maxarea, remainingdepth) as geom LOOP + IF NOT ST_IsEmpty(geo.geom) AND ST_Dimension(geo.geom) = 2 THEN + RETURN NEXT geo.geom; + END IF; + END LOOP; END IF; END LOOP; RETURN; END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION split_geometry(geometry GEOMETRY) @@ -467,14 +464,26 @@ CREATE OR REPLACE FUNCTION split_geometry(geometry GEOMETRY) DECLARE geo RECORD; BEGIN - -- 10000000000 is ~~ 1x1 degree - FOR geo IN select quad_split_geometry(geometry, 0.25, 20) as geom LOOP - RETURN NEXT geo.geom; - END LOOP; + IF ST_GeometryType(geometry) = 'ST_MultiPolygon' + and ST_Area(geometry) * 10 > ST_Area(Box2D(geometry)) + THEN + FOR geo IN + SELECT quad_split_geometry(g, 0.25, 20) as geom + FROM (SELECT (ST_Dump(geometry)).geom::geometry(Polygon, 4326) AS g) xx + LOOP + RETURN NEXT geo.geom; + END LOOP; + ELSE + FOR geo IN + SELECT quad_split_geometry(geometry, 0.25, 20) as geom + LOOP + RETURN NEXT geo.geom; + END LOOP; + END IF; RETURN; END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION simplify_large_polygons(geometry GEOMETRY) RETURNS GEOMETRY @@ -488,7 +497,7 @@ BEGIN RETURN geometry; END; $$ -LANGUAGE plpgsql IMMUTABLE; +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION place_force_delete(placeid BIGINT) diff --git a/lib-sql/tokenizer/icu_tokenizer.sql b/lib-sql/tokenizer/icu_tokenizer.sql index 04fcedcb..8cf13120 100644 --- a/lib-sql/tokenizer/icu_tokenizer.sql +++ b/lib-sql/tokenizer/icu_tokenizer.sql @@ -12,7 +12,7 @@ CREATE OR REPLACE FUNCTION token_get_name_search_tokens(info JSONB) RETURNS INTEGER[] AS $$ SELECT (info->>'names')::INTEGER[] -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; -- Get tokens for matching the place name against others. @@ -22,7 +22,7 @@ CREATE OR REPLACE FUNCTION token_get_name_match_tokens(info JSONB) RETURNS INTEGER[] AS $$ SELECT (info->>'names')::INTEGER[] -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; -- Return the housenumber tokens applicable for the place. @@ -30,7 +30,7 @@ CREATE OR REPLACE FUNCTION token_get_housenumber_search_tokens(info JSONB) RETURNS INTEGER[] AS $$ SELECT (info->>'hnr_tokens')::INTEGER[] -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; -- Return the housenumber in the form that it can be matched during search. @@ -38,77 +38,77 @@ CREATE OR REPLACE FUNCTION token_normalized_housenumber(info JSONB) RETURNS TEXT AS $$ SELECT info->>'hnr'; -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION token_is_street_address(info JSONB) RETURNS BOOLEAN AS $$ SELECT info->>'street' is not null or info->>'place' is null; -$$ LANGUAGE SQL IMMUTABLE; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION token_has_addr_street(info JSONB) RETURNS BOOLEAN AS $$ SELECT info->>'street' is not null and info->>'street' != '{}'; -$$ LANGUAGE SQL IMMUTABLE; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION token_has_addr_place(info JSONB) RETURNS BOOLEAN AS $$ SELECT info->>'place' is not null; -$$ LANGUAGE SQL IMMUTABLE; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION token_matches_street(info JSONB, street_tokens INTEGER[]) RETURNS BOOLEAN AS $$ SELECT (info->>'street')::INTEGER[] && street_tokens -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION token_matches_place(info JSONB, place_tokens INTEGER[]) RETURNS BOOLEAN AS $$ SELECT (info->>'place')::INTEGER[] <@ place_tokens -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION token_addr_place_search_tokens(info JSONB) RETURNS INTEGER[] AS $$ SELECT (info->>'place')::INTEGER[] -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION token_get_address_keys(info JSONB) RETURNS SETOF TEXT AS $$ SELECT * FROM jsonb_object_keys(info->'addr'); -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION token_get_address_search_tokens(info JSONB, key TEXT) RETURNS INTEGER[] AS $$ SELECT (info->'addr'->>key)::INTEGER[]; -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION token_matches_address(info JSONB, key TEXT, tokens INTEGER[]) RETURNS BOOLEAN AS $$ SELECT (info->'addr'->>key)::INTEGER[] <@ tokens; -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION token_get_postcode(info JSONB) RETURNS TEXT AS $$ SELECT info->>'postcode'; -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; -- Return token info that should be saved permanently in the database. @@ -116,7 +116,7 @@ CREATE OR REPLACE FUNCTION token_strip_info(info JSONB) RETURNS JSONB AS $$ SELECT NULL::JSONB; -$$ LANGUAGE SQL IMMUTABLE STRICT; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; --------------- private functions ---------------------------------------------- @@ -128,16 +128,14 @@ DECLARE partial_terms TEXT[] = '{}'::TEXT[]; term TEXT; term_id INTEGER; - term_count INTEGER; BEGIN SELECT min(word_id) INTO full_token FROM word WHERE word = norm_term and type = 'W'; IF full_token IS NULL THEN full_token := nextval('seq_word'); - INSERT INTO word (word_id, word_token, type, word, info) - SELECT full_token, lookup_term, 'W', norm_term, - json_build_object('count', 0) + INSERT INTO word (word_id, word_token, type, word) + SELECT full_token, lookup_term, 'W', norm_term FROM unnest(lookup_terms) as lookup_term; END IF; @@ -150,14 +148,67 @@ BEGIN partial_tokens := '{}'::INT[]; FOR term IN SELECT unnest(partial_terms) LOOP - SELECT min(word_id), max(info->>'count') INTO term_id, term_count + SELECT min(word_id) INTO term_id + FROM word WHERE word_token = term and type = 'w'; + + IF term_id IS NULL THEN + term_id := nextval('seq_word'); + INSERT INTO word (word_id, word_token, type) + VALUES (term_id, term, 'w'); + END IF; + + partial_tokens := array_merge(partial_tokens, ARRAY[term_id]); + END LOOP; +END; +$$ +LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION getorcreate_full_word(norm_term TEXT, + lookup_terms TEXT[], + lookup_norm_terms TEXT[], + OUT full_token INT, + OUT partial_tokens INT[]) + AS $$ +DECLARE + partial_terms TEXT[] = '{}'::TEXT[]; + term TEXT; + term_id INTEGER; +BEGIN + SELECT min(word_id) INTO full_token + FROM word WHERE word = norm_term and type = 'W'; + + IF full_token IS NULL THEN + full_token := nextval('seq_word'); + IF lookup_norm_terms IS NULL THEN + INSERT INTO word (word_id, word_token, type, word) + SELECT full_token, lookup_term, 'W', norm_term + FROM unnest(lookup_terms) as lookup_term; + ELSE + INSERT INTO word (word_id, word_token, type, word, info) + SELECT full_token, t.lookup, 'W', norm_term, + CASE WHEN norm_term = t.norm THEN null + ELSE json_build_object('lookup', t.norm) END + FROM unnest(lookup_terms, lookup_norm_terms) as t(lookup, norm); + END IF; + END IF; + + FOR term IN SELECT unnest(string_to_array(unnest(lookup_terms), ' ')) LOOP + term := trim(term); + IF NOT (ARRAY[term] <@ partial_terms) THEN + partial_terms := partial_terms || term; + END IF; + END LOOP; + + partial_tokens := '{}'::INT[]; + FOR term IN SELECT unnest(partial_terms) LOOP + SELECT min(word_id) INTO term_id FROM word WHERE word_token = term and type = 'w'; IF term_id IS NULL THEN term_id := nextval('seq_word'); - term_count := 0; - INSERT INTO word (word_id, word_token, type, info) - VALUES (term_id, term, 'w', json_build_object('count', term_count)); + INSERT INTO word (word_id, word_token, type) + VALUES (term_id, term, 'w'); END IF; partial_tokens := array_merge(partial_tokens, ARRAY[term_id]); diff --git a/nominatim-cli.py b/nominatim-cli.py index 1f3c1210..7a1aadb8 100755 --- a/nominatim-cli.py +++ b/nominatim-cli.py @@ -3,7 +3,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2024 by the Nominatim developer community. +# Copyright (C) 2025 by the Nominatim developer community. # For a full list of authors see the git log. """ Helper script for development to run nominatim from the source directory. @@ -15,4 +15,4 @@ sys.path.insert(1, str((Path(__file__) / '..' / 'src').resolve())) from nominatim_db import cli -exit(cli.nominatim(module_dir=None, osm2pgsql_path=None)) +exit(cli.nominatim()) diff --git a/packaging/nominatim-api/extra_src/paths.py b/packaging/nominatim-api/extra_src/paths.py index 797acbb5..7d186da1 100644 --- a/packaging/nominatim-api/extra_src/paths.py +++ b/packaging/nominatim-api/extra_src/paths.py @@ -11,4 +11,5 @@ from pathlib import Path DATA_DIR = None SQLLIB_DIR = None +LUALIB_DIR = None CONFIG_DIR = (Path(__file__) / '..' / 'resources' / 'settings').resolve() diff --git a/packaging/nominatim-api/pyproject.toml b/packaging/nominatim-api/pyproject.toml index d4ea8d8c..e467e8ac 100644 --- a/packaging/nominatim-api/pyproject.toml +++ b/packaging/nominatim-api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nominatim-api" -version = "4.5.0.post6" +version = "5.1.0.post7" description = "A tool for building a database of OpenStreetMap for geocoding and for searching the database. Search library." readme = "README.md" requires-python = ">=3.7" @@ -16,13 +16,9 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "python-dotenv==1.0.1", - "pyYAML==6.0.2", - "SQLAlchemy==2.0.36", - "psycopg[binary]==3.2.3", - "PyICU==2.14", + "SQLAlchemy==2.0.41", "falcon==4.0.2", - "uvicorn==0.32.0", + "uvicorn==0.34.2", "gunicorn==23.0.0" ] diff --git a/packaging/nominatim-db/extra_src/nominatim_db/paths.py b/packaging/nominatim-db/extra_src/nominatim_db/paths.py index 796ff08b..02df5047 100644 --- a/packaging/nominatim-db/extra_src/nominatim_db/paths.py +++ b/packaging/nominatim-db/extra_src/nominatim_db/paths.py @@ -11,4 +11,5 @@ from pathlib import Path DATA_DIR = (Path(__file__) / '..' / 'resources').resolve() SQLLIB_DIR = (DATA_DIR / 'lib-sql') +LUALIB_DIR = (DATA_DIR / 'lib-lua') CONFIG_DIR = (DATA_DIR / 'settings') diff --git a/packaging/nominatim-db/lib-lua b/packaging/nominatim-db/lib-lua new file mode 120000 index 00000000..e4e1bd04 --- /dev/null +++ b/packaging/nominatim-db/lib-lua @@ -0,0 +1 @@ +../../lib-lua \ No newline at end of file diff --git a/packaging/nominatim-db/pyproject.toml b/packaging/nominatim-db/pyproject.toml index 17065eb6..a695f28d 100644 --- a/packaging/nominatim-db/pyproject.toml +++ b/packaging/nominatim-db/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nominatim-db" -version = "4.5.0.post6" +version = "5.1.0.post7" description = "A tool for building a database of OpenStreetMap for geocoding and for searching the database. Database backend." readme = "README.md" requires-python = ">=3.7" @@ -16,13 +16,12 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "psycopg[binary]==3.2.3", - "python-dotenv==1.0.1", - "jinja2==3.1.4", + "psycopg[binary]==3.2.9", + "python-dotenv==1.1.0", + "jinja2==3.1.6", "pyYAML==6.0.2", - "datrie==0.8.2", - "psutil==6.1.0", - "PyICU==2.14", + "psutil==7.0.0", + "PyICU==2.15.2", "osmium==4.0.2", ] @@ -45,6 +44,7 @@ include = [ "src/nominatim_db", "scripts", "lib-sql/**/*.sql", + "lib-lua/**/*.lua", "settings", "data/words.sql", "extra_src/nominatim_db/paths.py" @@ -66,6 +66,7 @@ packages = ["src/nominatim_db"] [tool.hatch.build.targets.wheel.force-include] "lib-sql" = "nominatim_db/resources/lib-sql" +"lib-lua" = "nominatim_db/resources/lib-lua" "settings" = "nominatim_db/resources/settings" "data/country_osm_grid.sql.gz" = "nominatim_db/resources/country_osm_grid.sql.gz" "data/words.sql" = "nominatim_db/resources/words.sql" diff --git a/packaging/nominatim-db/scripts/nominatim b/packaging/nominatim-db/scripts/nominatim index 184ab4c6..bc384b02 100755 --- a/packaging/nominatim-db/scripts/nominatim +++ b/packaging/nominatim-db/scripts/nominatim @@ -2,4 +2,4 @@ from nominatim_db import cli -exit(cli.nominatim(osm2pgsql_path=None)) +exit(cli.nominatim()) diff --git a/settings/address-levels.json b/settings/address-levels.json index b63eac4e..1b1ef7e7 100644 --- a/settings/address-levels.json +++ b/settings/address-levels.json @@ -23,8 +23,8 @@ "allotments" : 22, "neighbourhood" : [20, 22], "quarter" : [20, 22], - "isolated_dwelling" : [22, 20], - "farm" : [22, 20], + "isolated_dwelling" : [22, 25], + "farm" : [22, 25], "city_block" : 25, "mountain_pass" : 25, "square" : 25, @@ -74,6 +74,14 @@ "stone" : 30, "" : [22, 0] }, + "water" : { + "lake" : [20, 0], + "reservoir" : [20, 0], + "wastewater" : [24, 0], + "pond" : [24, 0], + "fountain" : [24, 0], + "" : [22, 0] + }, "waterway" : { "river" : [19, 0], "stream" : [22, 0], @@ -208,6 +216,14 @@ } } }, +{ "countries" : ["sa"], + "tags" : { + "place" : { + "province" : 12, + "municipality" : 18 + } + } +}, { "countries" : ["sk"], "tags" : { "boundary" : { diff --git a/settings/country_settings.yaml b/settings/country_settings.yaml index 667684c6..88ace911 100644 --- a/settings/country_settings.yaml +++ b/settings/country_settings.yaml @@ -944,7 +944,7 @@ kp: # South Korea (대한민국) kr: partition: 49 - languages: ko, en + languages: ko names: !include country-names/kr.yaml postcode: pattern: "ddddd" @@ -1809,7 +1809,8 @@ us: languages: en names: !include country-names/us.yaml postcode: - pattern: "ddddd" + pattern: "(ddddd)(?:-dddd)?" + output: \1 # Uruguay (Uruguay) diff --git a/settings/env.defaults b/settings/env.defaults index b8c66667..3ebb288f 100644 --- a/settings/env.defaults +++ b/settings/env.defaults @@ -192,6 +192,13 @@ NOMINATIM_REQUEST_TIMEOUT=60 # to geocode" instead. NOMINATIM_SEARCH_WITHIN_COUNTRIES=False +# Specifies the order in which different name tags are used. +# The values in this list determine the preferred order of name variants, +# including language-specific names. +# Comma-separated list, where :XX stands for language-specific tags +# (e.g. name:en) and no :XX stands for general tags (e.g. name). +NOMINATIM_OUTPUT_NAMES=name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref + ### Log settings # # The following options allow to enable logging of API requests. diff --git a/settings/flex-base.lua b/settings/flex-base.lua deleted file mode 100644 index 7860737f..00000000 --- a/settings/flex-base.lua +++ /dev/null @@ -1,557 +0,0 @@ --- Core functions for Nominatim import flex style. --- - -local module = {} - -local PRE_DELETE = nil -local PRE_EXTRAS = nil -local POST_DELETE = nil -local MAIN_KEYS = nil -local NAMES = nil -local ADDRESS_TAGS = nil -local SAVE_EXTRA_MAINS = false -local POSTCODE_FALLBACK = true - --- tables required for taginfo -module.TAGINFO_MAIN = {keys = {}, delete_tags = {}} -module.TAGINFO_NAME_KEYS = {} -module.TAGINFO_ADDRESS_KEYS = {} - - --- The single place table. -local place_table = osm2pgsql.define_table{ - name = "place", - ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, - columns = { - { column = 'class', type = 'text', not_null = true }, - { column = 'type', type = 'text', not_null = true }, - { column = 'admin_level', type = 'smallint' }, - { column = 'name', type = 'hstore' }, - { column = 'address', type = 'hstore' }, - { column = 'extratags', type = 'hstore' }, - { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true }, - }, - data_tablespace = os.getenv("NOMINATIM_TABLESPACE_PLACE_DATA"), - index_tablespace = os.getenv("NOMINATIM_TABLESPACE_PLACE_INDEX"), - indexes = {} -} - ------------- Geometry functions for relations --------------------- - -function module.relation_as_multipolygon(o) - return o:as_multipolygon() -end - -function module.relation_as_multiline(o) - return o:as_multilinestring():line_merge() -end - - -module.RELATION_TYPES = { - multipolygon = module.relation_as_multipolygon, - boundary = module.relation_as_multipolygon, - waterway = module.relation_as_multiline -} - -------------- Place class ------------------------------------------ - -local Place = {} -Place.__index = Place - -function Place.new(object, geom_func) - local self = setmetatable({}, Place) - self.object = object - self.geom_func = geom_func - - self.admin_level = tonumber(self.object:grab_tag('admin_level')) - if self.admin_level == nil - or self.admin_level <= 0 or self.admin_level > 15 - or math.floor(self.admin_level) ~= self.admin_level then - self.admin_level = 15 - end - - self.num_entries = 0 - self.has_name = false - self.names = {} - self.address = {} - self.extratags = {} - - return self -end - -function Place:clean(data) - for k, v in pairs(self.object.tags) do - if data.delete ~= nil and data.delete(k, v) then - self.object.tags[k] = nil - elseif data.extra ~= nil and data.extra(k, v) then - self.extratags[k] = v - self.object.tags[k] = nil - end - end -end - -function Place:delete(data) - if data.match ~= nil then - for k, v in pairs(self.object.tags) do - if data.match(k, v) then - self.object.tags[k] = nil - end - end - end -end - -function Place:grab_extratags(data) - local count = 0 - - if data.match ~= nil then - for k, v in pairs(self.object.tags) do - if data.match(k, v) then - self.object.tags[k] = nil - self.extratags[k] = v - count = count + 1 - end - end - end - - return count -end - -local function strip_address_prefix(k) - if k:sub(1, 5) == 'addr:' then - return k:sub(6) - end - - if k:sub(1, 6) == 'is_in:' then - return k:sub(7) - end - - return k -end - - -function Place:grab_address_parts(data) - local count = 0 - - if data.groups ~= nil then - for k, v in pairs(self.object.tags) do - local atype = data.groups(k, v) - - if atype ~= nil then - if atype == 'main' then - self.has_name = true - self.address[strip_address_prefix(k)] = v - count = count + 1 - elseif atype == 'extra' then - self.address[strip_address_prefix(k)] = v - else - self.address[atype] = v - end - self.object.tags[k] = nil - end - end - end - - return count -end - - -function Place:grab_name_parts(data) - local fallback = nil - - if data.groups ~= nil then - for k, v in pairs(self.object.tags) do - local atype = data.groups(k, v) - - if atype ~= nil then - self.names[k] = v - self.object.tags[k] = nil - if atype == 'main' then - self.has_name = true - elseif atype == 'house' then - self.has_name = true - fallback = {'place', 'house', 'always'} - end - end - end - end - - return fallback -end - - -function Place:write_place(k, v, mtype, save_extra_mains) - if mtype == nil then - return 0 - end - - v = v or self.object.tags[k] - if v == nil then - return 0 - end - - if type(mtype) == 'table' then - mtype = mtype[v] or mtype[1] - end - - if mtype == 'always' or (self.has_name and mtype == 'named') then - return self:write_row(k, v, save_extra_mains) - end - - if mtype == 'named_with_key' then - local names = {} - local prefix = k .. ':name' - for namek, namev in pairs(self.object.tags) do - if namek:sub(1, #prefix) == prefix - and (#namek == #prefix - or namek:sub(#prefix + 1, #prefix + 1) == ':') then - names[namek:sub(#k + 2)] = namev - end - end - - if next(names) ~= nil then - local saved_names = self.names - self.names = names - - local results = self:write_row(k, v, save_extra_mains) - - self.names = saved_names - - return results - end - end - - return 0 -end - -function Place:write_row(k, v, save_extra_mains) - if self.geometry == nil then - self.geometry = self.geom_func(self.object) - end - if self.geometry == nil or self.geometry:is_null() then - return 0 - end - - if save_extra_mains ~= nil then - for extra_k, extra_v in pairs(self.object.tags) do - if extra_k ~= k and save_extra_mains(extra_k, extra_v) then - self.extratags[extra_k] = extra_v - end - end - end - - place_table:insert{ - class = k, - type = v, - admin_level = self.admin_level, - name = next(self.names) and self.names, - address = next(self.address) and self.address, - extratags = next(self.extratags) and self.extratags, - geometry = self.geometry - } - - if save_extra_mains then - for tk, tv in pairs(self.object.tags) do - if save_extra_mains(tk, tv) then - self.extratags[tk] = nil - end - end - end - - self.num_entries = self.num_entries + 1 - - return 1 -end - - -function module.tag_match(data) - if data == nil or next(data) == nil then - return nil - end - - local fullmatches = {} - local key_prefixes = {} - local key_suffixes = {} - - if data.keys ~= nil then - for _, key in pairs(data.keys) do - if key:sub(1, 1) == '*' then - if #key > 1 then - if key_suffixes[#key - 1] == nil then - key_suffixes[#key - 1] = {} - end - key_suffixes[#key - 1][key:sub(2)] = true - end - elseif key:sub(#key, #key) == '*' then - if key_prefixes[#key - 1] == nil then - key_prefixes[#key - 1] = {} - end - key_prefixes[#key - 1][key:sub(1, #key - 1)] = true - else - fullmatches[key] = true - end - end - end - - if data.tags ~= nil then - for k, vlist in pairs(data.tags) do - if fullmatches[k] == nil then - fullmatches[k] = {} - for _, v in pairs(vlist) do - fullmatches[k][v] = true - end - end - end - end - - return function (k, v) - if fullmatches[k] ~= nil and (fullmatches[k] == true or fullmatches[k][v] ~= nil) then - return true - end - - for slen, slist in pairs(key_suffixes) do - if #k >= slen and slist[k:sub(-slen)] ~= nil then - return true - end - end - - for slen, slist in pairs(key_prefixes) do - if #k >= slen and slist[k:sub(1, slen)] ~= nil then - return true - end - end - - return false - end -end - - -function module.tag_group(data) - if data == nil or next(data) == nil then - return nil - end - - local fullmatches = {} - local key_prefixes = {} - local key_suffixes = {} - - for group, tags in pairs(data) do - for _, key in pairs(tags) do - if key:sub(1, 1) == '*' then - if #key > 1 then - if key_suffixes[#key - 1] == nil then - key_suffixes[#key - 1] = {} - end - key_suffixes[#key - 1][key:sub(2)] = group - end - elseif key:sub(#key, #key) == '*' then - if key_prefixes[#key - 1] == nil then - key_prefixes[#key - 1] = {} - end - key_prefixes[#key - 1][key:sub(1, #key - 1)] = group - else - fullmatches[key] = group - end - end - end - - return function (k, v) - local val = fullmatches[k] - if val ~= nil then - return val - end - - for slen, slist in pairs(key_suffixes) do - if #k >= slen then - val = slist[k:sub(-slen)] - if val ~= nil then - return val - end - end - end - - for slen, slist in pairs(key_prefixes) do - if #k >= slen then - val = slist[k:sub(1, slen)] - if val ~= nil then - return val - end - end - end - end -end - --- Returns prefix part of the keys, and reject suffix matching keys -local function process_key(key) - if key:sub(1, 1) == '*' then - return nil - end - if key:sub(#key, #key) == '*' then - return key:sub(1, #key - 2) - end - return key -end - --- Process functions for all data types -function module.process_node(object) - - local function geom_func(o) - return o:as_point() - end - - module.process_tags(Place.new(object, geom_func)) -end - -function module.process_way(object) - - local function geom_func(o) - local geom = o:as_polygon() - - if geom:is_null() then - geom = o:as_linestring() - if not geom:is_null() and geom:length() > 30 then - return nil - end - end - - return geom - end - - module.process_tags(Place.new(object, geom_func)) -end - -function module.process_relation(object) - local geom_func = module.RELATION_TYPES[object.tags.type] - - if geom_func ~= nil then - module.process_tags(Place.new(object, geom_func)) - end -end - --- The process functions are used by default by osm2pgsql. -osm2pgsql.process_node = module.process_node -osm2pgsql.process_way = module.process_way -osm2pgsql.process_relation = module.process_relation - -function module.process_tags(o) - o:clean{delete = PRE_DELETE, extra = PRE_EXTRAS} - - -- Exception for boundary/place double tagging - if o.object.tags.boundary == 'administrative' then - o:grab_extratags{match = function (k, v) - return k == 'place' and v:sub(1,3) ~= 'isl' - end} - end - - -- name keys - local fallback = o:grab_name_parts{groups=NAMES} - - -- address keys - if o:grab_address_parts{groups=ADDRESS_TAGS} > 0 and fallback == nil then - fallback = {'place', 'house', 'always'} - end - if o.address.country ~= nil and #o.address.country ~= 2 then - o.address['country'] = nil - end - if POSTCODE_FALLBACK and fallback == nil and o.address.postcode ~= nil then - fallback = {'place', 'postcode', 'always'} - end - - if o.address.interpolation ~= nil then - o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS) - return - end - - o:clean{delete = POST_DELETE} - - -- collect main keys - for k, v in pairs(o.object.tags) do - local ktype = MAIN_KEYS[k] - if ktype == 'fallback' then - if o.has_name then - fallback = {k, v, 'named'} - end - elseif ktype ~= nil then - o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS) - end - end - - if fallback ~= nil and o.num_entries == 0 then - o:write_place(fallback[1], fallback[2], fallback[3], SAVE_EXTRA_MAINS) - end -end - ---------- Convenience functions for simple style configuration ----------------- - - -function module.set_prefilters(data) - PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} - PRE_EXTRAS = module.tag_match{keys = data.extra_keys, - tags = data.extra_tags} - module.TAGINFO_MAIN.delete_tags = data.delete_tags -end - -function module.set_main_tags(data) - MAIN_KEYS = data - local keys = {} - for k, _ in pairs(data) do - table.insert(keys, k) - end - module.TAGINFO_MAIN.keys = keys -end - -function module.set_name_tags(data) - NAMES = module.tag_group(data) - - for _, lst in pairs(data) do - for _, k in ipairs(lst) do - local key = process_key(k) - if key ~= nil then - module.TAGINFO_NAME_KEYS[key] = true - end - end - end -end - -function module.set_address_tags(data) - if data.postcode_fallback ~= nil then - POSTCODE_FALLBACK = data.postcode_fallback - data.postcode_fallback = nil - end - ADDRESS_TAGS = module.tag_group(data) - - for _, lst in pairs(data) do - if lst ~= nil then - for _, k in ipairs(lst) do - local key = process_key(k) - if key ~= nil then - module.TAGINFO_ADDRESS_KEYS[key] = true - end - end - end - end -end - -function module.set_unused_handling(data) - if data.extra_keys == nil and data.extra_tags == nil then - POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} - SAVE_EXTRA_MAINS = function() return true end - elseif data.delete_keys == nil and data.delete_tags == nil then - POST_DELETE = nil - SAVE_EXTRA_MAINS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags} - else - error("unused handler can have only 'extra_keys' or 'delete_keys' set.") - end -end - -function module.set_relation_types(data) - module.RELATION_TYPES = {} - for k, v in data do - if v == 'multipolygon' then - module.RELATION_TYPES[k] = module.relation_as_multipolygon - elseif v == 'multiline' then - module.RELATION_TYPES[k] = module.relation_as_multiline - end - end -end - -return module diff --git a/settings/icu-rules/variants-ca.yaml b/settings/icu-rules/variants-ca.yaml index fab36dec..943ead51 100644 --- a/settings/icu-rules/variants-ca.yaml +++ b/settings/icu-rules/variants-ca.yaml @@ -4,7 +4,7 @@ - aparcament -> aparc - apartament -> apmt - apartat -> apt - - àtic -> àt + - àtic -> àt - autopista -> auto - autopista -> autop - autovia -> autov @@ -19,7 +19,6 @@ - biblioteca -> bibl - bloc -> bl - carrer -> c - - carrer -> c/ - carreró -> cró - carretera -> ctra - cantonada -> cant @@ -58,7 +57,6 @@ - número -> n - sense número -> s/n - parada -> par - - parcel·la -> parc - passadís -> pdís - passatge -> ptge - passeig -> pg diff --git a/settings/icu-rules/variants-en.yaml b/settings/icu-rules/variants-en.yaml index 5508fcd1..54a7b475 100644 --- a/settings/icu-rules/variants-en.yaml +++ b/settings/icu-rules/variants-en.yaml @@ -1,438 +1,393 @@ # Source: https://wiki.openstreetmap.org/wiki/Name_finder:Abbreviations#English +# Source: https://pe.usps.com/text/pub28/28apc_002.htm - lang: en words: - Access -> Accs - Air Force Base -> AFB - Air National Guard Base -> ANGB - Airport -> Aprt - - Alley -> Al - - Alley -> All - - Alley -> Ally - - Alley -> Aly + - Alley -> Al,All,Ally,Aly - Alleyway -> Alwy - Amble -> Ambl + - Anex -> Anx - Apartments -> Apts - - Approach -> Apch - - Approach -> App + - Approach -> Apch,App - Arcade -> Arc - Arterial -> Artl - Artery -> Arty - - Avenue -> Av - - Avenue -> Ave + - Avenue -> Av,Ave - Back -> Bk - Banan -> Ba - - Basin -> Basn - - Basin -> Bsn + - Basin -> Basn,Bsn + - Bayou -> Byu - Beach -> Bch - - Bend -> Bend - Bend -> Bnd - Block -> Blk + - Bluff -> Blf + - Bluffs -> Blfs - Boardwalk -> Bwlk - - Boulevard -> Blvd - - Boulevard -> Bvd + - Bottom -> Btm + - Boulevard -> Blvd,Bvd - Boundary -> Bdy - Bowl -> Bl - Brace -> Br - Brae -> Br - - Brae -> Brae + - Branch -> Br - Break -> Brk - - Bridge -> Bdge - - Bridge -> Br - - Bridge -> Brdg - - Bridge -> Bri - - Broadway -> Bdwy - - Broadway -> Bway - - Broadway -> Bwy + - Bridge$ -> Bdge,Br,Brdg,Brg,Bri + - Broadway -> Bdwy,Bway,Bwy - Brook -> Brk + - Brooks -> Brks - Brow -> Brw - - Brow -> Brow - - Buildings -> Bldgs - - Buildings -> Bldngs + - Buildings -> Bldgs,Bldngs - Business -> Bus - - Bypass -> Bps - - Bypass -> Byp - - Bypass -> Bypa + - Burg -> Bg + - Burgs -> Bgs + - Bypass -> Bps,Byp,Bypa - Byway -> Bywy + - Camp -> Cp + - Canyon -> Cyn + - Cape -> Cpe - Caravan -> Cvn - - Causeway -> Caus - - Causeway -> Cswy - - Causeway -> Cway - - Center -> Cen - - Center -> Ctr + - Causeway -> Caus,Cswy,Cway + - Center,Centre -> Cen,Ctr + - Centers -> Ctrs - Central -> Ctrl - - Centre -> Cen - - Centre -> Ctr - Centreway -> Cnwy - Chase -> Ch - Church -> Ch - Circle -> Cir - - Circuit -> Cct - - Circuit -> Ci - - Circus -> Crc - - Circus -> Crcs + - Circles -> Cirs + - Circuit -> Cct,Ci + - Circus -> Crc,Crcs - City -> Cty + - Cliff -> Clf + - Cliffs -> Clfs - Close -> Cl - - Common -> Cmn - - Common -> Comm + - Club -> Clb + - Common -> Cmn,Comm + - Commons -> Cmns - Community -> Comm - Concourse -> Cnc - Concourse -> Con - Copse -> Cps - - Corner -> Cnr - - Corner -> Crn + - Corner -> Cor,Cnr,Crn + - Corners -> Cors - Corso -> Cso - Cottages -> Cotts - County -> Co - County Road -> CR - County Route -> CR - - Court -> Crt - - Court -> Ct + - Course -> Crse + - Court -> Crt,Ct + - Courts -> Cts - Courtyard -> Cyd - Courtyard -> Ctyd - - Cove -> Ce - - Cove -> Cov - - Cove -> Cove - - Cove -> Cv - - Creek -> Ck - - Creek -> Cr - - Creek -> Crk + - Cove$ -> Ce,Cov,Cv + - Coves -> Cvs + - Creek$ -> Ck,Cr,Crk - Crescent -> Cr - Crescent -> Cres - - Crest -> Crst - - Crest -> Cst + - Crest -> Crst,Cst - Croft -> Cft - - Cross -> Cs - - Cross -> Crss - - Crossing -> Crsg - - Crossing -> Csg - - Crossing -> Xing - - Crossroad -> Crd + - Cross -> Cs,Crss + - Crossing -> Crsg,Csg,Xing + - Crossroad -> Crd,Xrd + - Crossroads -> Xrds - Crossway -> Cowy - - Cul-de-sac -> Cds - - Cul-de-sac -> Csac - - Curve -> Cve + - Cul-de-sac -> Cds,Csac + - Curve -> Cve,Curv - Cutting -> Cutt - Dale -> Dle - - Dale -> Dale + - Dam -> Dm - Deviation -> Devn - - Dip -> Dip - Distributor -> Dstr + - Divide -> Dv - Down -> Dn - Downs -> Dn - - Drive -> Dr - - Drive -> Drv - - Drive -> Dv + - Drive -> Dr,Drv,Dv + - Drives -> Drs - Drive-In => Drive-In # prevent abbreviation here - - Driveway -> Drwy - - Driveway -> Dvwy - - Driveway -> Dwy + - Driveway -> Drwy,Dvwy,Dwy - East -> E - Edge -> Edg - - Edge -> Edge - Elbow -> Elb - - End -> End - Entrance -> Ent - Esplanade -> Esp - Estate -> Est - - Expressway -> Exp - - Expressway -> Expy - - Expressway -> Expwy - - Expressway -> Xway + - Estates -> Ests + - Expressway -> Exp,Expy,Expwy,Xway - Extension -> Ex - - Fairway -> Fawy - - Fairway -> Fy + - Extensions -> Exts + - Fairway -> Fawy,Fy + - Falls -> Fls - Father -> Fr - - Ferry -> Fy - - Field -> Fd + - Ferry -> Fy,Fry + - Field -> Fd,Fld + - Fields -> Flds - Fire Track -> Ftrk - Firetrail -> Fit - - Flat -> Fl - - Flat -> Flat + - Flat -> Fl,Flt + - Flats -> Flts - Follow -> Folw - Footway -> Ftwy + - Ford -> Frd + - Fords -> Frds - Foreshore -> Fshr + - Forest -> Frst - Forest Service Road -> FSR + - Forge -> Frg + - Forges -> Frgs - Formation -> Form + - Fork -> Frk + - Forks -> Frks - Fort -> Ft - - Freeway -> Frwy - - Freeway -> Fwy + - Freeway -> Frwy,Fwy - Front -> Frnt - - Frontage -> Fr - - Frontage -> Frtg - - Gap -> Gap + - Frontage -> Fr,Frtg - Garden -> Gdn - - Gardens -> Gdn - - Gardens -> Gdns - - Gate -> Ga - - Gate -> Gte - - Gates -> Ga - - Gates -> Gte - - Gateway -> Gwy + - Gardens -> Gdn,Gdns + - Gate,Gates -> Ga,Gte + - Gateway -> Gwy,Gtwy - George -> Geo - - Glade -> Gl - - Glade -> Gld - - Glade -> Glde + - Glade$ -> Gl,Gld,Glde - Glen -> Gln - - Glen -> Glen + - Glens -> Glns - Grange -> Gra - - Green -> Gn - - Green -> Grn + - Green -> Gn,Grn + - Greens -> Grns - Ground -> Grnd - - Grove -> Gr - - Grove -> Gro + - Grove$ -> Gr,Gro,Grv + - Groves -> Grvs - Grovet -> Gr - Gully -> Gly - - Harbor -> Hbr - - Harbour -> Hbr + - Harbor -> Hbr,Harbour + - Harbors -> Hbrs + - Harbour -> Hbr,Harbor - Haven -> Hvn - Head -> Hd - Heads -> Hd - - Heights -> Hgts - - Heights -> Ht - - Heights -> Hts + - Heights -> Hgts,Ht,Hts - High School -> HS - - Highroad -> Hird - - Highroad -> Hrd + - Highroad -> Hird,Hrd - Highway -> Hwy - - Hill -> Hill - Hill -> Hl - - Hills -> Hl - - Hills -> Hls + - Hills -> Hl,Hls + - Hollow -> Holw - Hospital -> Hosp - - House -> Ho - - House -> Hse + - House -> Ho,Hse - Industrial -> Ind + - Inlet -> Inlt - Interchange -> Intg - International -> Intl - - Island -> I - - Island -> Is - - Junction -> Jctn - - Junction -> Jnc + - Island -> I,Is + - Islands -> Iss + - Junction -> Jct,Jctn,Jnc + - Junctions -> Jcts - Junior -> Jr - - Key -> Key + - Key -> Ky + - Keys -> Kys + - Knoll -> Knl + - Knolls -> Knls - Lagoon -> Lgn - - Lakes -> L - - Landing -> Ldg - - Lane -> La - - Lane -> Lane - - Lane -> Ln + - Lake -> Lk + - Lakes -> L,Lks + - Landing -> Ldg,Lndg + - Lane -> La,Ln - Laneway -> Lnwy - - Line -> Line + - Light -> Lgt + - Lights -> Lgts - Line -> Ln - - Link -> Link - Link -> Lk - - Little -> Lit - - Little -> Lt + - Little -> Lit,Lt + - Loaf -> Lf + - Lock -> Lck + - Locks -> Lcks - Lodge -> Ldg - Lookout -> Lkt - - Loop -> Loop - Loop -> Lp - - Lower -> Low - - Lower -> Lr - - Lower -> Lwr - - Mall -> Mall + - Lower -> Low,Lr,Lwr - Mall -> Ml - Manor -> Mnr + - Manors -> Mnrs - Mansions -> Mans - Market -> Mkt - Meadow -> Mdw - - Meadows -> Mdw - - Meadows -> Mdws + - Meadows -> Mdw,Mdws - Mead -> Md - - Meander -> Mdr - - Meander -> Mndr - - Meander -> Mr + - Meander -> Mdr,Mndr,Mr - Medical -> Med - Memorial -> Mem - - Mews -> Mews - Mews -> Mw - Middle -> Mid - Middle School -> MS - Mile -> Mi - Military -> Mil - - Motorway -> Mtwy - - Motorway -> Mwy + - Mill -> Ml + - Mills -> Mls + - Mission -> Msn + - Motorway -> Mtwy,Mwy - Mount -> Mt - Mountain -> Mtn - - Mountains -> Mtn + - Mountains$ -> Mtn,Mtns - Municipal -> Mun - Museum -> Mus - National Park -> NP - National Recreation Area -> NRA - National Wildlife Refuge Area -> NWRA + - Neck -> Nck - Nook -> Nk - - Nook -> Nook - North -> N - Northeast -> NE - Northwest -> NW - - Outlook -> Out - - Outlook -> Otlk + - Orchard -> Orch + - Outlook -> Out,Otlk + - Overpass -> Opas - Parade -> Pde - Paradise -> Pdse - - Park -> Park - Park -> Pk - Parklands -> Pkld - - Parkway -> Pkwy - - Parkway -> Pky - - Parkway -> Pwy - - Pass -> Pass + - Parkway -> Pkwy,Pky,Pwy + - Parkways -> Pkwy - Pass -> Ps - Passage -> Psge - - Path -> Path - - Pathway -> Phwy - - Pathway -> Pway - - Pathway -> Pwy + - Pathway -> Phwy,Pway,Pwy - Piazza -> Piaz - Pike -> Pk + - Pine -> Pne + - Pines -> Pnes - Place -> Pl - - Plain -> Pl - - Plains -> Pl + - Plain -> Pl,Pln + - Plains -> Pl,Plns - Plateau -> Plat - - Plaza -> Pl - - Plaza -> Plz - - Plaza -> Plza + - Plaza -> Pl,Plz,Plza - Pocket -> Pkt - - Point -> Pnt - - Point -> Pt - - Port -> Port - - Port -> Pt + - Point -> Pnt,Pt + - Points -> Pts + - Port -> Prt,Pt + - Ports -> Prts - Post Office -> PO + - Prairie -> Pr - Precinct -> Pct - - Promenade -> Prm - - Promenade -> Prom - - Quad -> Quad + - Promenade -> Prm,Prom - Quadrangle -> Qdgl - - Quadrant -> Qdrt - - Quadrant -> Qd + - Quadrant -> Qdrt,Qd - Quay -> Qy - Quays -> Qy - Quays -> Qys + - Radial -> Radl - Ramble -> Ra - Ramble -> Rmbl - - Range -> Rge - - Range -> Rnge + - Ranch -> Rnch + - Range -> Rge,Rnge + - Rapid -> Rpd + - Rapids -> Rpds - Reach -> Rch - Reservation -> Res - Reserve -> Res - Reservoir -> Res - - Rest -> Rest - Rest -> Rst - - Retreat -> Rt - - Retreat -> Rtt + - Retreat -> Rt,Rtt - Return -> Rtn - - Ridge -> Rdg - - Ridge -> Rdge + - Ridge -> Rdg,Rdge + - Ridges -> Rdgs - Ridgeway -> Rgwy - Right of Way -> Rowy - Rise -> Ri - - Rise -> Rise - - River -> R - - River -> Riv - - River -> Rvr + - ^River -> R,Riv,Rvr + - River$ -> R,Riv,Rvr - Riverway -> Rvwy - Riviera -> Rvra - Road -> Rd - Roads -> Rds - Roadside -> Rdsd - - Roadway -> Rdwy - - Roadway -> Rdy - - Robert -> Robt + - Roadway -> Rdwy,Rdy - Rocks -> Rks - Ronde -> Rnde - Rosebowl -> Rsbl - Rotary -> Rty - Round -> Rnd - - Route -> Rt - - Route -> Rte - - Row -> Row - - Rue -> Rue - - Run -> Run + - Route -> Rt,Rte - Saint -> St - Saints -> SS - Senior -> Sr - - Serviceway -> Swy - - Serviceway -> Svwy + - Serviceway -> Swy,Svwy + - Shoal -> Shl + - Shore -> Shr + - Shores -> Shrs - Shunt -> Shun - Siding -> Sdng - Sister -> Sr + - Skyway -> Skwy - Slope -> Slpe - Sound -> Snd - - South -> S - - South -> Sth + - South -> S,Sth - Southeast -> SE - Southwest -> SW - - Spur -> Spur + - Spring -> Spg + - Springs -> Spgs + - Spurs -> Spur - Square -> Sq + - Squares -> Sqs - Stairway -> Strwy - - State Highway -> SH - - State Highway -> SHwy + - State Highway -> SH,SHwy - State Route -> SR - - Station -> Sta - - Station -> Stn - - Strand -> Sd - - Strand -> Stra + - Station -> Sta,Stn + - Strand -> Sd,Stra + - Stravenue -> Stra + - Stream -> Strm - Street -> St + - Streets -> Sts - Strip -> Strp - Subway -> Sbwy + - Summit -> Smt - Tarn -> Tn - - Tarn -> Tarn - Terminal -> Term - - Terrace -> Tce - - Terrace -> Ter - - Terrace -> Terr - - Thoroughfare -> Thfr - - Thoroughfare -> Thor - - Tollway -> Tlwy - - Tollway -> Twy - - Top -> Top - - Tor -> Tor + - Terrace -> Tce,Ter,Terr + - Thoroughfare -> Thfr,Thor + - Throughway -> Trwy + - Tollway -> Tlwy,Twy - Towers -> Twrs - Township -> Twp - Trace -> Trce - - Track -> Tr - - Track -> Trk + - Track -> Tr,Trak,Trk + - Trafficway -> Trfy - Trail -> Trl - Trailer -> Trlr - Triangle -> Tri - Trunkway -> Tkwy - - Tunnel -> Tun - - Turn -> Tn - - Turn -> Trn - - Turn -> Turn - - Turnpike -> Tpk - - Turnpike -> Tpke - - Underpass -> Upas - - Underpass -> Ups - - University -> Uni - - University -> Univ + - Tunnel -> Tun,Tunl + - Turn -> Tn,Trn + - Turnpike -> Tpk,Tpke + - Underpass -> Upas,Ups + - Union -> Un + - Unions -> Uns + - University -> Uni,Univ - Upper -> Up - Upper -> Upr - Vale -> Va - - Vale -> Vale + - Valley -> Vly - Valley -> Vy - - Viaduct -> Vdct - - Viaduct -> Via - - Viaduct -> Viad + - Valleys -> Vlys + - Viaduct$ -> Vdct,Via,Viad - View -> Vw - - View -> View - - Village -> Vill + - Views -> Vws + - Village -> Vill,Vlg + - Villages -> Vlgs - Villas -> Vlls - - Vista -> Vst - - Vista -> Vsta - - Walk -> Walk - - Walk -> Wk - - Walk -> Wlk - - Walkway -> Wkwy - - Walkway -> Wky + - Ville -> Vl + - Vista -> Vis,Vst,Vsta + - Walk -> Wk,Wlk + - Walks -> Walk + - Walkway -> Wkwy,Wky - Waters -> Wtr - - Way -> Way - Way -> Wy + - Well -> Wl + - Wells -> Wls - West -> W - Wharf -> Whrf - William -> Wm - Wynd -> Wyn - - Wynd -> Wynd - - Yard -> Yard - Yard -> Yd - lang: en country: ca diff --git a/settings/icu-rules/variants-es.yaml b/settings/icu-rules/variants-es.yaml index 5c5e6abe..fbe40b54 100644 --- a/settings/icu-rules/variants-es.yaml +++ b/settings/icu-rules/variants-es.yaml @@ -30,7 +30,6 @@ - Bloque -> Blq - Bulevar -> Blvr - Boulevard -> Blvd - - Calle -> C/ - Calle -> C - Calle -> Cl - Calleja -> Cllja diff --git a/settings/icu-rules/variants-fr.yaml b/settings/icu-rules/variants-fr.yaml index 0fd9c337..b9cfc493 100644 --- a/settings/icu-rules/variants-fr.yaml +++ b/settings/icu-rules/variants-fr.yaml @@ -3,20 +3,16 @@ words: - Abbaye -> ABE - Agglomération -> AGL - - Aire -> AIRE - Aires -> AIRE - Allée -> ALL - - Allée -> All - Allées -> ALL - Ancien chemin -> ACH - Ancienne route -> ART - Anciennes routes -> ART - - Anse -> ANSE - Arcade -> ARC - Arcades -> ARC - Autoroute -> AUT - Avenue -> AV - - Avenue -> Av - Barrière -> BRE - Barrières -> BRE - Bas chemin -> BCH @@ -28,16 +24,11 @@ - Berges -> BER - Bois -> BOIS - Boucle -> BCLE - - Boulevard -> Bd - Boulevard -> BD - Bourg -> BRG - Butte -> BUT - - Cité -> CITE - Cités -> CITE - - Côte -> COTE - Côteau -> COTE - - Cale -> CALE - - Camp -> CAMP - Campagne -> CGNE - Camping -> CPG - Carreau -> CAU @@ -56,17 +47,13 @@ - Chaussées -> CHS - Chemin -> Ch - Chemin -> CHE - - Chemin -> Che - Chemin vicinal -> CHV - Cheminement -> CHEM - Cheminements -> CHEM - Chemins -> CHE - Chemins vicinaux -> CHV - - Chez -> CHEZ - Château -> CHT - Cloître -> CLOI - - Clos -> CLOS - - Col -> COL - Colline -> COLI - Collines -> COLI - Contour -> CTR @@ -74,9 +61,7 @@ - Corniches -> COR - Cottage -> COTT - Cottages -> COTT - - Cour -> COUR - Cours -> CRS - - Cours -> Crs - Darse -> DARS - Degré -> DEG - Degrés -> DEG @@ -87,11 +72,8 @@ - Domaine -> DOM - Domaines -> DOM - Écluse -> ECL - - Écluse -> ÉCL - Écluses -> ECL - - Écluses -> ÉCL - Église -> EGL - - Église -> ÉGL - Enceinte -> EN - Enclave -> ENV - Enclos -> ENC @@ -100,21 +82,16 @@ - Espace -> ESPA - Esplanade -> ESP - Esplanades -> ESP - - Étang -> ETANG - - Étang -> ÉTANG - Faubourg -> FG - - Faubourg -> Fg - Ferme -> FRM - Fermes -> FRM - Fontaine -> FON - - Fort -> FORT - Forum -> FORM - Fosse -> FOS - Fosses -> FOS - Foyer -> FOYR - Galerie -> GAL - Galeries -> GAL - - Gare -> GARE - Garenne -> GARN - Grand boulevard -> GBD - Grand ensemble -> GDEN @@ -134,13 +111,9 @@ - Haut chemin -> HCH - Hauts chemins -> HCH - Hippodrome -> HIP - - HLM -> HLM - - Île -> ILE - - Île -> ÎLE - Immeuble -> IMM - Immeubles -> IMM - Impasse -> IMP - - Impasse -> Imp - Impasses -> IMP - Jardin -> JARD - Jardins -> JARD @@ -150,13 +123,11 @@ - Lieu-dit -> LD - Lotissement -> LOT - Lotissements -> LOT - - Mail -> MAIL - Maison forestière -> MF - Manoir -> MAN - Marche -> MAR - Marches -> MAR - Maréchal -> MAL - - Mas -> MAS - Monseigneur -> Mgr - Mont -> Mt - Montée -> MTE @@ -168,13 +139,9 @@ - Métro -> MÉT - Nouvelle route -> NTE - Palais -> PAL - - Parc -> PARC - - Parcs -> PARC - Parking -> PKG - Parvis -> PRV - Passage -> PAS - - Passage -> Pas - - Passage -> Pass - Passage à niveau -> PN - Passe -> PASS - Passerelle -> PLE @@ -191,19 +158,14 @@ - Petite rue -> PTR - Petites allées -> PTA - Place -> PL - - Place -> Pl - Placis -> PLCI - Plage -> PLAG - Plages -> PLAG - Plaine -> PLN - - Plan -> PLAN - Plateau -> PLT - Plateaux -> PLT - Pointe -> PNT - - Pont -> PONT - - Ponts -> PONT - Porche -> PCH - - Port -> PORT - Porte -> PTE - Portique -> PORQ - Portiques -> PORQ @@ -211,25 +173,19 @@ - Pourtour -> POUR - Presqu’île -> PRQ - Promenade -> PROM - - Promenade -> Prom - - Pré -> PRE - - Pré -> PRÉ - Périphérique -> PERI - Péristyle -> PSTY - Quai -> QU - - Quai -> Qu - Quartier -> QUA - Raccourci -> RAC - Raidillon -> RAID - Rampe -> RPE - Rempart -> REM - - Roc -> ROC - Rocade -> ROC - Rond point -> RPT - Roquet -> ROQT - Rotonde -> RTD - Route -> RTE - - Route -> Rte - Routes -> RTE - Rue -> R - Rue -> R @@ -245,7 +201,6 @@ - Sentier -> SEN - Sentiers -> SEN - Square -> SQ - - Square -> Sq - Stade -> STDE - Station -> STA - Terrain -> TRN @@ -254,13 +209,11 @@ - Terre plein -> TPL - Tertre -> TRT - Tertres -> TRT - - Tour -> TOUR - Traverse -> TRA - Vallon -> VAL - Vallée -> VAL - Venelle -> VEN - Venelles -> VEN - - Via -> VIA - Vieille route -> VTE - Vieux chemin -> VCHE - Villa -> VLA @@ -269,7 +222,6 @@ - Villas -> VLA - Voie -> VOI - Voies -> VOI - - Zone -> ZONE - Zone artisanale -> ZA - Zone d'aménagement concerté -> ZAC - Zone d'aménagement différé -> ZAD @@ -289,7 +241,6 @@ - Esplanade -> ESPL - Passage -> PASS - Plateau -> PLAT - - Rang -> RANG - Rond-point -> RDPT - Sentier -> SENT - Subdivision -> SUBDIV diff --git a/settings/icu-rules/variants-it.yaml b/settings/icu-rules/variants-it.yaml index 3e45521e..55678e1a 100644 --- a/settings/icu-rules/variants-it.yaml +++ b/settings/icu-rules/variants-it.yaml @@ -29,7 +29,6 @@ - Prima -> I - Primo -> I - Primo -> 1 - - Primo -> 1° - Quarta -> IV - Quarto -> IV - Quattro -> IV diff --git a/settings/icu-rules/variants-no.yaml b/settings/icu-rules/variants-no.yaml index de75e3c3..dd17408f 100644 --- a/settings/icu-rules/variants-no.yaml +++ b/settings/icu-rules/variants-no.yaml @@ -1,11 +1,10 @@ # Source: https://wiki.openstreetmap.org/wiki/Name_finder:Abbreviations#Norsk_-_Norwegian -- lang: no +- lang: "no" words: # convert between Nynorsk and Bookmal here - - vei, veg => v,vn,vei,veg - - veien, vegen -> v,vn,veien,vegen - - gate -> g,gt + - ~vei, ~veg -> v,vei,veg + - ~veien, ~vegen -> vn,veien,vegen # convert between the two female forms - - gaten, gata => g,gt,gaten,gata + - gate, gaten, gata -> g,gt - plass, plassen -> pl - sving, svingen -> sv diff --git a/settings/icu-rules/variants-ru.yaml b/settings/icu-rules/variants-ru.yaml index 2092e32e..87f872c5 100644 --- a/settings/icu-rules/variants-ru.yaml +++ b/settings/icu-rules/variants-ru.yaml @@ -1,14 +1,128 @@ # Source: https://wiki.openstreetmap.org/wiki/Name_finder:Abbreviations#.D0.A0.D1.83.D1.81.D1.81.D0.BA.D0.B8.D0.B9_-_Russian +# Source: https://www.plantarium.ru/page/help/topic/abbreviations.html +# Source: https://dic.academic.ru/dic.nsf/ruwiki/1871310 - lang: ru words: + - Академик, Академика -> Ак + - акционерное общество -> АО - аллея -> ал + - архипелаг -> арх + - атомная электростанция -> АЭС + - аэродром -> аэрд + - аэропорт -> аэрп + - Башкирский, Башкирская, Башкирское, Башкирские -> Баш, Башк, Башкир + - Белый, Белая, Белое. Белые -> Бел + - болото -> бол + - больница -> больн + - Большой, Большая, Большое, Большие -> Б, Бол + - брод -> бр - бульвар -> бул + - бухта -> бух + - бывший, бывшая, бывшее, бывшие -> бывш + - Великий, Великая, Великое, Великие -> Вел + - Верхний, Верхняя, Верхнее, Верхние -> В, Верх + - водокачка -> вдкч + - водопад -> вдп + - водохранилище -> вдхр + - вокзал -> вкз, вокз + - Восточный, Восточная, Восточное, Восточные -> В, Вост + - вулкан -> влк + - гидроэлектростанция -> ГЭС + - гора -> г + - город -> г + - дворец культуры, дом культуры -> ДК + - дворец спорта -> ДС + - деревня -> д, дер + - детский оздоровительный лагерь -> ДОЛ + - дом -> д + - дом отдыха -> Д О + - железная дорога -> ж д + - железнодорожный, железнодорожная, железнодорожное -> ж-д + - железобетонных изделий -> ЖБИ + - жилой комплекс -> ЖК + - завод -> з-д + - закрытое административно-территориальное образование -> ЗАТО + - залив -> зал + - Западный, Западная, Западное, Западные -> З, Зап, Запад + - заповедник -> запов + - имени -> им + - институт -> инст + - исправительная колония -> ИК + - километр -> км + - Красный, Красная, Красное, Красные -> Кр, Крас + - лагерь -> лаг + - Левый, Левая,Левое, Левые -> Л, Лев + - ледник -> ледн + - лесничество -> леснич + - лесной, лесная, лесное -> лес + - линия электропередачи -> ЛЭП + - Малый, Малая, Малое, Малые -> М, Мал + - Мордовский, Мордовская, Мордовское, Мордовские -> Мордов + - морской, морская, морское -> мор + - Московский, Московская, Московское, Московские -> Мос, Моск + - мыс -> м - набережная -> наб + - Нижний, Нижняя, Нижнее, Нижние -> Ниж, Н + - Новый, Новая, Новое, Новые -> Нов, Н + - обгонный пункт -> обг п + - область -> обл + - озеро -> оз + - особо охраняемая природная территория -> ООПТ + - остановочный пункт -> о п + - остров -> о + - острова -> о-ва + - парк культуры и отдыха -> ПКиО + - перевал -> пер - переулок -> пер + - пещера -> пещ + - пионерский лагерь -> пионерлаг + - платформа -> пл, платф - площадь -> пл + - подсобное хозяйство -> подсоб хоз + - полуостров -> п-ов + - посёлок -> пос, п + - посёлок городского типа -> п г т, пгт + - Правый, Правая, Правое, Правые -> П, Пр, Прав - проезд -> пр - проспект -> просп - - шоссе -> ш + - пруд -> пр + - пустыня -> пуст + - разъезд -> рзд + - район -> р-н + - резинотехнических изделий -> РТИ + - река -> р + - речной, речная, речное -> реч, речн + - Российский, Российская, Российское, Российские -> Рос + - Русский, Русская, Русское, Русские -> Рус, Русск + - ручей -> руч + - садовое некоммерческое товарищество -> СНТ + - садовые участки -> сад уч + - санаторий -> сан + - сарай -> сар + - Северный, Северная, Северное, Северные -> С, Сев + - село -> с + - Сибирский, Сибирская, Сибирское, Сибирские -> Сиб + - Советский, Советская, Советское, Советские -> Сов + - совхоз -> свх + - Сортировочный, Сортировочная, Сортировочное, Сортировочные -> Сорт + - станция -> ст + - Старый, Старая, Среднее, Средние -> Ср + - Татарский, Татарская, Татарское, Татарские -> Тат, Татар + - теплоэлекстростанция -> ТЭС + - теплоэлектроцентраль -> ТЭЦ + - техникум -> техн + - тоннель, туннель -> тун - тупик -> туп - улица -> ул - - область -> обл + - Уральский, Уральская, Уральское, Уральские -> Ур, Урал + - урочище -> ур + - хозяйство -> хоз, хоз-во + - хребет -> хр + - хутор -> хут + - Чёрный, Чёрная, Чёрное, Чёрные -> Черн + - Чувашский, Чувашская, Чувашское, Чувашские -> Чуваш + - шахта -> шах + - школа -> шк + - шоссе -> ш + - элеватор -> элев + - Южный, Южная, Южное, Южные -> Ю, Юж, Южн \ No newline at end of file diff --git a/settings/icu_tokenizer.yaml b/settings/icu_tokenizer.yaml index c5a809c6..bb81f80b 100644 --- a/settings/icu_tokenizer.yaml +++ b/settings/icu_tokenizer.yaml @@ -1,3 +1,6 @@ +query-preprocessing: + - step: split_japanese_phrases + - step: normalize normalization: - ":: lower ()" - ":: Hans-Hant" @@ -7,16 +10,17 @@ normalization: - "'nº' > 'no'" - "ª > a" - "º > o" - - "[[:Punctuation:][:Symbol:]\u02bc] > ' '" + - "[[:Punctuation:][:Symbol:][\u02bc] - [-:]]+ > '-'" - "ß > 'ss'" # German szet is unambiguously equal to double ss - - "[^[:alnum:] [:Canonical_Combining_Class=Virama:] [:Space:]] >" + - "[^[:alnum:] [:Canonical_Combining_Class=Virama:] [:Space:] [-:]] >" - "[:Lm:] >" - ":: [[:Number:]] Latin ()" - ":: [[:Number:]] Ascii ();" - ":: [[:Number:]] NFD ();" - "[[:Nonspacing Mark:] [:Cf:]] >;" - - "[:Space:]+ > ' '" + - "[-:]?[:Space:]+[-:]? > ' '" transliteration: + - "[-:] > ' '" - ":: Latin ()" - !include icu-rules/extended-unicode-to-asccii.yaml - ":: Ascii ()" @@ -42,7 +46,7 @@ sanitizers: - step: strip-brace-terms - step: tag-analyzer-by-language filter-kind: [".*name.*"] - whitelist: [bg,ca,cs,da,de,el,en,es,et,eu,fi,fr,gl,hu,it,ja,mg,ms,nl,no,pl,pt,ro,ru,sk,sl,sv,tr,uk,vi] + whitelist: [bg,ca,cs,da,de,el,en,es,et,eu,fi,fr,gl,hu,it,ja,mg,ms,nl,"no",pl,pt,ro,ru,sk,sl,sv,tr,uk,vi] use-defaults: all mode: append - step: tag-japanese @@ -154,7 +158,7 @@ token-analysis: mode: variant-only variants: - !include icu-rules/variants-nl.yaml - - id: no + - id: "no" analyzer: generic mode: variant-only variants: diff --git a/settings/import-address.lua b/settings/import-address.lua deleted file mode 100644 index b177b73c..00000000 --- a/settings/import-address.lua +++ /dev/null @@ -1,74 +0,0 @@ -local flex = require('flex-base') - -flex.set_main_tags{ - highway = {motorway = 'always', - trunk = 'always', - primary = 'always', - secondary = 'always', - tertiary = 'always', - unclassified = 'always', - residential = 'always', - road = 'always', - living_street = 'always', - pedestrian = 'always', - service = 'named', - cycleway = 'named', - path = 'named', - footway = 'named', - steps = 'named', - bridleway = 'named', - track = 'named', - motorway_link = 'named', - trunk_link = 'named', - primary_link = 'named', - secondary_link = 'named', - tertiary_link = 'named'}, - boundary = {administrative = 'named', - postal_code = 'always'}, - landuse = 'fallback', - place = 'always' -} - -flex.set_prefilters{delete_keys = {'building', 'source', - 'source', '*source', 'type', - 'is_in:postcode', '*:wikidata', '*:wikipedia', - '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', - 'name:etymology', 'name:signed', 'name:botanical', - 'addr:street:name', 'addr:street:type'}, - delete_tags = {landuse = {'cemetry', 'no'}, - boundary = {'place'}}, - extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'} - } - -flex.set_name_tags{main = {'name', 'name:*', - 'int_name', 'int_name:*', - 'nat_name', 'nat_name:*', - 'reg_name', 'reg_name:*', - 'loc_name', 'loc_name:*', - 'old_name', 'old_name:*', - 'alt_name', 'alt_name:*', 'alt_name_*', - 'official_name', 'official_name:*', - 'place_name', 'place_name:*', - 'short_name', 'short_name:*'}, - extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref', - 'loc_ref', 'old_ref', - 'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}, - house = {'addr:housename'} - } - -flex.set_address_tags{main = {'addr:housenumber', - 'addr:conscriptionnumber', - 'addr:streetnumber'}, - extra = {'addr:*', 'is_in:*', 'tiger:county'}, - postcode = {'postal_code', 'postcode', 'addr:postcode', - 'tiger:zip_left', 'tiger:zip_right'}, - country = {'country_code', 'ISO3166-1', - 'addr:country_code', 'is_in:country_code', - 'addr:country', 'is_in:country'}, - interpolation = {'addr:interpolation'} - } - - -flex.set_unused_handling{extra_keys = {'place'}} - -return flex diff --git a/settings/import-admin.lua b/settings/import-admin.lua deleted file mode 100644 index 78eac5f5..00000000 --- a/settings/import-admin.lua +++ /dev/null @@ -1,72 +0,0 @@ -local flex = require('flex-base') - -flex.set_main_tags{ - boundary = {administrative = 'named'}, - landuse = {residential = 'fallback', - farm = 'fallback', - farmyard = 'fallback', - industrial = 'fallback', - commercial = 'fallback', - allotments = 'fallback', - retail = 'fallback'}, - place = {county = 'always', - district = 'always', - municipality = 'always', - city = 'always', - town = 'always', - borough = 'always', - village = 'always', - suburb = 'always', - hamlet = 'always', - croft = 'always', - subdivision = 'always', - allotments = 'always', - neighbourhood = 'always', - quarter = 'always', - isolated_dwelling = 'always', - farm = 'always', - city_block = 'always', - mountain_pass = 'always', - square = 'always', - locality = 'always'} -} - -flex.set_prefilters{delete_keys = {'building', 'source', 'highway', - 'addr:housenumber', 'addr:street', 'addr:city', - 'addr:interpolation', - 'source', '*source', 'type', - 'is_in:postcode', '*:wikidata', '*:wikipedia', - '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', - 'name:etymology', 'name:signed', 'name:botanical', - 'addr:street:name', 'addr:street:type'}, - delete_tags = {landuse = {'cemetry', 'no'}, - boundary = {'place'}}, - extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'} - } - -flex.set_name_tags{main = {'name', 'name:*', - 'int_name', 'int_name:*', - 'nat_name', 'nat_name:*', - 'reg_name', 'reg_name:*', - 'loc_name', 'loc_name:*', - 'old_name', 'old_name:*', - 'alt_name', 'alt_name:*', 'alt_name_*', - 'official_name', 'official_name:*', - 'place_name', 'place_name:*', - 'short_name', 'short_name:*'}, - extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref', - 'loc_ref', 'old_ref', - 'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'} - } - -flex.set_address_tags{extra = {'addr:*', 'is_in:*'}, - postcode = {'postal_code', 'postcode', 'addr:postcode'}, - country = {'country_code', 'ISO3166-1', - 'addr:country_code', 'is_in:country_code', - 'addr:country', 'is_in:country'}, - postcode_fallback = false - } - -flex.set_unused_handling{extra_keys = {'place'}} - -return flex diff --git a/settings/import-extratags.lua b/settings/import-extratags.lua deleted file mode 100644 index 0d242b0f..00000000 --- a/settings/import-extratags.lua +++ /dev/null @@ -1,126 +0,0 @@ -local flex = require('flex-base') - -flex.set_main_tags{ - building = 'fallback', - emergency = 'always', - healthcare = 'fallback', - historic = 'always', - military = 'always', - natural = 'named', - highway = {'always', - street_lamp = 'named', - traffic_signals = 'named', - service = 'named', - cycleway = 'named', - path = 'named', - footway = 'named', - steps = 'named', - bridleway = 'named', - track = 'named', - motorway_link = 'named', - trunk_link = 'named', - primary_link = 'named', - secondary_link = 'named', - tertiary_link = 'named'}, - railway = 'named', - man_made = {'none', - pier = 'always', - tower = 'always', - bridge = 'always', - works = 'named', - water_tower = 'always', - dyke = 'named', - adit = 'named', - lighthouse = 'always', - watermill = 'always', - tunnel = 'always'}, - aerialway = 'always', - boundary = {'named', - postal_code = 'always'}, - aeroway = 'always', - amenity = 'always', - club = 'always', - craft = 'always', - junction = 'fallback', - landuse = 'fallback', - leisure = {'always', - nature_reserve = 'fallback'}, - office = 'always', - mountain_pass = 'always', - shop = 'always', - tourism = 'always', - bridge = 'named_with_key', - tunnel = 'named_with_key', - waterway = 'named', - place = 'always' -} - -flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attribution', - 'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*', - 'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*', - 'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*', - 'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type', - 'ref:linz:*', 'is_in:postcode'}, - delete_tags = {emergency = {'yes', 'no', 'fire_hydrant'}, - historic = {'yes', 'no'}, - military = {'yes', 'no'}, - natural = {'yes', 'no', 'coastline'}, - highway = {'no', 'turning_circle', 'mini_roundabout', - 'noexit', 'crossing', 'give_way', 'stop'}, - railway = {'level_crossing', 'no', 'rail', 'switch', - 'abandoned', 'signal', 'buffer_stop', 'razed'}, - aerialway = {'pylon', 'no'}, - aeroway = {'no'}, - amenity = {'no', 'parking_space', 'parking_entrance', - 'waste_disposal', 'hunting_stand'}, - club = {'no'}, - craft = {'no'}, - leisure = {'no'}, - office = {'no'}, - mountain_pass = {'no'}, - shop = {'no'}, - tourism = {'yes', 'no'}, - bridge = {'no'}, - tunnel = {'no'}, - waterway = {'riverbank'}, - building = {'no'}, - boundary = {'place', 'land_area'}}, - extra_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', - 'name:etymology', 'name:signed', 'name:botanical', - 'wikidata', '*:wikidata', - '*:wikipedia', 'brand:wikipedia:*', - 'addr:street:name', 'addr:street:type'} - } - -flex.set_name_tags{main = {'name', 'name:*', - 'int_name', 'int_name:*', - 'nat_name', 'nat_name:*', - 'reg_name', 'reg_name:*', - 'loc_name', 'loc_name:*', - 'old_name', 'old_name:*', - 'alt_name', 'alt_name:*', 'alt_name_*', - 'official_name', 'official_name:*', - 'place_name', 'place_name:*', - 'short_name', 'short_name:*', 'brand'}, - extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref', - 'loc_ref', 'old_ref', - 'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}, - house = {'addr:housename'} - } - -flex.set_address_tags{main = {'addr:housenumber', - 'addr:conscriptionnumber', - 'addr:streetnumber'}, - extra = {'addr:*', 'is_in:*', 'tiger:county'}, - postcode = {'postal_code', 'postcode', 'addr:postcode', - 'tiger:zip_left', 'tiger:zip_right'}, - country = {'country_code', 'ISO3166-1', - 'addr:country_code', 'is_in:country_code', - 'addr:country', 'is_in:country'}, - interpolation = {'addr:interpolation'} - } - - -flex.set_unused_handling{delete_keys = {'tiger:*'}} - -return flex diff --git a/settings/import-full.lua b/settings/import-full.lua deleted file mode 100644 index 11bd1f3a..00000000 --- a/settings/import-full.lua +++ /dev/null @@ -1,126 +0,0 @@ -local flex = require('flex-base') - -flex.set_main_tags{ - building = 'fallback', - emergency = 'always', - healthcare = 'fallback', - historic = 'always', - military = 'always', - natural = 'named', - highway = {'always', - street_lamp = 'named', - traffic_signals = 'named', - service = 'named', - cycleway = 'named', - path = 'named', - footway = 'named', - steps = 'named', - bridleway = 'named', - track = 'named', - motorway_link = 'named', - trunk_link = 'named', - primary_link = 'named', - secondary_link = 'named', - tertiary_link = 'named'}, - railway = 'named', - man_made = {'none', - pier = 'always', - tower = 'always', - bridge = 'always', - works = 'named', - water_tower = 'always', - dyke = 'named', - adit = 'named', - lighthouse = 'always', - watermill = 'always', - tunnel = 'always'}, - aerialway = 'always', - boundary = {'named', - postal_code = 'always'}, - aeroway = 'always', - amenity = 'always', - club = 'always', - craft = 'always', - junction = 'fallback', - landuse = 'fallback', - leisure = {'always', - nature_reserve = 'fallback'}, - office = 'always', - mountain_pass = 'always', - shop = 'always', - tourism = 'always', - bridge = 'named_with_key', - tunnel = 'named_with_key', - waterway = 'named', - place = 'always' -} - -flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attribution', - 'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*', - 'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*', - 'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*', - 'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type', - 'ref:linz:*', 'is_in:postcode', - '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', - 'name:etymology', 'name:signed', 'name:botanical', - '*:wikidata', '*:wikipedia', 'brand:wikipedia:*', - 'addr:street:name', 'addr:street:type'}, - delete_tags = {emergency = {'yes', 'no', 'fire_hydrant'}, - historic = {'yes', 'no'}, - military = {'yes', 'no'}, - natural = {'yes', 'no', 'coastline'}, - highway = {'no', 'turning_circle', 'mini_roundabout', - 'noexit', 'crossing', 'give_way', 'stop'}, - railway = {'level_crossing', 'no', 'rail', 'switch', - 'abandoned', 'signal', 'buffer_stop', 'razed'}, - aerialway = {'pylon', 'no'}, - aeroway = {'no'}, - amenity = {'no', 'parking_space', 'parking_entrance', - 'waste_disposal', 'hunting_stand'}, - club = {'no'}, - craft = {'no'}, - leisure = {'no'}, - office = {'no'}, - mountain_pass = {'no'}, - shop = {'no'}, - tourism = {'yes', 'no'}, - bridge = {'no'}, - tunnel = {'no'}, - waterway = {'riverbank'}, - building = {'no'}, - boundary = {'place', 'land_area'}}, - extra_keys = {'wikidata', 'wikipedia', 'wikipedia:*'} - } - -flex.set_name_tags{main = {'name', 'name:*', - 'int_name', 'int_name:*', - 'nat_name', 'nat_name:*', - 'reg_name', 'reg_name:*', - 'loc_name', 'loc_name:*', - 'old_name', 'old_name:*', - 'alt_name', 'alt_name:*', 'alt_name_*', - 'official_name', 'official_name:*', - 'place_name', 'place_name:*', - 'short_name', 'short_name:*', 'brand'}, - extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref', - 'loc_ref', 'old_ref', - 'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}, - house = {'addr:housename'} - } - -flex.set_address_tags{main = {'addr:housenumber', - 'addr:conscriptionnumber', - 'addr:streetnumber'}, - extra = {'addr:*', 'is_in:*', 'tiger:county'}, - postcode = {'postal_code', 'postcode', 'addr:postcode', - 'tiger:zip_left', 'tiger:zip_right'}, - country = {'country_code', 'ISO3166-1', - 'addr:country_code', 'is_in:country_code', - 'addr:country', 'is_in:country'}, - interpolation = {'addr:interpolation'} - } - - -flex.set_unused_handling{extra_keys = {'place'}} - -return flex diff --git a/settings/import-street.lua b/settings/import-street.lua deleted file mode 100644 index 60b76dfb..00000000 --- a/settings/import-street.lua +++ /dev/null @@ -1,74 +0,0 @@ -local flex = require('flex-base') - -flex.set_main_tags{ - highway = {motorway = 'always', - trunk = 'always', - primary = 'always', - secondary = 'always', - tertiary = 'always', - unclassified = 'always', - residential = 'always', - road = 'always', - living_street = 'always', - pedestrian = 'always', - service = 'named', - cycleway = 'named', - path = 'named', - footway = 'named', - steps = 'named', - bridleway = 'named', - track = 'named', - motorway_link = 'named', - trunk_link = 'named', - primary_link = 'named', - secondary_link = 'named', - tertiary_link = 'named'}, - boundary = {administrative = 'named', - postal_code = 'always'}, - landuse = 'fallback', - place = 'always' -} - -flex.set_prefilters{delete_keys = {'building', 'source', - 'addr:housenumber', 'addr:street', - 'source', '*source', 'type', - 'is_in:postcode', '*:wikidata', '*:wikipedia', - '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', - 'name:etymology', 'name:signed', 'name:botanical', - 'addr:street:name', 'addr:street:type'}, - delete_tags = {landuse = {'cemetry', 'no'}, - boundary = {'place'}}, - extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'} - } - -flex.set_name_tags{main = {'name', 'name:*', - 'int_name', 'int_name:*', - 'nat_name', 'nat_name:*', - 'reg_name', 'reg_name:*', - 'loc_name', 'loc_name:*', - 'old_name', 'old_name:*', - 'alt_name', 'alt_name:*', 'alt_name_*', - 'official_name', 'official_name:*', - 'place_name', 'place_name:*', - 'short_name', 'short_name:*'}, - extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref', - 'loc_ref', 'old_ref', - 'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'} - } - -flex.set_address_tags{main = {'addr:housenumber', - 'addr:conscriptionnumber', - 'addr:streetnumber'}, - extra = {'addr:*', 'is_in:*', 'tiger:county'}, - postcode = {'postal_code', 'postcode', 'addr:postcode', - 'tiger:zip_left', 'tiger:zip_right'}, - country = {'country_code', 'ISO3166-1', - 'addr:country_code', 'is_in:country_code', - 'addr:country', 'is_in:country'}, - interpolation = {'addr:interpolation'}, - postcode_fallback = false - } - -flex.set_unused_handling{extra_keys = {'place'}} - -return flex diff --git a/settings/taginfo.lua b/settings/taginfo.lua deleted file mode 100644 index ef2ad2a6..00000000 --- a/settings/taginfo.lua +++ /dev/null @@ -1,74 +0,0 @@ --- Prints taginfo project description in the standard output --- - --- create fake "osm2pgsql" table for flex-base, originally created by the main C++ program -osm2pgsql = {} -function osm2pgsql.define_table(...) end - --- provide path to flex-style lua file -flex = require('import-extratags') -local json = require ('dkjson') - - ------------- helper functions --------------------- - -function get_key_description(key, description) - local desc = {} - desc.key = key - desc.description = description - set_keyorder(desc, {'key', 'description'}) - return desc -end - --- Sets the key order for the resulting JSON table -function set_keyorder(table, order) - setmetatable(table, { - __jsonorder = order - }) -end - - --- Prints the collected tags in the required format in JSON -function print_taginfo() - local tags = {} - - for _, k in ipairs(flex.TAGINFO_MAIN.keys) do - local desc = get_key_description(k, 'POI/feature in the search database') - if flex.TAGINFO_MAIN.delete_tags[k] ~= nil then - desc.description = string.format('%s (except for values: %s).', desc.description, - table.concat(flex.TAGINFO_MAIN.delete_tags[k], ', ')) - end - table.insert(tags, desc) - end - - for k, _ in pairs(flex.TAGINFO_NAME_KEYS) do - local desc = get_key_description(k, 'Searchable name of the place.') - table.insert(tags, desc) - end - for k, _ in pairs(flex.TAGINFO_ADDRESS_KEYS) do - local desc = get_key_description(k, 'Used to determine the address of a place.') - table.insert(tags, desc) - end - - local format = { - data_format = 1, - data_url = 'https://nominatim.openstreetmap.org/taginfo.json', - project = { - name = 'Nominatim', - description = 'OSM search engine.', - project_url = 'https://nominatim.openstreetmap.org', - doc_url = 'https://nominatim.org/release-docs/develop/', - contact_name = 'Sarah Hoffmann', - contact_email = 'lonvia@denofr.de' - } - } - format.tags = tags - - set_keyorder(format, {'data_format', 'data_url', 'project', 'tags'}) - set_keyorder(format.project, {'name', 'description', 'project_url', 'doc_url', - 'contact_name', 'contact_email'}) - - print(json.encode(format)) -end - -print_taginfo() diff --git a/src/nominatim_api/connection.py b/src/nominatim_api/connection.py index e104745e..04268dc3 100644 --- a/src/nominatim_api/connection.py +++ b/src/nominatim_api/connection.py @@ -18,6 +18,7 @@ from .typing import SaFromClause from .sql.sqlalchemy_schema import SearchTables from .sql.sqlalchemy_types import Geometry from .logging import log +from .config import Configuration T = TypeVar('T') @@ -31,9 +32,11 @@ class SearchConnection: def __init__(self, conn: AsyncConnection, tables: SearchTables, - properties: Dict[str, Any]) -> None: + properties: Dict[str, Any], + config: Configuration) -> None: self.connection = conn self.t = tables + self.config = config self._property_cache = properties self._classtables: Optional[Set[str]] = None self.query_timeout: Optional[int] = None diff --git a/src/nominatim_api/core.py b/src/nominatim_api/core.py index 3cf9e989..f8941bcc 100644 --- a/src/nominatim_api/core.py +++ b/src/nominatim_api/core.py @@ -26,7 +26,7 @@ from .connection import SearchConnection from .status import get_status, StatusResult from .lookup import get_places, get_detailed_place from .reverse import ReverseGeocoder -from .search import ForwardGeocoder, Phrase, PhraseType, make_query_analyzer +from . import search as nsearch from . import types as ntyp from .results import DetailedResult, ReverseResult, SearchResults @@ -184,7 +184,7 @@ class NominatimAPIAsync: assert self._tables is not None async with self._engine.begin() as conn: - yield SearchConnection(conn, self._tables, self._property_cache) + yield SearchConnection(conn, self._tables, self._property_cache, self.config) async def status(self) -> StatusResult: """ Return the status of the database. @@ -207,7 +207,7 @@ class NominatimAPIAsync: async with self.begin() as conn: conn.set_query_timeout(self.query_timeout) if details.keywords: - await make_query_analyzer(conn) + await nsearch.make_query_analyzer(conn) return await get_detailed_place(conn, place, details) async def lookup(self, places: Sequence[ntyp.PlaceRef], **params: Any) -> SearchResults: @@ -219,7 +219,7 @@ class NominatimAPIAsync: async with self.begin() as conn: conn.set_query_timeout(self.query_timeout) if details.keywords: - await make_query_analyzer(conn) + await nsearch.make_query_analyzer(conn) return await get_places(conn, places, details) async def reverse(self, coord: ntyp.AnyPoint, **params: Any) -> Optional[ReverseResult]: @@ -237,7 +237,7 @@ class NominatimAPIAsync: async with self.begin() as conn: conn.set_query_timeout(self.query_timeout) if details.keywords: - await make_query_analyzer(conn) + await nsearch.make_query_analyzer(conn) geocoder = ReverseGeocoder(conn, details, self.reverse_restrict_to_country_area) return await geocoder.lookup(coord) @@ -251,10 +251,10 @@ class NominatimAPIAsync: async with self.begin() as conn: conn.set_query_timeout(self.query_timeout) - geocoder = ForwardGeocoder(conn, ntyp.SearchDetails.from_kwargs(params), - self.config.get_int('REQUEST_TIMEOUT') - if self.config.REQUEST_TIMEOUT else None) - phrases = [Phrase(PhraseType.NONE, p.strip()) for p in query.split(',')] + geocoder = nsearch.ForwardGeocoder(conn, ntyp.SearchDetails.from_kwargs(params), + self.config.get_int('REQUEST_TIMEOUT') + if self.config.REQUEST_TIMEOUT else None) + phrases = [nsearch.Phrase(nsearch.PHRASE_ANY, p.strip()) for p in query.split(',')] return await geocoder.lookup(phrases) async def search_address(self, amenity: Optional[str] = None, @@ -271,22 +271,22 @@ class NominatimAPIAsync: conn.set_query_timeout(self.query_timeout) details = ntyp.SearchDetails.from_kwargs(params) - phrases: List[Phrase] = [] + phrases: List[nsearch.Phrase] = [] if amenity: - phrases.append(Phrase(PhraseType.AMENITY, amenity)) + phrases.append(nsearch.Phrase(nsearch.PHRASE_AMENITY, amenity)) if street: - phrases.append(Phrase(PhraseType.STREET, street)) + phrases.append(nsearch.Phrase(nsearch.PHRASE_STREET, street)) if city: - phrases.append(Phrase(PhraseType.CITY, city)) + phrases.append(nsearch.Phrase(nsearch.PHRASE_CITY, city)) if county: - phrases.append(Phrase(PhraseType.COUNTY, county)) + phrases.append(nsearch.Phrase(nsearch.PHRASE_COUNTY, county)) if state: - phrases.append(Phrase(PhraseType.STATE, state)) + phrases.append(nsearch.Phrase(nsearch.PHRASE_STATE, state)) if postalcode: - phrases.append(Phrase(PhraseType.POSTCODE, postalcode)) + phrases.append(nsearch.Phrase(nsearch.PHRASE_POSTCODE, postalcode)) if country: - phrases.append(Phrase(PhraseType.COUNTRY, country)) + phrases.append(nsearch.Phrase(nsearch.PHRASE_COUNTRY, country)) if not phrases: raise UsageError('Nothing to search for.') @@ -304,14 +304,14 @@ class NominatimAPIAsync: else: details.restrict_min_max_rank(4, 4) - if 'layers' not in params: + if details.layers is None: details.layers = ntyp.DataLayer.ADDRESS if amenity: details.layers |= ntyp.DataLayer.POI - geocoder = ForwardGeocoder(conn, details, - self.config.get_int('REQUEST_TIMEOUT') - if self.config.REQUEST_TIMEOUT else None) + geocoder = nsearch.ForwardGeocoder(conn, details, + self.config.get_int('REQUEST_TIMEOUT') + if self.config.REQUEST_TIMEOUT else None) return await geocoder.lookup(phrases) async def search_category(self, categories: List[Tuple[str, str]], @@ -328,15 +328,15 @@ class NominatimAPIAsync: async with self.begin() as conn: conn.set_query_timeout(self.query_timeout) if near_query: - phrases = [Phrase(PhraseType.NONE, p) for p in near_query.split(',')] + phrases = [nsearch.Phrase(nsearch.PHRASE_ANY, p) for p in near_query.split(',')] else: phrases = [] if details.keywords: - await make_query_analyzer(conn) + await nsearch.make_query_analyzer(conn) - geocoder = ForwardGeocoder(conn, details, - self.config.get_int('REQUEST_TIMEOUT') - if self.config.REQUEST_TIMEOUT else None) + geocoder = nsearch.ForwardGeocoder(conn, details, + self.config.get_int('REQUEST_TIMEOUT') + if self.config.REQUEST_TIMEOUT else None) return await geocoder.lookup_pois(categories, phrases) diff --git a/src/nominatim_api/localization.py b/src/nominatim_api/localization.py index bbf9225b..3414286e 100644 --- a/src/nominatim_api/localization.py +++ b/src/nominatim_api/localization.py @@ -8,6 +8,7 @@ Helper functions for localizing names of results. """ from typing import Mapping, List, Optional +from .config import Configuration import re @@ -20,14 +21,18 @@ class Locales: """ def __init__(self, langs: Optional[List[str]] = None): + self.config = Configuration(None) self.languages = langs or [] self.name_tags: List[str] = [] - # Build the list of supported tags. It is currently hard-coded. - self._add_lang_tags('name') - self._add_tags('name', 'brand') - self._add_lang_tags('official_name', 'short_name') - self._add_tags('official_name', 'short_name', 'ref') + parts = self.config.OUTPUT_NAMES.split(',') + + for part in parts: + part = part.strip() + if part.endswith(":XX"): + self._add_lang_tags(part[:-3]) + else: + self._add_tags(part) def __bool__(self) -> bool: return len(self.languages) > 0 diff --git a/src/nominatim_api/logging.py b/src/nominatim_api/logging.py index 1a6aef9b..64d43fdc 100644 --- a/src/nominatim_api/logging.py +++ b/src/nominatim_api/logging.py @@ -342,7 +342,8 @@ HTML_HEADER: str = """ Nominatim - Debug