From: Sarah Hoffmann Date: Thu, 5 Jan 2023 10:39:20 +0000 (+0100) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~80 X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/d485aef7019206bc02167d391c91cb1aa47fe6dc?hp=0cdbb97f6a4953971d60a46ddabc5df24be5f8ad Merge remote-tracking branch 'upstream/master' --- diff --git a/.github/actions/build-nominatim/action.yml b/.github/actions/build-nominatim/action.yml index acc9b8b6..48cbf1bc 100644 --- a/.github/actions/build-nominatim/action.yml +++ b/.github/actions/build-nominatim/action.yml @@ -27,9 +27,10 @@ runs: run: | sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION} if [ "x$UBUNTUVER" == "x18" ]; then - pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 datrie + pip3 install MarkupSafe==2.0.1 python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4 datrie asyncpg else - sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml + sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml python3-asyncpg + pip3 install sqlalchemy fi shell: bash env: diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index cdc7ea1e..35e6306a 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -42,17 +42,14 @@ jobs: - ubuntu: 18 postgresql: 9.6 postgis: 2.5 - pytest: pytest php: 7.2 - ubuntu: 20 postgresql: 13 postgis: 3 - pytest: py.test-3 php: 7.4 - ubuntu: 22 postgresql: 15 postgis: 3 - pytest: py.test-3 php: 8.1 runs-on: ubuntu-${{ matrix.ubuntu }}.04 @@ -74,7 +71,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: 3.6 + python-version: 3.7 if: matrix.ubuntu == 18 - uses: ./Nominatim/.github/actions/setup-postgresql @@ -86,50 +83,58 @@ jobs: with: ubuntu: ${{ matrix.ubuntu }} - - name: Install test prerequsites - run: sudo apt-get install -y -qq python3-pytest python3-behave + - name: Install test prerequsites (behave from apt) + run: sudo apt-get install -y -qq python3-behave if: matrix.ubuntu == 20 - - name: Install test prerequsites - run: pip3 install pylint pytest behave==1.2.6 + - name: Install test prerequsites (behave from pip) + run: pip3 install behave==1.2.6 if: ${{ (matrix.ubuntu == 18) || (matrix.ubuntu == 22) }} - - name: Install test prerequsites - run: sudo apt-get install -y -qq python3-pytest - if: matrix.ubuntu == 22 + - name: Install test prerequsites (from apt for Ununtu 2x) + run: sudo apt-get install -y -qq python3-pytest uvicorn + if: matrix.ubuntu >= 20 + + - name: Install test prerequsites (from pip for Ubuntu 18) + run: pip3 install pytest uvicorn + if: matrix.ubuntu == 18 + + - name: Install Python webservers + run: pip3 install falcon sanic sanic-testing starlette - name: Install latest pylint/mypy - run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions + run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions asgi_lifespan sqlalchemy2-stubs - name: PHP linting run: phpcs --report-width=120 . working-directory: Nominatim - name: Python linting - run: pylint nominatim + run: python3 -m pylint nominatim working-directory: Nominatim - - name: Python static typechecking - run: mypy --strict nominatim - working-directory: Nominatim - - - name: PHP unit tests run: phpunit ./ working-directory: Nominatim/test/php if: ${{ (matrix.ubuntu == 20) || (matrix.ubuntu == 22) }} - name: Python unit tests - run: $PYTEST test/python + run: python3 -m pytest test/python working-directory: Nominatim - env: - PYTEST: ${{ matrix.pytest }} - name: BDD tests run: | - behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3 + python3 -m behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3 working-directory: Nominatim/test/bdd + - name: Install newer Python packages (for typechecking info) + run: pip3 install -U osmium uvicorn + if: matrix.ubuntu >= 20 + + - name: Python static typechecking + run: python3 -m mypy --strict nominatim + working-directory: Nominatim + if: matrix.ubuntu >= 20 legacy-test: needs: create-archive @@ -166,7 +171,7 @@ jobs: - name: BDD tests (legacy tokenizer) run: | - behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DTOKENIZER=legacy --format=progress3 + python3 -m behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DTOKENIZER=legacy --format=progress3 working-directory: Nominatim/test/bdd @@ -176,20 +181,13 @@ jobs: strategy: matrix: - name: [Ubuntu-18, Ubuntu-20, Ubuntu-22] + name: [Ubuntu-20, Ubuntu-22] include: - - name: Ubuntu-18 - flavour: ubuntu - image: "ubuntu:18.04" - ubuntu: 18 - install_mode: install-nginx - name: Ubuntu-20 - flavour: ubuntu image: "ubuntu:20.04" ubuntu: 20 install_mode: install-apache - name: Ubuntu-22 - flavour: ubuntu image: "ubuntu:22.04" ubuntu: 22 install_mode: install-apache @@ -212,14 +210,6 @@ jobs: apt-get install -y git sudo wget ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone shell: bash - if: matrix.flavour == 'ubuntu' - - - name: Prepare container (CentOS) - run: | - dnf update -y - dnf install -y sudo glibc-langpack-en - shell: bash - if: matrix.flavour == 'centos' - name: Setup import user run: | @@ -253,14 +243,6 @@ jobs: mkdir data-env-reverse working-directory: /home/nominatim - - name: Prepare import environment (CentOS) - run: | - sudo ln -s /usr/local/bin/nominatim /usr/bin/nominatim - echo NOMINATIM_DATABASE_WEBUSER="apache" > nominatim-project/.env - cp nominatim-project/.env data-env-reverse/.env - working-directory: /home/nominatim - if: matrix.flavour == 'centos' - - name: Print version run: nominatim --version working-directory: /home/nominatim/nominatim-project @@ -268,7 +250,7 @@ jobs: - name: Collect host OS information run: nominatim admin --collect-os-info working-directory: /home/nominatim/nominatim-project - + - name: Import run: nominatim import --osm-file ../test.pbf working-directory: /home/nominatim/nominatim-project @@ -288,7 +270,6 @@ jobs: - name: Prepare update (Ubuntu) run: apt-get install -y python3-pip shell: bash - if: matrix.flavour == 'ubuntu' - name: Run update run: | diff --git a/.mypy.ini b/.mypy.ini index 81a5c2e7..ef2057d4 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -1,9 +1,10 @@ [mypy] +plugins = sqlalchemy.ext.mypy.plugin [mypy-icu.*] ignore_missing_imports = True -[mypy-osmium.*] +[mypy-asyncpg.*] ignore_missing_imports = True [mypy-datrie.*] @@ -11,3 +12,6 @@ ignore_missing_imports = True [mypy-dotenv.*] ignore_missing_imports = True + +[mypy-falcon.*] +ignore_missing_imports = True diff --git a/.pylintrc b/.pylintrc index e62371c6..881c1e76 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,6 +1,6 @@ [MASTER] -extension-pkg-whitelist=osmium +extension-pkg-whitelist=osmium,falcon ignored-modules=icu,datrie [MESSAGES CONTROL] diff --git a/CMakeLists.txt b/CMakeLists.txt index c8a57521..3f68012f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,7 +73,7 @@ endif() #----------------------------------------------------------------------------- if (BUILD_IMPORTER) - find_package(PythonInterp 3.6 REQUIRED) + find_package(PythonInterp 3.7 REQUIRED) endif() #----------------------------------------------------------------------------- diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 4fa860ad..edfc8829 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -23,7 +23,6 @@ foreach (src ${DOC_SOURCES}) endforeach() ADD_CUSTOM_TARGET(doc - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-18.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-18.md COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-20.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-20.md COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-22.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-22.md COMMAND PYTHONPATH=${PROJECT_SOURCE_DIR} mkdocs build -d ${CMAKE_CURRENT_BINARY_DIR}/../site-html -f ${CMAKE_CURRENT_BINARY_DIR}/../mkdocs.yml diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index 73fb3dae..90b2cb39 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -6,7 +6,6 @@ the following operating systems: * [Ubuntu 22.04](../appendix/Install-on-Ubuntu-22.md) * [Ubuntu 20.04](../appendix/Install-on-Ubuntu-20.md) - * [Ubuntu 18.04](../appendix/Install-on-Ubuntu-18.md) These OS-specific instructions can also be found in executable form in the `vagrant/` directory. @@ -44,11 +43,13 @@ For running Nominatim: * [PostgreSQL](https://www.postgresql.org) (9.6+ will work, 11+ strongly recommended) * [PostGIS](https://postgis.net) (2.2+ will work, 3.0+ strongly recommended) - * [Python 3](https://www.python.org/) (3.6+) + * [Python 3](https://www.python.org/) (3.7+) * [Psycopg2](https://www.psycopg.org) (2.7+) * [Python Dotenv](https://github.com/theskumar/python-dotenv) * [psutil](https://github.com/giampaolo/psutil) * [Jinja2](https://palletsprojects.com/p/jinja/) + * [SQLAlchemy](https://www.sqlalchemy.org/) (1.4+ with greenlet support) + * [asyncpg](https://magicstack.github.io/asyncpg) (0.8+) * [PyICU](https://pypi.org/project/PyICU/) * [PyYaml](https://pyyaml.org/) (5.1+) * [datrie](https://github.com/pytries/datrie) @@ -61,6 +62,14 @@ For running continuous updates: * [pyosmium](https://osmcode.org/pyosmium/) +For running the experimental Python frontend: + + * one of the following web frameworks: + * [falcon](https://falconframework.org/) (3.0+) + * [sanic](https://sanic.dev) + * [starlette](https://www.starlette.io/) + * [uvicorn](https://www.uvicorn.org/) (only with falcon and starlette framworks) + For dependencies for running tests and building documentation, see the [Development section](../develop/Development-Environment.md). diff --git a/docs/develop/Development-Environment.md b/docs/develop/Development-Environment.md index c6515d2c..0e1bbf61 100644 --- a/docs/develop/Development-Environment.md +++ b/docs/develop/Development-Environment.md @@ -37,6 +37,13 @@ It has the following additional requirements: * [Python Typing Extensions](https://github.com/python/typing_extensions) (for Python < 3.9) * [pytest](https://pytest.org) +For testing the Python search frontend, you need to install extra dependencies +depending on your choice of webserver framework: + +* [sanic-testing](https://sanic.dev/en/plugins/sanic-testing/getting-started.html) (sanic only) +* [httpx](https://www.python-httpx.org/) (starlette only) +* [asgi-lifespan](https://github.com/florimondmanca/asgi-lifespan) (starlette only) + The documentation is built with mkdocs: * [mkdocs](https://www.mkdocs.org/) >= 1.1.2 @@ -56,7 +63,8 @@ sudo apt install php-cgi phpunit php-codesniffer \ python3-pip python3-setuptools python3-dev pip3 install --user behave mkdocs mkdocstrings pytest pylint \ - mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil + mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil \ + sanic-testing httpx asgi-lifespan ``` The `mkdocs` executable will be located in `.local/bin`. You may have to add diff --git a/docs/develop/Testing.md b/docs/develop/Testing.md index 20c9d165..be13d949 100644 --- a/docs/develop/Testing.md +++ b/docs/develop/Testing.md @@ -84,6 +84,8 @@ The tests can be configured with a set of environment variables (`behave -D key= * `TEST_DB` - name of test database (db tests) * `API_TEST_DB` - name of the database containing the API test data (api tests) * `API_TEST_FILE` - OSM file to be imported into the API test database (api tests) + * `API_ENGINE` - webframe to use for running search queries, same values as + `nominatim serve --engine` parameter * `DB_HOST` - (optional) hostname of database host * `DB_PORT` - (optional) port of database on host * `DB_USER` - (optional) username of database login @@ -120,7 +122,7 @@ and compromises the following data: API tests should only be testing the functionality of the website PHP code. Most tests should be formulated as BDD DB creation tests (see below) instead. -#### Code Coverage +#### Code Coverage (PHP engine only) The API tests also support code coverage tests. You need to install [PHP_CodeCoverage](https://github.com/sebastianbergmann/php-code-coverage). @@ -153,7 +155,3 @@ needs superuser rights for postgres. These tests check that data is imported correctly into the place table. They use the same template database as the DB Creation tests, so the same remarks apply. - -Note that most testing of the gazetteer output of osm2pgsql is done in the tests -of osm2pgsql itself. The BDD tests are just there to ensure compatibility of -the osm2pgsql and Nominatim code. diff --git a/lib-php/DB.php b/lib-php/DB.php index 6b6326b0..553d9452 100644 --- a/lib-php/DB.php +++ b/lib-php/DB.php @@ -38,23 +38,25 @@ class DB // https://secure.php.net/manual/en/ref.pdo-pgsql.connection.php try { - $conn = new \PDO($this->sDSN, null, null, $aConnOptions); + $this->connection = new \PDO($this->sDSN, null, null, $aConnOptions); } catch (\PDOException $e) { $sMsg = 'Failed to establish database connection:' . $e->getMessage(); throw new \Nominatim\DatabaseError($sMsg, 500, null, $e->getMessage()); } - $conn->exec("SET DateStyle TO 'sql,european'"); - $conn->exec("SET client_encoding TO 'utf-8'"); + $this->connection->exec("SET DateStyle TO 'sql,european'"); + $this->connection->exec("SET client_encoding TO 'utf-8'"); // Disable JIT and parallel workers. They interfere badly with search SQL. - $conn->exec("UPDATE pg_settings SET setting = -1 WHERE name = 'jit_above_cost'"); - $conn->exec("UPDATE pg_settings SET setting = 0 WHERE name = 'max_parallel_workers_per_gather'"); + $this->connection->exec('SET max_parallel_workers_per_gather TO 0'); + if ($this->getPostgresVersion() >= 11) { + $this->connection->exec('SET jit_above_cost TO -1'); + } + $iMaxExecution = ini_get('max_execution_time'); if ($iMaxExecution > 0) { - $conn->setAttribute(\PDO::ATTR_TIMEOUT, $iMaxExecution); // seconds + $this->connection->setAttribute(\PDO::ATTR_TIMEOUT, $iMaxExecution); // seconds } - $this->connection = $conn; return true; } diff --git a/lib-sql/functions/place_triggers.sql b/lib-sql/functions/place_triggers.sql index 489ecb35..6a52021b 100644 --- a/lib-sql/functions/place_triggers.sql +++ b/lib-sql/functions/place_triggers.sql @@ -384,7 +384,19 @@ BEGIN -- Mark for delete in the placex table UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted - WHERE placex.osm_type = place_to_be_deleted.osm_type + WHERE placex.osm_type = 'N' and place_to_be_deleted.osm_type = 'N' + and placex.osm_id = place_to_be_deleted.osm_id + and placex.class = place_to_be_deleted.class + and placex.type = place_to_be_deleted.type + and not deferred; + UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted + WHERE placex.osm_type = 'W' and place_to_be_deleted.osm_type = 'W' + and placex.osm_id = place_to_be_deleted.osm_id + and placex.class = place_to_be_deleted.class + and placex.type = place_to_be_deleted.type + and not deferred; + UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted + WHERE placex.osm_type = 'R' and place_to_be_deleted.osm_type = 'R' and placex.osm_id = place_to_be_deleted.osm_id and placex.class = place_to_be_deleted.class and placex.type = place_to_be_deleted.type diff --git a/nominatim/api.py b/nominatim/api.py new file mode 100644 index 00000000..10cca533 --- /dev/null +++ b/nominatim/api.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Implementation of classes for API access via libraries. +""" +from typing import Mapping, Optional, cast, Any +import asyncio +from pathlib import Path + +from sqlalchemy import text, event +from sqlalchemy.engine.url import URL +from sqlalchemy.ext.asyncio import create_async_engine +import asyncpg + +from nominatim.config import Configuration +from nominatim.apicmd.status import get_status, StatusResult + +class NominatimAPIAsync: + """ API loader asynchornous version. + """ + def __init__(self, project_dir: Path, + environ: Optional[Mapping[str, str]] = None) -> None: + self.config = Configuration(project_dir, environ) + + dsn = self.config.get_database_params() + + dburl = URL.create( + 'postgresql+asyncpg', + database=dsn.get('dbname'), + username=dsn.get('user'), password=dsn.get('password'), + host=dsn.get('host'), port=int(dsn['port']) if 'port' in dsn else None, + query={k: v for k, v in dsn.items() + if k not in ('user', 'password', 'dbname', 'host', 'port')}) + self.engine = create_async_engine( + dburl, future=True, + connect_args={'server_settings': { + 'DateStyle': 'sql,european', + 'max_parallel_workers_per_gather': '0' + }}) + asyncio.get_event_loop().run_until_complete(self._query_server_version()) + asyncio.get_event_loop().run_until_complete(self.close()) + + if self.server_version >= 110000: + @event.listens_for(self.engine.sync_engine, "connect") # type: ignore[misc] + def _on_connect(dbapi_con: Any, _: Any) -> None: + cursor = dbapi_con.cursor() + cursor.execute("SET jit_above_cost TO '-1'") + + + async def _query_server_version(self) -> None: + try: + async with self.engine.begin() as conn: + result = await conn.scalar(text('SHOW server_version_num')) + self.server_version = int(cast(str, result)) + except asyncpg.PostgresError: + self.server_version = 0 + + async def close(self) -> None: + """ Close all active connections to the database. The NominatimAPIAsync + object remains usable after closing. If a new API functions is + called, new connections are created. + """ + await self.engine.dispose() + + + async def status(self) -> StatusResult: + """ Return the status of the database. + """ + return await get_status(self.engine) + + +class NominatimAPI: + """ API loader, synchronous version. + """ + + def __init__(self, project_dir: Path, + environ: Optional[Mapping[str, str]] = None) -> None: + self.async_api = NominatimAPIAsync(project_dir, environ) + + + def close(self) -> None: + """ Close all active connections to the database. The NominatimAPIAsync + object remains usable after closing. If a new API functions is + called, new connections are created. + """ + asyncio.get_event_loop().run_until_complete(self.async_api.close()) + + + def status(self) -> StatusResult: + """ Return the status of the database. + """ + return asyncio.get_event_loop().run_until_complete(self.async_api.status()) diff --git a/nominatim/apicmd/__init__.py b/nominatim/apicmd/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nominatim/apicmd/status.py b/nominatim/apicmd/status.py new file mode 100644 index 00000000..85071db9 --- /dev/null +++ b/nominatim/apicmd/status.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Classes and function releated to status call. +""" +from typing import Optional, cast +import datetime as dt + +import sqlalchemy as sqla +from sqlalchemy.ext.asyncio.engine import AsyncEngine, AsyncConnection +import asyncpg + +from nominatim import version + +class StatusResult: + """ Result of a call to the status API. + """ + + def __init__(self, status: int, msg: str): + self.status = status + self.message = msg + self.software_version = version.NOMINATIM_VERSION + self.data_updated: Optional[dt.datetime] = None + self.database_version: Optional[version.NominatimVersion] = None + + +async def _get_database_date(conn: AsyncConnection) -> Optional[dt.datetime]: + """ Query the database date. + """ + sql = sqla.text('SELECT lastimportdate FROM import_status LIMIT 1') + result = await conn.execute(sql) + + for row in result: + return cast(dt.datetime, row[0]) + + return None + + +async def _get_database_version(conn: AsyncConnection) -> Optional[version.NominatimVersion]: + sql = sqla.text("""SELECT value FROM nominatim_properties + WHERE property = 'database_version'""") + result = await conn.execute(sql) + + for row in result: + return version.parse_version(cast(str, row[0])) + + return None + + +async def get_status(engine: AsyncEngine) -> StatusResult: + """ Execute a status API call. + """ + status = StatusResult(0, 'OK') + try: + async with engine.begin() as conn: + status.data_updated = await _get_database_date(conn) + status.database_version = await _get_database_version(conn) + except asyncpg.PostgresError: + return StatusResult(700, 'Database connection failed') + + return status diff --git a/nominatim/cli.py b/nominatim/cli.py index 56ed6a07..cedbdb4a 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -9,6 +9,7 @@ Command-line interface to the Nominatim functions for import, update, database administration and querying. """ from typing import Optional, Any, List, Union +import importlib import logging import os import sys @@ -60,7 +61,7 @@ class CommandlineParser: def nominatim_version_text(self) -> str: """ Program name and version number as string """ - text = f'Nominatim version {version.version_str()}' + text = f'Nominatim version {version.NOMINATIM_VERSION!s}' if version.GIT_COMMIT_HASH is not None: text += f' ({version.GIT_COMMIT_HASH})' return text @@ -197,10 +198,15 @@ class AdminServe: """\ Start a simple web server for serving the API. - This command starts the built-in PHP webserver to serve the website + This command starts a built-in webserver to serve the website from the current project directory. This webserver is only suitable for testing and development. Do not use it in production setups! + There are different webservers available. The default 'php' engine + runs the classic PHP frontend. The other engines are Python servers + which run the new Python frontend code. This is highly experimental + at the moment and may not include the full API. + By the default, the webserver can be accessed at: http://127.0.0.1:8088 """ @@ -208,10 +214,40 @@ class AdminServe: group = parser.add_argument_group('Server arguments') group.add_argument('--server', default='127.0.0.1:8088', help='The address the server will listen to.') + group.add_argument('--engine', default='php', + choices=('php', 'sanic', 'falcon', 'starlette'), + help='Webserver framework to run. (default: php)') def run(self, args: NominatimArgs) -> int: - run_php_server(args.server, args.project_dir / 'website') + if args.engine == 'php': + run_php_server(args.server, args.project_dir / 'website') + else: + server_info = args.server.split(':', 1) + host = server_info[0] + if len(server_info) > 1: + if not server_info[1].isdigit(): + raise UsageError('Invalid format for --server parameter. Use :') + port = int(server_info[1]) + else: + port = 8088 + + if args.engine == 'sanic': + server_module = importlib.import_module('nominatim.server.sanic.server') + + app = server_module.get_application(args.project_dir) + app.run(host=host, port=port, debug=True) + else: + import uvicorn # pylint: disable=import-outside-toplevel + + if args.engine == 'falcon': + server_module = importlib.import_module('nominatim.server.falcon.server') + elif args.engine == 'starlette': + server_module = importlib.import_module('nominatim.server.starlette.server') + + app = server_module.get_application(args.project_dir) + uvicorn.run(app, host=host, port=port) + return 0 diff --git a/nominatim/clicmd/api.py b/nominatim/clicmd/api.py index b899afad..9a4828b8 100644 --- a/nominatim/clicmd/api.py +++ b/nominatim/clicmd/api.py @@ -14,6 +14,9 @@ import logging from nominatim.tools.exec_utils import run_api_script from nominatim.errors import UsageError from nominatim.clicmd.args import NominatimArgs +from nominatim.api import NominatimAPI +from nominatim.apicmd.status import StatusResult +import nominatim.result_formatter.v1 as formatting # Do not repeat documentation of subcommand classes. # pylint: disable=C0111 @@ -264,7 +267,7 @@ class APIDetails: class APIStatus: - """\ + """ Execute API status query. This command works exactly the same as if calling the /status endpoint on @@ -274,10 +277,13 @@ class APIStatus: """ def add_args(self, parser: argparse.ArgumentParser) -> None: + formats = formatting.create(StatusResult).list_formats() group = parser.add_argument_group('API parameters') - group.add_argument('--format', default='text', choices=['text', 'json'], + group.add_argument('--format', default=formats[0], choices=formats, help='Format of result') def run(self, args: NominatimArgs) -> int: - return _run_api('status', args, dict(format=args.format)) + status = NominatimAPI(args.project_dir).status() + print(formatting.create(StatusResult).format(status, args.format)) + return 0 diff --git a/nominatim/clicmd/args.py b/nominatim/clicmd/args.py index 15de72a5..e47287b3 100644 --- a/nominatim/clicmd/args.py +++ b/nominatim/clicmd/args.py @@ -127,6 +127,7 @@ class NominatimArgs: # Arguments to 'serve' server: str + engine: str # Arguments to 'special-phrases import_from_wiki: bool diff --git a/nominatim/clicmd/setup.py b/nominatim/clicmd/setup.py index 68884fe1..8464e151 100644 --- a/nominatim/clicmd/setup.py +++ b/nominatim/clicmd/setup.py @@ -18,7 +18,7 @@ from nominatim.config import Configuration from nominatim.db.connection import connect from nominatim.db import status, properties from nominatim.tokenizer.base import AbstractTokenizer -from nominatim.version import version_str +from nominatim.version import NOMINATIM_VERSION from nominatim.clicmd.args import NominatimArgs from nominatim.errors import UsageError @@ -205,4 +205,4 @@ class SetupAll: except Exception as exc: # pylint: disable=broad-except LOG.error('Cannot determine date of database: %s', exc) - properties.set_property(conn, 'database_version', version_str()) + properties.set_property(conn, 'database_version', str(NOMINATIM_VERSION)) diff --git a/nominatim/config.py b/nominatim/config.py index e0f19b04..3a4c3a6b 100644 --- a/nominatim/config.py +++ b/nominatim/config.py @@ -17,6 +17,7 @@ import json import yaml from dotenv import dotenv_values +from psycopg2.extensions import parse_dsn from nominatim.typing import StrPath from nominatim.errors import UsageError @@ -51,7 +52,7 @@ class Configuration: Nominatim uses dotenv to configure the software. Configuration options are resolved in the following order: - * from the OS environment (or the dirctionary given in `environ` + * from the OS environment (or the dictionary given in `environ`) * from the .env file in the project directory of the installation * from the default installation in the configuration directory @@ -164,6 +165,18 @@ class Configuration: return dsn + def get_database_params(self) -> Mapping[str, str]: + """ Get the configured parameters for the database connection + as a mapping. + """ + dsn = self.DATABASE_DSN + + if dsn.startswith('pgsql:'): + return dict((p.split('=', 1) for p in dsn[6:].split(';'))) + + return parse_dsn(dsn) + + def get_import_style_file(self) -> Path: """ Return the import style file as a path object. Translates the name of the standard styles automatically into a file in the diff --git a/nominatim/result_formatter/__init__.py b/nominatim/result_formatter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nominatim/result_formatter/base.py b/nominatim/result_formatter/base.py new file mode 100644 index 00000000..d77f4db8 --- /dev/null +++ b/nominatim/result_formatter/base.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Helper classes and function for writing result formatting modules. +""" +from typing import Type, TypeVar, Dict, Mapping, List, Callable, Generic, Any +from collections import defaultdict + +T = TypeVar('T') # pylint: disable=invalid-name +FormatFunc = Callable[[T], str] + +class ResultFormatter(Generic[T]): + """ This class dispatches format calls to the appropriate formatting + function previously defined with the `format_func` decorator. + """ + + def __init__(self, funcs: Mapping[str, FormatFunc[T]]) -> None: + self.functions = funcs + + + def list_formats(self) -> List[str]: + """ Return a list of formats supported by this formatter. + """ + return list(self.functions.keys()) + + + def supports_format(self, fmt: str) -> bool: + """ Check if the given format is supported by this formatter. + """ + return fmt in self.functions + + + def format(self, result: T, fmt: str) -> str: + """ Convert the given result into a string using the given format. + + The format is expected to be in the list returned by + `list_formats()`. + """ + return self.functions[fmt](result) + + +class FormatDispatcher: + """ A factory class for result formatters. + """ + + def __init__(self) -> None: + self.format_functions: Dict[Type[Any], Dict[str, FormatFunc[Any]]] = defaultdict(dict) + + + def format_func(self, result_class: Type[T], + fmt: str) -> Callable[[FormatFunc[T]], FormatFunc[T]]: + """ Decorator for a function that formats a given type of result into the + selected format. + """ + def decorator(func: FormatFunc[T]) -> FormatFunc[T]: + self.format_functions[result_class][fmt] = func + return func + + return decorator + + + def __call__(self, result_class: Type[T]) -> ResultFormatter[T]: + """ Create an instance of a format class for the given result type. + """ + return ResultFormatter(self.format_functions[result_class]) diff --git a/nominatim/result_formatter/v1.py b/nominatim/result_formatter/v1.py new file mode 100644 index 00000000..1d437af7 --- /dev/null +++ b/nominatim/result_formatter/v1.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Output formatters for API version v1. +""" +from typing import Dict, Any +from collections import OrderedDict +import json + +from nominatim.result_formatter.base import FormatDispatcher +from nominatim.apicmd.status import StatusResult + +create = FormatDispatcher() + +@create.format_func(StatusResult, 'text') +def _format_status_text(result: StatusResult) -> str: + if result.status: + return f"ERROR: {result.message}" + + return 'OK' + + +@create.format_func(StatusResult, 'json') +def _format_status_json(result: StatusResult) -> str: + out: Dict[str, Any] = OrderedDict() + out['status'] = result.status + out['message'] = result.message + if result.data_updated is not None: + out['data_updated'] = result.data_updated.isoformat() + out['software_version'] = str(result.software_version) + if result.database_version is not None: + out['database_version'] = str(result.database_version) + + return json.dumps(out) diff --git a/nominatim/server/__init__.py b/nominatim/server/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nominatim/server/falcon/__init__.py b/nominatim/server/falcon/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nominatim/server/falcon/server.py b/nominatim/server/falcon/server.py new file mode 100644 index 00000000..81e6ed39 --- /dev/null +++ b/nominatim/server/falcon/server.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Server implementation using the falcon webserver framework. +""" +from typing import Type, Any, Optional, Mapping +from pathlib import Path + +import falcon +import falcon.asgi + +from nominatim.api import NominatimAPIAsync +from nominatim.apicmd.status import StatusResult +import nominatim.result_formatter.v1 as formatting + +CONTENT_TYPE = { + 'text': falcon.MEDIA_TEXT, + 'xml': falcon.MEDIA_XML +} + +class NominatimV1: + """ Implementation of V1 version of the Nominatim API. + """ + + def __init__(self, project_dir: Path, environ: Optional[Mapping[str, str]]) -> None: + self.api = NominatimAPIAsync(project_dir, environ) + self.formatters = {} + + for rtype in (StatusResult, ): + self.formatters[rtype] = formatting.create(rtype) + + + def parse_format(self, req: falcon.asgi.Request, rtype: Type[Any], default: str) -> None: + """ Get and check the 'format' parameter and prepare the formatter. + `rtype` describes the expected return type and `default` the + format value to assume when no parameter is present. + """ + req.context.format = req.get_param('format', default=default) + req.context.formatter = self.formatters[rtype] + + if not req.context.formatter.supports_format(req.context.format): + raise falcon.HTTPBadRequest( + description="Parameter 'format' must be one of: " + + ', '.join(req.context.formatter.list_formats())) + + + def format_response(self, req: falcon.asgi.Request, resp: falcon.asgi.Response, + result: Any) -> None: + """ Render response into a string according to the formatter + set in `parse_format()`. + """ + resp.text = req.context.formatter.format(result, req.context.format) + resp.content_type = CONTENT_TYPE.get(req.context.format, falcon.MEDIA_JSON) + + + async def on_get_status(self, req: falcon.asgi.Request, resp: falcon.asgi.Response) -> None: + """ Implementation of status endpoint. + """ + self.parse_format(req, StatusResult, 'text') + + result = await self.api.status() + + self.format_response(req, resp, result) + if result.status and req.context.format == 'text': + resp.status = 500 + + +def get_application(project_dir: Path, + environ: Optional[Mapping[str, str]] = None) -> falcon.asgi.App: + """ Create a Nominatim falcon ASGI application. + """ + app = falcon.asgi.App() + + api = NominatimV1(project_dir, environ) + + app.add_route('/status', api, suffix='status') + + return app diff --git a/nominatim/server/sanic/__init__.py b/nominatim/server/sanic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nominatim/server/sanic/server.py b/nominatim/server/sanic/server.py new file mode 100644 index 00000000..74841f3f --- /dev/null +++ b/nominatim/server/sanic/server.py @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Server implementation using the sanic webserver framework. +""" +from typing import Any, Optional, Mapping +from pathlib import Path + +import sanic + +from nominatim.api import NominatimAPIAsync +from nominatim.apicmd.status import StatusResult +import nominatim.result_formatter.v1 as formatting + +api = sanic.Blueprint('NominatimAPI') + +CONTENT_TYPE = { + 'text': 'text/plain; charset=utf-8', + 'xml': 'text/xml; charset=utf-8' +} + +def usage_error(msg: str) -> sanic.HTTPResponse: + """ Format the response for an error with the query parameters. + """ + return sanic.response.text(msg, status=400) + + +def api_response(request: sanic.Request, result: Any) -> sanic.HTTPResponse: + """ Render a response from the query results using the configured + formatter. + """ + body = request.ctx.formatter.format(result, request.ctx.format) + return sanic.response.text(body, + content_type=CONTENT_TYPE.get(request.ctx.format, + 'application/json')) + + +@api.on_request # type: ignore[misc] +async def extract_format(request: sanic.Request) -> Optional[sanic.HTTPResponse]: + """ Get and check the 'format' parameter and prepare the formatter. + `ctx.result_type` describes the expected return type and + `ctx.default_format` the format value to assume when no parameter + is present. + """ + assert request.route is not None + request.ctx.formatter = request.app.ctx.formatters[request.route.ctx.result_type] + + request.ctx.format = request.args.get('format', request.route.ctx.default_format) + if not request.ctx.formatter.supports_format(request.ctx.format): + return usage_error("Parameter 'format' must be one of: " + + ', '.join(request.ctx.formatter.list_formats())) + + return None + + +@api.get('/status', ctx_result_type=StatusResult, ctx_default_format='text') +async def status(request: sanic.Request) -> sanic.HTTPResponse: + """ Implementation of status endpoint. + """ + result = await request.app.ctx.api.status() + response = api_response(request, result) + + if request.ctx.format == 'text' and result.status: + response.status = 500 + + return response + + +def get_application(project_dir: Path, + environ: Optional[Mapping[str, str]] = None) -> sanic.Sanic: + """ Create a Nominatim sanic ASGI application. + """ + app = sanic.Sanic("NominatimInstance") + + app.ctx.api = NominatimAPIAsync(project_dir, environ) + app.ctx.formatters = {} + for rtype in (StatusResult, ): + app.ctx.formatters[rtype] = formatting.create(rtype) + + app.blueprint(api) + + return app diff --git a/nominatim/server/starlette/__init__.py b/nominatim/server/starlette/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nominatim/server/starlette/server.py b/nominatim/server/starlette/server.py new file mode 100644 index 00000000..41ad899c --- /dev/null +++ b/nominatim/server/starlette/server.py @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Server implementation using the starlette webserver framework. +""" +from typing import Any, Type, Optional, Mapping +from pathlib import Path + +from starlette.applications import Starlette +from starlette.routing import Route +from starlette.exceptions import HTTPException +from starlette.responses import Response +from starlette.requests import Request + +from nominatim.api import NominatimAPIAsync +from nominatim.apicmd.status import StatusResult +import nominatim.result_formatter.v1 as formatting + +CONTENT_TYPE = { + 'text': 'text/plain; charset=utf-8', + 'xml': 'text/xml; charset=utf-8' +} + +FORMATTERS = { + StatusResult: formatting.create(StatusResult) +} + + +def parse_format(request: Request, rtype: Type[Any], default: str) -> None: + """ Get and check the 'format' parameter and prepare the formatter. + `rtype` describes the expected return type and `default` the + format value to assume when no parameter is present. + """ + fmt = request.query_params.get('format', default=default) + fmtter = FORMATTERS[rtype] + + if not fmtter.supports_format(fmt): + raise HTTPException(400, detail="Parameter 'format' must be one of: " + + ', '.join(fmtter.list_formats())) + + request.state.format = fmt + request.state.formatter = fmtter + + +def format_response(request: Request, result: Any) -> Response: + """ Render response into a string according to the formatter + set in `parse_format()`. + """ + fmt = request.state.format + return Response(request.state.formatter.format(result, fmt), + media_type=CONTENT_TYPE.get(fmt, 'application/json')) + + +async def on_status(request: Request) -> Response: + """ Implementation of status endpoint. + """ + parse_format(request, StatusResult, 'text') + result = await request.app.state.API.status() + response = format_response(request, result) + + if request.state.format == 'text' and result.status: + response.status_code = 500 + + return response + + +V1_ROUTES = [ + Route('/status', endpoint=on_status) +] + +def get_application(project_dir: Path, + environ: Optional[Mapping[str, str]] = None) -> Starlette: + """ Create a Nominatim falcon ASGI application. + """ + app = Starlette(debug=True, routes=V1_ROUTES) + + app.state.API = NominatimAPIAsync(project_dir, environ) + + return app diff --git a/nominatim/tools/collect_os_info.py b/nominatim/tools/collect_os_info.py index 9d76f229..29e1cd53 100644 --- a/nominatim/tools/collect_os_info.py +++ b/nominatim/tools/collect_os_info.py @@ -20,7 +20,7 @@ from psycopg2.extensions import make_dsn, parse_dsn from nominatim.config import Configuration from nominatim.db.connection import connect from nominatim.typing import DictCursorResults -from nominatim.version import version_str +from nominatim.version import NOMINATIM_VERSION def convert_version(ver_tup: Tuple[int, int]) -> str: @@ -135,8 +135,8 @@ def report_system_information(config: Configuration) -> None: **Software Environment:** - Python version: {sys.version} - - Nominatim version: {version_str()} - - PostgreSQL version: {postgresql_ver} + - Nominatim version: {NOMINATIM_VERSION!s} + - PostgreSQL version: {postgresql_ver} - PostGIS version: {postgis_ver} - OS: {os_name_info()} diff --git a/nominatim/tools/exec_utils.py b/nominatim/tools/exec_utils.py index ab2ccc7c..566ac06e 100644 --- a/nominatim/tools/exec_utils.py +++ b/nominatim/tools/exec_utils.py @@ -17,7 +17,7 @@ from urllib.parse import urlencode from nominatim.config import Configuration from nominatim.typing import StrPath -from nominatim.version import version_str +from nominatim.version import NOMINATIM_VERSION from nominatim.db.connection import get_pg_env LOG = logging.getLogger() @@ -119,7 +119,7 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None: cmd = [str(options['osm2pgsql']), '--hstore', '--latlon', '--slim', '--log-progress', 'true', - '--number-processes', str(options['threads']), + '--number-processes', '1' if options['append'] else str(options['threads']), '--cache', str(options['osm2pgsql_cache']), '--style', str(options['osm2pgsql_style']) ] @@ -162,7 +162,7 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None: def get_url(url: str) -> str: """ Get the contents from the given URL and return it as a UTF-8 string. """ - headers = {"User-Agent": f"Nominatim/{version_str()}"} + headers = {"User-Agent": f"Nominatim/{NOMINATIM_VERSION!s}"} try: request = urlrequest.Request(url, headers=headers) diff --git a/nominatim/tools/migration.py b/nominatim/tools/migration.py index 10bca15c..545f5c48 100644 --- a/nominatim/tools/migration.py +++ b/nominatim/tools/migration.py @@ -15,16 +15,14 @@ from psycopg2 import sql as pysql from nominatim.config import Configuration from nominatim.db import properties from nominatim.db.connection import connect, Connection -from nominatim.version import NOMINATIM_VERSION, version_str +from nominatim.version import NominatimVersion, NOMINATIM_VERSION, parse_version from nominatim.tools import refresh from nominatim.tokenizer import factory as tokenizer_factory from nominatim.errors import UsageError LOG = logging.getLogger() -VersionTuple = Tuple[int, int, int, int] - -_MIGRATION_FUNCTIONS : List[Tuple[VersionTuple, Callable[..., None]]] = [] +_MIGRATION_FUNCTIONS : List[Tuple[NominatimVersion, Callable[..., None]]] = [] def migrate(config: Configuration, paths: Any) -> int: """ Check for the current database version and execute migrations, @@ -37,8 +35,7 @@ def migrate(config: Configuration, paths: Any) -> int: db_version_str = None if db_version_str is not None: - parts = db_version_str.split('.') - db_version = tuple(int(x) for x in parts[:2] + parts[2].split('-')) + db_version = parse_version(db_version_str) if db_version == NOMINATIM_VERSION: LOG.warning("Database already at latest version (%s)", db_version_str) @@ -53,8 +50,7 @@ def migrate(config: Configuration, paths: Any) -> int: for version, func in _MIGRATION_FUNCTIONS: if db_version <= version: title = func.__doc__ or '' - LOG.warning("Running: %s (%s)", title.split('\n', 1)[0], - version_str(version)) + LOG.warning("Running: %s (%s)", title.split('\n', 1)[0], version) kwargs = dict(conn=conn, config=config, paths=paths) func(**kwargs) conn.commit() @@ -66,14 +62,14 @@ def migrate(config: Configuration, paths: Any) -> int: tokenizer = tokenizer_factory.get_tokenizer_for_db(config) tokenizer.update_sql_functions(config) - properties.set_property(conn, 'database_version', version_str()) + properties.set_property(conn, 'database_version', str(NOMINATIM_VERSION)) conn.commit() return 0 -def _guess_version(conn: Connection) -> VersionTuple: +def _guess_version(conn: Connection) -> NominatimVersion: """ Guess a database version when there is no property table yet. Only migrations for 3.6 and later are supported, so bail out when the version seems older. @@ -89,7 +85,7 @@ def _guess_version(conn: Connection) -> VersionTuple: 'prior to 3.6.0. Automatic migration not possible.') raise UsageError('Migration not possible.') - return (3, 5, 0, 99) + return NominatimVersion(3, 5, 0, 99) @@ -108,7 +104,8 @@ def _migration(major: int, minor: int, patch: int = 0, there. """ def decorator(func: Callable[..., None]) -> Callable[..., None]: - _MIGRATION_FUNCTIONS.append(((major, minor, patch, dbpatch), func)) + version = (NominatimVersion(major, minor, patch, dbpatch)) + _MIGRATION_FUNCTIONS.append((version, func)) return func return decorator diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py index b35d3aae..45796014 100644 --- a/nominatim/tools/refresh.py +++ b/nominatim/tools/refresh.py @@ -18,7 +18,7 @@ from nominatim.config import Configuration from nominatim.db.connection import Connection, connect from nominatim.db.utils import execute_file from nominatim.db.sql_preprocessor import SQLPreprocessor -from nominatim.version import version_str +from nominatim.version import NOMINATIM_VERSION LOG = logging.getLogger() @@ -223,7 +223,7 @@ def setup_website(basedir: Path, config: Configuration, conn: Connection) -> Non @define('CONST_Debug', $_GET['debug'] ?? false); @define('CONST_LibDir', '{config.lib_dir.php}'); @define('CONST_TokenizerDir', '{config.project_dir / 'tokenizer'}'); - @define('CONST_NominatimVersion', '{version_str()}'); + @define('CONST_NominatimVersion', '{NOMINATIM_VERSION!s}'); """) diff --git a/nominatim/version.py b/nominatim/version.py index 43a30d9e..40e3bda4 100644 --- a/nominatim/version.py +++ b/nominatim/version.py @@ -7,25 +7,34 @@ """ Version information for Nominatim. """ -from typing import Optional, Tuple +from typing import Optional, NamedTuple -# Version information: major, minor, patch level, database patch level -# -# The first three numbers refer to the last released version. -# -# The database patch level tracks important changes between releases -# and must always be increased when there is a change to the database or code -# that requires a migration. -# -# When adding a migration on the development branch, raise the patch level -# to 99 to make sure that the migration is applied when updating from a -# patch release to the next minor version. Patch releases usually shouldn't -# have migrations in them. When they are needed, then make sure that the -# migration can be reapplied and set the migration version to the appropriate -# patch level when cherry-picking the commit with the migration. -# -# Released versions always have a database patch level of 0. -NOMINATIM_VERSION = (4, 2, 99, 0) +class NominatimVersion(NamedTuple): + """ Version information for Nominatim. We follow semantic versioning. + + Major, minor and patch_level refer to the last released version. + The database patch level tracks important changes between releases + and must always be increased when there is a change to the database or code + that requires a migration. + + When adding a migration on the development branch, raise the patch level + to 99 to make sure that the migration is applied when updating from a + patch release to the next minor version. Patch releases usually shouldn't + have migrations in them. When they are needed, then make sure that the + migration can be reapplied and set the migration version to the appropriate + patch level when cherry-picking the commit with the migration. + """ + + major: int + minor: int + patch_level: int + db_patch_level: int + + def __str__(self) -> str: + return f"{self.major}.{self.minor}.{self.patch_level}-{self.db_patch_level}" + + +NOMINATIM_VERSION = NominatimVersion(4, 2, 99, 0) POSTGRESQL_REQUIRED_VERSION = (9, 6) POSTGIS_REQUIRED_VERSION = (2, 2) @@ -37,9 +46,11 @@ POSTGIS_REQUIRED_VERSION = (2, 2) GIT_COMMIT_HASH : Optional[str] = None -# pylint: disable=consider-using-f-string -def version_str(version:Tuple[int, int, int, int] = NOMINATIM_VERSION) -> str: - """ - Return a human-readable string of the version. +def parse_version(version: str) -> NominatimVersion: + """ Parse a version string into a version consisting of a tuple of + four ints: major, minor, patch level, database patch level + + This is the reverse operation of `version_str()`. """ - return '{}.{}.{}-{}'.format(*version) + parts = version.split('.') + return NominatimVersion(*[int(x) for x in parts[:2] + parts[2].split('-')]) diff --git a/test/bdd/environment.py b/test/bdd/environment.py index 7a7b943d..305c88e9 100644 --- a/test/bdd/environment.py +++ b/test/bdd/environment.py @@ -28,6 +28,7 @@ userconfig = { 'SERVER_MODULE_PATH' : None, 'TOKENIZER' : None, # Test with a custom tokenizer 'STYLE' : 'extratags', + 'API_ENGINE': 'php', 'PHPCOV' : False, # set to output directory to enable code coverage } diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index 6179ca34..e156c60c 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -5,6 +5,7 @@ # Copyright (C) 2022 by the Nominatim developer community. # For a full list of authors see the git log. from pathlib import Path +import importlib import sys import tempfile @@ -49,6 +50,12 @@ class NominatimEnvironment: self.api_db_done = False self.website_dir = None + self.api_engine = None + if config['API_ENGINE'] != 'php': + if not hasattr(self, f"create_api_request_func_{config['API_ENGINE']}"): + raise RuntimeError(f"Unknown API engine '{config['API_ENGINE']}'") + self.api_engine = getattr(self, f"create_api_request_func_{config['API_ENGINE']}")() + def connect_database(self, dbname): """ Return a connection to the database with the given name. Uses configured host, user and port. @@ -323,3 +330,51 @@ class NominatimEnvironment: WHERE class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'""") + + + def create_api_request_func_starlette(self): + import nominatim.server.starlette.server + from asgi_lifespan import LifespanManager + import httpx + + async def _request(endpoint, params, project_dir, environ): + app = nominatim.server.starlette.server.get_application(project_dir, environ) + + async with LifespanManager(app): + async with httpx.AsyncClient(app=app, base_url="http://nominatim.test") as client: + response = await client.get(f"/{endpoint}", params=params) + + return response.text, response.status_code + + return _request + + + def create_api_request_func_sanic(self): + import nominatim.server.sanic.server + + async def _request(endpoint, params, project_dir, environ): + app = nominatim.server.sanic.server.get_application(project_dir, environ) + + _, response = await app.asgi_client.get(f"/{endpoint}", params=params) + + return response.text, response.status_code + + return _request + + + def create_api_request_func_falcon(self): + import nominatim.server.falcon.server + import falcon.testing + + async def _request(endpoint, params, project_dir, environ): + app = nominatim.server.falcon.server.get_application(project_dir, environ) + + async with falcon.testing.ASGIConductor(app) as conductor: + response = await conductor.get(f"/{endpoint}", params=params) + + return response.text, response.status_code + + return _request + + + diff --git a/test/bdd/steps/steps_api_queries.py b/test/bdd/steps/steps_api_queries.py index 22517338..7bf38d14 100644 --- a/test/bdd/steps/steps_api_queries.py +++ b/test/bdd/steps/steps_api_queries.py @@ -9,10 +9,12 @@ Queries may either be run directly via PHP using the query script or via the HTTP interface using php-cgi. """ +from pathlib import Path import json import os import re import logging +import asyncio from urllib.parse import urlencode from utils import run_script @@ -72,6 +74,16 @@ def send_api_query(endpoint, params, fmt, context): for h in context.table.headings: params[h] = context.table[0][h] + if context.nominatim.api_engine is None: + return send_api_query_php(endpoint, params, context) + + return asyncio.run(context.nominatim.api_engine(endpoint, params, + Path(context.nominatim.website_dir.name), + context.nominatim.test_env)) + + + +def send_api_query_php(endpoint, params, context): env = dict(BASE_SERVER_ENV) env['QUERY_STRING'] = urlencode(params) diff --git a/test/python/api/conftest.py b/test/python/api/conftest.py new file mode 100644 index 00000000..4c2e0cc0 --- /dev/null +++ b/test/python/api/conftest.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Helper fixtures for API call tests. +""" +from pathlib import Path +import pytest +import time + +from nominatim.api import NominatimAPI + +@pytest.fixture +def apiobj(temp_db): + """ Create an asynchronous SQLAlchemy engine for the test DB. + """ + api = NominatimAPI(Path('/invalid'), {}) + yield api + api.close() diff --git a/test/python/api/test_api_status.py b/test/python/api/test_api_status.py new file mode 100644 index 00000000..6bc1fccc --- /dev/null +++ b/test/python/api/test_api_status.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Tests for the status API call. +""" +from pathlib import Path +import datetime as dt +import pytest + +from nominatim.version import NOMINATIM_VERSION, NominatimVersion +from nominatim.api import NominatimAPI + +def test_status_no_extra_info(apiobj, table_factory): + table_factory('import_status', + definition="lastimportdate timestamp with time zone NOT NULL") + table_factory('nominatim_properties', + definition='property TEXT, value TEXT') + + result = apiobj.status() + + assert result.status == 0 + assert result.message == 'OK' + assert result.software_version == NOMINATIM_VERSION + assert result.database_version is None + assert result.data_updated is None + + +def test_status_full(apiobj, table_factory): + table_factory('import_status', + definition="lastimportdate timestamp with time zone NOT NULL", + content=(('2022-12-07 15:14:46+01',),)) + table_factory('nominatim_properties', + definition='property TEXT, value TEXT', + content=(('database_version', '99.5.4-2'), )) + + result = apiobj.status() + + assert result.status == 0 + assert result.message == 'OK' + assert result.software_version == NOMINATIM_VERSION + assert result.database_version == NominatimVersion(99, 5, 4, 2) + assert result.data_updated == dt.datetime(2022, 12, 7, 14, 14, 46, 0, tzinfo=dt.timezone.utc) + + +def test_status_database_not_found(monkeypatch): + monkeypatch.setenv('NOMINATIM_DATABASE_DSN', 'dbname=rgjdfkgjedkrgdfkngdfkg') + + api = NominatimAPI(Path('/invalid'), {}) + + result = api.status() + + assert result.status == 700 + assert result.message == 'Database connection failed' + assert result.software_version == NOMINATIM_VERSION + assert result.database_version is None + assert result.data_updated is None diff --git a/test/python/cli/test_cli.py b/test/python/cli/test_cli.py index 1072f6c9..d0e3307e 100644 --- a/test/python/cli/test_cli.py +++ b/test/python/cli/test_cli.py @@ -11,6 +11,7 @@ These tests just check that the various command line parameters route to the correct functionionality. They use a lot of monkeypatching to avoid executing the actual functions. """ +import importlib import pytest import nominatim.indexer.indexer @@ -59,7 +60,7 @@ def test_cli_add_data_tiger_data(cli_call, cli_tokenizer_mock, mock_func_factory assert mock.called == 1 -def test_cli_serve_command(cli_call, mock_func_factory): +def test_cli_serve_php(cli_call, mock_func_factory): func = mock_func_factory(nominatim.cli, 'run_php_server') cli_call('serve') == 0 @@ -67,6 +68,47 @@ def test_cli_serve_command(cli_call, mock_func_factory): assert func.called == 1 +def test_cli_serve_sanic(cli_call, mock_func_factory): + mod = pytest.importorskip("sanic") + func = mock_func_factory(mod.Sanic, "run") + + cli_call('serve', '--engine', 'sanic') == 0 + + assert func.called == 1 + + +def test_cli_serve_starlette_custom_server(cli_call, mock_func_factory): + pytest.importorskip("starlette") + mod = pytest.importorskip("uvicorn") + func = mock_func_factory(mod, "run") + + cli_call('serve', '--engine', 'starlette', '--server', 'foobar:4545') == 0 + + assert func.called == 1 + assert func.last_kwargs['host'] == 'foobar' + assert func.last_kwargs['port'] == 4545 + + +def test_cli_serve_starlette_custom_server_bad_port(cli_call, mock_func_factory): + pytest.importorskip("starlette") + mod = pytest.importorskip("uvicorn") + func = mock_func_factory(mod, "run") + + cli_call('serve', '--engine', 'starlette', '--server', 'foobar:45:45') == 1 + + +@pytest.mark.parametrize("engine", ['falcon', 'starlette']) +def test_cli_serve_uvicorn_based(cli_call, engine, mock_func_factory): + pytest.importorskip(engine) + mod = pytest.importorskip("uvicorn") + func = mock_func_factory(mod, "run") + + cli_call('serve', '--engine', engine) == 0 + + assert func.called == 1 + assert func.last_kwargs['host'] == '127.0.0.1' + assert func.last_kwargs['port'] == 8088 + def test_cli_export_command(cli_call, mock_run_legacy): assert cli_call('export', '--output-all-postcodes') == 0 diff --git a/test/python/cli/test_cmd_api.py b/test/python/cli/test_cmd_api.py index 96415938..4031441f 100644 --- a/test/python/cli/test_cmd_api.py +++ b/test/python/cli/test_cmd_api.py @@ -7,9 +7,12 @@ """ Tests for API access commands of command-line interface wrapper. """ +import json import pytest import nominatim.clicmd.api +import nominatim.api +from nominatim.apicmd.status import StatusResult @pytest.mark.parametrize("endpoint", (('search', 'reverse', 'lookup', 'details', 'status'))) @@ -27,9 +30,8 @@ def test_no_api_without_phpcgi(endpoint): ('details', '--node', '1'), ('details', '--way', '1'), ('details', '--relation', '1'), - ('details', '--place_id', '10001'), - ('status',)]) -class TestCliApiCall: + ('details', '--place_id', '10001')]) +class TestCliApiCallPhp: @pytest.fixture(autouse=True) def setup_cli_call(self, params, cli_call, mock_func_factory, tmp_path): @@ -55,6 +57,29 @@ class TestCliApiCall: assert self.run_nominatim() == 1 +class TestCliStatusCall: + + @pytest.fixture(autouse=True) + def setup_status_mock(self, monkeypatch): + monkeypatch.setattr(nominatim.api.NominatimAPI, 'status', + lambda self: StatusResult(200, 'OK')) + + + def test_status_simple(self, cli_call, tmp_path): + result = cli_call('status', '--project-dir', str(tmp_path)) + + assert result == 0 + + + def test_status_json_format(self, cli_call, tmp_path, capsys): + result = cli_call('status', '--project-dir', str(tmp_path), + '--format', 'json') + + assert result == 0 + + json.loads(capsys.readouterr().out) + + QUERY_PARAMS = { 'search': ('--query', 'somewhere'), 'reverse': ('--lat', '20', '--lon', '30'), diff --git a/test/python/result_formatter/test_v1.py b/test/python/result_formatter/test_v1.py new file mode 100644 index 00000000..919f5b80 --- /dev/null +++ b/test/python/result_formatter/test_v1.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Tests for formatting results for the V1 API. +""" +import datetime as dt +import pytest + +import nominatim.result_formatter.v1 as format_module +from nominatim.apicmd.status import StatusResult +from nominatim.version import NOMINATIM_VERSION + +STATUS_FORMATS = {'text', 'json'} + +class TestStatusResultFormat: + + + @pytest.fixture(autouse=True) + def make_formatter(self): + self.formatter = format_module.create(StatusResult) + + + def test_format_list(self): + assert set(self.formatter.list_formats()) == STATUS_FORMATS + + + @pytest.mark.parametrize('fmt', list(STATUS_FORMATS)) + def test_supported(self, fmt): + assert self.formatter.supports_format(fmt) + + + def test_unsupported(self): + assert not self.formatter.supports_format('gagaga') + + + def test_format_text(self): + assert self.formatter.format(StatusResult(0, 'message here'), 'text') == 'OK' + + + def test_format_text(self): + assert self.formatter.format(StatusResult(500, 'message here'), 'text') == 'ERROR: message here' + + + def test_format_json_minimal(self): + status = StatusResult(700, 'Bad format.') + + result = self.formatter.format(status, 'json') + + assert result == '{"status": 700, "message": "Bad format.", "software_version": "%s"}' % (NOMINATIM_VERSION, ) + + + def test_format_json_full(self): + status = StatusResult(0, 'OK') + status.data_updated = dt.datetime(2010, 2, 7, 20, 20, 3, 0, tzinfo=dt.timezone.utc) + status.database_version = '5.6' + + result = self.formatter.format(status, 'json') + + assert result == '{"status": 0, "message": "OK", "data_updated": "2010-02-07T20:20:03+00:00", "software_version": "%s", "database_version": "5.6"}' % (NOMINATIM_VERSION, ) diff --git a/vagrant/Install-on-Ubuntu-18.sh b/vagrant/Install-on-Ubuntu-18.sh deleted file mode 100755 index 09de9747..00000000 --- a/vagrant/Install-on-Ubuntu-18.sh +++ /dev/null @@ -1,277 +0,0 @@ -#!/bin/bash -e -# -# hacks for broken vagrant box #DOCS: -sudo rm -f /var/lib/dpkg/lock #DOCS: -export APT_LISTCHANGES_FRONTEND=none #DOCS: -export DEBIAN_FRONTEND=noninteractive #DOCS: - -# -# *Note:* these installation instructions are also available in executable -# form for use with vagrant under vagrant/Install-on-Ubuntu-18.sh. -# -# Installing the Required Software -# ================================ -# -# These instructions expect that you have a freshly installed Ubuntu 18.04. -# -# Make sure all packages are up-to-date by running: -# - - sudo apt update -qq - -# Now you can install all packages needed for Nominatim: - - sudo apt install -y php-cgi - sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \ - libboost-filesystem-dev libexpat1-dev zlib1g-dev\ - libbz2-dev libpq-dev liblua5.3-dev lua5.3\ - postgresql-10-postgis-2.4 \ - postgresql-contrib-10 postgresql-10-postgis-scripts \ - php-cli php-pgsql php-intl libicu-dev python3-pip \ - python3-psutil python3-jinja2 python3-yaml python3-icu git - -# Some of the Python packages that come with Ubuntu 18.04 are too old, so -# install the latest version from pip: - - pip3 install --user python-dotenv datrie pyyaml psycopg2-binary - -# -# System Configuration -# ==================== -# -# The following steps are meant to configure a fresh Ubuntu installation -# for use with Nominatim. You may skip some of the steps if you have your -# OS already configured. -# -# Creating Dedicated User Accounts -# -------------------------------- -# -# Nominatim will run as a global service on your machine. It is therefore -# best to install it under its own separate user account. In the following -# we assume this user is called nominatim and the installation will be in -# /srv/nominatim. To create the user and directory run: -# -# sudo useradd -d /srv/nominatim -s /bin/bash -m nominatim -# -# You may find a more suitable location if you wish. -# -# To be able to copy and paste instructions from this manual, export -# user name and home directory now like this: -# -if [ "x$USERNAME" == "x" ]; then #DOCS: - export USERNAME=vagrant #DOCS: export USERNAME=nominatim - export USERHOME=/home/vagrant #DOCS: export USERHOME=/srv/nominatim -fi #DOCS: -# -# **Never, ever run the installation as a root user.** You have been warned. -# -# Make sure that system servers can read from the home directory: - - chmod a+x $USERHOME - -# Setting up PostgreSQL -# --------------------- -# -# Tune the postgresql configuration, which is located in -# `/etc/postgresql/10/main/postgresql.conf`. See section *Postgres Tuning* in -# [the installation page](../admin/Installation.md#postgresql-tuning) -# for the parameters to change. -# -# Restart the postgresql service after updating this config file. - -if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS: - sudo pg_ctlcluster 10 main start #DOCS: -else #DOCS: - sudo systemctl restart postgresql -fi #DOCS: - -# -# Finally, we need to add two postgres users: one for the user that does -# the import and another for the webserver which should access the database -# for reading only: -# - - sudo -u postgres createuser -s $USERNAME - sudo -u postgres createuser www-data - -# -# Installing Nominatim -# ==================== -# -# Building and Configuration -# -------------------------- -# -# Get the source code from Github and change into the source directory -# -if [ "x$1" == "xyes" ]; then #DOCS: :::sh - cd $USERHOME - git clone --recursive https://github.com/openstreetmap/Nominatim.git - cd Nominatim -else #DOCS: - cd $USERHOME/Nominatim #DOCS: -fi #DOCS: - -# When installing the latest source from github, you also need to -# download the country grid: - -if [ ! -f data/country_osm_grid.sql.gz ]; then #DOCS: :::sh - wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz -fi #DOCS: - -# The code must be built in a separate directory. Create this directory, -# then configure and build Nominatim in there: - - mkdir $USERHOME/build - cd $USERHOME/build - cmake $USERHOME/Nominatim - make - sudo make install - - -# Nominatim is now ready to use. You can continue with -# [importing a database from OSM data](../admin/Import.md). If you want to set up -# a webserver first, continue reading. -# -# Setting up a webserver -# ====================== -# -# The webserver should serve the php scripts from the website directory of your -# [project directory](../admin/Import.md#creating-the-project-directory). -# This directory needs to exist when being configured. -# Therefore set up a project directory and create the website directory: - - mkdir $USERHOME/nominatim-project - mkdir $USERHOME/nominatim-project/website - -# The import process will populate the directory later. -# -# Option 1: Using Apache -# ---------------------- -# -if [ "x$2" == "xinstall-apache" ]; then #DOCS: -# -# Apache has a PHP module that can be used to serve Nominatim. To install them -# run: - - sudo apt install -y apache2 libapache2-mod-php - -# You need to create an alias to the website directory in your apache -# configuration. Add a separate nominatim configuration to your webserver: - -#DOCS:```sh -sudo tee /etc/apache2/conf-available/nominatim.conf << EOFAPACHECONF - - Options FollowSymLinks MultiViews - AddType text/html .php - DirectoryIndex search.php - Require all granted - - -Alias /nominatim $USERHOME/nominatim-project/website -EOFAPACHECONF -#DOCS:``` - -# -# Then enable the configuration with -# - - sudo a2enconf nominatim - -# and restart apache: - -if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS: - sudo apache2ctl start #DOCS: -else #DOCS: - sudo systemctl restart apache2 -fi #DOCS: - -# The Nominatim API is now available at `http://localhost/nominatim/`. - -fi #DOCS: - -# -# Option 2: Using nginx -# --------------------- -# -if [ "x$2" == "xinstall-nginx" ]; then #DOCS: - -# Nginx has no native support for php scripts. You need to set up php-fpm for -# this purpose. First install nginx and php-fpm: - - sudo apt install -y nginx php-fpm - -# You need to configure php-fpm to listen on a Unix socket. - -#DOCS:```sh -sudo tee /etc/php/7.2/fpm/pool.d/www.conf << EOF_PHP_FPM_CONF -[www] -; Replace the tcp listener and add the unix socket -listen = /var/run/php-fpm-nominatim.sock - -; Ensure that the daemon runs as the correct user -listen.owner = www-data -listen.group = www-data -listen.mode = 0666 - -; Unix user of FPM processes -user = www-data -group = www-data - -; Choose process manager type (static, dynamic, ondemand) -pm = ondemand -pm.max_children = 5 -EOF_PHP_FPM_CONF -#DOCS:``` - -# Then create a Nginx configuration to forward http requests to that socket. - -#DOCS:```sh -sudo tee /etc/nginx/sites-available/default << EOF_NGINX_CONF -server { - listen 80 default_server; - listen [::]:80 default_server; - - root $USERHOME/nominatim-project/website; - index search.php index.html; - location / { - try_files \$uri \$uri/ @php; - } - - location @php { - fastcgi_param SCRIPT_FILENAME "\$document_root\$uri.php"; - fastcgi_param PATH_TRANSLATED "\$document_root\$uri.php"; - fastcgi_param QUERY_STRING \$args; - fastcgi_pass unix:/var/run/php-fpm-nominatim.sock; - fastcgi_index index.php; - include fastcgi_params; - } - - location ~ [^/]\.php(/|$) { - fastcgi_split_path_info ^(.+?\.php)(/.*)$; - if (!-f \$document_root\$fastcgi_script_name) { - return 404; - } - fastcgi_pass unix:/var/run/php-fpm-nominatim.sock; - fastcgi_index search.php; - include fastcgi.conf; - } -} -EOF_NGINX_CONF -#DOCS:``` - -# -# Enable the configuration and restart Nginx -# - -if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS: - sudo /usr/sbin/php-fpm7.2 --nodaemonize --fpm-config /etc/php/7.2/fpm/php-fpm.conf & #DOCS: - sudo /usr/sbin/nginx & #DOCS: -else #DOCS: - sudo systemctl restart php7.2-fpm nginx -fi #DOCS: - -# The Nominatim API is now available at `http://localhost/`. - - - -fi #DOCS: diff --git a/vagrant/Install-on-Ubuntu-20.sh b/vagrant/Install-on-Ubuntu-20.sh index 2d4eaa71..34e81637 100755 --- a/vagrant/Install-on-Ubuntu-20.sh +++ b/vagrant/Install-on-Ubuntu-20.sh @@ -27,9 +27,15 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: postgresql-12-postgis-3 \ postgresql-contrib-12 postgresql-12-postgis-3-scripts \ php-cli php-pgsql php-intl libicu-dev python3-dotenv \ - python3-psycopg2 python3-psutil python3-jinja2 \ + python3-psycopg2 python3-psutil python3-jinja2 python3-pip \ python3-icu python3-datrie python3-yaml git +# Some of the Python packages that come with Ubuntu 20.04 are too old, so +# install the latest version from pip: + + pip3 install --user sqlalchemy asyncpg + + # # System Configuration # ==================== diff --git a/vagrant/Install-on-Ubuntu-22.sh b/vagrant/Install-on-Ubuntu-22.sh index c45ac55e..82e706c9 100755 --- a/vagrant/Install-on-Ubuntu-22.sh +++ b/vagrant/Install-on-Ubuntu-22.sh @@ -28,7 +28,8 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: postgresql-contrib-14 postgresql-14-postgis-3-scripts \ php-cli php-pgsql php-intl libicu-dev python3-dotenv \ python3-psycopg2 python3-psutil python3-jinja2 \ - python3-icu python3-datrie git + python3-icu python3-datrie python3-sqlalchemy \ + python3-asyncpg git # # System Configuration