run: |
sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION}
if [ "x$UBUNTUVER" == "x18" ]; then
- pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 datrie
+ pip3 install MarkupSafe==2.0.1 python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4 datrie asyncpg
else
- sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml
+ sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml python3-asyncpg
+ pip3 install sqlalchemy
fi
shell: bash
env:
- ubuntu: 18
postgresql: 9.6
postgis: 2.5
- pytest: pytest
php: 7.2
- ubuntu: 20
postgresql: 13
postgis: 3
- pytest: py.test-3
php: 7.4
- ubuntu: 22
postgresql: 15
postgis: 3
- pytest: py.test-3
php: 8.1
runs-on: ubuntu-${{ matrix.ubuntu }}.04
- uses: actions/setup-python@v4
with:
- python-version: 3.6
+ python-version: 3.7
if: matrix.ubuntu == 18
- uses: ./Nominatim/.github/actions/setup-postgresql
with:
ubuntu: ${{ matrix.ubuntu }}
- - name: Install test prerequsites
- run: sudo apt-get install -y -qq python3-pytest python3-behave
+ - name: Install test prerequsites (behave from apt)
+ run: sudo apt-get install -y -qq python3-behave
if: matrix.ubuntu == 20
- - name: Install test prerequsites
- run: pip3 install pylint pytest behave==1.2.6
+ - name: Install test prerequsites (behave from pip)
+ run: pip3 install behave==1.2.6
if: ${{ (matrix.ubuntu == 18) || (matrix.ubuntu == 22) }}
- - name: Install test prerequsites
- run: sudo apt-get install -y -qq python3-pytest
- if: matrix.ubuntu == 22
+ - name: Install test prerequsites (from apt for Ununtu 2x)
+ run: sudo apt-get install -y -qq python3-pytest uvicorn
+ if: matrix.ubuntu >= 20
+
+ - name: Install test prerequsites (from pip for Ubuntu 18)
+ run: pip3 install pytest uvicorn
+ if: matrix.ubuntu == 18
+
+ - name: Install Python webservers
+ run: pip3 install falcon sanic sanic-testing starlette
- name: Install latest pylint/mypy
- run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions
+ run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions asgi_lifespan sqlalchemy2-stubs
- name: PHP linting
run: phpcs --report-width=120 .
working-directory: Nominatim
- name: Python linting
- run: pylint nominatim
+ run: python3 -m pylint nominatim
working-directory: Nominatim
- - name: Python static typechecking
- run: mypy --strict nominatim
- working-directory: Nominatim
-
-
- name: PHP unit tests
run: phpunit ./
working-directory: Nominatim/test/php
if: ${{ (matrix.ubuntu == 20) || (matrix.ubuntu == 22) }}
- name: Python unit tests
- run: $PYTEST test/python
+ run: python3 -m pytest test/python
working-directory: Nominatim
- env:
- PYTEST: ${{ matrix.pytest }}
- name: BDD tests
run: |
- behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3
+ python3 -m behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3
working-directory: Nominatim/test/bdd
+ - name: Install newer Python packages (for typechecking info)
+ run: pip3 install -U osmium uvicorn
+ if: matrix.ubuntu >= 20
+
+ - name: Python static typechecking
+ run: python3 -m mypy --strict nominatim
+ working-directory: Nominatim
+ if: matrix.ubuntu >= 20
legacy-test:
needs: create-archive
- name: BDD tests (legacy tokenizer)
run: |
- behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DTOKENIZER=legacy --format=progress3
+ python3 -m behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DTOKENIZER=legacy --format=progress3
working-directory: Nominatim/test/bdd
strategy:
matrix:
- name: [Ubuntu-18, Ubuntu-20, Ubuntu-22]
+ name: [Ubuntu-20, Ubuntu-22]
include:
- - name: Ubuntu-18
- flavour: ubuntu
- image: "ubuntu:18.04"
- ubuntu: 18
- install_mode: install-nginx
- name: Ubuntu-20
- flavour: ubuntu
image: "ubuntu:20.04"
ubuntu: 20
install_mode: install-apache
- name: Ubuntu-22
- flavour: ubuntu
image: "ubuntu:22.04"
ubuntu: 22
install_mode: install-apache
apt-get install -y git sudo wget
ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone
shell: bash
- if: matrix.flavour == 'ubuntu'
-
- - name: Prepare container (CentOS)
- run: |
- dnf update -y
- dnf install -y sudo glibc-langpack-en
- shell: bash
- if: matrix.flavour == 'centos'
- name: Setup import user
run: |
mkdir data-env-reverse
working-directory: /home/nominatim
- - name: Prepare import environment (CentOS)
- run: |
- sudo ln -s /usr/local/bin/nominatim /usr/bin/nominatim
- echo NOMINATIM_DATABASE_WEBUSER="apache" > nominatim-project/.env
- cp nominatim-project/.env data-env-reverse/.env
- working-directory: /home/nominatim
- if: matrix.flavour == 'centos'
-
- name: Print version
run: nominatim --version
working-directory: /home/nominatim/nominatim-project
- name: Collect host OS information
run: nominatim admin --collect-os-info
working-directory: /home/nominatim/nominatim-project
-
+
- name: Import
run: nominatim import --osm-file ../test.pbf
working-directory: /home/nominatim/nominatim-project
- name: Prepare update (Ubuntu)
run: apt-get install -y python3-pip
shell: bash
- if: matrix.flavour == 'ubuntu'
- name: Run update
run: |
[mypy]
+plugins = sqlalchemy.ext.mypy.plugin
[mypy-icu.*]
ignore_missing_imports = True
-[mypy-osmium.*]
+[mypy-asyncpg.*]
ignore_missing_imports = True
[mypy-datrie.*]
[mypy-dotenv.*]
ignore_missing_imports = True
+
+[mypy-falcon.*]
+ignore_missing_imports = True
[MASTER]
-extension-pkg-whitelist=osmium
+extension-pkg-whitelist=osmium,falcon
ignored-modules=icu,datrie
[MESSAGES CONTROL]
#-----------------------------------------------------------------------------
if (BUILD_IMPORTER)
- find_package(PythonInterp 3.6 REQUIRED)
+ find_package(PythonInterp 3.7 REQUIRED)
endif()
#-----------------------------------------------------------------------------
endforeach()
ADD_CUSTOM_TARGET(doc
- COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-18.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-18.md
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-20.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-20.md
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-22.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-22.md
COMMAND PYTHONPATH=${PROJECT_SOURCE_DIR} mkdocs build -d ${CMAKE_CURRENT_BINARY_DIR}/../site-html -f ${CMAKE_CURRENT_BINARY_DIR}/../mkdocs.yml
* [Ubuntu 22.04](../appendix/Install-on-Ubuntu-22.md)
* [Ubuntu 20.04](../appendix/Install-on-Ubuntu-20.md)
- * [Ubuntu 18.04](../appendix/Install-on-Ubuntu-18.md)
These OS-specific instructions can also be found in executable form
in the `vagrant/` directory.
* [PostgreSQL](https://www.postgresql.org) (9.6+ will work, 11+ strongly recommended)
* [PostGIS](https://postgis.net) (2.2+ will work, 3.0+ strongly recommended)
- * [Python 3](https://www.python.org/) (3.6+)
+ * [Python 3](https://www.python.org/) (3.7+)
* [Psycopg2](https://www.psycopg.org) (2.7+)
* [Python Dotenv](https://github.com/theskumar/python-dotenv)
* [psutil](https://github.com/giampaolo/psutil)
* [Jinja2](https://palletsprojects.com/p/jinja/)
+ * [SQLAlchemy](https://www.sqlalchemy.org/) (1.4+ with greenlet support)
+ * [asyncpg](https://magicstack.github.io/asyncpg) (0.8+)
* [PyICU](https://pypi.org/project/PyICU/)
* [PyYaml](https://pyyaml.org/) (5.1+)
* [datrie](https://github.com/pytries/datrie)
* [pyosmium](https://osmcode.org/pyosmium/)
+For running the experimental Python frontend:
+
+ * one of the following web frameworks:
+ * [falcon](https://falconframework.org/) (3.0+)
+ * [sanic](https://sanic.dev)
+ * [starlette](https://www.starlette.io/)
+ * [uvicorn](https://www.uvicorn.org/) (only with falcon and starlette framworks)
+
For dependencies for running tests and building documentation, see
the [Development section](../develop/Development-Environment.md).
* [Python Typing Extensions](https://github.com/python/typing_extensions) (for Python < 3.9)
* [pytest](https://pytest.org)
+For testing the Python search frontend, you need to install extra dependencies
+depending on your choice of webserver framework:
+
+* [sanic-testing](https://sanic.dev/en/plugins/sanic-testing/getting-started.html) (sanic only)
+* [httpx](https://www.python-httpx.org/) (starlette only)
+* [asgi-lifespan](https://github.com/florimondmanca/asgi-lifespan) (starlette only)
+
The documentation is built with mkdocs:
* [mkdocs](https://www.mkdocs.org/) >= 1.1.2
python3-pip python3-setuptools python3-dev
pip3 install --user behave mkdocs mkdocstrings pytest pylint \
- mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil
+ mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil \
+ sanic-testing httpx asgi-lifespan
```
The `mkdocs` executable will be located in `.local/bin`. You may have to add
* `TEST_DB` - name of test database (db tests)
* `API_TEST_DB` - name of the database containing the API test data (api tests)
* `API_TEST_FILE` - OSM file to be imported into the API test database (api tests)
+ * `API_ENGINE` - webframe to use for running search queries, same values as
+ `nominatim serve --engine` parameter
* `DB_HOST` - (optional) hostname of database host
* `DB_PORT` - (optional) port of database on host
* `DB_USER` - (optional) username of database login
API tests should only be testing the functionality of the website PHP code.
Most tests should be formulated as BDD DB creation tests (see below) instead.
-#### Code Coverage
+#### Code Coverage (PHP engine only)
The API tests also support code coverage tests. You need to install
[PHP_CodeCoverage](https://github.com/sebastianbergmann/php-code-coverage).
These tests check that data is imported correctly into the place table. They
use the same template database as the DB Creation tests, so the same remarks apply.
-
-Note that most testing of the gazetteer output of osm2pgsql is done in the tests
-of osm2pgsql itself. The BDD tests are just there to ensure compatibility of
-the osm2pgsql and Nominatim code.
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Implementation of classes for API access via libraries.
+"""
+from typing import Mapping, Optional, cast, Any
+import asyncio
+from pathlib import Path
+
+from sqlalchemy import text, event
+from sqlalchemy.engine.url import URL
+from sqlalchemy.ext.asyncio import create_async_engine
+import asyncpg
+
+from nominatim.config import Configuration
+from nominatim.apicmd.status import get_status, StatusResult
+
+class NominatimAPIAsync:
+ """ API loader asynchornous version.
+ """
+ def __init__(self, project_dir: Path,
+ environ: Optional[Mapping[str, str]] = None) -> None:
+ self.config = Configuration(project_dir, environ)
+
+ dsn = self.config.get_database_params()
+
+ dburl = URL.create(
+ 'postgresql+asyncpg',
+ database=dsn.get('dbname'),
+ username=dsn.get('user'), password=dsn.get('password'),
+ host=dsn.get('host'), port=int(dsn['port']) if 'port' in dsn else None,
+ query={k: v for k, v in dsn.items()
+ if k not in ('user', 'password', 'dbname', 'host', 'port')})
+ self.engine = create_async_engine(
+ dburl, future=True,
+ connect_args={'server_settings': {
+ 'DateStyle': 'sql,european',
+ 'max_parallel_workers_per_gather': '0'
+ }})
+ asyncio.get_event_loop().run_until_complete(self._query_server_version())
+ asyncio.get_event_loop().run_until_complete(self.close())
+
+ if self.server_version >= 110000:
+ @event.listens_for(self.engine.sync_engine, "connect") # type: ignore[misc]
+ def _on_connect(dbapi_con: Any, _: Any) -> None:
+ cursor = dbapi_con.cursor()
+ cursor.execute("SET jit_above_cost TO '-1'")
+
+
+ async def _query_server_version(self) -> None:
+ try:
+ async with self.engine.begin() as conn:
+ result = await conn.scalar(text('SHOW server_version_num'))
+ self.server_version = int(cast(str, result))
+ except asyncpg.PostgresError:
+ self.server_version = 0
+
+ async def close(self) -> None:
+ """ Close all active connections to the database. The NominatimAPIAsync
+ object remains usable after closing. If a new API functions is
+ called, new connections are created.
+ """
+ await self.engine.dispose()
+
+
+ async def status(self) -> StatusResult:
+ """ Return the status of the database.
+ """
+ return await get_status(self.engine)
+
+
+class NominatimAPI:
+ """ API loader, synchronous version.
+ """
+
+ def __init__(self, project_dir: Path,
+ environ: Optional[Mapping[str, str]] = None) -> None:
+ self.async_api = NominatimAPIAsync(project_dir, environ)
+
+
+ def close(self) -> None:
+ """ Close all active connections to the database. The NominatimAPIAsync
+ object remains usable after closing. If a new API functions is
+ called, new connections are created.
+ """
+ asyncio.get_event_loop().run_until_complete(self.async_api.close())
+
+
+ def status(self) -> StatusResult:
+ """ Return the status of the database.
+ """
+ return asyncio.get_event_loop().run_until_complete(self.async_api.status())
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Classes and function releated to status call.
+"""
+from typing import Optional, cast
+import datetime as dt
+
+import sqlalchemy as sqla
+from sqlalchemy.ext.asyncio.engine import AsyncEngine, AsyncConnection
+import asyncpg
+
+from nominatim import version
+
+class StatusResult:
+ """ Result of a call to the status API.
+ """
+
+ def __init__(self, status: int, msg: str):
+ self.status = status
+ self.message = msg
+ self.software_version = version.NOMINATIM_VERSION
+ self.data_updated: Optional[dt.datetime] = None
+ self.database_version: Optional[version.NominatimVersion] = None
+
+
+async def _get_database_date(conn: AsyncConnection) -> Optional[dt.datetime]:
+ """ Query the database date.
+ """
+ sql = sqla.text('SELECT lastimportdate FROM import_status LIMIT 1')
+ result = await conn.execute(sql)
+
+ for row in result:
+ return cast(dt.datetime, row[0])
+
+ return None
+
+
+async def _get_database_version(conn: AsyncConnection) -> Optional[version.NominatimVersion]:
+ sql = sqla.text("""SELECT value FROM nominatim_properties
+ WHERE property = 'database_version'""")
+ result = await conn.execute(sql)
+
+ for row in result:
+ return version.parse_version(cast(str, row[0]))
+
+ return None
+
+
+async def get_status(engine: AsyncEngine) -> StatusResult:
+ """ Execute a status API call.
+ """
+ status = StatusResult(0, 'OK')
+ try:
+ async with engine.begin() as conn:
+ status.data_updated = await _get_database_date(conn)
+ status.database_version = await _get_database_version(conn)
+ except asyncpg.PostgresError:
+ return StatusResult(700, 'Database connection failed')
+
+ return status
database administration and querying.
"""
from typing import Optional, Any, List, Union
+import importlib
import logging
import os
import sys
def nominatim_version_text(self) -> str:
""" Program name and version number as string
"""
- text = f'Nominatim version {version.version_str()}'
+ text = f'Nominatim version {version.NOMINATIM_VERSION!s}'
if version.GIT_COMMIT_HASH is not None:
text += f' ({version.GIT_COMMIT_HASH})'
return text
"""\
Start a simple web server for serving the API.
- This command starts the built-in PHP webserver to serve the website
+ This command starts a built-in webserver to serve the website
from the current project directory. This webserver is only suitable
for testing and development. Do not use it in production setups!
+ There are different webservers available. The default 'php' engine
+ runs the classic PHP frontend. The other engines are Python servers
+ which run the new Python frontend code. This is highly experimental
+ at the moment and may not include the full API.
+
By the default, the webserver can be accessed at: http://127.0.0.1:8088
"""
group = parser.add_argument_group('Server arguments')
group.add_argument('--server', default='127.0.0.1:8088',
help='The address the server will listen to.')
+ group.add_argument('--engine', default='php',
+ choices=('php', 'sanic', 'falcon', 'starlette'),
+ help='Webserver framework to run. (default: php)')
def run(self, args: NominatimArgs) -> int:
- run_php_server(args.server, args.project_dir / 'website')
+ if args.engine == 'php':
+ run_php_server(args.server, args.project_dir / 'website')
+ else:
+ server_info = args.server.split(':', 1)
+ host = server_info[0]
+ if len(server_info) > 1:
+ if not server_info[1].isdigit():
+ raise UsageError('Invalid format for --server parameter. Use <host>:<port>')
+ port = int(server_info[1])
+ else:
+ port = 8088
+
+ if args.engine == 'sanic':
+ server_module = importlib.import_module('nominatim.server.sanic.server')
+
+ app = server_module.get_application(args.project_dir)
+ app.run(host=host, port=port, debug=True)
+ else:
+ import uvicorn # pylint: disable=import-outside-toplevel
+
+ if args.engine == 'falcon':
+ server_module = importlib.import_module('nominatim.server.falcon.server')
+ elif args.engine == 'starlette':
+ server_module = importlib.import_module('nominatim.server.starlette.server')
+
+ app = server_module.get_application(args.project_dir)
+ uvicorn.run(app, host=host, port=port)
+
return 0
from nominatim.tools.exec_utils import run_api_script
from nominatim.errors import UsageError
from nominatim.clicmd.args import NominatimArgs
+from nominatim.api import NominatimAPI
+from nominatim.apicmd.status import StatusResult
+import nominatim.result_formatter.v1 as formatting
# Do not repeat documentation of subcommand classes.
# pylint: disable=C0111
class APIStatus:
- """\
+ """
Execute API status query.
This command works exactly the same as if calling the /status endpoint on
"""
def add_args(self, parser: argparse.ArgumentParser) -> None:
+ formats = formatting.create(StatusResult).list_formats()
group = parser.add_argument_group('API parameters')
- group.add_argument('--format', default='text', choices=['text', 'json'],
+ group.add_argument('--format', default=formats[0], choices=formats,
help='Format of result')
def run(self, args: NominatimArgs) -> int:
- return _run_api('status', args, dict(format=args.format))
+ status = NominatimAPI(args.project_dir).status()
+ print(formatting.create(StatusResult).format(status, args.format))
+ return 0
# Arguments to 'serve'
server: str
+ engine: str
# Arguments to 'special-phrases
import_from_wiki: bool
from nominatim.db.connection import connect
from nominatim.db import status, properties
from nominatim.tokenizer.base import AbstractTokenizer
-from nominatim.version import version_str
+from nominatim.version import NOMINATIM_VERSION
from nominatim.clicmd.args import NominatimArgs
from nominatim.errors import UsageError
except Exception as exc: # pylint: disable=broad-except
LOG.error('Cannot determine date of database: %s', exc)
- properties.set_property(conn, 'database_version', version_str())
+ properties.set_property(conn, 'database_version', str(NOMINATIM_VERSION))
import yaml
from dotenv import dotenv_values
+from psycopg2.extensions import parse_dsn
from nominatim.typing import StrPath
from nominatim.errors import UsageError
Nominatim uses dotenv to configure the software. Configuration options
are resolved in the following order:
- * from the OS environment (or the dirctionary given in `environ`
+ * from the OS environment (or the dictionary given in `environ`)
* from the .env file in the project directory of the installation
* from the default installation in the configuration directory
return dsn
+ def get_database_params(self) -> Mapping[str, str]:
+ """ Get the configured parameters for the database connection
+ as a mapping.
+ """
+ dsn = self.DATABASE_DSN
+
+ if dsn.startswith('pgsql:'):
+ return dict((p.split('=', 1) for p in dsn[6:].split(';')))
+
+ return parse_dsn(dsn)
+
+
def get_import_style_file(self) -> Path:
""" Return the import style file as a path object. Translates the
name of the standard styles automatically into a file in the
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Helper classes and function for writing result formatting modules.
+"""
+from typing import Type, TypeVar, Dict, Mapping, List, Callable, Generic, Any
+from collections import defaultdict
+
+T = TypeVar('T') # pylint: disable=invalid-name
+FormatFunc = Callable[[T], str]
+
+class ResultFormatter(Generic[T]):
+ """ This class dispatches format calls to the appropriate formatting
+ function previously defined with the `format_func` decorator.
+ """
+
+ def __init__(self, funcs: Mapping[str, FormatFunc[T]]) -> None:
+ self.functions = funcs
+
+
+ def list_formats(self) -> List[str]:
+ """ Return a list of formats supported by this formatter.
+ """
+ return list(self.functions.keys())
+
+
+ def supports_format(self, fmt: str) -> bool:
+ """ Check if the given format is supported by this formatter.
+ """
+ return fmt in self.functions
+
+
+ def format(self, result: T, fmt: str) -> str:
+ """ Convert the given result into a string using the given format.
+
+ The format is expected to be in the list returned by
+ `list_formats()`.
+ """
+ return self.functions[fmt](result)
+
+
+class FormatDispatcher:
+ """ A factory class for result formatters.
+ """
+
+ def __init__(self) -> None:
+ self.format_functions: Dict[Type[Any], Dict[str, FormatFunc[Any]]] = defaultdict(dict)
+
+
+ def format_func(self, result_class: Type[T],
+ fmt: str) -> Callable[[FormatFunc[T]], FormatFunc[T]]:
+ """ Decorator for a function that formats a given type of result into the
+ selected format.
+ """
+ def decorator(func: FormatFunc[T]) -> FormatFunc[T]:
+ self.format_functions[result_class][fmt] = func
+ return func
+
+ return decorator
+
+
+ def __call__(self, result_class: Type[T]) -> ResultFormatter[T]:
+ """ Create an instance of a format class for the given result type.
+ """
+ return ResultFormatter(self.format_functions[result_class])
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Output formatters for API version v1.
+"""
+from typing import Dict, Any
+from collections import OrderedDict
+import json
+
+from nominatim.result_formatter.base import FormatDispatcher
+from nominatim.apicmd.status import StatusResult
+
+create = FormatDispatcher()
+
+@create.format_func(StatusResult, 'text')
+def _format_status_text(result: StatusResult) -> str:
+ if result.status:
+ return f"ERROR: {result.message}"
+
+ return 'OK'
+
+
+@create.format_func(StatusResult, 'json')
+def _format_status_json(result: StatusResult) -> str:
+ out: Dict[str, Any] = OrderedDict()
+ out['status'] = result.status
+ out['message'] = result.message
+ if result.data_updated is not None:
+ out['data_updated'] = result.data_updated.isoformat()
+ out['software_version'] = str(result.software_version)
+ if result.database_version is not None:
+ out['database_version'] = str(result.database_version)
+
+ return json.dumps(out)
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Server implementation using the falcon webserver framework.
+"""
+from typing import Type, Any, Optional, Mapping
+from pathlib import Path
+
+import falcon
+import falcon.asgi
+
+from nominatim.api import NominatimAPIAsync
+from nominatim.apicmd.status import StatusResult
+import nominatim.result_formatter.v1 as formatting
+
+CONTENT_TYPE = {
+ 'text': falcon.MEDIA_TEXT,
+ 'xml': falcon.MEDIA_XML
+}
+
+class NominatimV1:
+ """ Implementation of V1 version of the Nominatim API.
+ """
+
+ def __init__(self, project_dir: Path, environ: Optional[Mapping[str, str]]) -> None:
+ self.api = NominatimAPIAsync(project_dir, environ)
+ self.formatters = {}
+
+ for rtype in (StatusResult, ):
+ self.formatters[rtype] = formatting.create(rtype)
+
+
+ def parse_format(self, req: falcon.asgi.Request, rtype: Type[Any], default: str) -> None:
+ """ Get and check the 'format' parameter and prepare the formatter.
+ `rtype` describes the expected return type and `default` the
+ format value to assume when no parameter is present.
+ """
+ req.context.format = req.get_param('format', default=default)
+ req.context.formatter = self.formatters[rtype]
+
+ if not req.context.formatter.supports_format(req.context.format):
+ raise falcon.HTTPBadRequest(
+ description="Parameter 'format' must be one of: " +
+ ', '.join(req.context.formatter.list_formats()))
+
+
+ def format_response(self, req: falcon.asgi.Request, resp: falcon.asgi.Response,
+ result: Any) -> None:
+ """ Render response into a string according to the formatter
+ set in `parse_format()`.
+ """
+ resp.text = req.context.formatter.format(result, req.context.format)
+ resp.content_type = CONTENT_TYPE.get(req.context.format, falcon.MEDIA_JSON)
+
+
+ async def on_get_status(self, req: falcon.asgi.Request, resp: falcon.asgi.Response) -> None:
+ """ Implementation of status endpoint.
+ """
+ self.parse_format(req, StatusResult, 'text')
+
+ result = await self.api.status()
+
+ self.format_response(req, resp, result)
+ if result.status and req.context.format == 'text':
+ resp.status = 500
+
+
+def get_application(project_dir: Path,
+ environ: Optional[Mapping[str, str]] = None) -> falcon.asgi.App:
+ """ Create a Nominatim falcon ASGI application.
+ """
+ app = falcon.asgi.App()
+
+ api = NominatimV1(project_dir, environ)
+
+ app.add_route('/status', api, suffix='status')
+
+ return app
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Server implementation using the sanic webserver framework.
+"""
+from typing import Any, Optional, Mapping
+from pathlib import Path
+
+import sanic
+
+from nominatim.api import NominatimAPIAsync
+from nominatim.apicmd.status import StatusResult
+import nominatim.result_formatter.v1 as formatting
+
+api = sanic.Blueprint('NominatimAPI')
+
+CONTENT_TYPE = {
+ 'text': 'text/plain; charset=utf-8',
+ 'xml': 'text/xml; charset=utf-8'
+}
+
+def usage_error(msg: str) -> sanic.HTTPResponse:
+ """ Format the response for an error with the query parameters.
+ """
+ return sanic.response.text(msg, status=400)
+
+
+def api_response(request: sanic.Request, result: Any) -> sanic.HTTPResponse:
+ """ Render a response from the query results using the configured
+ formatter.
+ """
+ body = request.ctx.formatter.format(result, request.ctx.format)
+ return sanic.response.text(body,
+ content_type=CONTENT_TYPE.get(request.ctx.format,
+ 'application/json'))
+
+
+@api.on_request # type: ignore[misc]
+async def extract_format(request: sanic.Request) -> Optional[sanic.HTTPResponse]:
+ """ Get and check the 'format' parameter and prepare the formatter.
+ `ctx.result_type` describes the expected return type and
+ `ctx.default_format` the format value to assume when no parameter
+ is present.
+ """
+ assert request.route is not None
+ request.ctx.formatter = request.app.ctx.formatters[request.route.ctx.result_type]
+
+ request.ctx.format = request.args.get('format', request.route.ctx.default_format)
+ if not request.ctx.formatter.supports_format(request.ctx.format):
+ return usage_error("Parameter 'format' must be one of: " +
+ ', '.join(request.ctx.formatter.list_formats()))
+
+ return None
+
+
+@api.get('/status', ctx_result_type=StatusResult, ctx_default_format='text')
+async def status(request: sanic.Request) -> sanic.HTTPResponse:
+ """ Implementation of status endpoint.
+ """
+ result = await request.app.ctx.api.status()
+ response = api_response(request, result)
+
+ if request.ctx.format == 'text' and result.status:
+ response.status = 500
+
+ return response
+
+
+def get_application(project_dir: Path,
+ environ: Optional[Mapping[str, str]] = None) -> sanic.Sanic:
+ """ Create a Nominatim sanic ASGI application.
+ """
+ app = sanic.Sanic("NominatimInstance")
+
+ app.ctx.api = NominatimAPIAsync(project_dir, environ)
+ app.ctx.formatters = {}
+ for rtype in (StatusResult, ):
+ app.ctx.formatters[rtype] = formatting.create(rtype)
+
+ app.blueprint(api)
+
+ return app
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Server implementation using the starlette webserver framework.
+"""
+from typing import Any, Type, Optional, Mapping
+from pathlib import Path
+
+from starlette.applications import Starlette
+from starlette.routing import Route
+from starlette.exceptions import HTTPException
+from starlette.responses import Response
+from starlette.requests import Request
+
+from nominatim.api import NominatimAPIAsync
+from nominatim.apicmd.status import StatusResult
+import nominatim.result_formatter.v1 as formatting
+
+CONTENT_TYPE = {
+ 'text': 'text/plain; charset=utf-8',
+ 'xml': 'text/xml; charset=utf-8'
+}
+
+FORMATTERS = {
+ StatusResult: formatting.create(StatusResult)
+}
+
+
+def parse_format(request: Request, rtype: Type[Any], default: str) -> None:
+ """ Get and check the 'format' parameter and prepare the formatter.
+ `rtype` describes the expected return type and `default` the
+ format value to assume when no parameter is present.
+ """
+ fmt = request.query_params.get('format', default=default)
+ fmtter = FORMATTERS[rtype]
+
+ if not fmtter.supports_format(fmt):
+ raise HTTPException(400, detail="Parameter 'format' must be one of: " +
+ ', '.join(fmtter.list_formats()))
+
+ request.state.format = fmt
+ request.state.formatter = fmtter
+
+
+def format_response(request: Request, result: Any) -> Response:
+ """ Render response into a string according to the formatter
+ set in `parse_format()`.
+ """
+ fmt = request.state.format
+ return Response(request.state.formatter.format(result, fmt),
+ media_type=CONTENT_TYPE.get(fmt, 'application/json'))
+
+
+async def on_status(request: Request) -> Response:
+ """ Implementation of status endpoint.
+ """
+ parse_format(request, StatusResult, 'text')
+ result = await request.app.state.API.status()
+ response = format_response(request, result)
+
+ if request.state.format == 'text' and result.status:
+ response.status_code = 500
+
+ return response
+
+
+V1_ROUTES = [
+ Route('/status', endpoint=on_status)
+]
+
+def get_application(project_dir: Path,
+ environ: Optional[Mapping[str, str]] = None) -> Starlette:
+ """ Create a Nominatim falcon ASGI application.
+ """
+ app = Starlette(debug=True, routes=V1_ROUTES)
+
+ app.state.API = NominatimAPIAsync(project_dir, environ)
+
+ return app
from nominatim.config import Configuration
from nominatim.db.connection import connect
from nominatim.typing import DictCursorResults
-from nominatim.version import version_str
+from nominatim.version import NOMINATIM_VERSION
def convert_version(ver_tup: Tuple[int, int]) -> str:
**Software Environment:**
- Python version: {sys.version}
- - Nominatim version: {version_str()}
- - PostgreSQL version: {postgresql_ver}
+ - Nominatim version: {NOMINATIM_VERSION!s}
+ - PostgreSQL version: {postgresql_ver}
- PostGIS version: {postgis_ver}
- OS: {os_name_info()}
from nominatim.config import Configuration
from nominatim.typing import StrPath
-from nominatim.version import version_str
+from nominatim.version import NOMINATIM_VERSION
from nominatim.db.connection import get_pg_env
LOG = logging.getLogger()
def get_url(url: str) -> str:
""" Get the contents from the given URL and return it as a UTF-8 string.
"""
- headers = {"User-Agent": f"Nominatim/{version_str()}"}
+ headers = {"User-Agent": f"Nominatim/{NOMINATIM_VERSION!s}"}
try:
request = urlrequest.Request(url, headers=headers)
from nominatim.config import Configuration
from nominatim.db import properties
from nominatim.db.connection import connect, Connection
-from nominatim.version import NOMINATIM_VERSION, version_str
+from nominatim.version import NominatimVersion, NOMINATIM_VERSION, parse_version
from nominatim.tools import refresh
from nominatim.tokenizer import factory as tokenizer_factory
from nominatim.errors import UsageError
LOG = logging.getLogger()
-VersionTuple = Tuple[int, int, int, int]
-
-_MIGRATION_FUNCTIONS : List[Tuple[VersionTuple, Callable[..., None]]] = []
+_MIGRATION_FUNCTIONS : List[Tuple[NominatimVersion, Callable[..., None]]] = []
def migrate(config: Configuration, paths: Any) -> int:
""" Check for the current database version and execute migrations,
db_version_str = None
if db_version_str is not None:
- parts = db_version_str.split('.')
- db_version = tuple(int(x) for x in parts[:2] + parts[2].split('-'))
+ db_version = parse_version(db_version_str)
if db_version == NOMINATIM_VERSION:
LOG.warning("Database already at latest version (%s)", db_version_str)
for version, func in _MIGRATION_FUNCTIONS:
if db_version <= version:
title = func.__doc__ or ''
- LOG.warning("Running: %s (%s)", title.split('\n', 1)[0],
- version_str(version))
+ LOG.warning("Running: %s (%s)", title.split('\n', 1)[0], version)
kwargs = dict(conn=conn, config=config, paths=paths)
func(**kwargs)
conn.commit()
tokenizer = tokenizer_factory.get_tokenizer_for_db(config)
tokenizer.update_sql_functions(config)
- properties.set_property(conn, 'database_version', version_str())
+ properties.set_property(conn, 'database_version', str(NOMINATIM_VERSION))
conn.commit()
return 0
-def _guess_version(conn: Connection) -> VersionTuple:
+def _guess_version(conn: Connection) -> NominatimVersion:
""" Guess a database version when there is no property table yet.
Only migrations for 3.6 and later are supported, so bail out
when the version seems older.
'prior to 3.6.0. Automatic migration not possible.')
raise UsageError('Migration not possible.')
- return (3, 5, 0, 99)
+ return NominatimVersion(3, 5, 0, 99)
there.
"""
def decorator(func: Callable[..., None]) -> Callable[..., None]:
- _MIGRATION_FUNCTIONS.append(((major, minor, patch, dbpatch), func))
+ version = (NominatimVersion(major, minor, patch, dbpatch))
+ _MIGRATION_FUNCTIONS.append((version, func))
return func
return decorator
from nominatim.db.connection import Connection, connect
from nominatim.db.utils import execute_file
from nominatim.db.sql_preprocessor import SQLPreprocessor
-from nominatim.version import version_str
+from nominatim.version import NOMINATIM_VERSION
LOG = logging.getLogger()
@define('CONST_Debug', $_GET['debug'] ?? false);
@define('CONST_LibDir', '{config.lib_dir.php}');
@define('CONST_TokenizerDir', '{config.project_dir / 'tokenizer'}');
- @define('CONST_NominatimVersion', '{version_str()}');
+ @define('CONST_NominatimVersion', '{NOMINATIM_VERSION!s}');
""")
"""
Version information for Nominatim.
"""
-from typing import Optional, Tuple
+from typing import Optional, NamedTuple
-# Version information: major, minor, patch level, database patch level
-#
-# The first three numbers refer to the last released version.
-#
-# The database patch level tracks important changes between releases
-# and must always be increased when there is a change to the database or code
-# that requires a migration.
-#
-# When adding a migration on the development branch, raise the patch level
-# to 99 to make sure that the migration is applied when updating from a
-# patch release to the next minor version. Patch releases usually shouldn't
-# have migrations in them. When they are needed, then make sure that the
-# migration can be reapplied and set the migration version to the appropriate
-# patch level when cherry-picking the commit with the migration.
-#
-# Released versions always have a database patch level of 0.
-NOMINATIM_VERSION = (4, 2, 99, 0)
+class NominatimVersion(NamedTuple):
+ """ Version information for Nominatim. We follow semantic versioning.
+
+ Major, minor and patch_level refer to the last released version.
+ The database patch level tracks important changes between releases
+ and must always be increased when there is a change to the database or code
+ that requires a migration.
+
+ When adding a migration on the development branch, raise the patch level
+ to 99 to make sure that the migration is applied when updating from a
+ patch release to the next minor version. Patch releases usually shouldn't
+ have migrations in them. When they are needed, then make sure that the
+ migration can be reapplied and set the migration version to the appropriate
+ patch level when cherry-picking the commit with the migration.
+ """
+
+ major: int
+ minor: int
+ patch_level: int
+ db_patch_level: int
+
+ def __str__(self) -> str:
+ return f"{self.major}.{self.minor}.{self.patch_level}-{self.db_patch_level}"
+
+
+NOMINATIM_VERSION = NominatimVersion(4, 2, 99, 0)
POSTGRESQL_REQUIRED_VERSION = (9, 6)
POSTGIS_REQUIRED_VERSION = (2, 2)
GIT_COMMIT_HASH : Optional[str] = None
-# pylint: disable=consider-using-f-string
-def version_str(version:Tuple[int, int, int, int] = NOMINATIM_VERSION) -> str:
- """
- Return a human-readable string of the version.
+def parse_version(version: str) -> NominatimVersion:
+ """ Parse a version string into a version consisting of a tuple of
+ four ints: major, minor, patch level, database patch level
+
+ This is the reverse operation of `version_str()`.
"""
- return '{}.{}.{}-{}'.format(*version)
+ parts = version.split('.')
+ return NominatimVersion(*[int(x) for x in parts[:2] + parts[2].split('-')])
'SERVER_MODULE_PATH' : None,
'TOKENIZER' : None, # Test with a custom tokenizer
'STYLE' : 'extratags',
+ 'API_ENGINE': 'php',
'PHPCOV' : False, # set to output directory to enable code coverage
}
# Copyright (C) 2022 by the Nominatim developer community.
# For a full list of authors see the git log.
from pathlib import Path
+import importlib
import sys
import tempfile
self.api_db_done = False
self.website_dir = None
+ self.api_engine = None
+ if config['API_ENGINE'] != 'php':
+ if not hasattr(self, f"create_api_request_func_{config['API_ENGINE']}"):
+ raise RuntimeError(f"Unknown API engine '{config['API_ENGINE']}'")
+ self.api_engine = getattr(self, f"create_api_request_func_{config['API_ENGINE']}")()
+
def connect_database(self, dbname):
""" Return a connection to the database with the given name.
Uses configured host, user and port.
WHERE class='place' and type='houses'
and osm_type='W'
and ST_GeometryType(geometry) = 'ST_LineString'""")
+
+
+ def create_api_request_func_starlette(self):
+ import nominatim.server.starlette.server
+ from asgi_lifespan import LifespanManager
+ import httpx
+
+ async def _request(endpoint, params, project_dir, environ):
+ app = nominatim.server.starlette.server.get_application(project_dir, environ)
+
+ async with LifespanManager(app):
+ async with httpx.AsyncClient(app=app, base_url="http://nominatim.test") as client:
+ response = await client.get(f"/{endpoint}", params=params)
+
+ return response.text, response.status_code
+
+ return _request
+
+
+ def create_api_request_func_sanic(self):
+ import nominatim.server.sanic.server
+
+ async def _request(endpoint, params, project_dir, environ):
+ app = nominatim.server.sanic.server.get_application(project_dir, environ)
+
+ _, response = await app.asgi_client.get(f"/{endpoint}", params=params)
+
+ return response.text, response.status_code
+
+ return _request
+
+
+ def create_api_request_func_falcon(self):
+ import nominatim.server.falcon.server
+ import falcon.testing
+
+ async def _request(endpoint, params, project_dir, environ):
+ app = nominatim.server.falcon.server.get_application(project_dir, environ)
+
+ async with falcon.testing.ASGIConductor(app) as conductor:
+ response = await conductor.get(f"/{endpoint}", params=params)
+
+ return response.text, response.status_code
+
+ return _request
+
+
+
Queries may either be run directly via PHP using the query script
or via the HTTP interface using php-cgi.
"""
+from pathlib import Path
import json
import os
import re
import logging
+import asyncio
from urllib.parse import urlencode
from utils import run_script
for h in context.table.headings:
params[h] = context.table[0][h]
+ if context.nominatim.api_engine is None:
+ return send_api_query_php(endpoint, params, context)
+
+ return asyncio.run(context.nominatim.api_engine(endpoint, params,
+ Path(context.nominatim.website_dir.name),
+ context.nominatim.test_env))
+
+
+
+def send_api_query_php(endpoint, params, context):
env = dict(BASE_SERVER_ENV)
env['QUERY_STRING'] = urlencode(params)
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Helper fixtures for API call tests.
+"""
+from pathlib import Path
+import pytest
+import time
+
+from nominatim.api import NominatimAPI
+
+@pytest.fixture
+def apiobj(temp_db):
+ """ Create an asynchronous SQLAlchemy engine for the test DB.
+ """
+ api = NominatimAPI(Path('/invalid'), {})
+ yield api
+ api.close()
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Tests for the status API call.
+"""
+from pathlib import Path
+import datetime as dt
+import pytest
+
+from nominatim.version import NOMINATIM_VERSION, NominatimVersion
+from nominatim.api import NominatimAPI
+
+def test_status_no_extra_info(apiobj, table_factory):
+ table_factory('import_status',
+ definition="lastimportdate timestamp with time zone NOT NULL")
+ table_factory('nominatim_properties',
+ definition='property TEXT, value TEXT')
+
+ result = apiobj.status()
+
+ assert result.status == 0
+ assert result.message == 'OK'
+ assert result.software_version == NOMINATIM_VERSION
+ assert result.database_version is None
+ assert result.data_updated is None
+
+
+def test_status_full(apiobj, table_factory):
+ table_factory('import_status',
+ definition="lastimportdate timestamp with time zone NOT NULL",
+ content=(('2022-12-07 15:14:46+01',),))
+ table_factory('nominatim_properties',
+ definition='property TEXT, value TEXT',
+ content=(('database_version', '99.5.4-2'), ))
+
+ result = apiobj.status()
+
+ assert result.status == 0
+ assert result.message == 'OK'
+ assert result.software_version == NOMINATIM_VERSION
+ assert result.database_version == NominatimVersion(99, 5, 4, 2)
+ assert result.data_updated == dt.datetime(2022, 12, 7, 14, 14, 46, 0, tzinfo=dt.timezone.utc)
+
+
+def test_status_database_not_found(monkeypatch):
+ monkeypatch.setenv('NOMINATIM_DATABASE_DSN', 'dbname=rgjdfkgjedkrgdfkngdfkg')
+
+ api = NominatimAPI(Path('/invalid'), {})
+
+ result = api.status()
+
+ assert result.status == 700
+ assert result.message == 'Database connection failed'
+ assert result.software_version == NOMINATIM_VERSION
+ assert result.database_version is None
+ assert result.data_updated is None
correct functionionality. They use a lot of monkeypatching to avoid executing
the actual functions.
"""
+import importlib
import pytest
import nominatim.indexer.indexer
assert mock.called == 1
-def test_cli_serve_command(cli_call, mock_func_factory):
+def test_cli_serve_php(cli_call, mock_func_factory):
func = mock_func_factory(nominatim.cli, 'run_php_server')
cli_call('serve') == 0
assert func.called == 1
+def test_cli_serve_sanic(cli_call, mock_func_factory):
+ mod = pytest.importorskip("sanic")
+ func = mock_func_factory(mod.Sanic, "run")
+
+ cli_call('serve', '--engine', 'sanic') == 0
+
+ assert func.called == 1
+
+
+def test_cli_serve_starlette_custom_server(cli_call, mock_func_factory):
+ pytest.importorskip("starlette")
+ mod = pytest.importorskip("uvicorn")
+ func = mock_func_factory(mod, "run")
+
+ cli_call('serve', '--engine', 'starlette', '--server', 'foobar:4545') == 0
+
+ assert func.called == 1
+ assert func.last_kwargs['host'] == 'foobar'
+ assert func.last_kwargs['port'] == 4545
+
+
+def test_cli_serve_starlette_custom_server_bad_port(cli_call, mock_func_factory):
+ pytest.importorskip("starlette")
+ mod = pytest.importorskip("uvicorn")
+ func = mock_func_factory(mod, "run")
+
+ cli_call('serve', '--engine', 'starlette', '--server', 'foobar:45:45') == 1
+
+
+@pytest.mark.parametrize("engine", ['falcon', 'starlette'])
+def test_cli_serve_uvicorn_based(cli_call, engine, mock_func_factory):
+ pytest.importorskip(engine)
+ mod = pytest.importorskip("uvicorn")
+ func = mock_func_factory(mod, "run")
+
+ cli_call('serve', '--engine', engine) == 0
+
+ assert func.called == 1
+ assert func.last_kwargs['host'] == '127.0.0.1'
+ assert func.last_kwargs['port'] == 8088
+
def test_cli_export_command(cli_call, mock_run_legacy):
assert cli_call('export', '--output-all-postcodes') == 0
"""
Tests for API access commands of command-line interface wrapper.
"""
+import json
import pytest
import nominatim.clicmd.api
+import nominatim.api
+from nominatim.apicmd.status import StatusResult
@pytest.mark.parametrize("endpoint", (('search', 'reverse', 'lookup', 'details', 'status')))
('details', '--node', '1'),
('details', '--way', '1'),
('details', '--relation', '1'),
- ('details', '--place_id', '10001'),
- ('status',)])
-class TestCliApiCall:
+ ('details', '--place_id', '10001')])
+class TestCliApiCallPhp:
@pytest.fixture(autouse=True)
def setup_cli_call(self, params, cli_call, mock_func_factory, tmp_path):
assert self.run_nominatim() == 1
+class TestCliStatusCall:
+
+ @pytest.fixture(autouse=True)
+ def setup_status_mock(self, monkeypatch):
+ monkeypatch.setattr(nominatim.api.NominatimAPI, 'status',
+ lambda self: StatusResult(200, 'OK'))
+
+
+ def test_status_simple(self, cli_call, tmp_path):
+ result = cli_call('status', '--project-dir', str(tmp_path))
+
+ assert result == 0
+
+
+ def test_status_json_format(self, cli_call, tmp_path, capsys):
+ result = cli_call('status', '--project-dir', str(tmp_path),
+ '--format', 'json')
+
+ assert result == 0
+
+ json.loads(capsys.readouterr().out)
+
+
QUERY_PARAMS = {
'search': ('--query', 'somewhere'),
'reverse': ('--lat', '20', '--lon', '30'),
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Tests for formatting results for the V1 API.
+"""
+import datetime as dt
+import pytest
+
+import nominatim.result_formatter.v1 as format_module
+from nominatim.apicmd.status import StatusResult
+from nominatim.version import NOMINATIM_VERSION
+
+STATUS_FORMATS = {'text', 'json'}
+
+class TestStatusResultFormat:
+
+
+ @pytest.fixture(autouse=True)
+ def make_formatter(self):
+ self.formatter = format_module.create(StatusResult)
+
+
+ def test_format_list(self):
+ assert set(self.formatter.list_formats()) == STATUS_FORMATS
+
+
+ @pytest.mark.parametrize('fmt', list(STATUS_FORMATS))
+ def test_supported(self, fmt):
+ assert self.formatter.supports_format(fmt)
+
+
+ def test_unsupported(self):
+ assert not self.formatter.supports_format('gagaga')
+
+
+ def test_format_text(self):
+ assert self.formatter.format(StatusResult(0, 'message here'), 'text') == 'OK'
+
+
+ def test_format_text(self):
+ assert self.formatter.format(StatusResult(500, 'message here'), 'text') == 'ERROR: message here'
+
+
+ def test_format_json_minimal(self):
+ status = StatusResult(700, 'Bad format.')
+
+ result = self.formatter.format(status, 'json')
+
+ assert result == '{"status": 700, "message": "Bad format.", "software_version": "%s"}' % (NOMINATIM_VERSION, )
+
+
+ def test_format_json_full(self):
+ status = StatusResult(0, 'OK')
+ status.data_updated = dt.datetime(2010, 2, 7, 20, 20, 3, 0, tzinfo=dt.timezone.utc)
+ status.database_version = '5.6'
+
+ result = self.formatter.format(status, 'json')
+
+ assert result == '{"status": 0, "message": "OK", "data_updated": "2010-02-07T20:20:03+00:00", "software_version": "%s", "database_version": "5.6"}' % (NOMINATIM_VERSION, )
+++ /dev/null
-#!/bin/bash -e
-#
-# hacks for broken vagrant box #DOCS:
-sudo rm -f /var/lib/dpkg/lock #DOCS:
-export APT_LISTCHANGES_FRONTEND=none #DOCS:
-export DEBIAN_FRONTEND=noninteractive #DOCS:
-
-#
-# *Note:* these installation instructions are also available in executable
-# form for use with vagrant under vagrant/Install-on-Ubuntu-18.sh.
-#
-# Installing the Required Software
-# ================================
-#
-# These instructions expect that you have a freshly installed Ubuntu 18.04.
-#
-# Make sure all packages are up-to-date by running:
-#
-
- sudo apt update -qq
-
-# Now you can install all packages needed for Nominatim:
-
- sudo apt install -y php-cgi
- sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
- libboost-filesystem-dev libexpat1-dev zlib1g-dev\
- libbz2-dev libpq-dev liblua5.3-dev lua5.3\
- postgresql-10-postgis-2.4 \
- postgresql-contrib-10 postgresql-10-postgis-scripts \
- php-cli php-pgsql php-intl libicu-dev python3-pip \
- python3-psutil python3-jinja2 python3-yaml python3-icu git
-
-# Some of the Python packages that come with Ubuntu 18.04 are too old, so
-# install the latest version from pip:
-
- pip3 install --user python-dotenv datrie pyyaml psycopg2-binary
-
-#
-# System Configuration
-# ====================
-#
-# The following steps are meant to configure a fresh Ubuntu installation
-# for use with Nominatim. You may skip some of the steps if you have your
-# OS already configured.
-#
-# Creating Dedicated User Accounts
-# --------------------------------
-#
-# Nominatim will run as a global service on your machine. It is therefore
-# best to install it under its own separate user account. In the following
-# we assume this user is called nominatim and the installation will be in
-# /srv/nominatim. To create the user and directory run:
-#
-# sudo useradd -d /srv/nominatim -s /bin/bash -m nominatim
-#
-# You may find a more suitable location if you wish.
-#
-# To be able to copy and paste instructions from this manual, export
-# user name and home directory now like this:
-#
-if [ "x$USERNAME" == "x" ]; then #DOCS:
- export USERNAME=vagrant #DOCS: export USERNAME=nominatim
- export USERHOME=/home/vagrant #DOCS: export USERHOME=/srv/nominatim
-fi #DOCS:
-#
-# **Never, ever run the installation as a root user.** You have been warned.
-#
-# Make sure that system servers can read from the home directory:
-
- chmod a+x $USERHOME
-
-# Setting up PostgreSQL
-# ---------------------
-#
-# Tune the postgresql configuration, which is located in
-# `/etc/postgresql/10/main/postgresql.conf`. See section *Postgres Tuning* in
-# [the installation page](../admin/Installation.md#postgresql-tuning)
-# for the parameters to change.
-#
-# Restart the postgresql service after updating this config file.
-
-if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
- sudo pg_ctlcluster 10 main start #DOCS:
-else #DOCS:
- sudo systemctl restart postgresql
-fi #DOCS:
-
-#
-# Finally, we need to add two postgres users: one for the user that does
-# the import and another for the webserver which should access the database
-# for reading only:
-#
-
- sudo -u postgres createuser -s $USERNAME
- sudo -u postgres createuser www-data
-
-#
-# Installing Nominatim
-# ====================
-#
-# Building and Configuration
-# --------------------------
-#
-# Get the source code from Github and change into the source directory
-#
-if [ "x$1" == "xyes" ]; then #DOCS: :::sh
- cd $USERHOME
- git clone --recursive https://github.com/openstreetmap/Nominatim.git
- cd Nominatim
-else #DOCS:
- cd $USERHOME/Nominatim #DOCS:
-fi #DOCS:
-
-# When installing the latest source from github, you also need to
-# download the country grid:
-
-if [ ! -f data/country_osm_grid.sql.gz ]; then #DOCS: :::sh
- wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
-fi #DOCS:
-
-# The code must be built in a separate directory. Create this directory,
-# then configure and build Nominatim in there:
-
- mkdir $USERHOME/build
- cd $USERHOME/build
- cmake $USERHOME/Nominatim
- make
- sudo make install
-
-
-# Nominatim is now ready to use. You can continue with
-# [importing a database from OSM data](../admin/Import.md). If you want to set up
-# a webserver first, continue reading.
-#
-# Setting up a webserver
-# ======================
-#
-# The webserver should serve the php scripts from the website directory of your
-# [project directory](../admin/Import.md#creating-the-project-directory).
-# This directory needs to exist when being configured.
-# Therefore set up a project directory and create the website directory:
-
- mkdir $USERHOME/nominatim-project
- mkdir $USERHOME/nominatim-project/website
-
-# The import process will populate the directory later.
-#
-# Option 1: Using Apache
-# ----------------------
-#
-if [ "x$2" == "xinstall-apache" ]; then #DOCS:
-#
-# Apache has a PHP module that can be used to serve Nominatim. To install them
-# run:
-
- sudo apt install -y apache2 libapache2-mod-php
-
-# You need to create an alias to the website directory in your apache
-# configuration. Add a separate nominatim configuration to your webserver:
-
-#DOCS:```sh
-sudo tee /etc/apache2/conf-available/nominatim.conf << EOFAPACHECONF
-<Directory "$USERHOME/nominatim-project/website">
- Options FollowSymLinks MultiViews
- AddType text/html .php
- DirectoryIndex search.php
- Require all granted
-</Directory>
-
-Alias /nominatim $USERHOME/nominatim-project/website
-EOFAPACHECONF
-#DOCS:```
-
-#
-# Then enable the configuration with
-#
-
- sudo a2enconf nominatim
-
-# and restart apache:
-
-if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
- sudo apache2ctl start #DOCS:
-else #DOCS:
- sudo systemctl restart apache2
-fi #DOCS:
-
-# The Nominatim API is now available at `http://localhost/nominatim/`.
-
-fi #DOCS:
-
-#
-# Option 2: Using nginx
-# ---------------------
-#
-if [ "x$2" == "xinstall-nginx" ]; then #DOCS:
-
-# Nginx has no native support for php scripts. You need to set up php-fpm for
-# this purpose. First install nginx and php-fpm:
-
- sudo apt install -y nginx php-fpm
-
-# You need to configure php-fpm to listen on a Unix socket.
-
-#DOCS:```sh
-sudo tee /etc/php/7.2/fpm/pool.d/www.conf << EOF_PHP_FPM_CONF
-[www]
-; Replace the tcp listener and add the unix socket
-listen = /var/run/php-fpm-nominatim.sock
-
-; Ensure that the daemon runs as the correct user
-listen.owner = www-data
-listen.group = www-data
-listen.mode = 0666
-
-; Unix user of FPM processes
-user = www-data
-group = www-data
-
-; Choose process manager type (static, dynamic, ondemand)
-pm = ondemand
-pm.max_children = 5
-EOF_PHP_FPM_CONF
-#DOCS:```
-
-# Then create a Nginx configuration to forward http requests to that socket.
-
-#DOCS:```sh
-sudo tee /etc/nginx/sites-available/default << EOF_NGINX_CONF
-server {
- listen 80 default_server;
- listen [::]:80 default_server;
-
- root $USERHOME/nominatim-project/website;
- index search.php index.html;
- location / {
- try_files \$uri \$uri/ @php;
- }
-
- location @php {
- fastcgi_param SCRIPT_FILENAME "\$document_root\$uri.php";
- fastcgi_param PATH_TRANSLATED "\$document_root\$uri.php";
- fastcgi_param QUERY_STRING \$args;
- fastcgi_pass unix:/var/run/php-fpm-nominatim.sock;
- fastcgi_index index.php;
- include fastcgi_params;
- }
-
- location ~ [^/]\.php(/|$) {
- fastcgi_split_path_info ^(.+?\.php)(/.*)$;
- if (!-f \$document_root\$fastcgi_script_name) {
- return 404;
- }
- fastcgi_pass unix:/var/run/php-fpm-nominatim.sock;
- fastcgi_index search.php;
- include fastcgi.conf;
- }
-}
-EOF_NGINX_CONF
-#DOCS:```
-
-#
-# Enable the configuration and restart Nginx
-#
-
-if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
- sudo /usr/sbin/php-fpm7.2 --nodaemonize --fpm-config /etc/php/7.2/fpm/php-fpm.conf & #DOCS:
- sudo /usr/sbin/nginx & #DOCS:
-else #DOCS:
- sudo systemctl restart php7.2-fpm nginx
-fi #DOCS:
-
-# The Nominatim API is now available at `http://localhost/`.
-
-
-
-fi #DOCS:
postgresql-12-postgis-3 \
postgresql-contrib-12 postgresql-12-postgis-3-scripts \
php-cli php-pgsql php-intl libicu-dev python3-dotenv \
- python3-psycopg2 python3-psutil python3-jinja2 \
+ python3-psycopg2 python3-psutil python3-jinja2 python3-pip \
python3-icu python3-datrie python3-yaml git
+# Some of the Python packages that come with Ubuntu 20.04 are too old, so
+# install the latest version from pip:
+
+ pip3 install --user sqlalchemy asyncpg
+
+
#
# System Configuration
# ====================
postgresql-contrib-14 postgresql-14-postgis-3-scripts \
php-cli php-pgsql php-intl libicu-dev python3-dotenv \
python3-psycopg2 python3-psutil python3-jinja2 \
- python3-icu python3-datrie git
+ python3-icu python3-datrie python3-sqlalchemy \
+ python3-asyncpg git
#
# System Configuration
projects / nominatim.git / commitdiff