From: Sarah Hoffmann Date: Sat, 13 Aug 2022 17:04:39 +0000 (+0200) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~94 X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/6b55ab5581a3f6541d16fb7544ce8fd57dc2a673?hp=bf95cafb9ade64e22566ee47acbc3f12e1baca37 Merge remote-tracking branch 'upstream/master' --- diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 481ec767..a26ad000 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -265,6 +265,10 @@ jobs: run: nominatim --version working-directory: /home/nominatim/nominatim-project + - name: Collect host OS information + run: nominatim admin --collect-os-info + working-directory: /home/nominatim/nominatim-project + - name: Import run: nominatim import --osm-file ../test.pbf working-directory: /home/nominatim/nominatim-project diff --git a/lib-sql/functions/interpolation.sql b/lib-sql/functions/interpolation.sql index 3a994711..96a105ae 100644 --- a/lib-sql/functions/interpolation.sql +++ b/lib-sql/functions/interpolation.sql @@ -15,7 +15,7 @@ DECLARE location RECORD; waynodes BIGINT[]; BEGIN - IF akeys(in_address) != ARRAY['interpolation'] THEN + IF in_address ? 'street' or in_address ? 'place' THEN RETURN in_address; END IF; diff --git a/nominatim/clicmd/admin.py b/nominatim/clicmd/admin.py index ad900579..c3ba9c0b 100644 --- a/nominatim/clicmd/admin.py +++ b/nominatim/clicmd/admin.py @@ -20,6 +20,7 @@ from nominatim.clicmd.args import NominatimArgs LOG = logging.getLogger() + class AdminFuncs: """\ Analyse and maintain the database. @@ -36,6 +37,8 @@ class AdminFuncs: help='Migrate the database to a new software version') objs.add_argument('--analyse-indexing', action='store_true', help='Print performance analysis of the indexing process') + objs.add_argument('--collect-os-info', action="store_true", + help="Generate a report about the host system information") group = parser.add_argument_group('Arguments for cache warming') group.add_argument('--search-only', action='store_const', dest='target', const='search', @@ -70,8 +73,13 @@ class AdminFuncs: from ..tools import migration return migration.migrate(args.config, args) - return 1 + if args.collect_os_info: + LOG.warning("Reporting System Information") + from ..tools import collect_os_info + collect_os_info.report_system_information(args.config) + return 0 + return 1 def _warm(self, args: NominatimArgs) -> int: LOG.warning('Warming database caches') diff --git a/nominatim/clicmd/args.py b/nominatim/clicmd/args.py index c976f394..4457db5f 100644 --- a/nominatim/clicmd/args.py +++ b/nominatim/clicmd/args.py @@ -76,6 +76,7 @@ class NominatimArgs: warm: bool check_database: bool migrate: bool + collect_os_info: bool analyse_indexing: bool target: Optional[str] osm_id: Optional[str] diff --git a/nominatim/tools/collect_os_info.py b/nominatim/tools/collect_os_info.py new file mode 100644 index 00000000..9d76f229 --- /dev/null +++ b/nominatim/tools/collect_os_info.py @@ -0,0 +1,167 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Collection of host system information including software versions, memory, +storage, and database configuration. +""" +import os +import subprocess +import sys +from pathlib import Path +from typing import List, Optional, Tuple, Union, cast + +import psutil +from psycopg2.extensions import make_dsn, parse_dsn + +from nominatim.config import Configuration +from nominatim.db.connection import connect +from nominatim.typing import DictCursorResults +from nominatim.version import version_str + + +def convert_version(ver_tup: Tuple[int, int]) -> str: + """converts tuple version (ver_tup) to a string representation""" + return ".".join(map(str, ver_tup)) + + +def friendly_memory_string(mem: float) -> str: + """Create a user friendly string for the amount of memory specified as mem""" + mem_magnitude = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") + mag = 0 + # determine order of magnitude + while mem > 1000: + mem /= 1000 + mag += 1 + + return f"{mem:.1f} {mem_magnitude[mag]}" + + +def run_command(cmd: Union[str, List[str]]) -> str: + """Runs a command using the shell and returns the output from stdout""" + try: + if sys.version_info < (3, 7): + cap_out = subprocess.run(cmd, stdout=subprocess.PIPE, check=False) + else: + cap_out = subprocess.run(cmd, capture_output=True, check=False) + return cap_out.stdout.decode("utf-8") + except FileNotFoundError: + # non-Linux system should end up here + return f"Unknown (unable to find the '{cmd}' command)" + + +def os_name_info() -> str: + """Obtain Operating System Name (and possibly the version)""" + os_info = None + # man page os-release(5) details meaning of the fields + if Path("/etc/os-release").is_file(): + os_info = from_file_find_line_portion( + "/etc/os-release", "PRETTY_NAME", "=") + # alternative location + elif Path("/usr/lib/os-release").is_file(): + os_info = from_file_find_line_portion( + "/usr/lib/os-release", "PRETTY_NAME", "=" + ) + + # fallback on Python's os name + if os_info is None or os_info == "": + os_info = os.name + + # if the above is insufficient, take a look at neofetch's approach to OS detection + return os_info + + +# Note: Intended to be used on informational files like /proc +def from_file_find_line_portion( + filename: str, start: str, sep: str, fieldnum: int = 1 +) -> Optional[str]: + """open filename, finds the line starting with the 'start' string. + Splits the line using seperator and returns a "fieldnum" from the split.""" + with open(filename, encoding='utf8') as file: + result = "" + for line in file: + if line.startswith(start): + result = line.split(sep)[fieldnum].strip() + return result + + +def get_postgresql_config(version: int) -> str: + """Retrieve postgres configuration file""" + try: + with open(f"/etc/postgresql/{version}/main/postgresql.conf", encoding='utf8') as file: + db_config = file.read() + file.close() + return db_config + except IOError: + return f"**Could not read '/etc/postgresql/{version}/main/postgresql.conf'**" + + +def report_system_information(config: Configuration) -> None: + """Generate a report about the host system including software versions, memory, + storage, and database configuration.""" + + with connect(make_dsn(config.get_libpq_dsn(), dbname='postgres')) as conn: + postgresql_ver: str = convert_version(conn.server_version_tuple()) + + with conn.cursor() as cur: + cur.execute(f""" + SELECT datname FROM pg_catalog.pg_database + WHERE datname='{parse_dsn(config.get_libpq_dsn())['dbname']}'""") + nominatim_db_exists = cast(Optional[DictCursorResults], cur.fetchall()) + if nominatim_db_exists: + with connect(config.get_libpq_dsn()) as conn: + postgis_ver: str = convert_version(conn.postgis_version_tuple()) + else: + postgis_ver = "Unable to connect to database" + + postgresql_config: str = get_postgresql_config(int(float(postgresql_ver))) + + # Note: psutil.disk_partitions() is similar to run_command("lsblk") + + # Note: run_command("systemd-detect-virt") only works on Linux, on other OSes + # should give a message: "Unknown (unable to find the 'systemd-detect-virt' command)" + + # Generates the Markdown report. + + report = f""" + **Instructions** + Use this information in your issue report at https://github.com/osm-search/Nominatim/issues + Redirect the output to a file: + $ ./collect_os_info.py > report.md + + + **Software Environment:** + - Python version: {sys.version} + - Nominatim version: {version_str()} + - PostgreSQL version: {postgresql_ver} + - PostGIS version: {postgis_ver} + - OS: {os_name_info()} + + + **Hardware Configuration:** + - RAM: {friendly_memory_string(psutil.virtual_memory().total)} + - number of CPUs: {psutil.cpu_count(logical=False)} + - bare metal/AWS/other cloud service (per systemd-detect-virt(1)): {run_command("systemd-detect-virt")} + - type and size of disks: + **`df -h` - df - report file system disk space usage: ** + ``` + {run_command(["df", "-h"])} + ``` + + **lsblk - list block devices: ** + ``` + {run_command("lsblk")} + ``` + + + **Postgresql Configuration:** + ``` + {postgresql_config} + ``` + **Notes** + Please add any notes about anything above anything above that is incorrect. +""" + print(report) diff --git a/test/bdd/db/import/interpolation.feature b/test/bdd/db/import/interpolation.feature index 8c136d07..b34d5adb 100644 --- a/test/bdd/db/import/interpolation.feature +++ b/test/bdd/db/import/interpolation.feature @@ -403,3 +403,33 @@ Feature: Import of address interpolations Then results contain | ID | osm_type | osm_id | type | display_name | | 0 | node | 1 | house | 0 | + + Scenario: Parenting of interpolation with additional tags + Given the grid + | 1 | | | | | | + | | | | | | | + | | 8 | | | 9 | | + | | | | | | | + | 2 | | | | | 3 | + Given the places + | osm | class | type | housenr | addr+street | + | N8 | place | house | 10 | Horiz St | + | N9 | place | house | 16 | Horiz St | + And the places + | osm | class | type | name | geometry | + | W1 | highway | residential | Vert St | 1,2 | + | W2 | highway | residential | Horiz St | 2,3 | + And the places + | osm | class | type | addr+interpolation | addr+inclusion | geometry | + | W10 | place | houses | even | actual | 8,9 | + And the ways + | id | nodes | + | 10 | 8,9 | + When importing + Then placex contains + | object | parent_place_id | + | N8 | W2 | + | N9 | W2 | + And W10 expands to interpolation + | start | end | parent_place_id | + | 12 | 14 | W2 | diff --git a/test/bdd/steps/steps_db_ops.py b/test/bdd/steps/steps_db_ops.py index 8fd918f8..2f598f3d 100644 --- a/test/bdd/steps/steps_db_ops.py +++ b/test/bdd/steps/steps_db_ops.py @@ -370,6 +370,6 @@ def check_location_property_osmline(context, oid, neg): DBRow(oid, res, context).assert_row(row, ('start', 'end')) - assert not todo + assert not todo, f"Unmatched lines in table: {list(context.table[i] for i in todo)}"