From e42349c963bfc1ef5f2db880804d7cbeff5a564f Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 25 Jul 2021 23:29:15 +0200 Subject: [PATCH] replace add-data function with native Python code --- nominatim/clicmd/add_data.py | 47 ++++++++++++++++++++------------- nominatim/tools/add_osm_data.py | 46 ++++++++++++++++++++++++++++++++ nominatim/tools/exec_utils.py | 9 +++++-- 3 files changed, 81 insertions(+), 21 deletions(-) create mode 100644 nominatim/tools/add_osm_data.py diff --git a/nominatim/clicmd/add_data.py b/nominatim/clicmd/add_data.py index 18d67ce0..d13f46d9 100644 --- a/nominatim/clicmd/add_data.py +++ b/nominatim/clicmd/add_data.py @@ -3,8 +3,6 @@ Implementation of the 'add-data' subcommand. """ import logging -from nominatim.tools.exec_utils import run_legacy_script - # Do not repeat documentation of subcommand classes. # pylint: disable=C0111 # Using non-top-level imports to avoid eventually unused imports. @@ -25,9 +23,9 @@ class UpdateAddData: group_name = parser.add_argument_group('Source') group = group_name.add_mutually_exclusive_group(required=True) group.add_argument('--file', metavar='FILE', - help='Import data from an OSM file') + help='Import data from an OSM file or diff file') group.add_argument('--diff', metavar='FILE', - help='Import data from an OSM diff file') + help='Import data from an OSM diff file (deprecated: use --file)') group.add_argument('--node', metavar='ID', type=int, help='Import a single node from the API') group.add_argument('--way', metavar='ID', type=int, @@ -39,11 +37,15 @@ class UpdateAddData: group = parser.add_argument_group('Extra arguments') group.add_argument('--use-main-api', action='store_true', help='Use OSM API instead of Overpass to download objects') + group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int, + help='Size of cache to be used by osm2pgsql (in MB)') + group.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60, + help='Set timeout for file downloads.') @staticmethod def run(args): from nominatim.tokenizer import factory as tokenizer_factory - from nominatim.tools import tiger_data + from nominatim.tools import tiger_data, add_osm_data if args.tiger_data: tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config) @@ -51,17 +53,24 @@ class UpdateAddData: args.config, args.threads or 1, tokenizer) - params = ['update.php'] - if args.file: - params.extend(('--import-file', args.file)) - elif args.diff: - params.extend(('--import-diff', args.diff)) - elif args.node: - params.extend(('--import-node', args.node)) - elif args.way: - params.extend(('--import-way', args.way)) - elif args.relation: - params.extend(('--import-relation', args.relation)) - if args.use_main_api: - params.append('--use-main-api') - return run_legacy_script(*params, nominatim_env=args) + osm2pgsql_params = args.osm2pgsql_options(default_cache=1000, default_threads=1) + if args.file or args.diff: + return add_osm_data.add_data_from_file(args.file or args.diff, + osm2pgsql_params) + + if args.node: + return add_osm_data.add_osm_object('node', args.node, + args.use_main_api, + osm2pgsql_params) + + if args.way: + return add_osm_data.add_osm_object('way', args.way, + args.use_main_api, + osm2pgsql_params) + + if args.relation: + return add_osm_data.add_osm_object('relation', args.relation, + args.use_main_api, + osm2pgsql_params) + + return 0 diff --git a/nominatim/tools/add_osm_data.py b/nominatim/tools/add_osm_data.py new file mode 100644 index 00000000..fa356673 --- /dev/null +++ b/nominatim/tools/add_osm_data.py @@ -0,0 +1,46 @@ +""" +Function to add additional OSM data from a file or the API into the database. +""" +from pathlib import Path +import logging +import urllib + +from nominatim.tools.exec_utils import run_osm2pgsql, get_url + +LOG = logging.getLogger() + +def add_data_from_file(fname, options): + """ Adds data from a OSM file to the database. The file may be a normal + OSM file or a diff file in all formats supported by libosmium. + """ + options['import_file'] = Path(fname) + options['append'] = True + run_osm2pgsql(options) + + # No status update. We don't know where the file came from. + return 0 + + +def add_osm_object(osm_type, osm_id, use_main_api, options): + """ Add or update a single OSM object from the latest version of the + API. + """ + if use_main_api: + base_url = f'https://www.openstreetmap.org/api/0.6/{osm_type}/{osm_id}' + if osm_type in ('way', 'relation'): + base_url += '/full' + else: + # use Overpass API + if osm_type == 'node': + data = f'node({osm_id});out meta;' + elif osm_type == 'way': + data = f'(way({osm_id});>;);out meta;' + else: + data = f'(rel(id:{osm_id});>;);out meta;' + base_url = 'https://overpass-api.de/api/interpreter?' \ + + urllib.parse.urlencode({'data': data}) + + options['append'] = True + options['import_data'] = get_url(base_url).encode('utf-8') + + run_osm2pgsql(options) diff --git a/nominatim/tools/exec_utils.py b/nominatim/tools/exec_utils.py index 72d252b7..6177b15f 100644 --- a/nominatim/tools/exec_utils.py +++ b/nominatim/tools/exec_utils.py @@ -128,9 +128,14 @@ def run_osm2pgsql(options): if options.get('disable_jit', False): env['PGOPTIONS'] = '-c jit=off -c max_parallel_workers_per_gather=0' - cmd.append(str(options['import_file'])) + if 'import_data' in options: + cmd.extend(('-r', 'xml', '-')) + else: + cmd.append(str(options['import_file'])) - subprocess.run(cmd, cwd=options.get('cwd', '.'), env=env, check=True) + subprocess.run(cmd, cwd=options.get('cwd', '.'), + input=options.get('import_data'), + env=env, check=True) def get_url(url): -- 2.43.2