replace add-data function with native Python code

diff --git a/nominatim/clicmd/add_data.py b/nominatim/clicmd/add_data.py
index 18d67ce08055c7ddf26f9337304b0f4329b96c17..d13f46d99ce5a7bb4f9255722becc4029c2108b8 100644 (file)
--- a/nominatim/clicmd/add_data.py
+++ b/nominatim/clicmd/add_data.py
@@ -3,8 +3,6 @@ Implementation of the 'add-data' subcommand.
 """
 import logging
 
 """
 import logging
 

-from nominatim.tools.exec_utils import run_legacy_script
-

 # Do not repeat documentation of subcommand classes.
 # pylint: disable=C0111
 # Using non-top-level imports to avoid eventually unused imports.
 # Do not repeat documentation of subcommand classes.
 # pylint: disable=C0111
 # Using non-top-level imports to avoid eventually unused imports.


@@ -25,9 +23,9 @@ class UpdateAddData:
         group_name = parser.add_argument_group('Source')
         group = group_name.add_mutually_exclusive_group(required=True)
         group.add_argument('--file', metavar='FILE',
         group_name = parser.add_argument_group('Source')
         group = group_name.add_mutually_exclusive_group(required=True)
         group.add_argument('--file', metavar='FILE',

-                           help='Import data from an OSM file')
+                           help='Import data from an OSM file or diff file')

         group.add_argument('--diff', metavar='FILE',
         group.add_argument('--diff', metavar='FILE',

-                           help='Import data from an OSM diff file')
+                           help='Import data from an OSM diff file (deprecated: use --file)')

         group.add_argument('--node', metavar='ID', type=int,
                            help='Import a single node from the API')
         group.add_argument('--way', metavar='ID', type=int,
         group.add_argument('--node', metavar='ID', type=int,
                            help='Import a single node from the API')
         group.add_argument('--way', metavar='ID', type=int,


@@ -39,11 +37,15 @@ class UpdateAddData:
         group = parser.add_argument_group('Extra arguments')
         group.add_argument('--use-main-api', action='store_true',
                            help='Use OSM API instead of Overpass to download objects')
         group = parser.add_argument_group('Extra arguments')
         group.add_argument('--use-main-api', action='store_true',
                            help='Use OSM API instead of Overpass to download objects')

+        group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
+                           help='Size of cache to be used by osm2pgsql (in MB)')
+        group.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60,
+                           help='Set timeout for file downloads.')

 
     @staticmethod
     def run(args):
         from nominatim.tokenizer import factory as tokenizer_factory
 
     @staticmethod
     def run(args):
         from nominatim.tokenizer import factory as tokenizer_factory

-        from nominatim.tools import tiger_data
+        from nominatim.tools import tiger_data, add_osm_data

 
         if args.tiger_data:
             tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)
 
         if args.tiger_data:
             tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)


@@ -51,17 +53,24 @@ class UpdateAddData:
                                              args.config, args.threads or 1,
                                              tokenizer)
 
                                              args.config, args.threads or 1,
                                              tokenizer)
 

-        params = ['update.php']
-        if args.file:
-            params.extend(('--import-file', args.file))
-        elif args.diff:
-            params.extend(('--import-diff', args.diff))
-        elif args.node:
-            params.extend(('--import-node', args.node))
-        elif args.way:
-            params.extend(('--import-way', args.way))
-        elif args.relation:
-            params.extend(('--import-relation', args.relation))
-        if args.use_main_api:
-            params.append('--use-main-api')
-        return run_legacy_script(*params, nominatim_env=args)
+        osm2pgsql_params = args.osm2pgsql_options(default_cache=1000, default_threads=1)
+        if args.file or args.diff:
+            return add_osm_data.add_data_from_file(args.file or args.diff,
+                                                   osm2pgsql_params)
+
+        if args.node:
+            return add_osm_data.add_osm_object('node', args.node,
+                                               args.use_main_api,
+                                               osm2pgsql_params)
+
+        if args.way:
+            return add_osm_data.add_osm_object('way', args.way,
+                                               args.use_main_api,
+                                               osm2pgsql_params)
+
+        if args.relation:
+            return add_osm_data.add_osm_object('relation', args.relation,
+                                               args.use_main_api,
+                                               osm2pgsql_params)
+
+        return 0

diff --git a/nominatim/tools/add_osm_data.py b/nominatim/tools/add_osm_data.py
new file mode 100644 (file)
index 0000000..fa35667
--- /dev/null
+++ b/nominatim/tools/add_osm_data.py
@@ -0,0 +1,46 @@
+"""
+Function to add additional OSM data from a file or the API into the database.
+"""
+from pathlib import Path
+import logging
+import urllib
+
+from nominatim.tools.exec_utils import run_osm2pgsql, get_url
+
+LOG = logging.getLogger()
+
+def add_data_from_file(fname, options):
+    """ Adds data from a OSM file to the database. The file may be a normal
+        OSM file or a diff file in all formats supported by libosmium.
+    """
+    options['import_file'] = Path(fname)
+    options['append'] = True
+    run_osm2pgsql(options)
+
+    # No status update. We don't know where the file came from.
+    return 0
+
+
+def add_osm_object(osm_type, osm_id, use_main_api, options):
+    """ Add or update a single OSM object from the latest version of the
+        API.
+    """
+    if use_main_api:
+        base_url = f'https://www.openstreetmap.org/api/0.6/{osm_type}/{osm_id}'
+        if osm_type in ('way', 'relation'):
+            base_url += '/full'
+    else:
+        # use Overpass API
+        if osm_type == 'node':
+            data = f'node({osm_id});out meta;'
+        elif osm_type == 'way':
+            data = f'(way({osm_id});>;);out meta;'
+        else:
+            data = f'(rel(id:{osm_id});>;);out meta;'
+        base_url = 'https://overpass-api.de/api/interpreter?' \
+                   + urllib.parse.urlencode({'data': data})
+
+    options['append'] = True
+    options['import_data'] = get_url(base_url).encode('utf-8')
+
+    run_osm2pgsql(options)

diff --git a/nominatim/tools/exec_utils.py b/nominatim/tools/exec_utils.py
index 72d252b7a83abdeb434912729bd595cfb10b3f7a..6177b15f000e4e429c9d8e4f27384d546abd49c1 100644 (file)
--- a/nominatim/tools/exec_utils.py
+++ b/nominatim/tools/exec_utils.py
@@ -128,9 +128,14 @@ def run_osm2pgsql(options):
     if options.get('disable_jit', False):
         env['PGOPTIONS'] = '-c jit=off -c max_parallel_workers_per_gather=0'
 
     if options.get('disable_jit', False):
         env['PGOPTIONS'] = '-c jit=off -c max_parallel_workers_per_gather=0'
 

-    cmd.append(str(options['import_file']))
+    if 'import_data' in options:
+        cmd.extend(('-r', 'xml', '-'))
+    else:
+        cmd.append(str(options['import_file']))

 
 

-    subprocess.run(cmd, cwd=options.get('cwd', '.'), env=env, check=True)
+    subprocess.run(cmd, cwd=options.get('cwd', '.'),
+                   input=options.get('import_data'),
+                   env=env, check=True)

 
 
 def get_url(url):
 
 
 def get_url(url):