reintroduce timeout for replication file download

[nominatim.git] / nominatim / cli.py

diff --git a/nominatim/cli.py b/nominatim/cli.py
index 68d3cca8ea302acfb007998e2065d9b243d24cba..e28ff29563f8edf9df2adef1a786f612520922a7 100644 (file)
--- a/nominatim/cli.py
+++ b/nominatim/cli.py
@@ -4,6 +4,7 @@ database administration and querying.
 """
 import datetime as dt
 import os
+import socket
 import sys
 import time
 import argparse
@@ -11,7 +12,7 @@ import logging
 from pathlib import Path
 
 from .config import Configuration
-from .tools.exec_utils import run_legacy_script, run_api_script
+from .tools.exec_utils import run_legacy_script, run_api_script, run_php_server
 from .db.connection import connect
 from .db import status
 from .errors import UsageError
@@ -81,7 +82,7 @@ class CommandlineParser:
 
         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'):
             setattr(args, arg, Path(kwargs[arg]))
-        args.project_dir = Path(args.project_dir)
+        args.project_dir = Path(args.project_dir).resolve()
 
         logging.basicConfig(stream=sys.stderr,
                             format='%(asctime)s: %(message)s',
@@ -90,10 +91,12 @@ class CommandlineParser:
 
         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
 
+        log = logging.getLogger()
+        log.warning('Using project directory: %s', str(args.project_dir))
+
         try:
             return args.command.run(args)
         except UsageError as exception:
-            log = logging.getLogger()
             if log.isEnabledFor(logging.DEBUG):
                 raise # use Python's exception printing
             log.fatal('FATAL: %s', exception)
@@ -259,11 +262,16 @@ class UpdateReplication:
                                    together with --once""")
         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
                            help='Size of cache to be used by osm2pgsql (in MB)')
+        group = parser.add_argument_group('Download parameters')
+        group.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60,
+                           help='Set timeout for file downloads.')
 
     @staticmethod
     def _init_replication(args):
         from .tools import replication, refresh
 
+        socket.setdefaulttimeout(args.socket_timeout)
+
         LOG.warning("Initialising replication updates")
         conn = connect(args.config.get_libpq_dsn())
         replication.init_replication(conn, base_url=args.config.REPLICATION_URL)
@@ -284,6 +292,18 @@ class UpdateReplication:
         conn.close()
         return ret
 
+    @staticmethod
+    def _report_update(batchdate, start_import, start_index):
+        def round_time(delta):
+            return dt.timedelta(seconds=int(delta.total_seconds()))
+
+        end = dt.datetime.now(dt.timezone.utc)
+        LOG.warning("Update completed. Import: %s. %sTotal: %s. Remaining backlog: %s.",
+                    round_time((start_index or end) - start_import),
+                    "Indexing: {} ".format(round_time(end - start_index))
+                    if start_index else '',
+                    round_time(end - start_import),
+                    round_time(end - batchdate))
 
     @staticmethod
     def _update(args):
@@ -317,10 +337,11 @@ class UpdateReplication:
             start = dt.datetime.now(dt.timezone.utc)
             state = replication.update(conn, params)
             status.log_status(conn, start, 'import')
+            batchdate, _, _ = status.get_status(conn)
             conn.close()
 
             if state is not replication.UpdateState.NO_CHANGES and args.do_index:
-                start = dt.datetime.now(dt.timezone.utc)
+                index_start = dt.datetime.now(dt.timezone.utc)
                 indexer = Indexer(args.config.get_libpq_dsn(),
                                   args.threads or 1)
                 indexer.index_boundaries(0, 30)
@@ -328,8 +349,13 @@ class UpdateReplication:
 
                 conn = connect(args.config.get_libpq_dsn())
                 status.set_indexed(conn, True)
-                status.log_status(conn, start, 'index')
+                status.log_status(conn, index_start, 'index')
                 conn.close()
+            else:
+                index_start = None
+
+            if LOG.isEnabledFor(logging.WARNING):
+                UpdateReplication._report_update(batchdate, start, index_start)
 
             if args.once:
                 break
@@ -613,6 +639,28 @@ class QueryExport:
 
         return run_legacy_script(*params, nominatim_env=args)
 
+
+class AdminServe:
+    """\
+    Start a simple web server for serving the API.
+
+    This command starts the built-in PHP webserver to serve the website
+    from the current project directory. This webserver is only suitable
+    for testing and develop. Do not use it in production setups!
+
+    By the default, the webserver can be accessed at: http://127.0.0.1:8088
+    """
+
+    @staticmethod
+    def add_args(parser):
+        group = parser.add_argument_group('Server arguments')
+        group.add_argument('--server', default='127.0.0.1:8088',
+                           help='The address the server will listen to.')
+
+    @staticmethod
+    def run(args):
+        run_php_server(args.server, args.project_dir / 'website')
+
 STRUCTURED_QUERY = (
     ('street', 'housenumber and street'),
     ('city', 'city, town or village'),
@@ -875,6 +923,7 @@ def nominatim(**kwargs):
     parser.add_subcommand('refresh', UpdateRefresh)
 
     parser.add_subcommand('export', QueryExport)
+    parser.add_subcommand('serve', AdminServe)
 
     if kwargs.get('phpcgi_path'):
         parser.add_subcommand('search', APISearch)