reintroduce timeout for replication file download

[nominatim.git] / nominatim / cli.py

diff --git a/nominatim/cli.py b/nominatim/cli.py
index 9d1e36d1f4fcf1897e48a79dbc7e8172b787df0f..e28ff29563f8edf9df2adef1a786f612520922a7 100644 (file)
--- a/nominatim/cli.py
+++ b/nominatim/cli.py
@@ -2,13 +2,31 @@
 Command-line interface to the Nominatim functions for import, update,
 database administration and querying.
 """
 Command-line interface to the Nominatim functions for import, update,
 database administration and querying.
 """

+import datetime as dt
+import os
+import socket

 import sys
 import sys

+import time

 import argparse
 import logging
 from pathlib import Path
 
 from .config import Configuration
 import argparse
 import logging
 from pathlib import Path
 
 from .config import Configuration

-from .admin.exec_utils import run_legacy_script
+from .tools.exec_utils import run_legacy_script, run_api_script, run_php_server
+from .db.connection import connect
+from .db import status
+from .errors import UsageError
+
+LOG = logging.getLogger()
+
+def _num_system_cpus():
+    try:
+        cpus = len(os.sched_getaffinity(0))
+    except NotImplementedError:
+        cpus = None
+
+    return cpus or os.cpu_count()
+

 
 class CommandlineParser:
     """ Wraps some of the common functions for parsing the command line
 
 class CommandlineParser:
     """ Wraps some of the common functions for parsing the command line


@@ -56,23 +74,60 @@ class CommandlineParser:
         """ Parse the command line arguments of the program and execute the
             appropriate subcommand.
         """
         """ Parse the command line arguments of the program and execute the
             appropriate subcommand.
         """

-        args = self.parser.parse_args()
+        args = self.parser.parse_args(args=kwargs.get('cli_args'))

 
         if args.subcommand is None:
 
         if args.subcommand is None:

-            return self.parser.print_help()
+            self.parser.print_help()
+            return 1

 
 

-        for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir'):
+        for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'):

             setattr(args, arg, Path(kwargs[arg]))
             setattr(args, arg, Path(kwargs[arg]))

-        args.project_dir = Path(args.project_dir)
+        args.project_dir = Path(args.project_dir).resolve()

 
         logging.basicConfig(stream=sys.stderr,
 
         logging.basicConfig(stream=sys.stderr,

-                            format='%(asctime)s %(levelname)s: %(message)s',
+                            format='%(asctime)s: %(message)s',

                             datefmt='%Y-%m-%d %H:%M:%S',
                             level=max(4 - args.verbose, 1) * 10)
 
         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
 
                             datefmt='%Y-%m-%d %H:%M:%S',
                             level=max(4 - args.verbose, 1) * 10)
 
         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
 

-        args.command.run(args)
+        log = logging.getLogger()
+        log.warning('Using project directory: %s', str(args.project_dir))
+
+        try:
+            return args.command.run(args)
+        except UsageError as exception:
+            if log.isEnabledFor(logging.DEBUG):
+                raise # use Python's exception printing
+            log.fatal('FATAL: %s', exception)
+
+        # If we get here, then execution has failed in some way.
+        return 1
+
+
+def _osm2pgsql_options_from_args(args, default_cache, default_threads):
+    """ Set up the stanadrd osm2pgsql from the command line arguments.
+    """
+    return dict(osm2pgsql=args.osm2pgsql_path,
+                osm2pgsql_cache=args.osm2pgsql_cache or default_cache,
+                osm2pgsql_style=args.config.get_import_style_file(),
+                threads=args.threads or default_threads,
+                dsn=args.config.get_libpq_dsn(),
+                flatnode_file=args.config.FLATNODE_FILE)
+
+##### Subcommand classes
+#
+# Each class needs to implement two functions: add_args() adds the CLI parameters
+# for the subfunction, run() executes the subcommand.
+#
+# The class documentation doubles as the help text for the command. The
+# first line is also used in the summary when calling the program without
+# a subcommand.
+#
+# No need to document the functions each time.
+# pylint: disable=C0111
+# Using non-top-level imports to make pyosmium optional for replication only.
+# pylint: disable=E0012,C0415

 
 
 class SetupAll:
 
 
 class SetupAll:


@@ -86,7 +141,7 @@ class SetupAll:
         group = group_name.add_mutually_exclusive_group(required=True)
         group.add_argument('--osm-file',
                            help='OSM file to be imported.')
         group = group_name.add_mutually_exclusive_group(required=True)
         group.add_argument('--osm-file',
                            help='OSM file to be imported.')

-        group.add_argument('--continue', nargs=1, dest='continue_at',
+        group.add_argument('--continue', dest='continue_at',

                            choices=['load-data', 'indexing', 'db-postprocess'],
                            help='Continue an import that was interrupted')
         group = parser.add_argument_group('Optional arguments')
                            choices=['load-data', 'indexing', 'db-postprocess'],
                            help='Continue an import that was interrupted')
         group = parser.add_argument_group('Optional arguments')


@@ -172,15 +227,14 @@ class SetupSpecialPhrases:
                            help='Pull special phrases from the OSM wiki.')
         group = parser.add_argument_group('Output arguments')
         group.add_argument('-o', '--output', default='-',
                            help='Pull special phrases from the OSM wiki.')
         group = parser.add_argument_group('Output arguments')
         group.add_argument('-o', '--output', default='-',

-                           type=argparse.FileType('w', encoding='UTF-8'),

                            help="""File to write the preprocessed phrases to.
                                    If omitted, it will be written to stdout.""")
 
     @staticmethod
     def run(args):
                            help="""File to write the preprocessed phrases to.
                                    If omitted, it will be written to stdout.""")
 
     @staticmethod
     def run(args):

-        if args.output.name != '<stdout>':
+        if args.output != '-':

             raise NotImplementedError('Only output to stdout is currently implemented.')
             raise NotImplementedError('Only output to stdout is currently implemented.')

-        return run_legacy_script('specialphrases.php', '--wiki-import' , nominatim_env=args)
+        return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)

 
 
 class UpdateReplication:
 
 
 class UpdateReplication:


@@ -206,26 +260,128 @@ class UpdateReplication:
         group.add_argument('--no-index', action='store_false', dest='do_index',
                            help="""Do not index the new data. Only applicable
                                    together with --once""")
         group.add_argument('--no-index', action='store_false', dest='do_index',
                            help="""Do not index the new data. Only applicable
                                    together with --once""")

+        group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
+                           help='Size of cache to be used by osm2pgsql (in MB)')
+        group = parser.add_argument_group('Download parameters')
+        group.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60,
+                           help='Set timeout for file downloads.')
+
+    @staticmethod
+    def _init_replication(args):
+        from .tools import replication, refresh
+
+        socket.setdefaulttimeout(args.socket_timeout)
+
+        LOG.warning("Initialising replication updates")
+        conn = connect(args.config.get_libpq_dsn())
+        replication.init_replication(conn, base_url=args.config.REPLICATION_URL)
+        if args.update_functions:
+            LOG.warning("Create functions")
+            refresh.create_functions(conn, args.config, args.data_dir,
+                                     True, False)
+        conn.close()
+        return 0
+
+
+    @staticmethod
+    def _check_for_updates(args):
+        from .tools import replication
+
+        conn = connect(args.config.get_libpq_dsn())
+        ret = replication.check_for_updates(conn, base_url=args.config.REPLICATION_URL)
+        conn.close()
+        return ret
+
+    @staticmethod
+    def _report_update(batchdate, start_import, start_index):
+        def round_time(delta):
+            return dt.timedelta(seconds=int(delta.total_seconds()))
+
+        end = dt.datetime.now(dt.timezone.utc)
+        LOG.warning("Update completed. Import: %s. %sTotal: %s. Remaining backlog: %s.",
+                    round_time((start_index or end) - start_import),
+                    "Indexing: {} ".format(round_time(end - start_index))
+                    if start_index else '',
+                    round_time(end - start_import),
+                    round_time(end - batchdate))
+
+    @staticmethod
+    def _update(args):
+        from .tools import replication
+        from .indexer.indexer import Indexer
+
+        params = _osm2pgsql_options_from_args(args, 2000, 1)
+        params.update(base_url=args.config.REPLICATION_URL,
+                      update_interval=args.config.get_int('REPLICATION_UPDATE_INTERVAL'),
+                      import_file=args.project_dir / 'osmosischange.osc',
+                      max_diff_size=args.config.get_int('REPLICATION_MAX_DIFF'),
+                      indexed_only=not args.once)
+
+        # Sanity check to not overwhelm the Geofabrik servers.
+        if 'download.geofabrik.de'in params['base_url']\
+           and params['update_interval'] < 86400:
+            LOG.fatal("Update interval too low for download.geofabrik.de.\n"
+                      "Please check install documentation "
+                      "(https://nominatim.org/release-docs/latest/admin/Import-and-Update#"
+                      "setting-up-the-update-process).")
+            raise UsageError("Invalid replication update interval setting.")
+
+        if not args.once:
+            if not args.do_index:
+                LOG.fatal("Indexing cannot be disabled when running updates continuously.")
+                raise UsageError("Bad argument '--no-index'.")
+            recheck_interval = args.config.get_int('REPLICATION_RECHECK_INTERVAL')
+
+        while True:
+            conn = connect(args.config.get_libpq_dsn())
+            start = dt.datetime.now(dt.timezone.utc)
+            state = replication.update(conn, params)
+            status.log_status(conn, start, 'import')
+            batchdate, _, _ = status.get_status(conn)
+            conn.close()
+
+            if state is not replication.UpdateState.NO_CHANGES and args.do_index:
+                index_start = dt.datetime.now(dt.timezone.utc)
+                indexer = Indexer(args.config.get_libpq_dsn(),
+                                  args.threads or 1)
+                indexer.index_boundaries(0, 30)
+                indexer.index_by_rank(0, 30)
+
+                conn = connect(args.config.get_libpq_dsn())
+                status.set_indexed(conn, True)
+                status.log_status(conn, index_start, 'index')
+                conn.close()
+            else:
+                index_start = None
+
+            if LOG.isEnabledFor(logging.WARNING):
+                UpdateReplication._report_update(batchdate, start, index_start)
+
+            if args.once:
+                break
+
+            if state is replication.UpdateState.NO_CHANGES:
+                LOG.warning("No new changes. Sleeping for %d sec.", recheck_interval)
+                time.sleep(recheck_interval)
+
+        return state.value

 
     @staticmethod
     def run(args):
 
     @staticmethod
     def run(args):

-        params = ['update.php']
+        try:
+            import osmium # pylint: disable=W0611
+        except ModuleNotFoundError:
+            LOG.fatal("pyosmium not installed. Replication functions not available.\n"
+                      "To install pyosmium via pip: pip3 install osmium")
+            return 1
+

         if args.init:
         if args.init:

-            params.append('--init-updates')
-            if not args.update_functions:
-                params.apend('--no-update-functions')
-        elif args.check_for_updates:
-            params.append('--check-for-updates')
-        else:
-            if args.once:
-                params.append('--import-osmosis')
-            else:
-                params.append('--import-osmosis-all')
-            if not args.do_index:
-                params.append('--no-index')
+            return UpdateReplication._init_replication(args)

 
 

-        return run_legacy_script(*params, nominatim_env=args)
+        if args.check_for_updates:
+            return UpdateReplication._check_for_updates(args)

 
 

+        return UpdateReplication._update(args)

 
 class UpdateAddData:
     """\
 
 class UpdateAddData:
     """\


@@ -258,9 +414,10 @@ class UpdateAddData:
     @staticmethod
     def run(args):
         if args.tiger_data:
     @staticmethod
     def run(args):
         if args.tiger_data:

+            os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data

             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 
             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 

-        params = [ 'update.php']
+        params = ['update.php']

         if args.file:
             params.extend(('--import-file', args.file))
         elif args.diff:
         if args.file:
             params.extend(('--import-file', args.file))
         elif args.diff:


@@ -270,7 +427,7 @@ class UpdateAddData:
         elif args.way:
             params.extend(('--import-way', args.way))
         elif args.relation:
         elif args.way:
             params.extend(('--import-way', args.way))
         elif args.relation:

-            params.extend(('--import-relation' , args.relation))
+            params.extend(('--import-relation', args.relation))

         if args.use_main_api:
             params.append('--use-main-api')
         return run_legacy_script(*params, nominatim_env=args)
         if args.use_main_api:
             params.append('--use-main-api')
         return run_legacy_script(*params, nominatim_env=args)


@@ -283,16 +440,40 @@ class UpdateIndex:
 
     @staticmethod
     def add_args(parser):
 
     @staticmethod
     def add_args(parser):

-        pass
+        group = parser.add_argument_group('Filter arguments')
+        group.add_argument('--boundaries-only', action='store_true',
+                           help="""Index only administrative boundaries.""")
+        group.add_argument('--no-boundaries', action='store_true',
+                           help="""Index everything except administrative boundaries.""")
+        group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
+                           help='Minimum/starting rank')
+        group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
+                           help='Maximum/finishing rank')

 
     @staticmethod
     def run(args):
 
     @staticmethod
     def run(args):

-        return run_legacy_script('update.php', '--index', nominatim_env=args)
+        from .indexer.indexer import Indexer
+
+        indexer = Indexer(args.config.get_libpq_dsn(),
+                          args.threads or _num_system_cpus() or 1)
+
+        if not args.no_boundaries:
+            indexer.index_boundaries(args.minrank, args.maxrank)
+        if not args.boundaries_only:
+            indexer.index_by_rank(args.minrank, args.maxrank)
+
+        if not args.no_boundaries and not args.boundaries_only \
+           and args.minrank == 0 and args.maxrank == 30:
+            conn = connect(args.config.get_libpq_dsn())
+            status.set_indexed(conn, True)
+            conn.close()
+
+        return 0

 
 
 class UpdateRefresh:
     """\
 
 
 class UpdateRefresh:
     """\

-    Recompute auxillary data used by the indexing process.
+    Recompute auxiliary data used by the indexing process.

 
     These functions must not be run in parallel with other update commands.
     """
 
     These functions must not be run in parallel with other update commands.
     """


@@ -306,12 +487,12 @@ class UpdateRefresh:
                            help='Compute frequency of full-word search terms')
         group.add_argument('--address-levels', action='store_true',
                            help='Reimport address level configuration')
                            help='Compute frequency of full-word search terms')
         group.add_argument('--address-levels', action='store_true',
                            help='Reimport address level configuration')

-        group.add_argument('--importance', action='store_true',
-                           help='Recompute place importances (expensive!)')

         group.add_argument('--functions', action='store_true',
                            help='Update the PL/pgSQL functions in the database')
         group.add_argument('--wiki-data', action='store_true',
                            help='Update Wikipedia/data importance numbers.')
         group.add_argument('--functions', action='store_true',
                            help='Update the PL/pgSQL functions in the database')
         group.add_argument('--wiki-data', action='store_true',
                            help='Update Wikipedia/data importance numbers.')

+        group.add_argument('--importance', action='store_true',
+                           help='Recompute place importances (expensive!)')

         group.add_argument('--website', action='store_true',
                            help='Refresh the directory that serves the scripts for the web API')
         group = parser.add_argument_group('Arguments for function refresh')
         group.add_argument('--website', action='store_true',
                            help='Refresh the directory that serves the scripts for the web API')
         group = parser.add_argument_group('Arguments for function refresh')


@@ -322,32 +503,47 @@ class UpdateRefresh:
 
     @staticmethod
     def run(args):
 
     @staticmethod
     def run(args):

+        from .tools import refresh
+

         if args.postcodes:
         if args.postcodes:

-            run_legacy_script('update.php', '--calculate-postcodes',
-                              nominatim_env=args, throw_on_fail=True)
+            LOG.warning("Update postcodes centroid")
+            conn = connect(args.config.get_libpq_dsn())
+            refresh.update_postcodes(conn, args.data_dir)
+            conn.close()
+

         if args.word_counts:
         if args.word_counts:

-            run_legacy_script('update.php', '--recompute-word-counts',
-                              nominatim_env=args, throw_on_fail=True)
+            LOG.warning('Recompute frequency of full-word search terms')
+            conn = connect(args.config.get_libpq_dsn())
+            refresh.recompute_word_counts(conn, args.data_dir)
+            conn.close()
+

         if args.address_levels:
         if args.address_levels:

-            run_legacy_script('update.php', '--update-address-levels',
-                              nominatim_env=args, throw_on_fail=True)
-        if args.importance:
-            run_legacy_script('update.php', '--recompute-importance',
-                              nominatim_env=args, throw_on_fail=True)
+            cfg = Path(args.config.ADDRESS_LEVEL_CONFIG)
+            LOG.warning('Updating address levels from %s', cfg)
+            conn = connect(args.config.get_libpq_dsn())
+            refresh.load_address_levels_from_file(conn, cfg)
+            conn.close()
+

         if args.functions:
         if args.functions:

-            params = ['setup.php', '--create-functions', '--create-partition-functions']
-            if args.diffs:
-                params.append('--enable-diff-updates')
-            if args.enable_debug_statements:
-                params.append('--enable-debug-statements')
-            run_legacy_script(*params, nominatim_env=args, throw_on_fail=True)
+            LOG.warning('Create functions')
+            conn = connect(args.config.get_libpq_dsn())
+            refresh.create_functions(conn, args.config, args.data_dir,
+                                     args.diffs, args.enable_debug_statements)
+            conn.close()
+

         if args.wiki_data:
             run_legacy_script('setup.php', '--import-wikipedia-articles',
                               nominatim_env=args, throw_on_fail=True)
         if args.wiki_data:
             run_legacy_script('setup.php', '--import-wikipedia-articles',
                               nominatim_env=args, throw_on_fail=True)

+        # Attention: importance MUST come after wiki data import.
+        if args.importance:
+            run_legacy_script('update.php', '--recompute-importance',
+                              nominatim_env=args, throw_on_fail=True)

         if args.website:
             run_legacy_script('setup.php', '--setup-website',
                               nominatim_env=args, throw_on_fail=True)
 
         if args.website:
             run_legacy_script('setup.php', '--setup-website',
                               nominatim_env=args, throw_on_fail=True)
 

+        return 0
+

 
 class AdminCheckDatabase:
     """\
 
 class AdminCheckDatabase:
     """\


@@ -390,7 +586,7 @@ class AdminWarm:
 
 class QueryExport:
     """\
 
 class QueryExport:
     """\

-    Export addresses as CSV file from a Nominatim database.
+    Export addresses as CSV file from the database.

     """
 
     @staticmethod
     """
 
     @staticmethod


@@ -435,24 +631,275 @@ class QueryExport:
         if args.restrict_to_country:
             params.extend(('--restrict-to-country', args.restrict_to_country))
         if args.restrict_to_osm_node:
         if args.restrict_to_country:
             params.extend(('--restrict-to-country', args.restrict_to_country))
         if args.restrict_to_osm_node:

-            params.exted(('--restrict-to-osm-node', args.restrict_to_osm_node))
+            params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))

         if args.restrict_to_osm_way:
         if args.restrict_to_osm_way:

-            params.exted(('--restrict-to-osm-way', args.restrict_to_osm_way))
+            params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))

         if args.restrict_to_osm_relation:
         if args.restrict_to_osm_relation:

-            params.exted(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
+            params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))

 
         return run_legacy_script(*params, nominatim_env=args)
 
 
         return run_legacy_script(*params, nominatim_env=args)
 

-class QueryTodo:
+
+class AdminServe:

     """\
     """\

-    Todo
+    Start a simple web server for serving the API.
+
+    This command starts the built-in PHP webserver to serve the website
+    from the current project directory. This webserver is only suitable
+    for testing and develop. Do not use it in production setups!
+
+    By the default, the webserver can be accessed at: http://127.0.0.1:8088
+    """
+
+    @staticmethod
+    def add_args(parser):
+        group = parser.add_argument_group('Server arguments')
+        group.add_argument('--server', default='127.0.0.1:8088',
+                           help='The address the server will listen to.')
+
+    @staticmethod
+    def run(args):
+        run_php_server(args.server, args.project_dir / 'website')
+
+STRUCTURED_QUERY = (
+    ('street', 'housenumber and street'),
+    ('city', 'city, town or village'),
+    ('county', 'county'),
+    ('state', 'state'),
+    ('country', 'country'),
+    ('postalcode', 'postcode')
+)
+
+EXTRADATA_PARAMS = (
+    ('addressdetails', 'Include a breakdown of the address into elements.'),
+    ('extratags', """Include additional information if available
+                     (e.g. wikipedia link, opening hours)."""),
+    ('namedetails', 'Include a list of alternative names.')
+)
+
+DETAILS_SWITCHES = (
+    ('addressdetails', 'Include a breakdown of the address into elements.'),
+    ('keywords', 'Include a list of name keywords and address keywords.'),
+    ('linkedplaces', 'Include a details of places that are linked with this one.'),
+    ('hierarchy', 'Include details of places lower in the address hierarchy.'),
+    ('group_hierarchy', 'Group the places by type.'),
+    ('polygon_geojson', 'Include geometry of result.')
+)
+
+def _add_api_output_arguments(parser):
+    group = parser.add_argument_group('Output arguments')
+    group.add_argument('--format', default='jsonv2',
+                       choices=['xml', 'json', 'jsonv2', 'geojson', 'geocodejson'],
+                       help='Format of result')
+    for name, desc in EXTRADATA_PARAMS:
+        group.add_argument('--' + name, action='store_true', help=desc)
+
+    group.add_argument('--lang', '--accept-language', metavar='LANGS',
+                       help='Preferred language order for presenting search results')
+    group.add_argument('--polygon-output',
+                       choices=['geojson', 'kml', 'svg', 'text'],
+                       help='Output geometry of results as a GeoJSON, KML, SVG or WKT.')
+    group.add_argument('--polygon-threshold', type=float, metavar='TOLERANCE',
+                       help="""Simplify output geometry.
+                               Parameter is difference tolerance in degrees.""")
+
+
+class APISearch:
+    """\
+    Execute API search query.
+    """
+
+    @staticmethod
+    def add_args(parser):
+        group = parser.add_argument_group('Query arguments')
+        group.add_argument('--query',
+                           help='Free-form query string')
+        for name, desc in STRUCTURED_QUERY:
+            group.add_argument('--' + name, help='Structured query: ' + desc)
+
+        _add_api_output_arguments(parser)
+
+        group = parser.add_argument_group('Result limitation')
+        group.add_argument('--countrycodes', metavar='CC,..',
+                           help='Limit search results to one or more countries.')
+        group.add_argument('--exclude_place_ids', metavar='ID,..',
+                           help='List of search object to be excluded')
+        group.add_argument('--limit', type=int,
+                           help='Limit the number of returned results')
+        group.add_argument('--viewbox', metavar='X1,Y1,X2,Y2',
+                           help='Preferred area to find search results')
+        group.add_argument('--bounded', action='store_true',
+                           help='Strictly restrict results to viewbox area')
+
+        group = parser.add_argument_group('Other arguments')
+        group.add_argument('--no-dedupe', action='store_false', dest='dedupe',
+                           help='Do not remove duplicates from the result list')
+
+
+    @staticmethod
+    def run(args):
+        if args.query:
+            params = dict(q=args.query)
+        else:
+            params = {k : getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)}
+
+        for param, _ in EXTRADATA_PARAMS:
+            if getattr(args, param):
+                params[param] = '1'
+        for param in ('format', 'countrycodes', 'exclude_place_ids', 'limit', 'viewbox'):
+            if getattr(args, param):
+                params[param] = getattr(args, param)
+        if args.lang:
+            params['accept-language'] = args.lang
+        if args.polygon_output:
+            params['polygon_' + args.polygon_output] = '1'
+        if args.polygon_threshold:
+            params['polygon_threshold'] = args.polygon_threshold
+        if args.bounded:
+            params['bounded'] = '1'
+        if not args.dedupe:
+            params['dedupe'] = '0'
+
+        return run_api_script('search', args.project_dir,
+                              phpcgi_bin=args.phpcgi_path, params=params)
+
+class APIReverse:
+    """\
+    Execute API reverse query.

     """
     """

+

     @staticmethod
     def add_args(parser):
     @staticmethod
     def add_args(parser):

-        pass
+        group = parser.add_argument_group('Query arguments')
+        group.add_argument('--lat', type=float, required=True,
+                           help='Latitude of coordinate to look up (in WGS84)')
+        group.add_argument('--lon', type=float, required=True,
+                           help='Longitude of coordinate to look up (in WGS84)')
+        group.add_argument('--zoom', type=int,
+                           help='Level of detail required for the address')
+
+        _add_api_output_arguments(parser)

 
 

+
+    @staticmethod

     def run(args):
     def run(args):

-        print("TODO: searching")
+        params = dict(lat=args.lat, lon=args.lon)
+        if args.zoom is not None:
+            params['zoom'] = args.zoom
+
+        for param, _ in EXTRADATA_PARAMS:
+            if getattr(args, param):
+                params[param] = '1'
+        if args.format:
+            params['format'] = args.format
+        if args.lang:
+            params['accept-language'] = args.lang
+        if args.polygon_output:
+            params['polygon_' + args.polygon_output] = '1'
+        if args.polygon_threshold:
+            params['polygon_threshold'] = args.polygon_threshold
+
+        return run_api_script('reverse', args.project_dir,
+                              phpcgi_bin=args.phpcgi_path, params=params)
+
+
+class APILookup:
+    """\
+    Execute API reverse query.
+    """
+
+    @staticmethod
+    def add_args(parser):
+        group = parser.add_argument_group('Query arguments')
+        group.add_argument('--id', metavar='OSMID',
+                           action='append', required=True, dest='ids',
+                           help='OSM id to lookup in format <NRW><id> (may be repeated)')
+
+        _add_api_output_arguments(parser)
+
+
+    @staticmethod
+    def run(args):
+        params = dict(osm_ids=','.join(args.ids))
+
+        for param, _ in EXTRADATA_PARAMS:
+            if getattr(args, param):
+                params[param] = '1'
+        if args.format:
+            params['format'] = args.format
+        if args.lang:
+            params['accept-language'] = args.lang
+        if args.polygon_output:
+            params['polygon_' + args.polygon_output] = '1'
+        if args.polygon_threshold:
+            params['polygon_threshold'] = args.polygon_threshold
+
+        return run_api_script('lookup', args.project_dir,
+                              phpcgi_bin=args.phpcgi_path, params=params)
+
+
+class APIDetails:
+    """\
+    Execute API lookup query.
+    """
+
+    @staticmethod
+    def add_args(parser):
+        group = parser.add_argument_group('Query arguments')
+        objs = group.add_mutually_exclusive_group(required=True)
+        objs.add_argument('--node', '-n', type=int,
+                          help="Look up the OSM node with the given ID.")
+        objs.add_argument('--way', '-w', type=int,
+                          help="Look up the OSM way with the given ID.")
+        objs.add_argument('--relation', '-r', type=int,
+                          help="Look up the OSM relation with the given ID.")
+        objs.add_argument('--place_id', '-p', type=int,
+                          help='Database internal identifier of the OSM object to look up.')
+        group.add_argument('--class', dest='object_class',
+                           help="""Class type to disambiguated multiple entries
+                                   of the same object.""")
+
+        group = parser.add_argument_group('Output arguments')
+        for name, desc in DETAILS_SWITCHES:
+            group.add_argument('--' + name, action='store_true', help=desc)
+        group.add_argument('--lang', '--accept-language', metavar='LANGS',
+                           help='Preferred language order for presenting search results')
+
+    @staticmethod
+    def run(args):
+        if args.node:
+            params = dict(osmtype='N', osmid=args.node)
+        elif args.way:
+            params = dict(osmtype='W', osmid=args.node)
+        elif args.relation:
+            params = dict(osmtype='R', osmid=args.node)
+        else:
+            params = dict(place_id=args.place_id)
+        if args.object_class:
+            params['class'] = args.object_class
+        for name, _ in DETAILS_SWITCHES:
+            params[name] = '1' if getattr(args, name) else '0'
+
+        return run_api_script('details', args.project_dir,
+                              phpcgi_bin=args.phpcgi_path, params=params)
+
+
+class APIStatus:
+    """\
+    Execute API status query.
+    """
+
+    @staticmethod
+    def add_args(parser):
+        group = parser.add_argument_group('API parameters')
+        group.add_argument('--format', default='text', choices=['text', 'json'],
+                           help='Format of result')
+
+    @staticmethod
+    def run(args):
+        return run_api_script('status', args.project_dir,
+                              phpcgi_bin=args.phpcgi_path,
+                              params=dict(format=args.format))

 
 
 def nominatim(**kwargs):
 
 
 def nominatim(**kwargs):


@@ -476,10 +923,15 @@ def nominatim(**kwargs):
     parser.add_subcommand('refresh', UpdateRefresh)
 
     parser.add_subcommand('export', QueryExport)
     parser.add_subcommand('refresh', UpdateRefresh)
 
     parser.add_subcommand('export', QueryExport)

-    parser.add_subcommand('search', QueryTodo)
-    parser.add_subcommand('reverse', QueryTodo)
-    parser.add_subcommand('lookup', QueryTodo)
-    parser.add_subcommand('details', QueryTodo)
-    parser.add_subcommand('status', QueryTodo)
-
-    parser.run(**kwargs)
+    parser.add_subcommand('serve', AdminServe)
+
+    if kwargs.get('phpcgi_path'):
+        parser.add_subcommand('search', APISearch)
+        parser.add_subcommand('reverse', APIReverse)
+        parser.add_subcommand('lookup', APILookup)
+        parser.add_subcommand('details', APIDetails)
+        parser.add_subcommand('status', APIStatus)
+    else:
+        parser.parser.epilog = 'php-cgi not found. Query commands not available.'
+
+    return parser.run(**kwargs)