nominatim/cli.py

   1 """
   2 Command-line interface to the Nominatim functions for import, update,
   3 database administration and querying.
   4 """
   5 import datetime as dt
   6 import os
   7 import socket
   8 import sys
   9 import time
  10 import argparse
  11 import logging
  12 from pathlib import Path
  13
  14 from .config import Configuration
  15 from .tools.exec_utils import run_legacy_script, run_api_script, run_php_server
  16 from .db.connection import connect
  17 from .db import status
  18 from .errors import UsageError
  19
  20 LOG = logging.getLogger()
  21
  22 def _num_system_cpus():
  23     try:
  24         cpus = len(os.sched_getaffinity(0))
  25     except NotImplementedError:
  26         cpus = None
  27
  28     return cpus or os.cpu_count()
  29
  30
  31 class CommandlineParser:
  32     """ Wraps some of the common functions for parsing the command line
  33         and setting up subcommands.
  34     """
  35     def __init__(self, prog, description):
  36         self.parser = argparse.ArgumentParser(
  37             prog=prog,
  38             description=description,
  39             formatter_class=argparse.RawDescriptionHelpFormatter)
  40
  41         self.subs = self.parser.add_subparsers(title='available commands',
  42                                                dest='subcommand')
  43
  44         # Arguments added to every sub-command
  45         self.default_args = argparse.ArgumentParser(add_help=False)
  46         group = self.default_args.add_argument_group('Default arguments')
  47         group.add_argument('-h', '--help', action='help',
  48                            help='Show this help message and exit')
  49         group.add_argument('-q', '--quiet', action='store_const', const=0,
  50                            dest='verbose', default=1,
  51                            help='Print only error messages')
  52         group.add_argument('-v', '--verbose', action='count', default=1,
  53                            help='Increase verboseness of output')
  54         group.add_argument('--project-dir', metavar='DIR', default='.',
  55                            help='Base directory of the Nominatim installation (default:.)')
  56         group.add_argument('-j', '--threads', metavar='NUM', type=int,
  57                            help='Number of parallel threads to use')
  58
  59
  60     def add_subcommand(self, name, cmd):
  61         """ Add a subcommand to the parser. The subcommand must be a class
  62             with a function add_args() that adds the parameters for the
  63             subcommand and a run() function that executes the command.
  64         """
  65         parser = self.subs.add_parser(name, parents=[self.default_args],
  66                                       help=cmd.__doc__.split('\n', 1)[0],
  67                                       description=cmd.__doc__,
  68                                       formatter_class=argparse.RawDescriptionHelpFormatter,
  69                                       add_help=False)
  70         parser.set_defaults(command=cmd)
  71         cmd.add_args(parser)
  72
  73     def run(self, **kwargs):
  74         """ Parse the command line arguments of the program and execute the
  75             appropriate subcommand.
  76         """
  77         args = self.parser.parse_args(args=kwargs.get('cli_args'))
  78
  79         if args.subcommand is None:
  80             self.parser.print_help()
  81             return 1
  82
  83         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'):
  84             setattr(args, arg, Path(kwargs[arg]))
  85         args.project_dir = Path(args.project_dir).resolve()
  86
  87         logging.basicConfig(stream=sys.stderr,
  88                             format='%(asctime)s: %(message)s',
  89                             datefmt='%Y-%m-%d %H:%M:%S',
  90                             level=max(4 - args.verbose, 1) * 10)
  91
  92         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
  93
  94         log = logging.getLogger()
  95         log.warning('Using project directory: %s', str(args.project_dir))
  96
  97         try:
  98             return args.command.run(args)
  99         except UsageError as exception:
 100             if log.isEnabledFor(logging.DEBUG):
 101                 raise # use Python's exception printing
 102             log.fatal('FATAL: %s', exception)
 103
 104         # If we get here, then execution has failed in some way.
 105         return 1
 106
 107
 108 def _osm2pgsql_options_from_args(args, default_cache, default_threads):
 109     """ Set up the stanadrd osm2pgsql from the command line arguments.
 110     """
 111     return dict(osm2pgsql=args.osm2pgsql_path,
 112                 osm2pgsql_cache=args.osm2pgsql_cache or default_cache,
 113                 osm2pgsql_style=args.config.get_import_style_file(),
 114                 threads=args.threads or default_threads,
 115                 dsn=args.config.get_libpq_dsn(),
 116                 flatnode_file=args.config.FLATNODE_FILE)
 117
 118 ##### Subcommand classes
 119 #
 120 # Each class needs to implement two functions: add_args() adds the CLI parameters
 121 # for the subfunction, run() executes the subcommand.
 122 #
 123 # The class documentation doubles as the help text for the command. The
 124 # first line is also used in the summary when calling the program without
 125 # a subcommand.
 126 #
 127 # No need to document the functions each time.
 128 # pylint: disable=C0111
 129 # Using non-top-level imports to make pyosmium optional for replication only.
 130 # pylint: disable=E0012,C0415
 131
 132
 133 class SetupAll:
 134     """\
 135     Create a new Nominatim database from an OSM file.
 136     """
 137
 138     @staticmethod
 139     def add_args(parser):
 140         group_name = parser.add_argument_group('Required arguments')
 141         group = group_name.add_mutually_exclusive_group(required=True)
 142         group.add_argument('--osm-file',
 143                            help='OSM file to be imported.')
 144         group.add_argument('--continue', dest='continue_at',
 145                            choices=['load-data', 'indexing', 'db-postprocess'],
 146                            help='Continue an import that was interrupted')
 147         group = parser.add_argument_group('Optional arguments')
 148         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 149                            help='Size of cache to be used by osm2pgsql (in MB)')
 150         group.add_argument('--reverse-only', action='store_true',
 151                            help='Do not create tables and indexes for searching')
 152         group.add_argument('--enable-debug-statements', action='store_true',
 153                            help='Include debug warning statements in SQL code')
 154         group.add_argument('--no-partitions', action='store_true',
 155                            help="""Do not partition search indices
 156                                    (speeds up import of single country extracts)""")
 157         group.add_argument('--no-updates', action='store_true',
 158                            help="""Do not keep tables that are only needed for
 159                                    updating the database later""")
 160         group = parser.add_argument_group('Expert options')
 161         group.add_argument('--ignore-errors', action='store_true',
 162                            help='Continue import even when errors in SQL are present')
 163         group.add_argument('--index-noanalyse', action='store_true',
 164                            help='Do not perform analyse operations during index')
 165
 166
 167     @staticmethod
 168     def run(args):
 169         params = ['setup.php']
 170         if args.osm_file:
 171             params.extend(('--all', '--osm-file', args.osm_file))
 172         else:
 173             if args.continue_at == 'load-data':
 174                 params.append('--load-data')
 175             if args.continue_at in ('load-data', 'indexing'):
 176                 params.append('--index')
 177             params.extend(('--create-search-indices', '--create-country-names',
 178                            '--setup-website'))
 179         if args.osm2pgsql_cache:
 180             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
 181         if args.reverse_only:
 182             params.append('--reverse-only')
 183         if args.enable_debug_statements:
 184             params.append('--enable-debug-statements')
 185         if args.no_partitions:
 186             params.append('--no-partitions')
 187         if args.no_updates:
 188             params.append('--drop')
 189         if args.ignore_errors:
 190             params.append('--ignore-errors')
 191         if args.index_noanalyse:
 192             params.append('--index-noanalyse')
 193
 194         return run_legacy_script(*params, nominatim_env=args)
 195
 196
 197 class SetupFreeze:
 198     """\
 199     Make database read-only.
 200
 201     About half of data in the Nominatim database is kept only to be able to
 202     keep the data up-to-date with new changes made in OpenStreetMap. This
 203     command drops all this data and only keeps the part needed for geocoding
 204     itself.
 205
 206     This command has the same effect as the `--no-updates` option for imports.
 207     """
 208
 209     @staticmethod
 210     def add_args(parser):
 211         pass # No options
 212
 213     @staticmethod
 214     def run(args):
 215         return run_legacy_script('setup.php', '--drop', nominatim_env=args)
 216
 217
 218 class SetupSpecialPhrases:
 219     """\
 220     Maintain special phrases.
 221     """
 222
 223     @staticmethod
 224     def add_args(parser):
 225         group = parser.add_argument_group('Input arguments')
 226         group.add_argument('--from-wiki', action='store_true',
 227                            help='Pull special phrases from the OSM wiki.')
 228         group = parser.add_argument_group('Output arguments')
 229         group.add_argument('-o', '--output', default='-',
 230                            help="""File to write the preprocessed phrases to.
 231                                    If omitted, it will be written to stdout.""")
 232
 233     @staticmethod
 234     def run(args):
 235         if args.output != '-':
 236             raise NotImplementedError('Only output to stdout is currently implemented.')
 237         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 238
 239
 240 class UpdateReplication:
 241     """\
 242     Update the database using an online replication service.
 243     """
 244
 245     @staticmethod
 246     def add_args(parser):
 247         group = parser.add_argument_group('Arguments for initialisation')
 248         group.add_argument('--init', action='store_true',
 249                            help='Initialise the update process')
 250         group.add_argument('--no-update-functions', dest='update_functions',
 251                            action='store_false',
 252                            help="""Do not update the trigger function to
 253                                    support differential updates.""")
 254         group = parser.add_argument_group('Arguments for updates')
 255         group.add_argument('--check-for-updates', action='store_true',
 256                            help='Check if new updates are available and exit')
 257         group.add_argument('--once', action='store_true',
 258                            help="""Download and apply updates only once. When
 259                                    not set, updates are continuously applied""")
 260         group.add_argument('--no-index', action='store_false', dest='do_index',
 261                            help="""Do not index the new data. Only applicable
 262                                    together with --once""")
 263         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 264                            help='Size of cache to be used by osm2pgsql (in MB)')
 265         group = parser.add_argument_group('Download parameters')
 266         group.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60,
 267                            help='Set timeout for file downloads.')
 268
 269     @staticmethod
 270     def _init_replication(args):
 271         from .tools import replication, refresh
 272
 273         socket.setdefaulttimeout(args.socket_timeout)
 274
 275         LOG.warning("Initialising replication updates")
 276         conn = connect(args.config.get_libpq_dsn())
 277         replication.init_replication(conn, base_url=args.config.REPLICATION_URL)
 278         if args.update_functions:
 279             LOG.warning("Create functions")
 280             refresh.create_functions(conn, args.config, args.data_dir,
 281                                      True, False)
 282         conn.close()
 283         return 0
 284
 285
 286     @staticmethod
 287     def _check_for_updates(args):
 288         from .tools import replication
 289
 290         conn = connect(args.config.get_libpq_dsn())
 291         ret = replication.check_for_updates(conn, base_url=args.config.REPLICATION_URL)
 292         conn.close()
 293         return ret
 294
 295     @staticmethod
 296     def _report_update(batchdate, start_import, start_index):
 297         def round_time(delta):
 298             return dt.timedelta(seconds=int(delta.total_seconds()))
 299
 300         end = dt.datetime.now(dt.timezone.utc)
 301         LOG.warning("Update completed. Import: %s. %sTotal: %s. Remaining backlog: %s.",
 302                     round_time((start_index or end) - start_import),
 303                     "Indexing: {} ".format(round_time(end - start_index))
 304                     if start_index else '',
 305                     round_time(end - start_import),
 306                     round_time(end - batchdate))
 307
 308     @staticmethod
 309     def _update(args):
 310         from .tools import replication
 311         from .indexer.indexer import Indexer
 312
 313         params = _osm2pgsql_options_from_args(args, 2000, 1)
 314         params.update(base_url=args.config.REPLICATION_URL,
 315                       update_interval=args.config.get_int('REPLICATION_UPDATE_INTERVAL'),
 316                       import_file=args.project_dir / 'osmosischange.osc',
 317                       max_diff_size=args.config.get_int('REPLICATION_MAX_DIFF'),
 318                       indexed_only=not args.once)
 319
 320         # Sanity check to not overwhelm the Geofabrik servers.
 321         if 'download.geofabrik.de'in params['base_url']\
 322            and params['update_interval'] < 86400:
 323             LOG.fatal("Update interval too low for download.geofabrik.de.\n"
 324                       "Please check install documentation "
 325                       "(https://nominatim.org/release-docs/latest/admin/Import-and-Update#"
 326                       "setting-up-the-update-process).")
 327             raise UsageError("Invalid replication update interval setting.")
 328
 329         if not args.once:
 330             if not args.do_index:
 331                 LOG.fatal("Indexing cannot be disabled when running updates continuously.")
 332                 raise UsageError("Bad argument '--no-index'.")
 333             recheck_interval = args.config.get_int('REPLICATION_RECHECK_INTERVAL')
 334
 335         while True:
 336             conn = connect(args.config.get_libpq_dsn())
 337             start = dt.datetime.now(dt.timezone.utc)
 338             state = replication.update(conn, params)
 339             if state is not replication.UpdateState.NO_CHANGES:
 340                 status.log_status(conn, start, 'import')
 341             batchdate, _, _ = status.get_status(conn)
 342             conn.close()
 343
 344             if state is not replication.UpdateState.NO_CHANGES and args.do_index:
 345                 index_start = dt.datetime.now(dt.timezone.utc)
 346                 indexer = Indexer(args.config.get_libpq_dsn(),
 347                                   args.threads or 1)
 348                 indexer.index_boundaries(0, 30)
 349                 indexer.index_by_rank(0, 30)
 350
 351                 conn = connect(args.config.get_libpq_dsn())
 352                 status.set_indexed(conn, True)
 353                 status.log_status(conn, index_start, 'index')
 354                 conn.close()
 355             else:
 356                 index_start = None
 357
 358             if LOG.isEnabledFor(logging.WARNING):
 359                 UpdateReplication._report_update(batchdate, start, index_start)
 360
 361             if args.once:
 362                 break
 363
 364             if state is replication.UpdateState.NO_CHANGES:
 365                 LOG.warning("No new changes. Sleeping for %d sec.", recheck_interval)
 366                 time.sleep(recheck_interval)
 367
 368         return state.value
 369
 370     @staticmethod
 371     def run(args):
 372         try:
 373             import osmium # pylint: disable=W0611
 374         except ModuleNotFoundError:
 375             LOG.fatal("pyosmium not installed. Replication functions not available.\n"
 376                       "To install pyosmium via pip: pip3 install osmium")
 377             return 1
 378
 379         if args.init:
 380             return UpdateReplication._init_replication(args)
 381
 382         if args.check_for_updates:
 383             return UpdateReplication._check_for_updates(args)
 384
 385         return UpdateReplication._update(args)
 386
 387 class UpdateAddData:
 388     """\
 389     Add additional data from a file or an online source.
 390
 391     Data is only imported, not indexed. You need to call `nominatim-update index`
 392     to complete the process.
 393     """
 394
 395     @staticmethod
 396     def add_args(parser):
 397         group_name = parser.add_argument_group('Source')
 398         group = group_name.add_mutually_exclusive_group(required=True)
 399         group.add_argument('--file', metavar='FILE',
 400                            help='Import data from an OSM file')
 401         group.add_argument('--diff', metavar='FILE',
 402                            help='Import data from an OSM diff file')
 403         group.add_argument('--node', metavar='ID', type=int,
 404                            help='Import a single node from the API')
 405         group.add_argument('--way', metavar='ID', type=int,
 406                            help='Import a single way from the API')
 407         group.add_argument('--relation', metavar='ID', type=int,
 408                            help='Import a single relation from the API')
 409         group.add_argument('--tiger-data', metavar='DIR',
 410                            help='Add housenumbers from the US TIGER census database.')
 411         group = parser.add_argument_group('Extra arguments')
 412         group.add_argument('--use-main-api', action='store_true',
 413                            help='Use OSM API instead of Overpass to download objects')
 414
 415     @staticmethod
 416     def run(args):
 417         if args.tiger_data:
 418             os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
 419             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 420
 421         params = ['update.php']
 422         if args.file:
 423             params.extend(('--import-file', args.file))
 424         elif args.diff:
 425             params.extend(('--import-diff', args.diff))
 426         elif args.node:
 427             params.extend(('--import-node', args.node))
 428         elif args.way:
 429             params.extend(('--import-way', args.way))
 430         elif args.relation:
 431             params.extend(('--import-relation', args.relation))
 432         if args.use_main_api:
 433             params.append('--use-main-api')
 434         return run_legacy_script(*params, nominatim_env=args)
 435
 436
 437 class UpdateIndex:
 438     """\
 439     Reindex all new and modified data.
 440     """
 441
 442     @staticmethod
 443     def add_args(parser):
 444         group = parser.add_argument_group('Filter arguments')
 445         group.add_argument('--boundaries-only', action='store_true',
 446                            help="""Index only administrative boundaries.""")
 447         group.add_argument('--no-boundaries', action='store_true',
 448                            help="""Index everything except administrative boundaries.""")
 449         group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
 450                            help='Minimum/starting rank')
 451         group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
 452                            help='Maximum/finishing rank')
 453
 454     @staticmethod
 455     def run(args):
 456         from .indexer.indexer import Indexer
 457
 458         indexer = Indexer(args.config.get_libpq_dsn(),
 459                           args.threads or _num_system_cpus() or 1)
 460
 461         if not args.no_boundaries:
 462             indexer.index_boundaries(args.minrank, args.maxrank)
 463         if not args.boundaries_only:
 464             indexer.index_by_rank(args.minrank, args.maxrank)
 465
 466         if not args.no_boundaries and not args.boundaries_only \
 467            and args.minrank == 0 and args.maxrank == 30:
 468             conn = connect(args.config.get_libpq_dsn())
 469             status.set_indexed(conn, True)
 470             conn.close()
 471
 472         return 0
 473
 474
 475 class UpdateRefresh:
 476     """\
 477     Recompute auxiliary data used by the indexing process.
 478
 479     These functions must not be run in parallel with other update commands.
 480     """
 481
 482     @staticmethod
 483     def add_args(parser):
 484         group = parser.add_argument_group('Data arguments')
 485         group.add_argument('--postcodes', action='store_true',
 486                            help='Update postcode centroid table')
 487         group.add_argument('--word-counts', action='store_true',
 488                            help='Compute frequency of full-word search terms')
 489         group.add_argument('--address-levels', action='store_true',
 490                            help='Reimport address level configuration')
 491         group.add_argument('--functions', action='store_true',
 492                            help='Update the PL/pgSQL functions in the database')
 493         group.add_argument('--wiki-data', action='store_true',
 494                            help='Update Wikipedia/data importance numbers.')
 495         group.add_argument('--importance', action='store_true',
 496                            help='Recompute place importances (expensive!)')
 497         group.add_argument('--website', action='store_true',
 498                            help='Refresh the directory that serves the scripts for the web API')
 499         group = parser.add_argument_group('Arguments for function refresh')
 500         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
 501                            help='Do not enable code for propagating updates')
 502         group.add_argument('--enable-debug-statements', action='store_true',
 503                            help='Enable debug warning statements in functions')
 504
 505     @staticmethod
 506     def run(args):
 507         from .tools import refresh
 508
 509         if args.postcodes:
 510             LOG.warning("Update postcodes centroid")
 511             conn = connect(args.config.get_libpq_dsn())
 512             refresh.update_postcodes(conn, args.data_dir)
 513             conn.close()
 514
 515         if args.word_counts:
 516             LOG.warning('Recompute frequency of full-word search terms')
 517             conn = connect(args.config.get_libpq_dsn())
 518             refresh.recompute_word_counts(conn, args.data_dir)
 519             conn.close()
 520
 521         if args.address_levels:
 522             cfg = Path(args.config.ADDRESS_LEVEL_CONFIG)
 523             LOG.warning('Updating address levels from %s', cfg)
 524             conn = connect(args.config.get_libpq_dsn())
 525             refresh.load_address_levels_from_file(conn, cfg)
 526             conn.close()
 527
 528         if args.functions:
 529             LOG.warning('Create functions')
 530             conn = connect(args.config.get_libpq_dsn())
 531             refresh.create_functions(conn, args.config, args.data_dir,
 532                                      args.diffs, args.enable_debug_statements)
 533             conn.close()
 534
 535         if args.wiki_data:
 536             run_legacy_script('setup.php', '--import-wikipedia-articles',
 537                               nominatim_env=args, throw_on_fail=True)
 538         # Attention: importance MUST come after wiki data import.
 539         if args.importance:
 540             run_legacy_script('update.php', '--recompute-importance',
 541                               nominatim_env=args, throw_on_fail=True)
 542         if args.website:
 543             run_legacy_script('setup.php', '--setup-website',
 544                               nominatim_env=args, throw_on_fail=True)
 545
 546         return 0
 547
 548
 549 class AdminCheckDatabase:
 550     """\
 551     Check that the database is complete and operational.
 552     """
 553
 554     @staticmethod
 555     def add_args(parser):
 556         pass # No options
 557
 558     @staticmethod
 559     def run(args):
 560         return run_legacy_script('check_import_finished.php', nominatim_env=args)
 561
 562
 563 class AdminWarm:
 564     """\
 565     Warm database caches for search and reverse queries.
 566     """
 567
 568     @staticmethod
 569     def add_args(parser):
 570         group = parser.add_argument_group('Target arguments')
 571         group.add_argument('--search-only', action='store_const', dest='target',
 572                            const='search',
 573                            help="Only pre-warm tables for search queries")
 574         group.add_argument('--reverse-only', action='store_const', dest='target',
 575                            const='reverse',
 576                            help="Only pre-warm tables for reverse queries")
 577
 578     @staticmethod
 579     def run(args):
 580         params = ['warm.php']
 581         if args.target == 'reverse':
 582             params.append('--reverse-only')
 583         if args.target == 'search':
 584             params.append('--search-only')
 585         return run_legacy_script(*params, nominatim_env=args)
 586
 587
 588 class QueryExport:
 589     """\
 590     Export addresses as CSV file from the database.
 591     """
 592
 593     @staticmethod
 594     def add_args(parser):
 595         group = parser.add_argument_group('Output arguments')
 596         group.add_argument('--output-type', default='street',
 597                            choices=('continent', 'country', 'state', 'county',
 598                                     'city', 'suburb', 'street', 'path'),
 599                            help='Type of places to output (default: street)')
 600         group.add_argument('--output-format',
 601                            default='street;suburb;city;county;state;country',
 602                            help="""Semicolon-separated list of address types
 603                                    (see --output-type). Multiple ranks can be
 604                                    merged into one column by simply using a
 605                                    comma-separated list.""")
 606         group.add_argument('--output-all-postcodes', action='store_true',
 607                            help="""List all postcodes for address instead of
 608                                    just the most likely one""")
 609         group.add_argument('--language',
 610                            help="""Preferred language for output
 611                                    (use local name, if omitted)""")
 612         group = parser.add_argument_group('Filter arguments')
 613         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
 614                            help='Export only objects within country')
 615         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
 616                            help='Export only children of this OSM node')
 617         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
 618                            help='Export only children of this OSM way')
 619         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
 620                            help='Export only children of this OSM relation')
 621
 622
 623     @staticmethod
 624     def run(args):
 625         params = ['export.php',
 626                   '--output-type', args.output_type,
 627                   '--output-format', args.output_format]
 628         if args.output_all_postcodes:
 629             params.append('--output-all-postcodes')
 630         if args.language:
 631             params.extend(('--language', args.language))
 632         if args.restrict_to_country:
 633             params.extend(('--restrict-to-country', args.restrict_to_country))
 634         if args.restrict_to_osm_node:
 635             params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
 636         if args.restrict_to_osm_way:
 637             params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
 638         if args.restrict_to_osm_relation:
 639             params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
 640
 641         return run_legacy_script(*params, nominatim_env=args)
 642
 643
 644 class AdminServe:
 645     """\
 646     Start a simple web server for serving the API.
 647
 648     This command starts the built-in PHP webserver to serve the website
 649     from the current project directory. This webserver is only suitable
 650     for testing and develop. Do not use it in production setups!
 651
 652     By the default, the webserver can be accessed at: http://127.0.0.1:8088
 653     """
 654
 655     @staticmethod
 656     def add_args(parser):
 657         group = parser.add_argument_group('Server arguments')
 658         group.add_argument('--server', default='127.0.0.1:8088',
 659                            help='The address the server will listen to.')
 660
 661     @staticmethod
 662     def run(args):
 663         run_php_server(args.server, args.project_dir / 'website')
 664
 665 STRUCTURED_QUERY = (
 666     ('street', 'housenumber and street'),
 667     ('city', 'city, town or village'),
 668     ('county', 'county'),
 669     ('state', 'state'),
 670     ('country', 'country'),
 671     ('postalcode', 'postcode')
 672 )
 673
 674 EXTRADATA_PARAMS = (
 675     ('addressdetails', 'Include a breakdown of the address into elements.'),
 676     ('extratags', """Include additional information if available
 677                      (e.g. wikipedia link, opening hours)."""),
 678     ('namedetails', 'Include a list of alternative names.')
 679 )
 680
 681 DETAILS_SWITCHES = (
 682     ('addressdetails', 'Include a breakdown of the address into elements.'),
 683     ('keywords', 'Include a list of name keywords and address keywords.'),
 684     ('linkedplaces', 'Include a details of places that are linked with this one.'),
 685     ('hierarchy', 'Include details of places lower in the address hierarchy.'),
 686     ('group_hierarchy', 'Group the places by type.'),
 687     ('polygon_geojson', 'Include geometry of result.')
 688 )
 689
 690 def _add_api_output_arguments(parser):
 691     group = parser.add_argument_group('Output arguments')
 692     group.add_argument('--format', default='jsonv2',
 693                        choices=['xml', 'json', 'jsonv2', 'geojson', 'geocodejson'],
 694                        help='Format of result')
 695     for name, desc in EXTRADATA_PARAMS:
 696         group.add_argument('--' + name, action='store_true', help=desc)
 697
 698     group.add_argument('--lang', '--accept-language', metavar='LANGS',
 699                        help='Preferred language order for presenting search results')
 700     group.add_argument('--polygon-output',
 701                        choices=['geojson', 'kml', 'svg', 'text'],
 702                        help='Output geometry of results as a GeoJSON, KML, SVG or WKT.')
 703     group.add_argument('--polygon-threshold', type=float, metavar='TOLERANCE',
 704                        help="""Simplify output geometry.
 705                                Parameter is difference tolerance in degrees.""")
 706
 707
 708 class APISearch:
 709     """\
 710     Execute API search query.
 711     """
 712
 713     @staticmethod
 714     def add_args(parser):
 715         group = parser.add_argument_group('Query arguments')
 716         group.add_argument('--query',
 717                            help='Free-form query string')
 718         for name, desc in STRUCTURED_QUERY:
 719             group.add_argument('--' + name, help='Structured query: ' + desc)
 720
 721         _add_api_output_arguments(parser)
 722
 723         group = parser.add_argument_group('Result limitation')
 724         group.add_argument('--countrycodes', metavar='CC,..',
 725                            help='Limit search results to one or more countries.')
 726         group.add_argument('--exclude_place_ids', metavar='ID,..',
 727                            help='List of search object to be excluded')
 728         group.add_argument('--limit', type=int,
 729                            help='Limit the number of returned results')
 730         group.add_argument('--viewbox', metavar='X1,Y1,X2,Y2',
 731                            help='Preferred area to find search results')
 732         group.add_argument('--bounded', action='store_true',
 733                            help='Strictly restrict results to viewbox area')
 734
 735         group = parser.add_argument_group('Other arguments')
 736         group.add_argument('--no-dedupe', action='store_false', dest='dedupe',
 737                            help='Do not remove duplicates from the result list')
 738
 739
 740     @staticmethod
 741     def run(args):
 742         if args.query:
 743             params = dict(q=args.query)
 744         else:
 745             params = {k : getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)}
 746
 747         for param, _ in EXTRADATA_PARAMS:
 748             if getattr(args, param):
 749                 params[param] = '1'
 750         for param in ('format', 'countrycodes', 'exclude_place_ids', 'limit', 'viewbox'):
 751             if getattr(args, param):
 752                 params[param] = getattr(args, param)
 753         if args.lang:
 754             params['accept-language'] = args.lang
 755         if args.polygon_output:
 756             params['polygon_' + args.polygon_output] = '1'
 757         if args.polygon_threshold:
 758             params['polygon_threshold'] = args.polygon_threshold
 759         if args.bounded:
 760             params['bounded'] = '1'
 761         if not args.dedupe:
 762             params['dedupe'] = '0'
 763
 764         return run_api_script('search', args.project_dir,
 765                               phpcgi_bin=args.phpcgi_path, params=params)
 766
 767 class APIReverse:
 768     """\
 769     Execute API reverse query.
 770     """
 771
 772     @staticmethod
 773     def add_args(parser):
 774         group = parser.add_argument_group('Query arguments')
 775         group.add_argument('--lat', type=float, required=True,
 776                            help='Latitude of coordinate to look up (in WGS84)')
 777         group.add_argument('--lon', type=float, required=True,
 778                            help='Longitude of coordinate to look up (in WGS84)')
 779         group.add_argument('--zoom', type=int,
 780                            help='Level of detail required for the address')
 781
 782         _add_api_output_arguments(parser)
 783
 784
 785     @staticmethod
 786     def run(args):
 787         params = dict(lat=args.lat, lon=args.lon)
 788         if args.zoom is not None:
 789             params['zoom'] = args.zoom
 790
 791         for param, _ in EXTRADATA_PARAMS:
 792             if getattr(args, param):
 793                 params[param] = '1'
 794         if args.format:
 795             params['format'] = args.format
 796         if args.lang:
 797             params['accept-language'] = args.lang
 798         if args.polygon_output:
 799             params['polygon_' + args.polygon_output] = '1'
 800         if args.polygon_threshold:
 801             params['polygon_threshold'] = args.polygon_threshold
 802
 803         return run_api_script('reverse', args.project_dir,
 804                               phpcgi_bin=args.phpcgi_path, params=params)
 805
 806
 807 class APILookup:
 808     """\
 809     Execute API reverse query.
 810     """
 811
 812     @staticmethod
 813     def add_args(parser):
 814         group = parser.add_argument_group('Query arguments')
 815         group.add_argument('--id', metavar='OSMID',
 816                            action='append', required=True, dest='ids',
 817                            help='OSM id to lookup in format <NRW><id> (may be repeated)')
 818
 819         _add_api_output_arguments(parser)
 820
 821
 822     @staticmethod
 823     def run(args):
 824         params = dict(osm_ids=','.join(args.ids))
 825
 826         for param, _ in EXTRADATA_PARAMS:
 827             if getattr(args, param):
 828                 params[param] = '1'
 829         if args.format:
 830             params['format'] = args.format
 831         if args.lang:
 832             params['accept-language'] = args.lang
 833         if args.polygon_output:
 834             params['polygon_' + args.polygon_output] = '1'
 835         if args.polygon_threshold:
 836             params['polygon_threshold'] = args.polygon_threshold
 837
 838         return run_api_script('lookup', args.project_dir,
 839                               phpcgi_bin=args.phpcgi_path, params=params)
 840
 841
 842 class APIDetails:
 843     """\
 844     Execute API lookup query.
 845     """
 846
 847     @staticmethod
 848     def add_args(parser):
 849         group = parser.add_argument_group('Query arguments')
 850         objs = group.add_mutually_exclusive_group(required=True)
 851         objs.add_argument('--node', '-n', type=int,
 852                           help="Look up the OSM node with the given ID.")
 853         objs.add_argument('--way', '-w', type=int,
 854                           help="Look up the OSM way with the given ID.")
 855         objs.add_argument('--relation', '-r', type=int,
 856                           help="Look up the OSM relation with the given ID.")
 857         objs.add_argument('--place_id', '-p', type=int,
 858                           help='Database internal identifier of the OSM object to look up.')
 859         group.add_argument('--class', dest='object_class',
 860                            help="""Class type to disambiguated multiple entries
 861                                    of the same object.""")
 862
 863         group = parser.add_argument_group('Output arguments')
 864         for name, desc in DETAILS_SWITCHES:
 865             group.add_argument('--' + name, action='store_true', help=desc)
 866         group.add_argument('--lang', '--accept-language', metavar='LANGS',
 867                            help='Preferred language order for presenting search results')
 868
 869     @staticmethod
 870     def run(args):
 871         if args.node:
 872             params = dict(osmtype='N', osmid=args.node)
 873         elif args.way:
 874             params = dict(osmtype='W', osmid=args.node)
 875         elif args.relation:
 876             params = dict(osmtype='R', osmid=args.node)
 877         else:
 878             params = dict(place_id=args.place_id)
 879         if args.object_class:
 880             params['class'] = args.object_class
 881         for name, _ in DETAILS_SWITCHES:
 882             params[name] = '1' if getattr(args, name) else '0'
 883
 884         return run_api_script('details', args.project_dir,
 885                               phpcgi_bin=args.phpcgi_path, params=params)
 886
 887
 888 class APIStatus:
 889     """\
 890     Execute API status query.
 891     """
 892
 893     @staticmethod
 894     def add_args(parser):
 895         group = parser.add_argument_group('API parameters')
 896         group.add_argument('--format', default='text', choices=['text', 'json'],
 897                            help='Format of result')
 898
 899     @staticmethod
 900     def run(args):
 901         return run_api_script('status', args.project_dir,
 902                               phpcgi_bin=args.phpcgi_path,
 903                               params=dict(format=args.format))
 904
 905
 906 def nominatim(**kwargs):
 907     """\
 908     Command-line tools for importing, updating, administrating and
 909     querying the Nominatim database.
 910     """
 911     parser = CommandlineParser('nominatim', nominatim.__doc__)
 912
 913     parser.add_subcommand('import', SetupAll)
 914     parser.add_subcommand('freeze', SetupFreeze)
 915     parser.add_subcommand('replication', UpdateReplication)
 916
 917     parser.add_subcommand('check-database', AdminCheckDatabase)
 918     parser.add_subcommand('warm', AdminWarm)
 919
 920     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
 921
 922     parser.add_subcommand('add-data', UpdateAddData)
 923     parser.add_subcommand('index', UpdateIndex)
 924     parser.add_subcommand('refresh', UpdateRefresh)
 925
 926     parser.add_subcommand('export', QueryExport)
 927     parser.add_subcommand('serve', AdminServe)
 928
 929     if kwargs.get('phpcgi_path'):
 930         parser.add_subcommand('search', APISearch)
 931         parser.add_subcommand('reverse', APIReverse)
 932         parser.add_subcommand('lookup', APILookup)
 933         parser.add_subcommand('details', APIDetails)
 934         parser.add_subcommand('status', APIStatus)
 935     else:
 936         parser.parser.epilog = 'php-cgi not found. Query commands not available.'
 937
 938     return parser.run(**kwargs)