nominatim/cli.py

   1 """
   2 Command-line interface to the Nominatim functions for import, update,
   3 database administration and querying.
   4 """
   5 import datetime as dt
   6 import os
   7 import sys
   8 import time
   9 import argparse
  10 import logging
  11 from pathlib import Path
  12
  13 from .config import Configuration
  14 from .tools.exec_utils import run_legacy_script, run_api_script
  15 from .db.connection import connect
  16 from .db import status
  17 from .errors import UsageError
  18
  19 LOG = logging.getLogger()
  20
  21 def _num_system_cpus():
  22     try:
  23         cpus = len(os.sched_getaffinity(0))
  24     except NotImplementedError:
  25         cpus = None
  26
  27     return cpus or os.cpu_count()
  28
  29
  30 class CommandlineParser:
  31     """ Wraps some of the common functions for parsing the command line
  32         and setting up subcommands.
  33     """
  34     def __init__(self, prog, description):
  35         self.parser = argparse.ArgumentParser(
  36             prog=prog,
  37             description=description,
  38             formatter_class=argparse.RawDescriptionHelpFormatter)
  39
  40         self.subs = self.parser.add_subparsers(title='available commands',
  41                                                dest='subcommand')
  42
  43         # Arguments added to every sub-command
  44         self.default_args = argparse.ArgumentParser(add_help=False)
  45         group = self.default_args.add_argument_group('Default arguments')
  46         group.add_argument('-h', '--help', action='help',
  47                            help='Show this help message and exit')
  48         group.add_argument('-q', '--quiet', action='store_const', const=0,
  49                            dest='verbose', default=1,
  50                            help='Print only error messages')
  51         group.add_argument('-v', '--verbose', action='count', default=1,
  52                            help='Increase verboseness of output')
  53         group.add_argument('--project-dir', metavar='DIR', default='.',
  54                            help='Base directory of the Nominatim installation (default:.)')
  55         group.add_argument('-j', '--threads', metavar='NUM', type=int,
  56                            help='Number of parallel threads to use')
  57
  58
  59     def add_subcommand(self, name, cmd):
  60         """ Add a subcommand to the parser. The subcommand must be a class
  61             with a function add_args() that adds the parameters for the
  62             subcommand and a run() function that executes the command.
  63         """
  64         parser = self.subs.add_parser(name, parents=[self.default_args],
  65                                       help=cmd.__doc__.split('\n', 1)[0],
  66                                       description=cmd.__doc__,
  67                                       formatter_class=argparse.RawDescriptionHelpFormatter,
  68                                       add_help=False)
  69         parser.set_defaults(command=cmd)
  70         cmd.add_args(parser)
  71
  72     def run(self, **kwargs):
  73         """ Parse the command line arguments of the program and execute the
  74             appropriate subcommand.
  75         """
  76         args = self.parser.parse_args(args=kwargs.get('cli_args'))
  77
  78         if args.subcommand is None:
  79             self.parser.print_help()
  80             return 1
  81
  82         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'):
  83             setattr(args, arg, Path(kwargs[arg]))
  84         args.project_dir = Path(args.project_dir)
  85
  86         logging.basicConfig(stream=sys.stderr,
  87                             format='%(asctime)s: %(message)s',
  88                             datefmt='%Y-%m-%d %H:%M:%S',
  89                             level=max(4 - args.verbose, 1) * 10)
  90
  91         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
  92
  93         try:
  94             return args.command.run(args)
  95         except UsageError as exception:
  96             log = logging.getLogger()
  97             if log.isEnabledFor(logging.DEBUG):
  98                 raise # use Python's exception printing
  99             log.fatal('FATAL: %s', exception)
 100
 101         # If we get here, then execution has failed in some way.
 102         return 1
 103
 104
 105 def _osm2pgsql_options_from_args(args, default_cache, default_threads):
 106     """ Set up the stanadrd osm2pgsql from the command line arguments.
 107     """
 108     return dict(osm2pgsql=args.osm2pgsql_path,
 109                 osm2pgsql_cache=args.osm2pgsql_cache or default_cache,
 110                 osm2pgsql_style=args.config.get_import_style_file(),
 111                 threads=args.threads or default_threads,
 112                 dsn=args.config.get_libpq_dsn(),
 113                 flatnode_file=args.config.FLATNODE_FILE)
 114
 115 ##### Subcommand classes
 116 #
 117 # Each class needs to implement two functions: add_args() adds the CLI parameters
 118 # for the subfunction, run() executes the subcommand.
 119 #
 120 # The class documentation doubles as the help text for the command. The
 121 # first line is also used in the summary when calling the program without
 122 # a subcommand.
 123 #
 124 # No need to document the functions each time.
 125 # pylint: disable=C0111
 126 # Using non-top-level imports to make pyosmium optional for replication only.
 127 # pylint: disable=E0012,C0415
 128
 129
 130 class SetupAll:
 131     """\
 132     Create a new Nominatim database from an OSM file.
 133     """
 134
 135     @staticmethod
 136     def add_args(parser):
 137         group_name = parser.add_argument_group('Required arguments')
 138         group = group_name.add_mutually_exclusive_group(required=True)
 139         group.add_argument('--osm-file',
 140                            help='OSM file to be imported.')
 141         group.add_argument('--continue', dest='continue_at',
 142                            choices=['load-data', 'indexing', 'db-postprocess'],
 143                            help='Continue an import that was interrupted')
 144         group = parser.add_argument_group('Optional arguments')
 145         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 146                            help='Size of cache to be used by osm2pgsql (in MB)')
 147         group.add_argument('--reverse-only', action='store_true',
 148                            help='Do not create tables and indexes for searching')
 149         group.add_argument('--enable-debug-statements', action='store_true',
 150                            help='Include debug warning statements in SQL code')
 151         group.add_argument('--no-partitions', action='store_true',
 152                            help="""Do not partition search indices
 153                                    (speeds up import of single country extracts)""")
 154         group.add_argument('--no-updates', action='store_true',
 155                            help="""Do not keep tables that are only needed for
 156                                    updating the database later""")
 157         group = parser.add_argument_group('Expert options')
 158         group.add_argument('--ignore-errors', action='store_true',
 159                            help='Continue import even when errors in SQL are present')
 160         group.add_argument('--index-noanalyse', action='store_true',
 161                            help='Do not perform analyse operations during index')
 162
 163
 164     @staticmethod
 165     def run(args):
 166         params = ['setup.php']
 167         if args.osm_file:
 168             params.extend(('--all', '--osm-file', args.osm_file))
 169         else:
 170             if args.continue_at == 'load-data':
 171                 params.append('--load-data')
 172             if args.continue_at in ('load-data', 'indexing'):
 173                 params.append('--index')
 174             params.extend(('--create-search-indices', '--create-country-names',
 175                            '--setup-website'))
 176         if args.osm2pgsql_cache:
 177             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
 178         if args.reverse_only:
 179             params.append('--reverse-only')
 180         if args.enable_debug_statements:
 181             params.append('--enable-debug-statements')
 182         if args.no_partitions:
 183             params.append('--no-partitions')
 184         if args.no_updates:
 185             params.append('--drop')
 186         if args.ignore_errors:
 187             params.append('--ignore-errors')
 188         if args.index_noanalyse:
 189             params.append('--index-noanalyse')
 190
 191         return run_legacy_script(*params, nominatim_env=args)
 192
 193
 194 class SetupFreeze:
 195     """\
 196     Make database read-only.
 197
 198     About half of data in the Nominatim database is kept only to be able to
 199     keep the data up-to-date with new changes made in OpenStreetMap. This
 200     command drops all this data and only keeps the part needed for geocoding
 201     itself.
 202
 203     This command has the same effect as the `--no-updates` option for imports.
 204     """
 205
 206     @staticmethod
 207     def add_args(parser):
 208         pass # No options
 209
 210     @staticmethod
 211     def run(args):
 212         return run_legacy_script('setup.php', '--drop', nominatim_env=args)
 213
 214
 215 class SetupSpecialPhrases:
 216     """\
 217     Maintain special phrases.
 218     """
 219
 220     @staticmethod
 221     def add_args(parser):
 222         group = parser.add_argument_group('Input arguments')
 223         group.add_argument('--from-wiki', action='store_true',
 224                            help='Pull special phrases from the OSM wiki.')
 225         group = parser.add_argument_group('Output arguments')
 226         group.add_argument('-o', '--output', default='-',
 227                            help="""File to write the preprocessed phrases to.
 228                                    If omitted, it will be written to stdout.""")
 229
 230     @staticmethod
 231     def run(args):
 232         if args.output != '-':
 233             raise NotImplementedError('Only output to stdout is currently implemented.')
 234         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 235
 236
 237 class UpdateReplication:
 238     """\
 239     Update the database using an online replication service.
 240     """
 241
 242     @staticmethod
 243     def add_args(parser):
 244         group = parser.add_argument_group('Arguments for initialisation')
 245         group.add_argument('--init', action='store_true',
 246                            help='Initialise the update process')
 247         group.add_argument('--no-update-functions', dest='update_functions',
 248                            action='store_false',
 249                            help="""Do not update the trigger function to
 250                                    support differential updates.""")
 251         group = parser.add_argument_group('Arguments for updates')
 252         group.add_argument('--check-for-updates', action='store_true',
 253                            help='Check if new updates are available and exit')
 254         group.add_argument('--once', action='store_true',
 255                            help="""Download and apply updates only once. When
 256                                    not set, updates are continuously applied""")
 257         group.add_argument('--no-index', action='store_false', dest='do_index',
 258                            help="""Do not index the new data. Only applicable
 259                                    together with --once""")
 260         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 261                            help='Size of cache to be used by osm2pgsql (in MB)')
 262
 263     @staticmethod
 264     def _init_replication(args):
 265         from .tools import replication, refresh
 266
 267         LOG.warning("Initialising replication updates")
 268         conn = connect(args.config.get_libpq_dsn())
 269         replication.init_replication(conn, base_url=args.config.REPLICATION_URL)
 270         if args.update_functions:
 271             LOG.warning("Create functions")
 272             refresh.create_functions(conn, args.config, args.data_dir,
 273                                      True, False)
 274         conn.close()
 275         return 0
 276
 277
 278     @staticmethod
 279     def _check_for_updates(args):
 280         from .tools import replication
 281
 282         conn = connect(args.config.get_libpq_dsn())
 283         ret = replication.check_for_updates(conn, base_url=args.config.REPLICATION_URL)
 284         conn.close()
 285         return ret
 286
 287     @staticmethod
 288     def _report_update(batchdate, start_import, start_index):
 289         def round_time(delta):
 290             return dt.timedelta(seconds=int(delta.total_seconds()))
 291
 292         end = dt.datetime.now(dt.timezone.utc)
 293         LOG.warning("Update completed. Import: %s. %sTotal: %s. Remaining backlog: %s.",
 294                     round_time((start_index or end) - start_import),
 295                     "Indexing: {} ".format(round_time(end - start_index))
 296                     if start_index else '',
 297                     round_time(end - start_import),
 298                     round_time(end - batchdate))
 299
 300     @staticmethod
 301     def _update(args):
 302         from .tools import replication
 303         from .indexer.indexer import Indexer
 304
 305         params = _osm2pgsql_options_from_args(args, 2000, 1)
 306         params.update(base_url=args.config.REPLICATION_URL,
 307                       update_interval=args.config.get_int('REPLICATION_UPDATE_INTERVAL'),
 308                       import_file=args.project_dir / 'osmosischange.osc',
 309                       max_diff_size=args.config.get_int('REPLICATION_MAX_DIFF'),
 310                       indexed_only=not args.once)
 311
 312         # Sanity check to not overwhelm the Geofabrik servers.
 313         if 'download.geofabrik.de'in params['base_url']\
 314            and params['update_interval'] < 86400:
 315             LOG.fatal("Update interval too low for download.geofabrik.de.\n"
 316                       "Please check install documentation "
 317                       "(https://nominatim.org/release-docs/latest/admin/Import-and-Update#"
 318                       "setting-up-the-update-process).")
 319             raise UsageError("Invalid replication update interval setting.")
 320
 321         if not args.once:
 322             if not args.do_index:
 323                 LOG.fatal("Indexing cannot be disabled when running updates continuously.")
 324                 raise UsageError("Bad argument '--no-index'.")
 325             recheck_interval = args.config.get_int('REPLICATION_RECHECK_INTERVAL')
 326
 327         while True:
 328             conn = connect(args.config.get_libpq_dsn())
 329             start = dt.datetime.now(dt.timezone.utc)
 330             state = replication.update(conn, params)
 331             status.log_status(conn, start, 'import')
 332             batchdate, _, _ = status.get_status(conn)
 333             conn.close()
 334
 335             if state is not replication.UpdateState.NO_CHANGES and args.do_index:
 336                 index_start = dt.datetime.now(dt.timezone.utc)
 337                 indexer = Indexer(args.config.get_libpq_dsn(),
 338                                   args.threads or 1)
 339                 indexer.index_boundaries(0, 30)
 340                 indexer.index_by_rank(0, 30)
 341
 342                 conn = connect(args.config.get_libpq_dsn())
 343                 status.set_indexed(conn, True)
 344                 status.log_status(conn, index_start, 'index')
 345                 conn.close()
 346             else:
 347                 index_start = None
 348
 349             if LOG.isEnabledFor(logging.WARNING):
 350                 UpdateReplication._report_update(batchdate, start, index_start)
 351
 352             if args.once:
 353                 break
 354
 355             if state is replication.UpdateState.NO_CHANGES:
 356                 LOG.warning("No new changes. Sleeping for %d sec.", recheck_interval)
 357                 time.sleep(recheck_interval)
 358
 359         return state.value
 360
 361     @staticmethod
 362     def run(args):
 363         try:
 364             import osmium # pylint: disable=W0611
 365         except ModuleNotFoundError:
 366             LOG.fatal("pyosmium not installed. Replication functions not available.\n"
 367                       "To install pyosmium via pip: pip3 install osmium")
 368             return 1
 369
 370         if args.init:
 371             return UpdateReplication._init_replication(args)
 372
 373         if args.check_for_updates:
 374             return UpdateReplication._check_for_updates(args)
 375
 376         return UpdateReplication._update(args)
 377
 378 class UpdateAddData:
 379     """\
 380     Add additional data from a file or an online source.
 381
 382     Data is only imported, not indexed. You need to call `nominatim-update index`
 383     to complete the process.
 384     """
 385
 386     @staticmethod
 387     def add_args(parser):
 388         group_name = parser.add_argument_group('Source')
 389         group = group_name.add_mutually_exclusive_group(required=True)
 390         group.add_argument('--file', metavar='FILE',
 391                            help='Import data from an OSM file')
 392         group.add_argument('--diff', metavar='FILE',
 393                            help='Import data from an OSM diff file')
 394         group.add_argument('--node', metavar='ID', type=int,
 395                            help='Import a single node from the API')
 396         group.add_argument('--way', metavar='ID', type=int,
 397                            help='Import a single way from the API')
 398         group.add_argument('--relation', metavar='ID', type=int,
 399                            help='Import a single relation from the API')
 400         group.add_argument('--tiger-data', metavar='DIR',
 401                            help='Add housenumbers from the US TIGER census database.')
 402         group = parser.add_argument_group('Extra arguments')
 403         group.add_argument('--use-main-api', action='store_true',
 404                            help='Use OSM API instead of Overpass to download objects')
 405
 406     @staticmethod
 407     def run(args):
 408         if args.tiger_data:
 409             os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
 410             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 411
 412         params = ['update.php']
 413         if args.file:
 414             params.extend(('--import-file', args.file))
 415         elif args.diff:
 416             params.extend(('--import-diff', args.diff))
 417         elif args.node:
 418             params.extend(('--import-node', args.node))
 419         elif args.way:
 420             params.extend(('--import-way', args.way))
 421         elif args.relation:
 422             params.extend(('--import-relation', args.relation))
 423         if args.use_main_api:
 424             params.append('--use-main-api')
 425         return run_legacy_script(*params, nominatim_env=args)
 426
 427
 428 class UpdateIndex:
 429     """\
 430     Reindex all new and modified data.
 431     """
 432
 433     @staticmethod
 434     def add_args(parser):
 435         group = parser.add_argument_group('Filter arguments')
 436         group.add_argument('--boundaries-only', action='store_true',
 437                            help="""Index only administrative boundaries.""")
 438         group.add_argument('--no-boundaries', action='store_true',
 439                            help="""Index everything except administrative boundaries.""")
 440         group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
 441                            help='Minimum/starting rank')
 442         group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
 443                            help='Maximum/finishing rank')
 444
 445     @staticmethod
 446     def run(args):
 447         from .indexer.indexer import Indexer
 448
 449         indexer = Indexer(args.config.get_libpq_dsn(),
 450                           args.threads or _num_system_cpus() or 1)
 451
 452         if not args.no_boundaries:
 453             indexer.index_boundaries(args.minrank, args.maxrank)
 454         if not args.boundaries_only:
 455             indexer.index_by_rank(args.minrank, args.maxrank)
 456
 457         if not args.no_boundaries and not args.boundaries_only \
 458            and args.minrank == 0 and args.maxrank == 30:
 459             conn = connect(args.config.get_libpq_dsn())
 460             status.set_indexed(conn, True)
 461             conn.close()
 462
 463         return 0
 464
 465
 466 class UpdateRefresh:
 467     """\
 468     Recompute auxiliary data used by the indexing process.
 469
 470     These functions must not be run in parallel with other update commands.
 471     """
 472
 473     @staticmethod
 474     def add_args(parser):
 475         group = parser.add_argument_group('Data arguments')
 476         group.add_argument('--postcodes', action='store_true',
 477                            help='Update postcode centroid table')
 478         group.add_argument('--word-counts', action='store_true',
 479                            help='Compute frequency of full-word search terms')
 480         group.add_argument('--address-levels', action='store_true',
 481                            help='Reimport address level configuration')
 482         group.add_argument('--functions', action='store_true',
 483                            help='Update the PL/pgSQL functions in the database')
 484         group.add_argument('--wiki-data', action='store_true',
 485                            help='Update Wikipedia/data importance numbers.')
 486         group.add_argument('--importance', action='store_true',
 487                            help='Recompute place importances (expensive!)')
 488         group.add_argument('--website', action='store_true',
 489                            help='Refresh the directory that serves the scripts for the web API')
 490         group = parser.add_argument_group('Arguments for function refresh')
 491         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
 492                            help='Do not enable code for propagating updates')
 493         group.add_argument('--enable-debug-statements', action='store_true',
 494                            help='Enable debug warning statements in functions')
 495
 496     @staticmethod
 497     def run(args):
 498         from .tools import refresh
 499
 500         if args.postcodes:
 501             LOG.warning("Update postcodes centroid")
 502             conn = connect(args.config.get_libpq_dsn())
 503             refresh.update_postcodes(conn, args.data_dir)
 504             conn.close()
 505
 506         if args.word_counts:
 507             LOG.warning('Recompute frequency of full-word search terms')
 508             conn = connect(args.config.get_libpq_dsn())
 509             refresh.recompute_word_counts(conn, args.data_dir)
 510             conn.close()
 511
 512         if args.address_levels:
 513             cfg = Path(args.config.ADDRESS_LEVEL_CONFIG)
 514             LOG.warning('Updating address levels from %s', cfg)
 515             conn = connect(args.config.get_libpq_dsn())
 516             refresh.load_address_levels_from_file(conn, cfg)
 517             conn.close()
 518
 519         if args.functions:
 520             LOG.warning('Create functions')
 521             conn = connect(args.config.get_libpq_dsn())
 522             refresh.create_functions(conn, args.config, args.data_dir,
 523                                      args.diffs, args.enable_debug_statements)
 524             conn.close()
 525
 526         if args.wiki_data:
 527             run_legacy_script('setup.php', '--import-wikipedia-articles',
 528                               nominatim_env=args, throw_on_fail=True)
 529         # Attention: importance MUST come after wiki data import.
 530         if args.importance:
 531             run_legacy_script('update.php', '--recompute-importance',
 532                               nominatim_env=args, throw_on_fail=True)
 533         if args.website:
 534             run_legacy_script('setup.php', '--setup-website',
 535                               nominatim_env=args, throw_on_fail=True)
 536
 537         return 0
 538
 539
 540 class AdminCheckDatabase:
 541     """\
 542     Check that the database is complete and operational.
 543     """
 544
 545     @staticmethod
 546     def add_args(parser):
 547         pass # No options
 548
 549     @staticmethod
 550     def run(args):
 551         return run_legacy_script('check_import_finished.php', nominatim_env=args)
 552
 553
 554 class AdminWarm:
 555     """\
 556     Warm database caches for search and reverse queries.
 557     """
 558
 559     @staticmethod
 560     def add_args(parser):
 561         group = parser.add_argument_group('Target arguments')
 562         group.add_argument('--search-only', action='store_const', dest='target',
 563                            const='search',
 564                            help="Only pre-warm tables for search queries")
 565         group.add_argument('--reverse-only', action='store_const', dest='target',
 566                            const='reverse',
 567                            help="Only pre-warm tables for reverse queries")
 568
 569     @staticmethod
 570     def run(args):
 571         params = ['warm.php']
 572         if args.target == 'reverse':
 573             params.append('--reverse-only')
 574         if args.target == 'search':
 575             params.append('--search-only')
 576         return run_legacy_script(*params, nominatim_env=args)
 577
 578
 579 class QueryExport:
 580     """\
 581     Export addresses as CSV file from the database.
 582     """
 583
 584     @staticmethod
 585     def add_args(parser):
 586         group = parser.add_argument_group('Output arguments')
 587         group.add_argument('--output-type', default='street',
 588                            choices=('continent', 'country', 'state', 'county',
 589                                     'city', 'suburb', 'street', 'path'),
 590                            help='Type of places to output (default: street)')
 591         group.add_argument('--output-format',
 592                            default='street;suburb;city;county;state;country',
 593                            help="""Semicolon-separated list of address types
 594                                    (see --output-type). Multiple ranks can be
 595                                    merged into one column by simply using a
 596                                    comma-separated list.""")
 597         group.add_argument('--output-all-postcodes', action='store_true',
 598                            help="""List all postcodes for address instead of
 599                                    just the most likely one""")
 600         group.add_argument('--language',
 601                            help="""Preferred language for output
 602                                    (use local name, if omitted)""")
 603         group = parser.add_argument_group('Filter arguments')
 604         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
 605                            help='Export only objects within country')
 606         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
 607                            help='Export only children of this OSM node')
 608         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
 609                            help='Export only children of this OSM way')
 610         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
 611                            help='Export only children of this OSM relation')
 612
 613
 614     @staticmethod
 615     def run(args):
 616         params = ['export.php',
 617                   '--output-type', args.output_type,
 618                   '--output-format', args.output_format]
 619         if args.output_all_postcodes:
 620             params.append('--output-all-postcodes')
 621         if args.language:
 622             params.extend(('--language', args.language))
 623         if args.restrict_to_country:
 624             params.extend(('--restrict-to-country', args.restrict_to_country))
 625         if args.restrict_to_osm_node:
 626             params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
 627         if args.restrict_to_osm_way:
 628             params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
 629         if args.restrict_to_osm_relation:
 630             params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
 631
 632         return run_legacy_script(*params, nominatim_env=args)
 633
 634 STRUCTURED_QUERY = (
 635     ('street', 'housenumber and street'),
 636     ('city', 'city, town or village'),
 637     ('county', 'county'),
 638     ('state', 'state'),
 639     ('country', 'country'),
 640     ('postalcode', 'postcode')
 641 )
 642
 643 EXTRADATA_PARAMS = (
 644     ('addressdetails', 'Include a breakdown of the address into elements.'),
 645     ('extratags', """Include additional information if available
 646                      (e.g. wikipedia link, opening hours)."""),
 647     ('namedetails', 'Include a list of alternative names.')
 648 )
 649
 650 DETAILS_SWITCHES = (
 651     ('addressdetails', 'Include a breakdown of the address into elements.'),
 652     ('keywords', 'Include a list of name keywords and address keywords.'),
 653     ('linkedplaces', 'Include a details of places that are linked with this one.'),
 654     ('hierarchy', 'Include details of places lower in the address hierarchy.'),
 655     ('group_hierarchy', 'Group the places by type.'),
 656     ('polygon_geojson', 'Include geometry of result.')
 657 )
 658
 659 def _add_api_output_arguments(parser):
 660     group = parser.add_argument_group('Output arguments')
 661     group.add_argument('--format', default='jsonv2',
 662                        choices=['xml', 'json', 'jsonv2', 'geojson', 'geocodejson'],
 663                        help='Format of result')
 664     for name, desc in EXTRADATA_PARAMS:
 665         group.add_argument('--' + name, action='store_true', help=desc)
 666
 667     group.add_argument('--lang', '--accept-language', metavar='LANGS',
 668                        help='Preferred language order for presenting search results')
 669     group.add_argument('--polygon-output',
 670                        choices=['geojson', 'kml', 'svg', 'text'],
 671                        help='Output geometry of results as a GeoJSON, KML, SVG or WKT.')
 672     group.add_argument('--polygon-threshold', type=float, metavar='TOLERANCE',
 673                        help="""Simplify output geometry.
 674                                Parameter is difference tolerance in degrees.""")
 675
 676
 677 class APISearch:
 678     """\
 679     Execute API search query.
 680     """
 681
 682     @staticmethod
 683     def add_args(parser):
 684         group = parser.add_argument_group('Query arguments')
 685         group.add_argument('--query',
 686                            help='Free-form query string')
 687         for name, desc in STRUCTURED_QUERY:
 688             group.add_argument('--' + name, help='Structured query: ' + desc)
 689
 690         _add_api_output_arguments(parser)
 691
 692         group = parser.add_argument_group('Result limitation')
 693         group.add_argument('--countrycodes', metavar='CC,..',
 694                            help='Limit search results to one or more countries.')
 695         group.add_argument('--exclude_place_ids', metavar='ID,..',
 696                            help='List of search object to be excluded')
 697         group.add_argument('--limit', type=int,
 698                            help='Limit the number of returned results')
 699         group.add_argument('--viewbox', metavar='X1,Y1,X2,Y2',
 700                            help='Preferred area to find search results')
 701         group.add_argument('--bounded', action='store_true',
 702                            help='Strictly restrict results to viewbox area')
 703
 704         group = parser.add_argument_group('Other arguments')
 705         group.add_argument('--no-dedupe', action='store_false', dest='dedupe',
 706                            help='Do not remove duplicates from the result list')
 707
 708
 709     @staticmethod
 710     def run(args):
 711         if args.query:
 712             params = dict(q=args.query)
 713         else:
 714             params = {k : getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)}
 715
 716         for param, _ in EXTRADATA_PARAMS:
 717             if getattr(args, param):
 718                 params[param] = '1'
 719         for param in ('format', 'countrycodes', 'exclude_place_ids', 'limit', 'viewbox'):
 720             if getattr(args, param):
 721                 params[param] = getattr(args, param)
 722         if args.lang:
 723             params['accept-language'] = args.lang
 724         if args.polygon_output:
 725             params['polygon_' + args.polygon_output] = '1'
 726         if args.polygon_threshold:
 727             params['polygon_threshold'] = args.polygon_threshold
 728         if args.bounded:
 729             params['bounded'] = '1'
 730         if not args.dedupe:
 731             params['dedupe'] = '0'
 732
 733         return run_api_script('search', args.project_dir,
 734                               phpcgi_bin=args.phpcgi_path, params=params)
 735
 736 class APIReverse:
 737     """\
 738     Execute API reverse query.
 739     """
 740
 741     @staticmethod
 742     def add_args(parser):
 743         group = parser.add_argument_group('Query arguments')
 744         group.add_argument('--lat', type=float, required=True,
 745                            help='Latitude of coordinate to look up (in WGS84)')
 746         group.add_argument('--lon', type=float, required=True,
 747                            help='Longitude of coordinate to look up (in WGS84)')
 748         group.add_argument('--zoom', type=int,
 749                            help='Level of detail required for the address')
 750
 751         _add_api_output_arguments(parser)
 752
 753
 754     @staticmethod
 755     def run(args):
 756         params = dict(lat=args.lat, lon=args.lon)
 757         if args.zoom is not None:
 758             params['zoom'] = args.zoom
 759
 760         for param, _ in EXTRADATA_PARAMS:
 761             if getattr(args, param):
 762                 params[param] = '1'
 763         if args.format:
 764             params['format'] = args.format
 765         if args.lang:
 766             params['accept-language'] = args.lang
 767         if args.polygon_output:
 768             params['polygon_' + args.polygon_output] = '1'
 769         if args.polygon_threshold:
 770             params['polygon_threshold'] = args.polygon_threshold
 771
 772         return run_api_script('reverse', args.project_dir,
 773                               phpcgi_bin=args.phpcgi_path, params=params)
 774
 775
 776 class APILookup:
 777     """\
 778     Execute API reverse query.
 779     """
 780
 781     @staticmethod
 782     def add_args(parser):
 783         group = parser.add_argument_group('Query arguments')
 784         group.add_argument('--id', metavar='OSMID',
 785                            action='append', required=True, dest='ids',
 786                            help='OSM id to lookup in format <NRW><id> (may be repeated)')
 787
 788         _add_api_output_arguments(parser)
 789
 790
 791     @staticmethod
 792     def run(args):
 793         params = dict(osm_ids=','.join(args.ids))
 794
 795         for param, _ in EXTRADATA_PARAMS:
 796             if getattr(args, param):
 797                 params[param] = '1'
 798         if args.format:
 799             params['format'] = args.format
 800         if args.lang:
 801             params['accept-language'] = args.lang
 802         if args.polygon_output:
 803             params['polygon_' + args.polygon_output] = '1'
 804         if args.polygon_threshold:
 805             params['polygon_threshold'] = args.polygon_threshold
 806
 807         return run_api_script('lookup', args.project_dir,
 808                               phpcgi_bin=args.phpcgi_path, params=params)
 809
 810
 811 class APIDetails:
 812     """\
 813     Execute API lookup query.
 814     """
 815
 816     @staticmethod
 817     def add_args(parser):
 818         group = parser.add_argument_group('Query arguments')
 819         objs = group.add_mutually_exclusive_group(required=True)
 820         objs.add_argument('--node', '-n', type=int,
 821                           help="Look up the OSM node with the given ID.")
 822         objs.add_argument('--way', '-w', type=int,
 823                           help="Look up the OSM way with the given ID.")
 824         objs.add_argument('--relation', '-r', type=int,
 825                           help="Look up the OSM relation with the given ID.")
 826         objs.add_argument('--place_id', '-p', type=int,
 827                           help='Database internal identifier of the OSM object to look up.')
 828         group.add_argument('--class', dest='object_class',
 829                            help="""Class type to disambiguated multiple entries
 830                                    of the same object.""")
 831
 832         group = parser.add_argument_group('Output arguments')
 833         for name, desc in DETAILS_SWITCHES:
 834             group.add_argument('--' + name, action='store_true', help=desc)
 835         group.add_argument('--lang', '--accept-language', metavar='LANGS',
 836                            help='Preferred language order for presenting search results')
 837
 838     @staticmethod
 839     def run(args):
 840         if args.node:
 841             params = dict(osmtype='N', osmid=args.node)
 842         elif args.way:
 843             params = dict(osmtype='W', osmid=args.node)
 844         elif args.relation:
 845             params = dict(osmtype='R', osmid=args.node)
 846         else:
 847             params = dict(place_id=args.place_id)
 848         if args.object_class:
 849             params['class'] = args.object_class
 850         for name, _ in DETAILS_SWITCHES:
 851             params[name] = '1' if getattr(args, name) else '0'
 852
 853         return run_api_script('details', args.project_dir,
 854                               phpcgi_bin=args.phpcgi_path, params=params)
 855
 856
 857 class APIStatus:
 858     """\
 859     Execute API status query.
 860     """
 861
 862     @staticmethod
 863     def add_args(parser):
 864         group = parser.add_argument_group('API parameters')
 865         group.add_argument('--format', default='text', choices=['text', 'json'],
 866                            help='Format of result')
 867
 868     @staticmethod
 869     def run(args):
 870         return run_api_script('status', args.project_dir,
 871                               phpcgi_bin=args.phpcgi_path,
 872                               params=dict(format=args.format))
 873
 874
 875 def nominatim(**kwargs):
 876     """\
 877     Command-line tools for importing, updating, administrating and
 878     querying the Nominatim database.
 879     """
 880     parser = CommandlineParser('nominatim', nominatim.__doc__)
 881
 882     parser.add_subcommand('import', SetupAll)
 883     parser.add_subcommand('freeze', SetupFreeze)
 884     parser.add_subcommand('replication', UpdateReplication)
 885
 886     parser.add_subcommand('check-database', AdminCheckDatabase)
 887     parser.add_subcommand('warm', AdminWarm)
 888
 889     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
 890
 891     parser.add_subcommand('add-data', UpdateAddData)
 892     parser.add_subcommand('index', UpdateIndex)
 893     parser.add_subcommand('refresh', UpdateRefresh)
 894
 895     parser.add_subcommand('export', QueryExport)
 896
 897     if kwargs.get('phpcgi_path'):
 898         parser.add_subcommand('search', APISearch)
 899         parser.add_subcommand('reverse', APIReverse)
 900         parser.add_subcommand('lookup', APILookup)
 901         parser.add_subcommand('details', APIDetails)
 902         parser.add_subcommand('status', APIStatus)
 903     else:
 904         parser.parser.epilog = 'php-cgi not found. Query commands not available.'
 905
 906     return parser.run(**kwargs)