nominatim/cli.py

   1 """
   2 Command-line interface to the Nominatim functions for import, update,
   3 database administration and querying.
   4 """
   5 import sys
   6 import os
   7 import argparse
   8 import logging
   9 from pathlib import Path
  10
  11 from .config import Configuration
  12 from .admin.exec_utils import run_legacy_script
  13
  14 from .indexer.indexer import Indexer
  15
  16 def _num_system_cpus():
  17     try:
  18         cpus = len(os.sched_getaffinity(0))
  19     except NotImplementedError:
  20         cpus = None
  21
  22     return cpus or os.cpu_count()
  23
  24
  25 class CommandlineParser:
  26     """ Wraps some of the common functions for parsing the command line
  27         and setting up subcommands.
  28     """
  29     def __init__(self, prog, description):
  30         self.parser = argparse.ArgumentParser(
  31             prog=prog,
  32             description=description,
  33             formatter_class=argparse.RawDescriptionHelpFormatter)
  34
  35         self.subs = self.parser.add_subparsers(title='available commands',
  36                                                dest='subcommand')
  37
  38         # Arguments added to every sub-command
  39         self.default_args = argparse.ArgumentParser(add_help=False)
  40         group = self.default_args.add_argument_group('Default arguments')
  41         group.add_argument('-h', '--help', action='help',
  42                            help='Show this help message and exit')
  43         group.add_argument('-q', '--quiet', action='store_const', const=0,
  44                            dest='verbose', default=1,
  45                            help='Print only error messages')
  46         group.add_argument('-v', '--verbose', action='count', default=1,
  47                            help='Increase verboseness of output')
  48         group.add_argument('--project-dir', metavar='DIR', default='.',
  49                            help='Base directory of the Nominatim installation (default:.)')
  50         group.add_argument('-j', '--threads', metavar='NUM', type=int,
  51                            help='Number of parallel threads to use')
  52
  53
  54     def add_subcommand(self, name, cmd):
  55         """ Add a subcommand to the parser. The subcommand must be a class
  56             with a function add_args() that adds the parameters for the
  57             subcommand and a run() function that executes the command.
  58         """
  59         parser = self.subs.add_parser(name, parents=[self.default_args],
  60                                       help=cmd.__doc__.split('\n', 1)[0],
  61                                       description=cmd.__doc__,
  62                                       formatter_class=argparse.RawDescriptionHelpFormatter,
  63                                       add_help=False)
  64         parser.set_defaults(command=cmd)
  65         cmd.add_args(parser)
  66
  67     def run(self, **kwargs):
  68         """ Parse the command line arguments of the program and execute the
  69             appropriate subcommand.
  70         """
  71         args = self.parser.parse_args()
  72
  73         if args.subcommand is None:
  74             return self.parser.print_help()
  75
  76         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir'):
  77             setattr(args, arg, Path(kwargs[arg]))
  78         args.project_dir = Path(args.project_dir)
  79
  80         logging.basicConfig(stream=sys.stderr,
  81                             format='%(asctime)s %(levelname)s: %(message)s',
  82                             datefmt='%Y-%m-%d %H:%M:%S',
  83                             level=max(4 - args.verbose, 1) * 10)
  84
  85         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
  86
  87         return args.command.run(args)
  88
  89 ##### Subcommand classes
  90 #
  91 # Each class needs to implement two functions: add_args() adds the CLI parameters
  92 # for the subfunction, run() executes the subcommand.
  93 #
  94 # The class documentation doubles as the help text for the command. The
  95 # first line is also used in the summary when calling the program without
  96 # a subcommand.
  97 #
  98 # No need to document the functions each time.
  99 # pylint: disable=C0111
 100
 101
 102 class SetupAll:
 103     """\
 104     Create a new Nominatim database from an OSM file.
 105     """
 106
 107     @staticmethod
 108     def add_args(parser):
 109         group_name = parser.add_argument_group('Required arguments')
 110         group = group_name.add_mutually_exclusive_group(required=True)
 111         group.add_argument('--osm-file',
 112                            help='OSM file to be imported.')
 113         group.add_argument('--continue', dest='continue_at',
 114                            choices=['load-data', 'indexing', 'db-postprocess'],
 115                            help='Continue an import that was interrupted')
 116         group = parser.add_argument_group('Optional arguments')
 117         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 118                            help='Size of cache to be used by osm2pgsql (in MB)')
 119         group.add_argument('--reverse-only', action='store_true',
 120                            help='Do not create tables and indexes for searching')
 121         group.add_argument('--enable-debug-statements', action='store_true',
 122                            help='Include debug warning statements in SQL code')
 123         group.add_argument('--no-partitions', action='store_true',
 124                            help="""Do not partition search indices
 125                                    (speeds up import of single country extracts)""")
 126         group.add_argument('--no-updates', action='store_true',
 127                            help="""Do not keep tables that are only needed for
 128                                    updating the database later""")
 129         group = parser.add_argument_group('Expert options')
 130         group.add_argument('--ignore-errors', action='store_true',
 131                            help='Continue import even when errors in SQL are present')
 132         group.add_argument('--index-noanalyse', action='store_true',
 133                            help='Do not perform analyse operations during index')
 134
 135
 136     @staticmethod
 137     def run(args):
 138         params = ['setup.php']
 139         if args.osm_file:
 140             params.extend(('--all', '--osm-file', args.osm_file))
 141         else:
 142             if args.continue_at == 'load-data':
 143                 params.append('--load-data')
 144             if args.continue_at in ('load-data', 'indexing'):
 145                 params.append('--index')
 146             params.extend(('--create-search-indices', '--create-country-names',
 147                            '--setup-website'))
 148         if args.osm2pgsql_cache:
 149             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
 150         if args.reverse_only:
 151             params.append('--reverse-only')
 152         if args.enable_debug_statements:
 153             params.append('--enable-debug-statements')
 154         if args.no_partitions:
 155             params.append('--no-partitions')
 156         if args.no_updates:
 157             params.append('--drop')
 158         if args.ignore_errors:
 159             params.append('--ignore-errors')
 160         if args.index_noanalyse:
 161             params.append('--index-noanalyse')
 162
 163         return run_legacy_script(*params, nominatim_env=args)
 164
 165
 166 class SetupFreeze:
 167     """\
 168     Make database read-only.
 169
 170     About half of data in the Nominatim database is kept only to be able to
 171     keep the data up-to-date with new changes made in OpenStreetMap. This
 172     command drops all this data and only keeps the part needed for geocoding
 173     itself.
 174
 175     This command has the same effect as the `--no-updates` option for imports.
 176     """
 177
 178     @staticmethod
 179     def add_args(parser):
 180         pass # No options
 181
 182     @staticmethod
 183     def run(args):
 184         return run_legacy_script('setup.php', '--drop', nominatim_env=args)
 185
 186
 187 class SetupSpecialPhrases:
 188     """\
 189     Maintain special phrases.
 190     """
 191
 192     @staticmethod
 193     def add_args(parser):
 194         group = parser.add_argument_group('Input arguments')
 195         group.add_argument('--from-wiki', action='store_true',
 196                            help='Pull special phrases from the OSM wiki.')
 197         group = parser.add_argument_group('Output arguments')
 198         group.add_argument('-o', '--output', default='-',
 199                            type=argparse.FileType('w', encoding='UTF-8'),
 200                            help="""File to write the preprocessed phrases to.
 201                                    If omitted, it will be written to stdout.""")
 202
 203     @staticmethod
 204     def run(args):
 205         if args.output.name != '<stdout>':
 206             raise NotImplementedError('Only output to stdout is currently implemented.')
 207         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 208
 209
 210 class UpdateReplication:
 211     """\
 212     Update the database using an online replication service.
 213     """
 214
 215     @staticmethod
 216     def add_args(parser):
 217         group = parser.add_argument_group('Arguments for initialisation')
 218         group.add_argument('--init', action='store_true',
 219                            help='Initialise the update process')
 220         group.add_argument('--no-update-functions', dest='update_functions',
 221                            action='store_false',
 222                            help="""Do not update the trigger function to
 223                                    support differential updates.""")
 224         group = parser.add_argument_group('Arguments for updates')
 225         group.add_argument('--check-for-updates', action='store_true',
 226                            help='Check if new updates are available and exit')
 227         group.add_argument('--once', action='store_true',
 228                            help="""Download and apply updates only once. When
 229                                    not set, updates are continuously applied""")
 230         group.add_argument('--no-index', action='store_false', dest='do_index',
 231                            help="""Do not index the new data. Only applicable
 232                                    together with --once""")
 233
 234     @staticmethod
 235     def run(args):
 236         params = ['update.php']
 237         if args.init:
 238             params.append('--init-updates')
 239             if not args.update_functions:
 240                 params.append('--no-update-functions')
 241         elif args.check_for_updates:
 242             params.append('--check-for-updates')
 243         else:
 244             if args.once:
 245                 params.append('--import-osmosis')
 246             else:
 247                 params.append('--import-osmosis-all')
 248             if not args.do_index:
 249                 params.append('--no-index')
 250
 251         return run_legacy_script(*params, nominatim_env=args)
 252
 253
 254 class UpdateAddData:
 255     """\
 256     Add additional data from a file or an online source.
 257
 258     Data is only imported, not indexed. You need to call `nominatim-update index`
 259     to complete the process.
 260     """
 261
 262     @staticmethod
 263     def add_args(parser):
 264         group_name = parser.add_argument_group('Source')
 265         group = group_name.add_mutually_exclusive_group(required=True)
 266         group.add_argument('--file', metavar='FILE',
 267                            help='Import data from an OSM file')
 268         group.add_argument('--diff', metavar='FILE',
 269                            help='Import data from an OSM diff file')
 270         group.add_argument('--node', metavar='ID', type=int,
 271                            help='Import a single node from the API')
 272         group.add_argument('--way', metavar='ID', type=int,
 273                            help='Import a single way from the API')
 274         group.add_argument('--relation', metavar='ID', type=int,
 275                            help='Import a single relation from the API')
 276         group.add_argument('--tiger-data', metavar='DIR',
 277                            help='Add housenumbers from the US TIGER census database.')
 278         group = parser.add_argument_group('Extra arguments')
 279         group.add_argument('--use-main-api', action='store_true',
 280                            help='Use OSM API instead of Overpass to download objects')
 281
 282     @staticmethod
 283     def run(args):
 284         if args.tiger_data:
 285             os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
 286             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 287
 288         params = ['update.php']
 289         if args.file:
 290             params.extend(('--import-file', args.file))
 291         elif args.diff:
 292             params.extend(('--import-diff', args.diff))
 293         elif args.node:
 294             params.extend(('--import-node', args.node))
 295         elif args.way:
 296             params.extend(('--import-way', args.way))
 297         elif args.relation:
 298             params.extend(('--import-relation', args.relation))
 299         if args.use_main_api:
 300             params.append('--use-main-api')
 301         return run_legacy_script(*params, nominatim_env=args)
 302
 303
 304 class UpdateIndex:
 305     """\
 306     Reindex all new and modified data.
 307     """
 308
 309     @staticmethod
 310     def add_args(parser):
 311         group = parser.add_argument_group('Filter arguments')
 312         group.add_argument('--boundaries-only', action='store_true',
 313                            help="""Index only administrative boundaries.""")
 314         group.add_argument('--no-boundaries', action='store_true',
 315                            help="""Index everything except administrative boundaries.""")
 316         group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
 317                            help='Minimum/starting rank')
 318         group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
 319                            help='Maximum/finishing rank')
 320
 321     @staticmethod
 322     def run(args):
 323         indexer = Indexer(args.config.get_libpq_dsn(),
 324                           args.threads or _num_system_cpus() or 1)
 325
 326         if not args.no_boundaries:
 327             indexer.index_boundaries(args.minrank, args.maxrank)
 328         if not args.boundaries_only:
 329             indexer.index_by_rank(args.minrank, args.maxrank)
 330
 331         return 0
 332
 333
 334 class UpdateRefresh:
 335     """\
 336     Recompute auxiliary data used by the indexing process.
 337
 338     These functions must not be run in parallel with other update commands.
 339     """
 340
 341     @staticmethod
 342     def add_args(parser):
 343         group = parser.add_argument_group('Data arguments')
 344         group.add_argument('--postcodes', action='store_true',
 345                            help='Update postcode centroid table')
 346         group.add_argument('--word-counts', action='store_true',
 347                            help='Compute frequency of full-word search terms')
 348         group.add_argument('--address-levels', action='store_true',
 349                            help='Reimport address level configuration')
 350         group.add_argument('--functions', action='store_true',
 351                            help='Update the PL/pgSQL functions in the database')
 352         group.add_argument('--wiki-data', action='store_true',
 353                            help='Update Wikipedia/data importance numbers.')
 354         group.add_argument('--importance', action='store_true',
 355                            help='Recompute place importances (expensive!)')
 356         group.add_argument('--website', action='store_true',
 357                            help='Refresh the directory that serves the scripts for the web API')
 358         group = parser.add_argument_group('Arguments for function refresh')
 359         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
 360                            help='Do not enable code for propagating updates')
 361         group.add_argument('--enable-debug-statements', action='store_true',
 362                            help='Enable debug warning statements in functions')
 363
 364     @staticmethod
 365     def run(args):
 366         if args.postcodes:
 367             run_legacy_script('update.php', '--calculate-postcodes',
 368                               nominatim_env=args, throw_on_fail=True)
 369         if args.word_counts:
 370             run_legacy_script('update.php', '--recompute-word-counts',
 371                               nominatim_env=args, throw_on_fail=True)
 372         if args.address_levels:
 373             run_legacy_script('update.php', '--update-address-levels',
 374                               nominatim_env=args, throw_on_fail=True)
 375         if args.functions:
 376             params = ['setup.php', '--create-functions', '--create-partition-functions']
 377             if args.diffs:
 378                 params.append('--enable-diff-updates')
 379             if args.enable_debug_statements:
 380                 params.append('--enable-debug-statements')
 381             run_legacy_script(*params, nominatim_env=args, throw_on_fail=True)
 382         if args.wiki_data:
 383             run_legacy_script('setup.php', '--import-wikipedia-articles',
 384                               nominatim_env=args, throw_on_fail=True)
 385         # Attention: importance MUST come after wiki data import.
 386         if args.importance:
 387             run_legacy_script('update.php', '--recompute-importance',
 388                               nominatim_env=args, throw_on_fail=True)
 389         if args.website:
 390             run_legacy_script('setup.php', '--setup-website',
 391                               nominatim_env=args, throw_on_fail=True)
 392
 393
 394 class AdminCheckDatabase:
 395     """\
 396     Check that the database is complete and operational.
 397     """
 398
 399     @staticmethod
 400     def add_args(parser):
 401         pass # No options
 402
 403     @staticmethod
 404     def run(args):
 405         return run_legacy_script('check_import_finished.php', nominatim_env=args)
 406
 407
 408 class AdminWarm:
 409     """\
 410     Warm database caches for search and reverse queries.
 411     """
 412
 413     @staticmethod
 414     def add_args(parser):
 415         group = parser.add_argument_group('Target arguments')
 416         group.add_argument('--search-only', action='store_const', dest='target',
 417                            const='search',
 418                            help="Only pre-warm tables for search queries")
 419         group.add_argument('--reverse-only', action='store_const', dest='target',
 420                            const='reverse',
 421                            help="Only pre-warm tables for reverse queries")
 422
 423     @staticmethod
 424     def run(args):
 425         params = ['warm.php']
 426         if args.target == 'reverse':
 427             params.append('--reverse-only')
 428         if args.target == 'search':
 429             params.append('--search-only')
 430         return run_legacy_script(*params, nominatim_env=args)
 431
 432
 433 class QueryExport:
 434     """\
 435     Export addresses as CSV file from a Nominatim database.
 436     """
 437
 438     @staticmethod
 439     def add_args(parser):
 440         group = parser.add_argument_group('Output arguments')
 441         group.add_argument('--output-type', default='street',
 442                            choices=('continent', 'country', 'state', 'county',
 443                                     'city', 'suburb', 'street', 'path'),
 444                            help='Type of places to output (default: street)')
 445         group.add_argument('--output-format',
 446                            default='street;suburb;city;county;state;country',
 447                            help="""Semicolon-separated list of address types
 448                                    (see --output-type). Multiple ranks can be
 449                                    merged into one column by simply using a
 450                                    comma-separated list.""")
 451         group.add_argument('--output-all-postcodes', action='store_true',
 452                            help="""List all postcodes for address instead of
 453                                    just the most likely one""")
 454         group.add_argument('--language',
 455                            help="""Preferred language for output
 456                                    (use local name, if omitted)""")
 457         group = parser.add_argument_group('Filter arguments')
 458         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
 459                            help='Export only objects within country')
 460         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
 461                            help='Export only children of this OSM node')
 462         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
 463                            help='Export only children of this OSM way')
 464         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
 465                            help='Export only children of this OSM relation')
 466
 467
 468     @staticmethod
 469     def run(args):
 470         params = ['export.php',
 471                   '--output-type', args.output_type,
 472                   '--output-format', args.output_format]
 473         if args.output_all_postcodes:
 474             params.append('--output-all-postcodes')
 475         if args.language:
 476             params.extend(('--language', args.language))
 477         if args.restrict_to_country:
 478             params.extend(('--restrict-to-country', args.restrict_to_country))
 479         if args.restrict_to_osm_node:
 480             params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
 481         if args.restrict_to_osm_way:
 482             params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
 483         if args.restrict_to_osm_relation:
 484             params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
 485
 486         return run_legacy_script(*params, nominatim_env=args)
 487
 488 class QueryTodo:
 489     """\
 490     Todo
 491     """
 492     @staticmethod
 493     def add_args(parser):
 494         pass
 495
 496     @staticmethod
 497     def run(args): # pylint: disable=W0613
 498         print("TODO: searching")
 499
 500
 501 def nominatim(**kwargs):
 502     """\
 503     Command-line tools for importing, updating, administrating and
 504     querying the Nominatim database.
 505     """
 506     parser = CommandlineParser('nominatim', nominatim.__doc__)
 507
 508     parser.add_subcommand('import', SetupAll)
 509     parser.add_subcommand('freeze', SetupFreeze)
 510     parser.add_subcommand('replication', UpdateReplication)
 511
 512     parser.add_subcommand('check-database', AdminCheckDatabase)
 513     parser.add_subcommand('warm', AdminWarm)
 514
 515     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
 516
 517     parser.add_subcommand('add-data', UpdateAddData)
 518     parser.add_subcommand('index', UpdateIndex)
 519     parser.add_subcommand('refresh', UpdateRefresh)
 520
 521     parser.add_subcommand('export', QueryExport)
 522     parser.add_subcommand('search', QueryTodo)
 523     parser.add_subcommand('reverse', QueryTodo)
 524     parser.add_subcommand('lookup', QueryTodo)
 525     parser.add_subcommand('details', QueryTodo)
 526     parser.add_subcommand('status', QueryTodo)
 527
 528     return parser.run(**kwargs)