nominatim/cli.py

   1 """
   2 Command-line interface to the Nominatim functions for import, update,
   3 database administration and querying.
   4 """
   5 import sys
   6 import os
   7 import argparse
   8 import logging
   9 from pathlib import Path
  10
  11 from .config import Configuration
  12 from .admin.exec_utils import run_legacy_script
  13
  14 from .indexer.indexer import Indexer
  15
  16 def _num_system_cpus():
  17     try:
  18         cpus = len(os.sched_getaffinity(0))
  19     except NotImplementedError:
  20         cpus = None
  21
  22     return cpus or os.cpu_count()
  23
  24
  25 class CommandlineParser:
  26     """ Wraps some of the common functions for parsing the command line
  27         and setting up subcommands.
  28     """
  29     def __init__(self, prog, description):
  30         self.parser = argparse.ArgumentParser(
  31             prog=prog,
  32             description=description,
  33             formatter_class=argparse.RawDescriptionHelpFormatter)
  34
  35         self.subs = self.parser.add_subparsers(title='available commands',
  36                                                dest='subcommand')
  37
  38         # Arguments added to every sub-command
  39         self.default_args = argparse.ArgumentParser(add_help=False)
  40         group = self.default_args.add_argument_group('Default arguments')
  41         group.add_argument('-h', '--help', action='help',
  42                            help='Show this help message and exit')
  43         group.add_argument('-q', '--quiet', action='store_const', const=0,
  44                            dest='verbose', default=1,
  45                            help='Print only error messages')
  46         group.add_argument('-v', '--verbose', action='count', default=1,
  47                            help='Increase verboseness of output')
  48         group.add_argument('--project-dir', metavar='DIR', default='.',
  49                            help='Base directory of the Nominatim installation (default:.)')
  50         group.add_argument('-j', '--threads', metavar='NUM', type=int,
  51                            help='Number of parallel threads to use')
  52
  53
  54     def add_subcommand(self, name, cmd):
  55         """ Add a subcommand to the parser. The subcommand must be a class
  56             with a function add_args() that adds the parameters for the
  57             subcommand and a run() function that executes the command.
  58         """
  59         parser = self.subs.add_parser(name, parents=[self.default_args],
  60                                       help=cmd.__doc__.split('\n', 1)[0],
  61                                       description=cmd.__doc__,
  62                                       formatter_class=argparse.RawDescriptionHelpFormatter,
  63                                       add_help=False)
  64         parser.set_defaults(command=cmd)
  65         cmd.add_args(parser)
  66
  67     def run(self, **kwargs):
  68         """ Parse the command line arguments of the program and execute the
  69             appropriate subcommand.
  70         """
  71         args = self.parser.parse_args()
  72
  73         if args.subcommand is None:
  74             return self.parser.print_help()
  75
  76         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir'):
  77             setattr(args, arg, Path(kwargs[arg]))
  78         args.project_dir = Path(args.project_dir)
  79
  80         logging.basicConfig(stream=sys.stderr,
  81                             format='%(asctime)s: %(message)s',
  82                             datefmt='%Y-%m-%d %H:%M:%S',
  83                             level=max(4 - args.verbose, 1) * 10)
  84
  85         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
  86
  87         return args.command.run(args)
  88
  89 ##### Subcommand classes
  90 #
  91 # Each class needs to implement two functions: add_args() adds the CLI parameters
  92 # for the subfunction, run() executes the subcommand.
  93 #
  94 # The class documentation doubles as the help text for the command. The
  95 # first line is also used in the summary when calling the program without
  96 # a subcommand.
  97 #
  98 # No need to document the functions each time.
  99 # pylint: disable=C0111
 100
 101
 102 class SetupAll:
 103     """\
 104     Create a new Nominatim database from an OSM file.
 105     """
 106
 107     @staticmethod
 108     def add_args(parser):
 109         group_name = parser.add_argument_group('Required arguments')
 110         group = group_name.add_mutually_exclusive_group(required=True)
 111         group.add_argument('--osm-file',
 112                            help='OSM file to be imported.')
 113         group.add_argument('--continue', dest='continue_at',
 114                            choices=['load-data', 'indexing', 'db-postprocess'],
 115                            help='Continue an import that was interrupted')
 116         group = parser.add_argument_group('Optional arguments')
 117         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 118                            help='Size of cache to be used by osm2pgsql (in MB)')
 119         group.add_argument('--reverse-only', action='store_true',
 120                            help='Do not create tables and indexes for searching')
 121         group.add_argument('--enable-debug-statements', action='store_true',
 122                            help='Include debug warning statements in SQL code')
 123         group.add_argument('--no-partitions', action='store_true',
 124                            help="""Do not partition search indices
 125                                    (speeds up import of single country extracts)""")
 126         group.add_argument('--no-updates', action='store_true',
 127                            help="""Do not keep tables that are only needed for
 128                                    updating the database later""")
 129         group = parser.add_argument_group('Expert options')
 130         group.add_argument('--ignore-errors', action='store_true',
 131                            help='Continue import even when errors in SQL are present')
 132         group.add_argument('--index-noanalyse', action='store_true',
 133                            help='Do not perform analyse operations during index')
 134
 135
 136     @staticmethod
 137     def run(args):
 138         params = ['setup.php']
 139         if args.osm_file:
 140             params.extend(('--all', '--osm-file', args.osm_file))
 141         else:
 142             if args.continue_at == 'load-data':
 143                 params.append('--load-data')
 144             if args.continue_at in ('load-data', 'indexing'):
 145                 params.append('--index')
 146             params.extend(('--create-search-indices', '--create-country-names',
 147                            '--setup-website'))
 148         if args.osm2pgsql_cache:
 149             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
 150         if args.reverse_only:
 151             params.append('--reverse-only')
 152         if args.enable_debug_statements:
 153             params.append('--enable-debug-statements')
 154         if args.no_partitions:
 155             params.append('--no-partitions')
 156         if args.no_updates:
 157             params.append('--drop')
 158         if args.ignore_errors:
 159             params.append('--ignore-errors')
 160         if args.index_noanalyse:
 161             params.append('--index-noanalyse')
 162
 163         return run_legacy_script(*params, nominatim_env=args)
 164
 165
 166 class SetupFreeze:
 167     """\
 168     Make database read-only.
 169
 170     About half of data in the Nominatim database is kept only to be able to
 171     keep the data up-to-date with new changes made in OpenStreetMap. This
 172     command drops all this data and only keeps the part needed for geocoding
 173     itself.
 174
 175     This command has the same effect as the `--no-updates` option for imports.
 176     """
 177
 178     @staticmethod
 179     def add_args(parser):
 180         pass # No options
 181
 182     @staticmethod
 183     def run(args):
 184         return run_legacy_script('setup.php', '--drop', nominatim_env=args)
 185
 186
 187 class SetupSpecialPhrases:
 188     """\
 189     Maintain special phrases.
 190     """
 191
 192     @staticmethod
 193     def add_args(parser):
 194         group = parser.add_argument_group('Input arguments')
 195         group.add_argument('--from-wiki', action='store_true',
 196                            help='Pull special phrases from the OSM wiki.')
 197         group = parser.add_argument_group('Output arguments')
 198         group.add_argument('-o', '--output', default='-',
 199                            type=argparse.FileType('w', encoding='UTF-8'),
 200                            help="""File to write the preprocessed phrases to.
 201                                    If omitted, it will be written to stdout.""")
 202
 203     @staticmethod
 204     def run(args):
 205         if args.output.name != '<stdout>':
 206             raise NotImplementedError('Only output to stdout is currently implemented.')
 207         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 208
 209
 210 class UpdateReplication:
 211     """\
 212     Update the database using an online replication service.
 213     """
 214
 215     @staticmethod
 216     def add_args(parser):
 217         group = parser.add_argument_group('Arguments for initialisation')
 218         group.add_argument('--init', action='store_true',
 219                            help='Initialise the update process')
 220         group.add_argument('--no-update-functions', dest='update_functions',
 221                            action='store_false',
 222                            help="""Do not update the trigger function to
 223                                    support differential updates.""")
 224         group = parser.add_argument_group('Arguments for updates')
 225         group.add_argument('--check-for-updates', action='store_true',
 226                            help='Check if new updates are available and exit')
 227         group.add_argument('--once', action='store_true',
 228                            help="""Download and apply updates only once. When
 229                                    not set, updates are continuously applied""")
 230         group.add_argument('--no-index', action='store_false', dest='do_index',
 231                            help="""Do not index the new data. Only applicable
 232                                    together with --once""")
 233
 234     @staticmethod
 235     def run(args):
 236         params = ['update.php']
 237         if args.init:
 238             params.append('--init-updates')
 239             if not args.update_functions:
 240                 params.append('--no-update-functions')
 241         elif args.check_for_updates:
 242             params.append('--check-for-updates')
 243         else:
 244             if args.once:
 245                 params.append('--import-osmosis')
 246             else:
 247                 params.append('--import-osmosis-all')
 248             if not args.do_index:
 249                 params.append('--no-index')
 250
 251         return run_legacy_script(*params, nominatim_env=args)
 252
 253
 254 class UpdateAddData:
 255     """\
 256     Add additional data from a file or an online source.
 257
 258     Data is only imported, not indexed. You need to call `nominatim-update index`
 259     to complete the process.
 260     """
 261
 262     @staticmethod
 263     def add_args(parser):
 264         group_name = parser.add_argument_group('Source')
 265         group = group_name.add_mutually_exclusive_group(required=True)
 266         group.add_argument('--file', metavar='FILE',
 267                            help='Import data from an OSM file')
 268         group.add_argument('--diff', metavar='FILE',
 269                            help='Import data from an OSM diff file')
 270         group.add_argument('--node', metavar='ID', type=int,
 271                            help='Import a single node from the API')
 272         group.add_argument('--way', metavar='ID', type=int,
 273                            help='Import a single way from the API')
 274         group.add_argument('--relation', metavar='ID', type=int,
 275                            help='Import a single relation from the API')
 276         group.add_argument('--tiger-data', metavar='DIR',
 277                            help='Add housenumbers from the US TIGER census database.')
 278         group = parser.add_argument_group('Extra arguments')
 279         group.add_argument('--use-main-api', action='store_true',
 280                            help='Use OSM API instead of Overpass to download objects')
 281
 282     @staticmethod
 283     def run(args):
 284         if args.tiger_data:
 285             os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
 286             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 287
 288         params = ['update.php']
 289         if args.file:
 290             params.extend(('--import-file', args.file))
 291         elif args.diff:
 292             params.extend(('--import-diff', args.diff))
 293         elif args.node:
 294             params.extend(('--import-node', args.node))
 295         elif args.way:
 296             params.extend(('--import-way', args.way))
 297         elif args.relation:
 298             params.extend(('--import-relation', args.relation))
 299         if args.use_main_api:
 300             params.append('--use-main-api')
 301         return run_legacy_script(*params, nominatim_env=args)
 302
 303
 304 class UpdateIndex:
 305     """\
 306     Reindex all new and modified data.
 307     """
 308
 309     @staticmethod
 310     def add_args(parser):
 311         group = parser.add_argument_group('Filter arguments')
 312         group.add_argument('--boundaries-only', action='store_true',
 313                            help="""Index only administrative boundaries.""")
 314         group.add_argument('--no-boundaries', action='store_true',
 315                            help="""Index everything except administrative boundaries.""")
 316         group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
 317                            help='Minimum/starting rank')
 318         group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
 319                            help='Maximum/finishing rank')
 320
 321     @staticmethod
 322     def run(args):
 323         indexer = Indexer(args.config.get_libpq_dsn(),
 324                           args.threads or _num_system_cpus() or 1)
 325
 326         if not args.no_boundaries:
 327             indexer.index_boundaries(args.minrank, args.maxrank)
 328         if not args.boundaries_only:
 329             indexer.index_by_rank(args.minrank, args.maxrank)
 330
 331         if not args.no_boundaries and not args.boundaries_only:
 332             indexer.update_status_table()
 333
 334         return 0
 335
 336
 337 class UpdateRefresh:
 338     """\
 339     Recompute auxiliary data used by the indexing process.
 340
 341     These functions must not be run in parallel with other update commands.
 342     """
 343
 344     @staticmethod
 345     def add_args(parser):
 346         group = parser.add_argument_group('Data arguments')
 347         group.add_argument('--postcodes', action='store_true',
 348                            help='Update postcode centroid table')
 349         group.add_argument('--word-counts', action='store_true',
 350                            help='Compute frequency of full-word search terms')
 351         group.add_argument('--address-levels', action='store_true',
 352                            help='Reimport address level configuration')
 353         group.add_argument('--functions', action='store_true',
 354                            help='Update the PL/pgSQL functions in the database')
 355         group.add_argument('--wiki-data', action='store_true',
 356                            help='Update Wikipedia/data importance numbers.')
 357         group.add_argument('--importance', action='store_true',
 358                            help='Recompute place importances (expensive!)')
 359         group.add_argument('--website', action='store_true',
 360                            help='Refresh the directory that serves the scripts for the web API')
 361         group = parser.add_argument_group('Arguments for function refresh')
 362         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
 363                            help='Do not enable code for propagating updates')
 364         group.add_argument('--enable-debug-statements', action='store_true',
 365                            help='Enable debug warning statements in functions')
 366
 367     @staticmethod
 368     def run(args):
 369         if args.postcodes:
 370             run_legacy_script('update.php', '--calculate-postcodes',
 371                               nominatim_env=args, throw_on_fail=True)
 372         if args.word_counts:
 373             run_legacy_script('update.php', '--recompute-word-counts',
 374                               nominatim_env=args, throw_on_fail=True)
 375         if args.address_levels:
 376             run_legacy_script('update.php', '--update-address-levels',
 377                               nominatim_env=args, throw_on_fail=True)
 378         if args.functions:
 379             params = ['setup.php', '--create-functions', '--create-partition-functions']
 380             if args.diffs:
 381                 params.append('--enable-diff-updates')
 382             if args.enable_debug_statements:
 383                 params.append('--enable-debug-statements')
 384             run_legacy_script(*params, nominatim_env=args, throw_on_fail=True)
 385         if args.wiki_data:
 386             run_legacy_script('setup.php', '--import-wikipedia-articles',
 387                               nominatim_env=args, throw_on_fail=True)
 388         # Attention: importance MUST come after wiki data import.
 389         if args.importance:
 390             run_legacy_script('update.php', '--recompute-importance',
 391                               nominatim_env=args, throw_on_fail=True)
 392         if args.website:
 393             run_legacy_script('setup.php', '--setup-website',
 394                               nominatim_env=args, throw_on_fail=True)
 395
 396
 397 class AdminCheckDatabase:
 398     """\
 399     Check that the database is complete and operational.
 400     """
 401
 402     @staticmethod
 403     def add_args(parser):
 404         pass # No options
 405
 406     @staticmethod
 407     def run(args):
 408         return run_legacy_script('check_import_finished.php', nominatim_env=args)
 409
 410
 411 class AdminWarm:
 412     """\
 413     Warm database caches for search and reverse queries.
 414     """
 415
 416     @staticmethod
 417     def add_args(parser):
 418         group = parser.add_argument_group('Target arguments')
 419         group.add_argument('--search-only', action='store_const', dest='target',
 420                            const='search',
 421                            help="Only pre-warm tables for search queries")
 422         group.add_argument('--reverse-only', action='store_const', dest='target',
 423                            const='reverse',
 424                            help="Only pre-warm tables for reverse queries")
 425
 426     @staticmethod
 427     def run(args):
 428         params = ['warm.php']
 429         if args.target == 'reverse':
 430             params.append('--reverse-only')
 431         if args.target == 'search':
 432             params.append('--search-only')
 433         return run_legacy_script(*params, nominatim_env=args)
 434
 435
 436 class QueryExport:
 437     """\
 438     Export addresses as CSV file from a Nominatim database.
 439     """
 440
 441     @staticmethod
 442     def add_args(parser):
 443         group = parser.add_argument_group('Output arguments')
 444         group.add_argument('--output-type', default='street',
 445                            choices=('continent', 'country', 'state', 'county',
 446                                     'city', 'suburb', 'street', 'path'),
 447                            help='Type of places to output (default: street)')
 448         group.add_argument('--output-format',
 449                            default='street;suburb;city;county;state;country',
 450                            help="""Semicolon-separated list of address types
 451                                    (see --output-type). Multiple ranks can be
 452                                    merged into one column by simply using a
 453                                    comma-separated list.""")
 454         group.add_argument('--output-all-postcodes', action='store_true',
 455                            help="""List all postcodes for address instead of
 456                                    just the most likely one""")
 457         group.add_argument('--language',
 458                            help="""Preferred language for output
 459                                    (use local name, if omitted)""")
 460         group = parser.add_argument_group('Filter arguments')
 461         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
 462                            help='Export only objects within country')
 463         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
 464                            help='Export only children of this OSM node')
 465         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
 466                            help='Export only children of this OSM way')
 467         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
 468                            help='Export only children of this OSM relation')
 469
 470
 471     @staticmethod
 472     def run(args):
 473         params = ['export.php',
 474                   '--output-type', args.output_type,
 475                   '--output-format', args.output_format]
 476         if args.output_all_postcodes:
 477             params.append('--output-all-postcodes')
 478         if args.language:
 479             params.extend(('--language', args.language))
 480         if args.restrict_to_country:
 481             params.extend(('--restrict-to-country', args.restrict_to_country))
 482         if args.restrict_to_osm_node:
 483             params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
 484         if args.restrict_to_osm_way:
 485             params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
 486         if args.restrict_to_osm_relation:
 487             params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
 488
 489         return run_legacy_script(*params, nominatim_env=args)
 490
 491 class QueryTodo:
 492     """\
 493     Todo
 494     """
 495     @staticmethod
 496     def add_args(parser):
 497         pass
 498
 499     @staticmethod
 500     def run(args): # pylint: disable=W0613
 501         print("TODO: searching")
 502
 503
 504 def nominatim(**kwargs):
 505     """\
 506     Command-line tools for importing, updating, administrating and
 507     querying the Nominatim database.
 508     """
 509     parser = CommandlineParser('nominatim', nominatim.__doc__)
 510
 511     parser.add_subcommand('import', SetupAll)
 512     parser.add_subcommand('freeze', SetupFreeze)
 513     parser.add_subcommand('replication', UpdateReplication)
 514
 515     parser.add_subcommand('check-database', AdminCheckDatabase)
 516     parser.add_subcommand('warm', AdminWarm)
 517
 518     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
 519
 520     parser.add_subcommand('add-data', UpdateAddData)
 521     parser.add_subcommand('index', UpdateIndex)
 522     parser.add_subcommand('refresh', UpdateRefresh)
 523
 524     parser.add_subcommand('export', QueryExport)
 525     parser.add_subcommand('search', QueryTodo)
 526     parser.add_subcommand('reverse', QueryTodo)
 527     parser.add_subcommand('lookup', QueryTodo)
 528     parser.add_subcommand('details', QueryTodo)
 529     parser.add_subcommand('status', QueryTodo)
 530
 531     return parser.run(**kwargs)