nominatim/cli.py

   1 """
   2 Command-line interface to the Nominatim functions for import, update,
   3 database administration and querying.
   4 """
   5 import sys
   6 import argparse
   7 import logging
   8 from pathlib import Path
   9
  10 from .config import Configuration
  11 from .admin.exec_utils import run_legacy_script
  12
  13 class CommandlineParser:
  14     """ Wraps some of the common functions for parsing the command line
  15         and setting up subcommands.
  16     """
  17     def __init__(self, prog, description):
  18         self.parser = argparse.ArgumentParser(
  19             prog=prog,
  20             description=description,
  21             formatter_class=argparse.RawDescriptionHelpFormatter)
  22
  23         self.subs = self.parser.add_subparsers(title='available commands',
  24                                                dest='subcommand')
  25
  26         # Arguments added to every sub-command
  27         self.default_args = argparse.ArgumentParser(add_help=False)
  28         group = self.default_args.add_argument_group('Default arguments')
  29         group.add_argument('-h', '--help', action='help',
  30                            help='Show this help message and exit')
  31         group.add_argument('-q', '--quiet', action='store_const', const=0,
  32                            dest='verbose', default=1,
  33                            help='Print only error messages')
  34         group.add_argument('-v', '--verbose', action='count', default=1,
  35                            help='Increase verboseness of output')
  36         group.add_argument('--project-dir', metavar='DIR', default='.',
  37                            help='Base directory of the Nominatim installation (default:.)')
  38         group.add_argument('-j', '--threads', metavar='NUM', type=int,
  39                            help='Number of parallel threads to use')
  40
  41
  42     def add_subcommand(self, name, cmd):
  43         """ Add a subcommand to the parser. The subcommand must be a class
  44             with a function add_args() that adds the parameters for the
  45             subcommand and a run() function that executes the command.
  46         """
  47         parser = self.subs.add_parser(name, parents=[self.default_args],
  48                                       help=cmd.__doc__.split('\n', 1)[0],
  49                                       description=cmd.__doc__,
  50                                       formatter_class=argparse.RawDescriptionHelpFormatter,
  51                                       add_help=False)
  52         parser.set_defaults(command=cmd)
  53         cmd.add_args(parser)
  54
  55     def run(self, **kwargs):
  56         """ Parse the command line arguments of the program and execute the
  57             appropriate subcommand.
  58         """
  59         args = self.parser.parse_args()
  60
  61         if args.subcommand is None:
  62             return self.parser.print_help()
  63
  64         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir'):
  65             setattr(args, arg, Path(kwargs[arg]))
  66         args.project_dir = Path(args.project_dir)
  67
  68         logging.basicConfig(stream=sys.stderr,
  69                             format='%(asctime)s %(levelname)s: %(message)s',
  70                             datefmt='%Y-%m-%d %H:%M:%S',
  71                             level=max(4 - args.verbose, 1) * 10)
  72
  73         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
  74
  75         args.command.run(args)
  76
  77
  78 class SetupAll:
  79     """\
  80     Create a new Nominatim database from an OSM file.
  81     """
  82
  83     @staticmethod
  84     def add_args(parser):
  85         group_name = parser.add_argument_group('Required arguments')
  86         group = group_name.add_mutually_exclusive_group(required=True)
  87         group.add_argument('--osm-file',
  88                            help='OSM file to be imported.')
  89         group.add_argument('--continue', nargs=1, dest='continue_at',
  90                            choices=['load-data', 'indexing', 'db-postprocess'],
  91                            help='Continue an import that was interrupted')
  92         group = parser.add_argument_group('Optional arguments')
  93         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
  94                            help='Size of cache to be used by osm2pgsql (in MB)')
  95         group.add_argument('--reverse-only', action='store_true',
  96                            help='Do not create tables and indexes for searching')
  97         group.add_argument('--enable-debug-statements', action='store_true',
  98                            help='Include debug warning statements in SQL code')
  99         group.add_argument('--no-partitions', action='store_true',
 100                            help="""Do not partition search indices
 101                                    (speeds up import of single country extracts)""")
 102         group.add_argument('--no-updates', action='store_true',
 103                            help="""Do not keep tables that are only needed for
 104                                    updating the database later""")
 105         group = parser.add_argument_group('Expert options')
 106         group.add_argument('--ignore-errors', action='store_true',
 107                            help='Continue import even when errors in SQL are present')
 108         group.add_argument('--index-noanalyse', action='store_true',
 109                            help='Do not perform analyse operations during index')
 110
 111
 112     @staticmethod
 113     def run(args):
 114         params = ['setup.php']
 115         if args.osm_file:
 116             params.extend(('--all', '--osm-file', args.osm_file))
 117         else:
 118             if args.continue_at == 'load-data':
 119                 params.append('--load-data')
 120             if args.continue_at in ('load-data', 'indexing'):
 121                 params.append('--index')
 122             params.extend(('--create-search-indices', '--create-country-names',
 123                            '--setup-website'))
 124         if args.osm2pgsql_cache:
 125             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
 126         if args.reverse_only:
 127             params.append('--reverse-only')
 128         if args.enable_debug_statements:
 129             params.append('--enable-debug-statements')
 130         if args.no_partitions:
 131             params.append('--no-partitions')
 132         if args.no_updates:
 133             params.append('--drop')
 134         if args.ignore_errors:
 135             params.append('--ignore-errors')
 136         if args.index_noanalyse:
 137             params.append('--index-noanalyse')
 138
 139         return run_legacy_script(*params, nominatim_env=args)
 140
 141
 142 class SetupFreeze:
 143     """\
 144     Make database read-only.
 145
 146     About half of data in the Nominatim database is kept only to be able to
 147     keep the data up-to-date with new changes made in OpenStreetMap. This
 148     command drops all this data and only keeps the part needed for geocoding
 149     itself.
 150
 151     This command has the same effect as the `--no-updates` option for imports.
 152     """
 153
 154     @staticmethod
 155     def add_args(parser):
 156         pass # No options
 157
 158     @staticmethod
 159     def run(args):
 160         return run_legacy_script('setup.php', '--drop', nominatim_env=args)
 161
 162
 163 class SetupSpecialPhrases:
 164     """\
 165     Maintain special phrases.
 166     """
 167
 168     @staticmethod
 169     def add_args(parser):
 170         group = parser.add_argument_group('Input arguments')
 171         group.add_argument('--from-wiki', action='store_true',
 172                            help='Pull special phrases from the OSM wiki.')
 173         group = parser.add_argument_group('Output arguments')
 174         group.add_argument('-o', '--output', default='-',
 175                            type=argparse.FileType('w', encoding='UTF-8'),
 176                            help="""File to write the preprocessed phrases to.
 177                                    If omitted, it will be written to stdout.""")
 178
 179     @staticmethod
 180     def run(args):
 181         if args.output.name != '<stdout>':
 182             raise NotImplementedError('Only output to stdout is currently implemented.')
 183         return run_legacy_script('specialphrases.php', '--wiki-import' , nominatim_env=args)
 184
 185
 186 class UpdateReplication:
 187     """\
 188     Update the database using an online replication service.
 189     """
 190
 191     @staticmethod
 192     def add_args(parser):
 193         group = parser.add_argument_group('Arguments for initialisation')
 194         group.add_argument('--init', action='store_true',
 195                            help='Initialise the update process')
 196         group.add_argument('--no-update-functions', dest='update_functions',
 197                            action='store_false',
 198                            help="""Do not update the trigger function to
 199                                    support differential updates.""")
 200         group = parser.add_argument_group('Arguments for updates')
 201         group.add_argument('--check-for-updates', action='store_true',
 202                            help='Check if new updates are available and exit')
 203         group.add_argument('--once', action='store_true',
 204                            help="""Download and apply updates only once. When
 205                                    not set, updates are continuously applied""")
 206         group.add_argument('--no-index', action='store_false', dest='do_index',
 207                            help="""Do not index the new data. Only applicable
 208                                    together with --once""")
 209
 210     @staticmethod
 211     def run(args):
 212         params = ['update.php']
 213         if args.init:
 214             params.append('--init-updates')
 215             if not args.update_functions:
 216                 params.apend('--no-update-functions')
 217         elif args.check_for_updates:
 218             params.append('--check-for-updates')
 219         else:
 220             if args.once:
 221                 params.append('--import-osmosis')
 222             else:
 223                 params.append('--import-osmosis-all')
 224             if not args.do_index:
 225                 params.append('--no-index')
 226
 227         return run_legacy_script(*params, nominatim_env=args)
 228
 229
 230 class UpdateAddData:
 231     """\
 232     Add additional data from a file or an online source.
 233
 234     Data is only imported, not indexed. You need to call `nominatim-update index`
 235     to complete the process.
 236     """
 237
 238     @staticmethod
 239     def add_args(parser):
 240         group_name = parser.add_argument_group('Source')
 241         group = group_name.add_mutually_exclusive_group(required=True)
 242         group.add_argument('--file', metavar='FILE',
 243                            help='Import data from an OSM file')
 244         group.add_argument('--diff', metavar='FILE',
 245                            help='Import data from an OSM diff file')
 246         group.add_argument('--node', metavar='ID', type=int,
 247                            help='Import a single node from the API')
 248         group.add_argument('--way', metavar='ID', type=int,
 249                            help='Import a single way from the API')
 250         group.add_argument('--relation', metavar='ID', type=int,
 251                            help='Import a single relation from the API')
 252         group.add_argument('--tiger-data', metavar='DIR',
 253                            help='Add housenumbers from the US TIGER census database.')
 254         group = parser.add_argument_group('Extra arguments')
 255         group.add_argument('--use-main-api', action='store_true',
 256                            help='Use OSM API instead of Overpass to download objects')
 257
 258     @staticmethod
 259     def run(args):
 260         if args.tiger_data:
 261             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 262
 263         params = [ 'update.php']
 264         if args.file:
 265             params.extend(('--import-file', args.file))
 266         elif args.diff:
 267             params.extend(('--import-diff', args.diff))
 268         elif args.node:
 269             params.extend(('--import-node', args.node))
 270         elif args.way:
 271             params.extend(('--import-way', args.way))
 272         elif args.relation:
 273             params.extend(('--import-relation' , args.relation))
 274         if args.use_main_api:
 275             params.append('--use-main-api')
 276         return run_legacy_script(*params, nominatim_env=args)
 277
 278
 279 class UpdateIndex:
 280     """\
 281     Reindex all new and modified data.
 282     """
 283
 284     @staticmethod
 285     def add_args(parser):
 286         pass
 287
 288     @staticmethod
 289     def run(args):
 290         return run_legacy_script('update.php', '--index', nominatim_env=args)
 291
 292
 293 class UpdateRefresh:
 294     """\
 295     Recompute auxillary data used by the indexing process.
 296
 297     These functions must not be run in parallel with other update commands.
 298     """
 299
 300     @staticmethod
 301     def add_args(parser):
 302         group = parser.add_argument_group('Data arguments')
 303         group.add_argument('--postcodes', action='store_true',
 304                            help='Update postcode centroid table')
 305         group.add_argument('--word-counts', action='store_true',
 306                            help='Compute frequency of full-word search terms')
 307         group.add_argument('--address-levels', action='store_true',
 308                            help='Reimport address level configuration')
 309         group.add_argument('--importance', action='store_true',
 310                            help='Recompute place importances (expensive!)')
 311         group.add_argument('--functions', action='store_true',
 312                            help='Update the PL/pgSQL functions in the database')
 313         group.add_argument('--wiki-data', action='store_true',
 314                            help='Update Wikipedia/data importance numbers.')
 315         group.add_argument('--website', action='store_true',
 316                            help='Refresh the directory that serves the scripts for the web API')
 317         group = parser.add_argument_group('Arguments for function refresh')
 318         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
 319                            help='Do not enable code for propagating updates')
 320         group.add_argument('--enable-debug-statements', action='store_true',
 321                            help='Enable debug warning statements in functions')
 322
 323     @staticmethod
 324     def run(args):
 325         if args.postcodes:
 326             run_legacy_script('update.php', '--calculate-postcodes',
 327                               nominatim_env=args, throw_on_fail=True)
 328         if args.word_counts:
 329             run_legacy_script('update.php', '--recompute-word-counts',
 330                               nominatim_env=args, throw_on_fail=True)
 331         if args.address_levels:
 332             run_legacy_script('update.php', '--update-address-levels',
 333                               nominatim_env=args, throw_on_fail=True)
 334         if args.importance:
 335             run_legacy_script('update.php', '--recompute-importance',
 336                               nominatim_env=args, throw_on_fail=True)
 337         if args.functions:
 338             params = ['setup.php', '--create-functions', '--create-partition-functions']
 339             if args.diffs:
 340                 params.append('--enable-diff-updates')
 341             if args.enable_debug_statements:
 342                 params.append('--enable-debug-statements')
 343             run_legacy_script(*params, nominatim_env=args, throw_on_fail=True)
 344         if args.wiki_data:
 345             run_legacy_script('setup.php', '--import-wikipedia-articles',
 346                               nominatim_env=args, throw_on_fail=True)
 347         if args.website:
 348             run_legacy_script('setup.php', '--setup-website',
 349                               nominatim_env=args, throw_on_fail=True)
 350
 351
 352 class AdminCheckDatabase:
 353     """\
 354     Check that the database is complete and operational.
 355     """
 356
 357     @staticmethod
 358     def add_args(parser):
 359         pass # No options
 360
 361     @staticmethod
 362     def run(args):
 363         return run_legacy_script('check_import_finished.php', nominatim_env=args)
 364
 365
 366 class AdminWarm:
 367     """\
 368     Warm database caches for search and reverse queries.
 369     """
 370
 371     @staticmethod
 372     def add_args(parser):
 373         group = parser.add_argument_group('Target arguments')
 374         group.add_argument('--search-only', action='store_const', dest='target',
 375                            const='search',
 376                            help="Only pre-warm tables for search queries")
 377         group.add_argument('--reverse-only', action='store_const', dest='target',
 378                            const='reverse',
 379                            help="Only pre-warm tables for reverse queries")
 380
 381     @staticmethod
 382     def run(args):
 383         params = ['warm.php']
 384         if args.target == 'reverse':
 385             params.append('--reverse-only')
 386         if args.target == 'search':
 387             params.append('--search-only')
 388         return run_legacy_script(*params, nominatim_env=args)
 389
 390
 391 class QueryExport:
 392     """\
 393     Export addresses as CSV file from a Nominatim database.
 394     """
 395
 396     @staticmethod
 397     def add_args(parser):
 398         group = parser.add_argument_group('Output arguments')
 399         group.add_argument('--output-type', default='street',
 400                            choices=('continent', 'country', 'state', 'county',
 401                                     'city', 'suburb', 'street', 'path'),
 402                            help='Type of places to output (default: street)')
 403         group.add_argument('--output-format',
 404                            default='street;suburb;city;county;state;country',
 405                            help="""Semicolon-separated list of address types
 406                                    (see --output-type). Multiple ranks can be
 407                                    merged into one column by simply using a
 408                                    comma-separated list.""")
 409         group.add_argument('--output-all-postcodes', action='store_true',
 410                            help="""List all postcodes for address instead of
 411                                    just the most likely one""")
 412         group.add_argument('--language',
 413                            help="""Preferred language for output
 414                                    (use local name, if omitted)""")
 415         group = parser.add_argument_group('Filter arguments')
 416         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
 417                            help='Export only objects within country')
 418         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
 419                            help='Export only children of this OSM node')
 420         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
 421                            help='Export only children of this OSM way')
 422         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
 423                            help='Export only children of this OSM relation')
 424
 425
 426     @staticmethod
 427     def run(args):
 428         params = ['export.php',
 429                   '--output-type', args.output_type,
 430                   '--output-format', args.output_format]
 431         if args.output_all_postcodes:
 432             params.append('--output-all-postcodes')
 433         if args.language:
 434             params.extend(('--language', args.language))
 435         if args.restrict_to_country:
 436             params.extend(('--restrict-to-country', args.restrict_to_country))
 437         if args.restrict_to_osm_node:
 438             params.exted(('--restrict-to-osm-node', args.restrict_to_osm_node))
 439         if args.restrict_to_osm_way:
 440             params.exted(('--restrict-to-osm-way', args.restrict_to_osm_way))
 441         if args.restrict_to_osm_relation:
 442             params.exted(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
 443
 444         return run_legacy_script(*params, nominatim_env=args)
 445
 446 class QueryTodo:
 447     """\
 448     Todo
 449     """
 450     @staticmethod
 451     def add_args(parser):
 452         pass
 453
 454     def run(args):
 455         print("TODO: searching")
 456
 457
 458 def nominatim(**kwargs):
 459     """\
 460     Command-line tools for importing, updating, administrating and
 461     querying the Nominatim database.
 462     """
 463     parser = CommandlineParser('nominatim', nominatim.__doc__)
 464
 465     parser.add_subcommand('import', SetupAll)
 466     parser.add_subcommand('freeze', SetupFreeze)
 467     parser.add_subcommand('replication', UpdateReplication)
 468
 469     parser.add_subcommand('check-database', AdminCheckDatabase)
 470     parser.add_subcommand('warm', AdminWarm)
 471
 472     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
 473
 474     parser.add_subcommand('add-data', UpdateAddData)
 475     parser.add_subcommand('index', UpdateIndex)
 476     parser.add_subcommand('refresh', UpdateRefresh)
 477
 478     parser.add_subcommand('export', QueryExport)
 479     parser.add_subcommand('search', QueryTodo)
 480     parser.add_subcommand('reverse', QueryTodo)
 481     parser.add_subcommand('lookup', QueryTodo)
 482     parser.add_subcommand('details', QueryTodo)
 483     parser.add_subcommand('status', QueryTodo)
 484
 485     parser.run(**kwargs)