nominatim/cli.py

   1 """
   2 Command-line interface to the Nominatim functions for import, update,
   3 database administration and querying.
   4 """
   5 import logging
   6 import os
   7 import sys
   8 import argparse
   9 from pathlib import Path
  10
  11 from .config import Configuration
  12 from .tools.exec_utils import run_legacy_script, run_php_server
  13 from .errors import UsageError
  14 from . import clicmd
  15 from .clicmd.args import NominatimArgs
  16
  17 LOG = logging.getLogger()
  18
  19
  20 class CommandlineParser:
  21     """ Wraps some of the common functions for parsing the command line
  22         and setting up subcommands.
  23     """
  24     def __init__(self, prog, description):
  25         self.parser = argparse.ArgumentParser(
  26             prog=prog,
  27             description=description,
  28             formatter_class=argparse.RawDescriptionHelpFormatter)
  29
  30         self.subs = self.parser.add_subparsers(title='available commands',
  31                                                dest='subcommand')
  32
  33         # Arguments added to every sub-command
  34         self.default_args = argparse.ArgumentParser(add_help=False)
  35         group = self.default_args.add_argument_group('Default arguments')
  36         group.add_argument('-h', '--help', action='help',
  37                            help='Show this help message and exit')
  38         group.add_argument('-q', '--quiet', action='store_const', const=0,
  39                            dest='verbose', default=1,
  40                            help='Print only error messages')
  41         group.add_argument('-v', '--verbose', action='count', default=1,
  42                            help='Increase verboseness of output')
  43         group.add_argument('--project-dir', metavar='DIR', default='.',
  44                            help='Base directory of the Nominatim installation (default:.)')
  45         group.add_argument('-j', '--threads', metavar='NUM', type=int,
  46                            help='Number of parallel threads to use')
  47
  48
  49     def add_subcommand(self, name, cmd):
  50         """ Add a subcommand to the parser. The subcommand must be a class
  51             with a function add_args() that adds the parameters for the
  52             subcommand and a run() function that executes the command.
  53         """
  54         parser = self.subs.add_parser(name, parents=[self.default_args],
  55                                       help=cmd.__doc__.split('\n', 1)[0],
  56                                       description=cmd.__doc__,
  57                                       formatter_class=argparse.RawDescriptionHelpFormatter,
  58                                       add_help=False)
  59         parser.set_defaults(command=cmd)
  60         cmd.add_args(parser)
  61
  62     def run(self, **kwargs):
  63         """ Parse the command line arguments of the program and execute the
  64             appropriate subcommand.
  65         """
  66         args = NominatimArgs()
  67         self.parser.parse_args(args=kwargs.get('cli_args'), namespace=args)
  68
  69         if args.subcommand is None:
  70             self.parser.print_help()
  71             return 1
  72
  73         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'sqllib_dir',
  74                     'data_dir', 'config_dir', 'phpcgi_path'):
  75             setattr(args, arg, Path(kwargs[arg]))
  76         args.project_dir = Path(args.project_dir).resolve()
  77
  78         logging.basicConfig(stream=sys.stderr,
  79                             format='%(asctime)s: %(message)s',
  80                             datefmt='%Y-%m-%d %H:%M:%S',
  81                             level=max(4 - args.verbose, 1) * 10)
  82
  83         args.config = Configuration(args.project_dir, args.config_dir)
  84
  85         log = logging.getLogger()
  86         log.warning('Using project directory: %s', str(args.project_dir))
  87
  88         try:
  89             return args.command.run(args)
  90         except UsageError as exception:
  91             if log.isEnabledFor(logging.DEBUG):
  92                 raise # use Python's exception printing
  93             log.fatal('FATAL: %s', exception)
  94
  95         # If we get here, then execution has failed in some way.
  96         return 1
  97
  98
  99 ##### Subcommand classes
 100 #
 101 # Each class needs to implement two functions: add_args() adds the CLI parameters
 102 # for the subfunction, run() executes the subcommand.
 103 #
 104 # The class documentation doubles as the help text for the command. The
 105 # first line is also used in the summary when calling the program without
 106 # a subcommand.
 107 #
 108 # No need to document the functions each time.
 109 # pylint: disable=C0111
 110 # Using non-top-level imports to make pyosmium optional for replication only.
 111 # pylint: disable=E0012,C0415
 112
 113
 114 class SetupAll:
 115     """\
 116     Create a new Nominatim database from an OSM file.
 117     """
 118
 119     @staticmethod
 120     def add_args(parser):
 121         group_name = parser.add_argument_group('Required arguments')
 122         group = group_name.add_mutually_exclusive_group(required=True)
 123         group.add_argument('--osm-file',
 124                            help='OSM file to be imported.')
 125         group.add_argument('--continue', dest='continue_at',
 126                            choices=['load-data', 'indexing', 'db-postprocess'],
 127                            help='Continue an import that was interrupted')
 128         group = parser.add_argument_group('Optional arguments')
 129         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 130                            help='Size of cache to be used by osm2pgsql (in MB)')
 131         group.add_argument('--reverse-only', action='store_true',
 132                            help='Do not create tables and indexes for searching')
 133         group.add_argument('--enable-debug-statements', action='store_true',
 134                            help='Include debug warning statements in SQL code')
 135         group.add_argument('--no-partitions', action='store_true',
 136                            help="""Do not partition search indices
 137                                    (speeds up import of single country extracts)""")
 138         group.add_argument('--no-updates', action='store_true',
 139                            help="""Do not keep tables that are only needed for
 140                                    updating the database later""")
 141         group = parser.add_argument_group('Expert options')
 142         group.add_argument('--ignore-errors', action='store_true',
 143                            help='Continue import even when errors in SQL are present')
 144         group.add_argument('--index-noanalyse', action='store_true',
 145                            help='Do not perform analyse operations during index')
 146
 147
 148     @staticmethod
 149     def run(args):
 150         params = ['setup.php']
 151         if args.osm_file:
 152             params.extend(('--all', '--osm-file', args.osm_file))
 153         else:
 154             if args.continue_at == 'load-data':
 155                 params.append('--load-data')
 156             if args.continue_at in ('load-data', 'indexing'):
 157                 params.append('--index')
 158             params.extend(('--create-search-indices', '--create-country-names',
 159                            '--setup-website'))
 160         if args.osm2pgsql_cache:
 161             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
 162         if args.reverse_only:
 163             params.append('--reverse-only')
 164         if args.enable_debug_statements:
 165             params.append('--enable-debug-statements')
 166         if args.no_partitions:
 167             params.append('--no-partitions')
 168         if args.no_updates:
 169             params.append('--drop')
 170         if args.ignore_errors:
 171             params.append('--ignore-errors')
 172         if args.index_noanalyse:
 173             params.append('--index-noanalyse')
 174
 175         return run_legacy_script(*params, nominatim_env=args)
 176
 177
 178 class SetupSpecialPhrases:
 179     """\
 180     Maintain special phrases.
 181     """
 182
 183     @staticmethod
 184     def add_args(parser):
 185         group = parser.add_argument_group('Input arguments')
 186         group.add_argument('--from-wiki', action='store_true',
 187                            help='Pull special phrases from the OSM wiki.')
 188         group = parser.add_argument_group('Output arguments')
 189         group.add_argument('-o', '--output', default='-',
 190                            help="""File to write the preprocessed phrases to.
 191                                    If omitted, it will be written to stdout.""")
 192
 193     @staticmethod
 194     def run(args):
 195         if args.output != '-':
 196             raise NotImplementedError('Only output to stdout is currently implemented.')
 197         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 198
 199
 200 class UpdateAddData:
 201     """\
 202     Add additional data from a file or an online source.
 203
 204     Data is only imported, not indexed. You need to call `nominatim-update index`
 205     to complete the process.
 206     """
 207
 208     @staticmethod
 209     def add_args(parser):
 210         group_name = parser.add_argument_group('Source')
 211         group = group_name.add_mutually_exclusive_group(required=True)
 212         group.add_argument('--file', metavar='FILE',
 213                            help='Import data from an OSM file')
 214         group.add_argument('--diff', metavar='FILE',
 215                            help='Import data from an OSM diff file')
 216         group.add_argument('--node', metavar='ID', type=int,
 217                            help='Import a single node from the API')
 218         group.add_argument('--way', metavar='ID', type=int,
 219                            help='Import a single way from the API')
 220         group.add_argument('--relation', metavar='ID', type=int,
 221                            help='Import a single relation from the API')
 222         group.add_argument('--tiger-data', metavar='DIR',
 223                            help='Add housenumbers from the US TIGER census database.')
 224         group = parser.add_argument_group('Extra arguments')
 225         group.add_argument('--use-main-api', action='store_true',
 226                            help='Use OSM API instead of Overpass to download objects')
 227
 228     @staticmethod
 229     def run(args):
 230         if args.tiger_data:
 231             os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
 232             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 233
 234         params = ['update.php']
 235         if args.file:
 236             params.extend(('--import-file', args.file))
 237         elif args.diff:
 238             params.extend(('--import-diff', args.diff))
 239         elif args.node:
 240             params.extend(('--import-node', args.node))
 241         elif args.way:
 242             params.extend(('--import-way', args.way))
 243         elif args.relation:
 244             params.extend(('--import-relation', args.relation))
 245         if args.use_main_api:
 246             params.append('--use-main-api')
 247         return run_legacy_script(*params, nominatim_env=args)
 248
 249
 250 class QueryExport:
 251     """\
 252     Export addresses as CSV file from the database.
 253     """
 254
 255     @staticmethod
 256     def add_args(parser):
 257         group = parser.add_argument_group('Output arguments')
 258         group.add_argument('--output-type', default='street',
 259                            choices=('continent', 'country', 'state', 'county',
 260                                     'city', 'suburb', 'street', 'path'),
 261                            help='Type of places to output (default: street)')
 262         group.add_argument('--output-format',
 263                            default='street;suburb;city;county;state;country',
 264                            help="""Semicolon-separated list of address types
 265                                    (see --output-type). Multiple ranks can be
 266                                    merged into one column by simply using a
 267                                    comma-separated list.""")
 268         group.add_argument('--output-all-postcodes', action='store_true',
 269                            help="""List all postcodes for address instead of
 270                                    just the most likely one""")
 271         group.add_argument('--language',
 272                            help="""Preferred language for output
 273                                    (use local name, if omitted)""")
 274         group = parser.add_argument_group('Filter arguments')
 275         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
 276                            help='Export only objects within country')
 277         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
 278                            help='Export only children of this OSM node')
 279         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
 280                            help='Export only children of this OSM way')
 281         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
 282                            help='Export only children of this OSM relation')
 283
 284
 285     @staticmethod
 286     def run(args):
 287         params = ['export.php',
 288                   '--output-type', args.output_type,
 289                   '--output-format', args.output_format]
 290         if args.output_all_postcodes:
 291             params.append('--output-all-postcodes')
 292         if args.language:
 293             params.extend(('--language', args.language))
 294         if args.restrict_to_country:
 295             params.extend(('--restrict-to-country', args.restrict_to_country))
 296         if args.restrict_to_osm_node:
 297             params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
 298         if args.restrict_to_osm_way:
 299             params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
 300         if args.restrict_to_osm_relation:
 301             params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
 302
 303         return run_legacy_script(*params, nominatim_env=args)
 304
 305
 306 class AdminServe:
 307     """\
 308     Start a simple web server for serving the API.
 309
 310     This command starts the built-in PHP webserver to serve the website
 311     from the current project directory. This webserver is only suitable
 312     for testing and develop. Do not use it in production setups!
 313
 314     By the default, the webserver can be accessed at: http://127.0.0.1:8088
 315     """
 316
 317     @staticmethod
 318     def add_args(parser):
 319         group = parser.add_argument_group('Server arguments')
 320         group.add_argument('--server', default='127.0.0.1:8088',
 321                            help='The address the server will listen to.')
 322
 323     @staticmethod
 324     def run(args):
 325         run_php_server(args.server, args.project_dir / 'website')
 326
 327
 328 def nominatim(**kwargs):
 329     """\
 330     Command-line tools for importing, updating, administrating and
 331     querying the Nominatim database.
 332     """
 333     parser = CommandlineParser('nominatim', nominatim.__doc__)
 334
 335     parser.add_subcommand('import', SetupAll)
 336     parser.add_subcommand('freeze', clicmd.SetupFreeze)
 337     parser.add_subcommand('replication', clicmd.UpdateReplication)
 338
 339     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
 340
 341     parser.add_subcommand('add-data', UpdateAddData)
 342     parser.add_subcommand('index', clicmd.UpdateIndex)
 343     parser.add_subcommand('refresh', clicmd.UpdateRefresh)
 344
 345     parser.add_subcommand('admin', clicmd.AdminFuncs)
 346
 347     parser.add_subcommand('export', QueryExport)
 348     parser.add_subcommand('serve', AdminServe)
 349
 350     if kwargs.get('phpcgi_path'):
 351         parser.add_subcommand('search', clicmd.APISearch)
 352         parser.add_subcommand('reverse', clicmd.APIReverse)
 353         parser.add_subcommand('lookup', clicmd.APILookup)
 354         parser.add_subcommand('details', clicmd.APIDetails)
 355         parser.add_subcommand('status', clicmd.APIStatus)
 356     else:
 357         parser.parser.epilog = 'php-cgi not found. Query commands not available.'
 358
 359     parser.add_subcommand('transition', clicmd.AdminTransition)
 360
 361     return parser.run(**kwargs)