]> git.openstreetmap.org Git - nominatim.git/blob - nominatim/cli.py
acb6839fa303937847bc54616f226713149ad836
[nominatim.git] / nominatim / cli.py
1 """
2 Command-line interface to the Nominatim functions for import, update,
3 database administration and querying.
4 """
5 import sys
6 import os
7 import argparse
8 import logging
9 from pathlib import Path
10
11 from .config import Configuration
12 from .admin.exec_utils import run_legacy_script
13
14 from .indexer.indexer import Indexer
15
16 def _num_system_cpus():
17     try:
18         cpus = len(os.sched_getaffinity(0))
19     except NotImplementedError:
20         cpus = None
21
22     return cpus or os.cpu_count()
23
24
25 class CommandlineParser:
26     """ Wraps some of the common functions for parsing the command line
27         and setting up subcommands.
28     """
29     def __init__(self, prog, description):
30         self.parser = argparse.ArgumentParser(
31             prog=prog,
32             description=description,
33             formatter_class=argparse.RawDescriptionHelpFormatter)
34
35         self.subs = self.parser.add_subparsers(title='available commands',
36                                                dest='subcommand')
37
38         # Arguments added to every sub-command
39         self.default_args = argparse.ArgumentParser(add_help=False)
40         group = self.default_args.add_argument_group('Default arguments')
41         group.add_argument('-h', '--help', action='help',
42                            help='Show this help message and exit')
43         group.add_argument('-q', '--quiet', action='store_const', const=0,
44                            dest='verbose', default=1,
45                            help='Print only error messages')
46         group.add_argument('-v', '--verbose', action='count', default=1,
47                            help='Increase verboseness of output')
48         group.add_argument('--project-dir', metavar='DIR', default='.',
49                            help='Base directory of the Nominatim installation (default:.)')
50         group.add_argument('-j', '--threads', metavar='NUM', type=int,
51                            help='Number of parallel threads to use')
52
53
54     def add_subcommand(self, name, cmd):
55         """ Add a subcommand to the parser. The subcommand must be a class
56             with a function add_args() that adds the parameters for the
57             subcommand and a run() function that executes the command.
58         """
59         parser = self.subs.add_parser(name, parents=[self.default_args],
60                                       help=cmd.__doc__.split('\n', 1)[0],
61                                       description=cmd.__doc__,
62                                       formatter_class=argparse.RawDescriptionHelpFormatter,
63                                       add_help=False)
64         parser.set_defaults(command=cmd)
65         cmd.add_args(parser)
66
67     def run(self, **kwargs):
68         """ Parse the command line arguments of the program and execute the
69             appropriate subcommand.
70         """
71         args = self.parser.parse_args()
72
73         if args.subcommand is None:
74             return self.parser.print_help()
75
76         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir'):
77             setattr(args, arg, Path(kwargs[arg]))
78         args.project_dir = Path(args.project_dir)
79
80         logging.basicConfig(stream=sys.stderr,
81                             format='%(asctime)s %(levelname)s: %(message)s',
82                             datefmt='%Y-%m-%d %H:%M:%S',
83                             level=max(4 - args.verbose, 1) * 10)
84
85         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
86
87         return args.command.run(args)
88
89 ##### Subcommand classes
90 #
91 # Each class needs to implement two functions: add_args() adds the CLI parameters
92 # for the subfunction, run() executes the subcommand.
93 #
94 # The class documentation doubles as the help text for the command. The
95 # first line is also used in the summary when calling the program without
96 # a subcommand.
97 #
98 # No need to document the functions each time.
99 # pylint: disable=C0111
100
101
102 class SetupAll:
103     """\
104     Create a new Nominatim database from an OSM file.
105     """
106
107     @staticmethod
108     def add_args(parser):
109         group_name = parser.add_argument_group('Required arguments')
110         group = group_name.add_mutually_exclusive_group(required=True)
111         group.add_argument('--osm-file',
112                            help='OSM file to be imported.')
113         group.add_argument('--continue', dest='continue_at',
114                            choices=['load-data', 'indexing', 'db-postprocess'],
115                            help='Continue an import that was interrupted')
116         group = parser.add_argument_group('Optional arguments')
117         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
118                            help='Size of cache to be used by osm2pgsql (in MB)')
119         group.add_argument('--reverse-only', action='store_true',
120                            help='Do not create tables and indexes for searching')
121         group.add_argument('--enable-debug-statements', action='store_true',
122                            help='Include debug warning statements in SQL code')
123         group.add_argument('--no-partitions', action='store_true',
124                            help="""Do not partition search indices
125                                    (speeds up import of single country extracts)""")
126         group.add_argument('--no-updates', action='store_true',
127                            help="""Do not keep tables that are only needed for
128                                    updating the database later""")
129         group = parser.add_argument_group('Expert options')
130         group.add_argument('--ignore-errors', action='store_true',
131                            help='Continue import even when errors in SQL are present')
132         group.add_argument('--index-noanalyse', action='store_true',
133                            help='Do not perform analyse operations during index')
134
135
136     @staticmethod
137     def run(args):
138         params = ['setup.php']
139         if args.osm_file:
140             params.extend(('--all', '--osm-file', args.osm_file))
141         else:
142             if args.continue_at == 'load-data':
143                 params.append('--load-data')
144             if args.continue_at in ('load-data', 'indexing'):
145                 params.append('--index')
146             params.extend(('--create-search-indices', '--create-country-names',
147                            '--setup-website'))
148         if args.osm2pgsql_cache:
149             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
150         if args.reverse_only:
151             params.append('--reverse-only')
152         if args.enable_debug_statements:
153             params.append('--enable-debug-statements')
154         if args.no_partitions:
155             params.append('--no-partitions')
156         if args.no_updates:
157             params.append('--drop')
158         if args.ignore_errors:
159             params.append('--ignore-errors')
160         if args.index_noanalyse:
161             params.append('--index-noanalyse')
162
163         return run_legacy_script(*params, nominatim_env=args)
164
165
166 class SetupFreeze:
167     """\
168     Make database read-only.
169
170     About half of data in the Nominatim database is kept only to be able to
171     keep the data up-to-date with new changes made in OpenStreetMap. This
172     command drops all this data and only keeps the part needed for geocoding
173     itself.
174
175     This command has the same effect as the `--no-updates` option for imports.
176     """
177
178     @staticmethod
179     def add_args(parser):
180         pass # No options
181
182     @staticmethod
183     def run(args):
184         return run_legacy_script('setup.php', '--drop', nominatim_env=args)
185
186
187 class SetupSpecialPhrases:
188     """\
189     Maintain special phrases.
190     """
191
192     @staticmethod
193     def add_args(parser):
194         group = parser.add_argument_group('Input arguments')
195         group.add_argument('--from-wiki', action='store_true',
196                            help='Pull special phrases from the OSM wiki.')
197         group = parser.add_argument_group('Output arguments')
198         group.add_argument('-o', '--output', default='-',
199                            type=argparse.FileType('w', encoding='UTF-8'),
200                            help="""File to write the preprocessed phrases to.
201                                    If omitted, it will be written to stdout.""")
202
203     @staticmethod
204     def run(args):
205         if args.output.name != '<stdout>':
206             raise NotImplementedError('Only output to stdout is currently implemented.')
207         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
208
209
210 class UpdateReplication:
211     """\
212     Update the database using an online replication service.
213     """
214
215     @staticmethod
216     def add_args(parser):
217         group = parser.add_argument_group('Arguments for initialisation')
218         group.add_argument('--init', action='store_true',
219                            help='Initialise the update process')
220         group.add_argument('--no-update-functions', dest='update_functions',
221                            action='store_false',
222                            help="""Do not update the trigger function to
223                                    support differential updates.""")
224         group = parser.add_argument_group('Arguments for updates')
225         group.add_argument('--check-for-updates', action='store_true',
226                            help='Check if new updates are available and exit')
227         group.add_argument('--once', action='store_true',
228                            help="""Download and apply updates only once. When
229                                    not set, updates are continuously applied""")
230         group.add_argument('--no-index', action='store_false', dest='do_index',
231                            help="""Do not index the new data. Only applicable
232                                    together with --once""")
233
234     @staticmethod
235     def run(args):
236         params = ['update.php']
237         if args.init:
238             params.append('--init-updates')
239             if not args.update_functions:
240                 params.append('--no-update-functions')
241         elif args.check_for_updates:
242             params.append('--check-for-updates')
243         else:
244             if args.once:
245                 params.append('--import-osmosis')
246             else:
247                 params.append('--import-osmosis-all')
248             if not args.do_index:
249                 params.append('--no-index')
250
251         return run_legacy_script(*params, nominatim_env=args)
252
253
254 class UpdateAddData:
255     """\
256     Add additional data from a file or an online source.
257
258     Data is only imported, not indexed. You need to call `nominatim-update index`
259     to complete the process.
260     """
261
262     @staticmethod
263     def add_args(parser):
264         group_name = parser.add_argument_group('Source')
265         group = group_name.add_mutually_exclusive_group(required=True)
266         group.add_argument('--file', metavar='FILE',
267                            help='Import data from an OSM file')
268         group.add_argument('--diff', metavar='FILE',
269                            help='Import data from an OSM diff file')
270         group.add_argument('--node', metavar='ID', type=int,
271                            help='Import a single node from the API')
272         group.add_argument('--way', metavar='ID', type=int,
273                            help='Import a single way from the API')
274         group.add_argument('--relation', metavar='ID', type=int,
275                            help='Import a single relation from the API')
276         group.add_argument('--tiger-data', metavar='DIR',
277                            help='Add housenumbers from the US TIGER census database.')
278         group = parser.add_argument_group('Extra arguments')
279         group.add_argument('--use-main-api', action='store_true',
280                            help='Use OSM API instead of Overpass to download objects')
281
282     @staticmethod
283     def run(args):
284         if args.tiger_data:
285             os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
286             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
287
288         params = ['update.php']
289         if args.file:
290             params.extend(('--import-file', args.file))
291         elif args.diff:
292             params.extend(('--import-diff', args.diff))
293         elif args.node:
294             params.extend(('--import-node', args.node))
295         elif args.way:
296             params.extend(('--import-way', args.way))
297         elif args.relation:
298             params.extend(('--import-relation', args.relation))
299         if args.use_main_api:
300             params.append('--use-main-api')
301         return run_legacy_script(*params, nominatim_env=args)
302
303
304 class UpdateIndex:
305     """\
306     Reindex all new and modified data.
307     """
308
309     @staticmethod
310     def add_args(parser):
311         group = parser.add_argument_group('Filter arguments')
312         group.add_argument('--boundaries-only', action='store_true',
313                            help="""Index only administrative boundaries.""")
314         group.add_argument('--no-boundaries', action='store_true',
315                            help="""Index everything except administrative boundaries.""")
316         group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
317                            help='Minimum/starting rank')
318         group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
319                            help='Maximum/finishing rank')
320
321     @staticmethod
322     def run(args):
323         indexer = Indexer(args.config.get_libpq_dsn(),
324                           args.threads or _num_system_cpus() or 1)
325
326         if not args.no_boundaries:
327             indexer.index_boundaries(args.minrank, args.maxrank)
328         if not args.boundaries_only:
329             indexer.index_by_rank(args.minrank, args.maxrank)
330
331         return 0
332
333
334 class UpdateRefresh:
335     """\
336     Recompute auxiliary data used by the indexing process.
337
338     These functions must not be run in parallel with other update commands.
339     """
340
341     @staticmethod
342     def add_args(parser):
343         group = parser.add_argument_group('Data arguments')
344         group.add_argument('--postcodes', action='store_true',
345                            help='Update postcode centroid table')
346         group.add_argument('--word-counts', action='store_true',
347                            help='Compute frequency of full-word search terms')
348         group.add_argument('--address-levels', action='store_true',
349                            help='Reimport address level configuration')
350         group.add_argument('--functions', action='store_true',
351                            help='Update the PL/pgSQL functions in the database')
352         group.add_argument('--wiki-data', action='store_true',
353                            help='Update Wikipedia/data importance numbers.')
354         group.add_argument('--importance', action='store_true',
355                            help='Recompute place importances (expensive!)')
356         group.add_argument('--website', action='store_true',
357                            help='Refresh the directory that serves the scripts for the web API')
358         group = parser.add_argument_group('Arguments for function refresh')
359         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
360                            help='Do not enable code for propagating updates')
361         group.add_argument('--enable-debug-statements', action='store_true',
362                            help='Enable debug warning statements in functions')
363
364     @staticmethod
365     def run(args):
366         if args.postcodes:
367             run_legacy_script('update.php', '--calculate-postcodes',
368                               nominatim_env=args, throw_on_fail=True)
369         if args.word_counts:
370             run_legacy_script('update.php', '--recompute-word-counts',
371                               nominatim_env=args, throw_on_fail=True)
372         if args.address_levels:
373             run_legacy_script('update.php', '--update-address-levels',
374                               nominatim_env=args, throw_on_fail=True)
375         if args.functions:
376             params = ['setup.php', '--create-functions', '--create-partition-functions']
377             if args.diffs:
378                 params.append('--enable-diff-updates')
379             if args.enable_debug_statements:
380                 params.append('--enable-debug-statements')
381             run_legacy_script(*params, nominatim_env=args, throw_on_fail=True)
382         if args.wiki_data:
383             run_legacy_script('setup.php', '--import-wikipedia-articles',
384                               nominatim_env=args, throw_on_fail=True)
385         # Attention: importance MUST come after wiki data import.
386         if args.importance:
387             run_legacy_script('update.php', '--recompute-importance',
388                               nominatim_env=args, throw_on_fail=True)
389         if args.website:
390             run_legacy_script('setup.php', '--setup-website',
391                               nominatim_env=args, throw_on_fail=True)
392
393
394 class AdminCheckDatabase:
395     """\
396     Check that the database is complete and operational.
397     """
398
399     @staticmethod
400     def add_args(parser):
401         pass # No options
402
403     @staticmethod
404     def run(args):
405         return run_legacy_script('check_import_finished.php', nominatim_env=args)
406
407
408 class AdminWarm:
409     """\
410     Warm database caches for search and reverse queries.
411     """
412
413     @staticmethod
414     def add_args(parser):
415         group = parser.add_argument_group('Target arguments')
416         group.add_argument('--search-only', action='store_const', dest='target',
417                            const='search',
418                            help="Only pre-warm tables for search queries")
419         group.add_argument('--reverse-only', action='store_const', dest='target',
420                            const='reverse',
421                            help="Only pre-warm tables for reverse queries")
422
423     @staticmethod
424     def run(args):
425         params = ['warm.php']
426         if args.target == 'reverse':
427             params.append('--reverse-only')
428         if args.target == 'search':
429             params.append('--search-only')
430         return run_legacy_script(*params, nominatim_env=args)
431
432
433 class QueryExport:
434     """\
435     Export addresses as CSV file from a Nominatim database.
436     """
437
438     @staticmethod
439     def add_args(parser):
440         group = parser.add_argument_group('Output arguments')
441         group.add_argument('--output-type', default='street',
442                            choices=('continent', 'country', 'state', 'county',
443                                     'city', 'suburb', 'street', 'path'),
444                            help='Type of places to output (default: street)')
445         group.add_argument('--output-format',
446                            default='street;suburb;city;county;state;country',
447                            help="""Semicolon-separated list of address types
448                                    (see --output-type). Multiple ranks can be
449                                    merged into one column by simply using a
450                                    comma-separated list.""")
451         group.add_argument('--output-all-postcodes', action='store_true',
452                            help="""List all postcodes for address instead of
453                                    just the most likely one""")
454         group.add_argument('--language',
455                            help="""Preferred language for output
456                                    (use local name, if omitted)""")
457         group = parser.add_argument_group('Filter arguments')
458         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
459                            help='Export only objects within country')
460         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
461                            help='Export only children of this OSM node')
462         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
463                            help='Export only children of this OSM way')
464         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
465                            help='Export only children of this OSM relation')
466
467
468     @staticmethod
469     def run(args):
470         params = ['export.php',
471                   '--output-type', args.output_type,
472                   '--output-format', args.output_format]
473         if args.output_all_postcodes:
474             params.append('--output-all-postcodes')
475         if args.language:
476             params.extend(('--language', args.language))
477         if args.restrict_to_country:
478             params.extend(('--restrict-to-country', args.restrict_to_country))
479         if args.restrict_to_osm_node:
480             params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
481         if args.restrict_to_osm_way:
482             params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
483         if args.restrict_to_osm_relation:
484             params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
485
486         return run_legacy_script(*params, nominatim_env=args)
487
488 class QueryTodo:
489     """\
490     Todo
491     """
492     @staticmethod
493     def add_args(parser):
494         pass
495
496     @staticmethod
497     def run(args): # pylint: disable=W0613
498         print("TODO: searching")
499
500
501 def nominatim(**kwargs):
502     """\
503     Command-line tools for importing, updating, administrating and
504     querying the Nominatim database.
505     """
506     parser = CommandlineParser('nominatim', nominatim.__doc__)
507
508     parser.add_subcommand('import', SetupAll)
509     parser.add_subcommand('freeze', SetupFreeze)
510     parser.add_subcommand('replication', UpdateReplication)
511
512     parser.add_subcommand('check-database', AdminCheckDatabase)
513     parser.add_subcommand('warm', AdminWarm)
514
515     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
516
517     parser.add_subcommand('add-data', UpdateAddData)
518     parser.add_subcommand('index', UpdateIndex)
519     parser.add_subcommand('refresh', UpdateRefresh)
520
521     parser.add_subcommand('export', QueryExport)
522     parser.add_subcommand('search', QueryTodo)
523     parser.add_subcommand('reverse', QueryTodo)
524     parser.add_subcommand('lookup', QueryTodo)
525     parser.add_subcommand('details', QueryTodo)
526     parser.add_subcommand('status', QueryTodo)
527
528     return parser.run(**kwargs)