]> git.openstreetmap.org Git - nominatim.git/blob - nominatim/cli.py
e28ff29563f8edf9df2adef1a786f612520922a7
[nominatim.git] / nominatim / cli.py
1 """
2 Command-line interface to the Nominatim functions for import, update,
3 database administration and querying.
4 """
5 import datetime as dt
6 import os
7 import socket
8 import sys
9 import time
10 import argparse
11 import logging
12 from pathlib import Path
13
14 from .config import Configuration
15 from .tools.exec_utils import run_legacy_script, run_api_script, run_php_server
16 from .db.connection import connect
17 from .db import status
18 from .errors import UsageError
19
20 LOG = logging.getLogger()
21
22 def _num_system_cpus():
23     try:
24         cpus = len(os.sched_getaffinity(0))
25     except NotImplementedError:
26         cpus = None
27
28     return cpus or os.cpu_count()
29
30
31 class CommandlineParser:
32     """ Wraps some of the common functions for parsing the command line
33         and setting up subcommands.
34     """
35     def __init__(self, prog, description):
36         self.parser = argparse.ArgumentParser(
37             prog=prog,
38             description=description,
39             formatter_class=argparse.RawDescriptionHelpFormatter)
40
41         self.subs = self.parser.add_subparsers(title='available commands',
42                                                dest='subcommand')
43
44         # Arguments added to every sub-command
45         self.default_args = argparse.ArgumentParser(add_help=False)
46         group = self.default_args.add_argument_group('Default arguments')
47         group.add_argument('-h', '--help', action='help',
48                            help='Show this help message and exit')
49         group.add_argument('-q', '--quiet', action='store_const', const=0,
50                            dest='verbose', default=1,
51                            help='Print only error messages')
52         group.add_argument('-v', '--verbose', action='count', default=1,
53                            help='Increase verboseness of output')
54         group.add_argument('--project-dir', metavar='DIR', default='.',
55                            help='Base directory of the Nominatim installation (default:.)')
56         group.add_argument('-j', '--threads', metavar='NUM', type=int,
57                            help='Number of parallel threads to use')
58
59
60     def add_subcommand(self, name, cmd):
61         """ Add a subcommand to the parser. The subcommand must be a class
62             with a function add_args() that adds the parameters for the
63             subcommand and a run() function that executes the command.
64         """
65         parser = self.subs.add_parser(name, parents=[self.default_args],
66                                       help=cmd.__doc__.split('\n', 1)[0],
67                                       description=cmd.__doc__,
68                                       formatter_class=argparse.RawDescriptionHelpFormatter,
69                                       add_help=False)
70         parser.set_defaults(command=cmd)
71         cmd.add_args(parser)
72
73     def run(self, **kwargs):
74         """ Parse the command line arguments of the program and execute the
75             appropriate subcommand.
76         """
77         args = self.parser.parse_args(args=kwargs.get('cli_args'))
78
79         if args.subcommand is None:
80             self.parser.print_help()
81             return 1
82
83         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'):
84             setattr(args, arg, Path(kwargs[arg]))
85         args.project_dir = Path(args.project_dir).resolve()
86
87         logging.basicConfig(stream=sys.stderr,
88                             format='%(asctime)s: %(message)s',
89                             datefmt='%Y-%m-%d %H:%M:%S',
90                             level=max(4 - args.verbose, 1) * 10)
91
92         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
93
94         log = logging.getLogger()
95         log.warning('Using project directory: %s', str(args.project_dir))
96
97         try:
98             return args.command.run(args)
99         except UsageError as exception:
100             if log.isEnabledFor(logging.DEBUG):
101                 raise # use Python's exception printing
102             log.fatal('FATAL: %s', exception)
103
104         # If we get here, then execution has failed in some way.
105         return 1
106
107
108 def _osm2pgsql_options_from_args(args, default_cache, default_threads):
109     """ Set up the stanadrd osm2pgsql from the command line arguments.
110     """
111     return dict(osm2pgsql=args.osm2pgsql_path,
112                 osm2pgsql_cache=args.osm2pgsql_cache or default_cache,
113                 osm2pgsql_style=args.config.get_import_style_file(),
114                 threads=args.threads or default_threads,
115                 dsn=args.config.get_libpq_dsn(),
116                 flatnode_file=args.config.FLATNODE_FILE)
117
118 ##### Subcommand classes
119 #
120 # Each class needs to implement two functions: add_args() adds the CLI parameters
121 # for the subfunction, run() executes the subcommand.
122 #
123 # The class documentation doubles as the help text for the command. The
124 # first line is also used in the summary when calling the program without
125 # a subcommand.
126 #
127 # No need to document the functions each time.
128 # pylint: disable=C0111
129 # Using non-top-level imports to make pyosmium optional for replication only.
130 # pylint: disable=E0012,C0415
131
132
133 class SetupAll:
134     """\
135     Create a new Nominatim database from an OSM file.
136     """
137
138     @staticmethod
139     def add_args(parser):
140         group_name = parser.add_argument_group('Required arguments')
141         group = group_name.add_mutually_exclusive_group(required=True)
142         group.add_argument('--osm-file',
143                            help='OSM file to be imported.')
144         group.add_argument('--continue', dest='continue_at',
145                            choices=['load-data', 'indexing', 'db-postprocess'],
146                            help='Continue an import that was interrupted')
147         group = parser.add_argument_group('Optional arguments')
148         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
149                            help='Size of cache to be used by osm2pgsql (in MB)')
150         group.add_argument('--reverse-only', action='store_true',
151                            help='Do not create tables and indexes for searching')
152         group.add_argument('--enable-debug-statements', action='store_true',
153                            help='Include debug warning statements in SQL code')
154         group.add_argument('--no-partitions', action='store_true',
155                            help="""Do not partition search indices
156                                    (speeds up import of single country extracts)""")
157         group.add_argument('--no-updates', action='store_true',
158                            help="""Do not keep tables that are only needed for
159                                    updating the database later""")
160         group = parser.add_argument_group('Expert options')
161         group.add_argument('--ignore-errors', action='store_true',
162                            help='Continue import even when errors in SQL are present')
163         group.add_argument('--index-noanalyse', action='store_true',
164                            help='Do not perform analyse operations during index')
165
166
167     @staticmethod
168     def run(args):
169         params = ['setup.php']
170         if args.osm_file:
171             params.extend(('--all', '--osm-file', args.osm_file))
172         else:
173             if args.continue_at == 'load-data':
174                 params.append('--load-data')
175             if args.continue_at in ('load-data', 'indexing'):
176                 params.append('--index')
177             params.extend(('--create-search-indices', '--create-country-names',
178                            '--setup-website'))
179         if args.osm2pgsql_cache:
180             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
181         if args.reverse_only:
182             params.append('--reverse-only')
183         if args.enable_debug_statements:
184             params.append('--enable-debug-statements')
185         if args.no_partitions:
186             params.append('--no-partitions')
187         if args.no_updates:
188             params.append('--drop')
189         if args.ignore_errors:
190             params.append('--ignore-errors')
191         if args.index_noanalyse:
192             params.append('--index-noanalyse')
193
194         return run_legacy_script(*params, nominatim_env=args)
195
196
197 class SetupFreeze:
198     """\
199     Make database read-only.
200
201     About half of data in the Nominatim database is kept only to be able to
202     keep the data up-to-date with new changes made in OpenStreetMap. This
203     command drops all this data and only keeps the part needed for geocoding
204     itself.
205
206     This command has the same effect as the `--no-updates` option for imports.
207     """
208
209     @staticmethod
210     def add_args(parser):
211         pass # No options
212
213     @staticmethod
214     def run(args):
215         return run_legacy_script('setup.php', '--drop', nominatim_env=args)
216
217
218 class SetupSpecialPhrases:
219     """\
220     Maintain special phrases.
221     """
222
223     @staticmethod
224     def add_args(parser):
225         group = parser.add_argument_group('Input arguments')
226         group.add_argument('--from-wiki', action='store_true',
227                            help='Pull special phrases from the OSM wiki.')
228         group = parser.add_argument_group('Output arguments')
229         group.add_argument('-o', '--output', default='-',
230                            help="""File to write the preprocessed phrases to.
231                                    If omitted, it will be written to stdout.""")
232
233     @staticmethod
234     def run(args):
235         if args.output != '-':
236             raise NotImplementedError('Only output to stdout is currently implemented.')
237         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
238
239
240 class UpdateReplication:
241     """\
242     Update the database using an online replication service.
243     """
244
245     @staticmethod
246     def add_args(parser):
247         group = parser.add_argument_group('Arguments for initialisation')
248         group.add_argument('--init', action='store_true',
249                            help='Initialise the update process')
250         group.add_argument('--no-update-functions', dest='update_functions',
251                            action='store_false',
252                            help="""Do not update the trigger function to
253                                    support differential updates.""")
254         group = parser.add_argument_group('Arguments for updates')
255         group.add_argument('--check-for-updates', action='store_true',
256                            help='Check if new updates are available and exit')
257         group.add_argument('--once', action='store_true',
258                            help="""Download and apply updates only once. When
259                                    not set, updates are continuously applied""")
260         group.add_argument('--no-index', action='store_false', dest='do_index',
261                            help="""Do not index the new data. Only applicable
262                                    together with --once""")
263         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
264                            help='Size of cache to be used by osm2pgsql (in MB)')
265         group = parser.add_argument_group('Download parameters')
266         group.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60,
267                            help='Set timeout for file downloads.')
268
269     @staticmethod
270     def _init_replication(args):
271         from .tools import replication, refresh
272
273         socket.setdefaulttimeout(args.socket_timeout)
274
275         LOG.warning("Initialising replication updates")
276         conn = connect(args.config.get_libpq_dsn())
277         replication.init_replication(conn, base_url=args.config.REPLICATION_URL)
278         if args.update_functions:
279             LOG.warning("Create functions")
280             refresh.create_functions(conn, args.config, args.data_dir,
281                                      True, False)
282         conn.close()
283         return 0
284
285
286     @staticmethod
287     def _check_for_updates(args):
288         from .tools import replication
289
290         conn = connect(args.config.get_libpq_dsn())
291         ret = replication.check_for_updates(conn, base_url=args.config.REPLICATION_URL)
292         conn.close()
293         return ret
294
295     @staticmethod
296     def _report_update(batchdate, start_import, start_index):
297         def round_time(delta):
298             return dt.timedelta(seconds=int(delta.total_seconds()))
299
300         end = dt.datetime.now(dt.timezone.utc)
301         LOG.warning("Update completed. Import: %s. %sTotal: %s. Remaining backlog: %s.",
302                     round_time((start_index or end) - start_import),
303                     "Indexing: {} ".format(round_time(end - start_index))
304                     if start_index else '',
305                     round_time(end - start_import),
306                     round_time(end - batchdate))
307
308     @staticmethod
309     def _update(args):
310         from .tools import replication
311         from .indexer.indexer import Indexer
312
313         params = _osm2pgsql_options_from_args(args, 2000, 1)
314         params.update(base_url=args.config.REPLICATION_URL,
315                       update_interval=args.config.get_int('REPLICATION_UPDATE_INTERVAL'),
316                       import_file=args.project_dir / 'osmosischange.osc',
317                       max_diff_size=args.config.get_int('REPLICATION_MAX_DIFF'),
318                       indexed_only=not args.once)
319
320         # Sanity check to not overwhelm the Geofabrik servers.
321         if 'download.geofabrik.de'in params['base_url']\
322            and params['update_interval'] < 86400:
323             LOG.fatal("Update interval too low for download.geofabrik.de.\n"
324                       "Please check install documentation "
325                       "(https://nominatim.org/release-docs/latest/admin/Import-and-Update#"
326                       "setting-up-the-update-process).")
327             raise UsageError("Invalid replication update interval setting.")
328
329         if not args.once:
330             if not args.do_index:
331                 LOG.fatal("Indexing cannot be disabled when running updates continuously.")
332                 raise UsageError("Bad argument '--no-index'.")
333             recheck_interval = args.config.get_int('REPLICATION_RECHECK_INTERVAL')
334
335         while True:
336             conn = connect(args.config.get_libpq_dsn())
337             start = dt.datetime.now(dt.timezone.utc)
338             state = replication.update(conn, params)
339             status.log_status(conn, start, 'import')
340             batchdate, _, _ = status.get_status(conn)
341             conn.close()
342
343             if state is not replication.UpdateState.NO_CHANGES and args.do_index:
344                 index_start = dt.datetime.now(dt.timezone.utc)
345                 indexer = Indexer(args.config.get_libpq_dsn(),
346                                   args.threads or 1)
347                 indexer.index_boundaries(0, 30)
348                 indexer.index_by_rank(0, 30)
349
350                 conn = connect(args.config.get_libpq_dsn())
351                 status.set_indexed(conn, True)
352                 status.log_status(conn, index_start, 'index')
353                 conn.close()
354             else:
355                 index_start = None
356
357             if LOG.isEnabledFor(logging.WARNING):
358                 UpdateReplication._report_update(batchdate, start, index_start)
359
360             if args.once:
361                 break
362
363             if state is replication.UpdateState.NO_CHANGES:
364                 LOG.warning("No new changes. Sleeping for %d sec.", recheck_interval)
365                 time.sleep(recheck_interval)
366
367         return state.value
368
369     @staticmethod
370     def run(args):
371         try:
372             import osmium # pylint: disable=W0611
373         except ModuleNotFoundError:
374             LOG.fatal("pyosmium not installed. Replication functions not available.\n"
375                       "To install pyosmium via pip: pip3 install osmium")
376             return 1
377
378         if args.init:
379             return UpdateReplication._init_replication(args)
380
381         if args.check_for_updates:
382             return UpdateReplication._check_for_updates(args)
383
384         return UpdateReplication._update(args)
385
386 class UpdateAddData:
387     """\
388     Add additional data from a file or an online source.
389
390     Data is only imported, not indexed. You need to call `nominatim-update index`
391     to complete the process.
392     """
393
394     @staticmethod
395     def add_args(parser):
396         group_name = parser.add_argument_group('Source')
397         group = group_name.add_mutually_exclusive_group(required=True)
398         group.add_argument('--file', metavar='FILE',
399                            help='Import data from an OSM file')
400         group.add_argument('--diff', metavar='FILE',
401                            help='Import data from an OSM diff file')
402         group.add_argument('--node', metavar='ID', type=int,
403                            help='Import a single node from the API')
404         group.add_argument('--way', metavar='ID', type=int,
405                            help='Import a single way from the API')
406         group.add_argument('--relation', metavar='ID', type=int,
407                            help='Import a single relation from the API')
408         group.add_argument('--tiger-data', metavar='DIR',
409                            help='Add housenumbers from the US TIGER census database.')
410         group = parser.add_argument_group('Extra arguments')
411         group.add_argument('--use-main-api', action='store_true',
412                            help='Use OSM API instead of Overpass to download objects')
413
414     @staticmethod
415     def run(args):
416         if args.tiger_data:
417             os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
418             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
419
420         params = ['update.php']
421         if args.file:
422             params.extend(('--import-file', args.file))
423         elif args.diff:
424             params.extend(('--import-diff', args.diff))
425         elif args.node:
426             params.extend(('--import-node', args.node))
427         elif args.way:
428             params.extend(('--import-way', args.way))
429         elif args.relation:
430             params.extend(('--import-relation', args.relation))
431         if args.use_main_api:
432             params.append('--use-main-api')
433         return run_legacy_script(*params, nominatim_env=args)
434
435
436 class UpdateIndex:
437     """\
438     Reindex all new and modified data.
439     """
440
441     @staticmethod
442     def add_args(parser):
443         group = parser.add_argument_group('Filter arguments')
444         group.add_argument('--boundaries-only', action='store_true',
445                            help="""Index only administrative boundaries.""")
446         group.add_argument('--no-boundaries', action='store_true',
447                            help="""Index everything except administrative boundaries.""")
448         group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
449                            help='Minimum/starting rank')
450         group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
451                            help='Maximum/finishing rank')
452
453     @staticmethod
454     def run(args):
455         from .indexer.indexer import Indexer
456
457         indexer = Indexer(args.config.get_libpq_dsn(),
458                           args.threads or _num_system_cpus() or 1)
459
460         if not args.no_boundaries:
461             indexer.index_boundaries(args.minrank, args.maxrank)
462         if not args.boundaries_only:
463             indexer.index_by_rank(args.minrank, args.maxrank)
464
465         if not args.no_boundaries and not args.boundaries_only \
466            and args.minrank == 0 and args.maxrank == 30:
467             conn = connect(args.config.get_libpq_dsn())
468             status.set_indexed(conn, True)
469             conn.close()
470
471         return 0
472
473
474 class UpdateRefresh:
475     """\
476     Recompute auxiliary data used by the indexing process.
477
478     These functions must not be run in parallel with other update commands.
479     """
480
481     @staticmethod
482     def add_args(parser):
483         group = parser.add_argument_group('Data arguments')
484         group.add_argument('--postcodes', action='store_true',
485                            help='Update postcode centroid table')
486         group.add_argument('--word-counts', action='store_true',
487                            help='Compute frequency of full-word search terms')
488         group.add_argument('--address-levels', action='store_true',
489                            help='Reimport address level configuration')
490         group.add_argument('--functions', action='store_true',
491                            help='Update the PL/pgSQL functions in the database')
492         group.add_argument('--wiki-data', action='store_true',
493                            help='Update Wikipedia/data importance numbers.')
494         group.add_argument('--importance', action='store_true',
495                            help='Recompute place importances (expensive!)')
496         group.add_argument('--website', action='store_true',
497                            help='Refresh the directory that serves the scripts for the web API')
498         group = parser.add_argument_group('Arguments for function refresh')
499         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
500                            help='Do not enable code for propagating updates')
501         group.add_argument('--enable-debug-statements', action='store_true',
502                            help='Enable debug warning statements in functions')
503
504     @staticmethod
505     def run(args):
506         from .tools import refresh
507
508         if args.postcodes:
509             LOG.warning("Update postcodes centroid")
510             conn = connect(args.config.get_libpq_dsn())
511             refresh.update_postcodes(conn, args.data_dir)
512             conn.close()
513
514         if args.word_counts:
515             LOG.warning('Recompute frequency of full-word search terms')
516             conn = connect(args.config.get_libpq_dsn())
517             refresh.recompute_word_counts(conn, args.data_dir)
518             conn.close()
519
520         if args.address_levels:
521             cfg = Path(args.config.ADDRESS_LEVEL_CONFIG)
522             LOG.warning('Updating address levels from %s', cfg)
523             conn = connect(args.config.get_libpq_dsn())
524             refresh.load_address_levels_from_file(conn, cfg)
525             conn.close()
526
527         if args.functions:
528             LOG.warning('Create functions')
529             conn = connect(args.config.get_libpq_dsn())
530             refresh.create_functions(conn, args.config, args.data_dir,
531                                      args.diffs, args.enable_debug_statements)
532             conn.close()
533
534         if args.wiki_data:
535             run_legacy_script('setup.php', '--import-wikipedia-articles',
536                               nominatim_env=args, throw_on_fail=True)
537         # Attention: importance MUST come after wiki data import.
538         if args.importance:
539             run_legacy_script('update.php', '--recompute-importance',
540                               nominatim_env=args, throw_on_fail=True)
541         if args.website:
542             run_legacy_script('setup.php', '--setup-website',
543                               nominatim_env=args, throw_on_fail=True)
544
545         return 0
546
547
548 class AdminCheckDatabase:
549     """\
550     Check that the database is complete and operational.
551     """
552
553     @staticmethod
554     def add_args(parser):
555         pass # No options
556
557     @staticmethod
558     def run(args):
559         return run_legacy_script('check_import_finished.php', nominatim_env=args)
560
561
562 class AdminWarm:
563     """\
564     Warm database caches for search and reverse queries.
565     """
566
567     @staticmethod
568     def add_args(parser):
569         group = parser.add_argument_group('Target arguments')
570         group.add_argument('--search-only', action='store_const', dest='target',
571                            const='search',
572                            help="Only pre-warm tables for search queries")
573         group.add_argument('--reverse-only', action='store_const', dest='target',
574                            const='reverse',
575                            help="Only pre-warm tables for reverse queries")
576
577     @staticmethod
578     def run(args):
579         params = ['warm.php']
580         if args.target == 'reverse':
581             params.append('--reverse-only')
582         if args.target == 'search':
583             params.append('--search-only')
584         return run_legacy_script(*params, nominatim_env=args)
585
586
587 class QueryExport:
588     """\
589     Export addresses as CSV file from the database.
590     """
591
592     @staticmethod
593     def add_args(parser):
594         group = parser.add_argument_group('Output arguments')
595         group.add_argument('--output-type', default='street',
596                            choices=('continent', 'country', 'state', 'county',
597                                     'city', 'suburb', 'street', 'path'),
598                            help='Type of places to output (default: street)')
599         group.add_argument('--output-format',
600                            default='street;suburb;city;county;state;country',
601                            help="""Semicolon-separated list of address types
602                                    (see --output-type). Multiple ranks can be
603                                    merged into one column by simply using a
604                                    comma-separated list.""")
605         group.add_argument('--output-all-postcodes', action='store_true',
606                            help="""List all postcodes for address instead of
607                                    just the most likely one""")
608         group.add_argument('--language',
609                            help="""Preferred language for output
610                                    (use local name, if omitted)""")
611         group = parser.add_argument_group('Filter arguments')
612         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
613                            help='Export only objects within country')
614         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
615                            help='Export only children of this OSM node')
616         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
617                            help='Export only children of this OSM way')
618         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
619                            help='Export only children of this OSM relation')
620
621
622     @staticmethod
623     def run(args):
624         params = ['export.php',
625                   '--output-type', args.output_type,
626                   '--output-format', args.output_format]
627         if args.output_all_postcodes:
628             params.append('--output-all-postcodes')
629         if args.language:
630             params.extend(('--language', args.language))
631         if args.restrict_to_country:
632             params.extend(('--restrict-to-country', args.restrict_to_country))
633         if args.restrict_to_osm_node:
634             params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
635         if args.restrict_to_osm_way:
636             params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
637         if args.restrict_to_osm_relation:
638             params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
639
640         return run_legacy_script(*params, nominatim_env=args)
641
642
643 class AdminServe:
644     """\
645     Start a simple web server for serving the API.
646
647     This command starts the built-in PHP webserver to serve the website
648     from the current project directory. This webserver is only suitable
649     for testing and develop. Do not use it in production setups!
650
651     By the default, the webserver can be accessed at: http://127.0.0.1:8088
652     """
653
654     @staticmethod
655     def add_args(parser):
656         group = parser.add_argument_group('Server arguments')
657         group.add_argument('--server', default='127.0.0.1:8088',
658                            help='The address the server will listen to.')
659
660     @staticmethod
661     def run(args):
662         run_php_server(args.server, args.project_dir / 'website')
663
664 STRUCTURED_QUERY = (
665     ('street', 'housenumber and street'),
666     ('city', 'city, town or village'),
667     ('county', 'county'),
668     ('state', 'state'),
669     ('country', 'country'),
670     ('postalcode', 'postcode')
671 )
672
673 EXTRADATA_PARAMS = (
674     ('addressdetails', 'Include a breakdown of the address into elements.'),
675     ('extratags', """Include additional information if available
676                      (e.g. wikipedia link, opening hours)."""),
677     ('namedetails', 'Include a list of alternative names.')
678 )
679
680 DETAILS_SWITCHES = (
681     ('addressdetails', 'Include a breakdown of the address into elements.'),
682     ('keywords', 'Include a list of name keywords and address keywords.'),
683     ('linkedplaces', 'Include a details of places that are linked with this one.'),
684     ('hierarchy', 'Include details of places lower in the address hierarchy.'),
685     ('group_hierarchy', 'Group the places by type.'),
686     ('polygon_geojson', 'Include geometry of result.')
687 )
688
689 def _add_api_output_arguments(parser):
690     group = parser.add_argument_group('Output arguments')
691     group.add_argument('--format', default='jsonv2',
692                        choices=['xml', 'json', 'jsonv2', 'geojson', 'geocodejson'],
693                        help='Format of result')
694     for name, desc in EXTRADATA_PARAMS:
695         group.add_argument('--' + name, action='store_true', help=desc)
696
697     group.add_argument('--lang', '--accept-language', metavar='LANGS',
698                        help='Preferred language order for presenting search results')
699     group.add_argument('--polygon-output',
700                        choices=['geojson', 'kml', 'svg', 'text'],
701                        help='Output geometry of results as a GeoJSON, KML, SVG or WKT.')
702     group.add_argument('--polygon-threshold', type=float, metavar='TOLERANCE',
703                        help="""Simplify output geometry.
704                                Parameter is difference tolerance in degrees.""")
705
706
707 class APISearch:
708     """\
709     Execute API search query.
710     """
711
712     @staticmethod
713     def add_args(parser):
714         group = parser.add_argument_group('Query arguments')
715         group.add_argument('--query',
716                            help='Free-form query string')
717         for name, desc in STRUCTURED_QUERY:
718             group.add_argument('--' + name, help='Structured query: ' + desc)
719
720         _add_api_output_arguments(parser)
721
722         group = parser.add_argument_group('Result limitation')
723         group.add_argument('--countrycodes', metavar='CC,..',
724                            help='Limit search results to one or more countries.')
725         group.add_argument('--exclude_place_ids', metavar='ID,..',
726                            help='List of search object to be excluded')
727         group.add_argument('--limit', type=int,
728                            help='Limit the number of returned results')
729         group.add_argument('--viewbox', metavar='X1,Y1,X2,Y2',
730                            help='Preferred area to find search results')
731         group.add_argument('--bounded', action='store_true',
732                            help='Strictly restrict results to viewbox area')
733
734         group = parser.add_argument_group('Other arguments')
735         group.add_argument('--no-dedupe', action='store_false', dest='dedupe',
736                            help='Do not remove duplicates from the result list')
737
738
739     @staticmethod
740     def run(args):
741         if args.query:
742             params = dict(q=args.query)
743         else:
744             params = {k : getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)}
745
746         for param, _ in EXTRADATA_PARAMS:
747             if getattr(args, param):
748                 params[param] = '1'
749         for param in ('format', 'countrycodes', 'exclude_place_ids', 'limit', 'viewbox'):
750             if getattr(args, param):
751                 params[param] = getattr(args, param)
752         if args.lang:
753             params['accept-language'] = args.lang
754         if args.polygon_output:
755             params['polygon_' + args.polygon_output] = '1'
756         if args.polygon_threshold:
757             params['polygon_threshold'] = args.polygon_threshold
758         if args.bounded:
759             params['bounded'] = '1'
760         if not args.dedupe:
761             params['dedupe'] = '0'
762
763         return run_api_script('search', args.project_dir,
764                               phpcgi_bin=args.phpcgi_path, params=params)
765
766 class APIReverse:
767     """\
768     Execute API reverse query.
769     """
770
771     @staticmethod
772     def add_args(parser):
773         group = parser.add_argument_group('Query arguments')
774         group.add_argument('--lat', type=float, required=True,
775                            help='Latitude of coordinate to look up (in WGS84)')
776         group.add_argument('--lon', type=float, required=True,
777                            help='Longitude of coordinate to look up (in WGS84)')
778         group.add_argument('--zoom', type=int,
779                            help='Level of detail required for the address')
780
781         _add_api_output_arguments(parser)
782
783
784     @staticmethod
785     def run(args):
786         params = dict(lat=args.lat, lon=args.lon)
787         if args.zoom is not None:
788             params['zoom'] = args.zoom
789
790         for param, _ in EXTRADATA_PARAMS:
791             if getattr(args, param):
792                 params[param] = '1'
793         if args.format:
794             params['format'] = args.format
795         if args.lang:
796             params['accept-language'] = args.lang
797         if args.polygon_output:
798             params['polygon_' + args.polygon_output] = '1'
799         if args.polygon_threshold:
800             params['polygon_threshold'] = args.polygon_threshold
801
802         return run_api_script('reverse', args.project_dir,
803                               phpcgi_bin=args.phpcgi_path, params=params)
804
805
806 class APILookup:
807     """\
808     Execute API reverse query.
809     """
810
811     @staticmethod
812     def add_args(parser):
813         group = parser.add_argument_group('Query arguments')
814         group.add_argument('--id', metavar='OSMID',
815                            action='append', required=True, dest='ids',
816                            help='OSM id to lookup in format <NRW><id> (may be repeated)')
817
818         _add_api_output_arguments(parser)
819
820
821     @staticmethod
822     def run(args):
823         params = dict(osm_ids=','.join(args.ids))
824
825         for param, _ in EXTRADATA_PARAMS:
826             if getattr(args, param):
827                 params[param] = '1'
828         if args.format:
829             params['format'] = args.format
830         if args.lang:
831             params['accept-language'] = args.lang
832         if args.polygon_output:
833             params['polygon_' + args.polygon_output] = '1'
834         if args.polygon_threshold:
835             params['polygon_threshold'] = args.polygon_threshold
836
837         return run_api_script('lookup', args.project_dir,
838                               phpcgi_bin=args.phpcgi_path, params=params)
839
840
841 class APIDetails:
842     """\
843     Execute API lookup query.
844     """
845
846     @staticmethod
847     def add_args(parser):
848         group = parser.add_argument_group('Query arguments')
849         objs = group.add_mutually_exclusive_group(required=True)
850         objs.add_argument('--node', '-n', type=int,
851                           help="Look up the OSM node with the given ID.")
852         objs.add_argument('--way', '-w', type=int,
853                           help="Look up the OSM way with the given ID.")
854         objs.add_argument('--relation', '-r', type=int,
855                           help="Look up the OSM relation with the given ID.")
856         objs.add_argument('--place_id', '-p', type=int,
857                           help='Database internal identifier of the OSM object to look up.')
858         group.add_argument('--class', dest='object_class',
859                            help="""Class type to disambiguated multiple entries
860                                    of the same object.""")
861
862         group = parser.add_argument_group('Output arguments')
863         for name, desc in DETAILS_SWITCHES:
864             group.add_argument('--' + name, action='store_true', help=desc)
865         group.add_argument('--lang', '--accept-language', metavar='LANGS',
866                            help='Preferred language order for presenting search results')
867
868     @staticmethod
869     def run(args):
870         if args.node:
871             params = dict(osmtype='N', osmid=args.node)
872         elif args.way:
873             params = dict(osmtype='W', osmid=args.node)
874         elif args.relation:
875             params = dict(osmtype='R', osmid=args.node)
876         else:
877             params = dict(place_id=args.place_id)
878         if args.object_class:
879             params['class'] = args.object_class
880         for name, _ in DETAILS_SWITCHES:
881             params[name] = '1' if getattr(args, name) else '0'
882
883         return run_api_script('details', args.project_dir,
884                               phpcgi_bin=args.phpcgi_path, params=params)
885
886
887 class APIStatus:
888     """\
889     Execute API status query.
890     """
891
892     @staticmethod
893     def add_args(parser):
894         group = parser.add_argument_group('API parameters')
895         group.add_argument('--format', default='text', choices=['text', 'json'],
896                            help='Format of result')
897
898     @staticmethod
899     def run(args):
900         return run_api_script('status', args.project_dir,
901                               phpcgi_bin=args.phpcgi_path,
902                               params=dict(format=args.format))
903
904
905 def nominatim(**kwargs):
906     """\
907     Command-line tools for importing, updating, administrating and
908     querying the Nominatim database.
909     """
910     parser = CommandlineParser('nominatim', nominatim.__doc__)
911
912     parser.add_subcommand('import', SetupAll)
913     parser.add_subcommand('freeze', SetupFreeze)
914     parser.add_subcommand('replication', UpdateReplication)
915
916     parser.add_subcommand('check-database', AdminCheckDatabase)
917     parser.add_subcommand('warm', AdminWarm)
918
919     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
920
921     parser.add_subcommand('add-data', UpdateAddData)
922     parser.add_subcommand('index', UpdateIndex)
923     parser.add_subcommand('refresh', UpdateRefresh)
924
925     parser.add_subcommand('export', QueryExport)
926     parser.add_subcommand('serve', AdminServe)
927
928     if kwargs.get('phpcgi_path'):
929         parser.add_subcommand('search', APISearch)
930         parser.add_subcommand('reverse', APIReverse)
931         parser.add_subcommand('lookup', APILookup)
932         parser.add_subcommand('details', APIDetails)
933         parser.add_subcommand('status', APIStatus)
934     else:
935         parser.parser.epilog = 'php-cgi not found. Query commands not available.'
936
937     return parser.run(**kwargs)