nominatim/clicmd/refresh.py

   1 """
   2 Implementation of 'refresh' subcommand.
   3 """
   4 import logging
   5 from pathlib import Path
   6
   7 from nominatim.db.connection import connect
   8
   9 # Do not repeat documentation of subcommand classes.
  10 # pylint: disable=C0111
  11 # Using non-top-level imports to avoid eventually unused imports.
  12 # pylint: disable=E0012,C0415
  13
  14 LOG = logging.getLogger()
  15
  16 class UpdateRefresh:
  17     """\
  18     Recompute auxiliary data used by the indexing process.
  19
  20     These functions must not be run in parallel with other update commands.
  21     """
  22
  23     @staticmethod
  24     def add_args(parser):
  25         group = parser.add_argument_group('Data arguments')
  26         group.add_argument('--postcodes', action='store_true',
  27                            help='Update postcode centroid table')
  28         group.add_argument('--word-counts', action='store_true',
  29                            help='Compute frequency of full-word search terms')
  30         group.add_argument('--address-levels', action='store_true',
  31                            help='Reimport address level configuration')
  32         group.add_argument('--functions', action='store_true',
  33                            help='Update the PL/pgSQL functions in the database')
  34         group.add_argument('--wiki-data', action='store_true',
  35                            help='Update Wikipedia/data importance numbers.')
  36         group.add_argument('--importance', action='store_true',
  37                            help='Recompute place importances (expensive!)')
  38         group.add_argument('--website', action='store_true',
  39                            help='Refresh the directory that serves the scripts for the web API')
  40         group = parser.add_argument_group('Arguments for function refresh')
  41         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
  42                            help='Do not enable code for propagating updates')
  43         group.add_argument('--enable-debug-statements', action='store_true',
  44                            help='Enable debug warning statements in functions')
  45
  46     @staticmethod
  47     def run(args):
  48         from ..tools import refresh, postcodes
  49         from ..tokenizer import factory as tokenizer_factory
  50         from ..indexer.indexer import Indexer
  51
  52         tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)
  53
  54         if args.postcodes:
  55             LOG.warning("Update postcodes centroid")
  56             postcodes.update_postcodes(args.config.get_libpq_dsn(),
  57                                        args.project_dir, tokenizer)
  58             indexer = Indexer(args.config.get_libpq_dsn(), tokenizer,
  59                               args.threads or 1)
  60             indexer.index_postcodes()
  61
  62         if args.word_counts:
  63             LOG.warning('Recompute frequency of full-word search terms')
  64             refresh.recompute_word_counts(args.config.get_libpq_dsn(), args.sqllib_dir)
  65
  66         if args.address_levels:
  67             cfg = Path(args.config.ADDRESS_LEVEL_CONFIG)
  68             LOG.warning('Updating address levels from %s', cfg)
  69             with connect(args.config.get_libpq_dsn()) as conn:
  70                 refresh.load_address_levels_from_file(conn, cfg)
  71
  72         if args.functions:
  73             LOG.warning('Create functions')
  74             with connect(args.config.get_libpq_dsn()) as conn:
  75                 refresh.create_functions(conn, args.config,
  76                                          args.diffs, args.enable_debug_statements)
  77                 tokenizer.update_sql_functions(args.config)
  78
  79         if args.wiki_data:
  80             data_path = Path(args.config.WIKIPEDIA_DATA_PATH
  81                              or args.project_dir)
  82             LOG.warning('Import wikipdia article importance from %s', data_path)
  83             if refresh.import_wikipedia_articles(args.config.get_libpq_dsn(),
  84                                                  data_path) > 0:
  85                 LOG.fatal('FATAL: Wikipedia importance dump file not found')
  86                 return 1
  87
  88         # Attention: importance MUST come after wiki data import.
  89         if args.importance:
  90             LOG.warning('Update importance values for database')
  91             with connect(args.config.get_libpq_dsn()) as conn:
  92                 refresh.recompute_importance(conn)
  93
  94         if args.website:
  95             webdir = args.project_dir / 'website'
  96             LOG.warning('Setting up website directory at %s', webdir)
  97             with connect(args.config.get_libpq_dsn()) as conn:
  98                 refresh.setup_website(webdir, args.config, conn)
  99         return 0