src/nominatim_db/indexer/progress.py

   1 # SPDX-License-Identifier: GPL-3.0-or-later
   2 #
   3 # This file is part of Nominatim. (https://nominatim.org)
   4 #
   5 # Copyright (C) 2024 by the Nominatim developer community.
   6 # For a full list of authors see the git log.
   7 """
   8 Helpers for progress logging.
   9 """
  10 import logging
  11 from datetime import datetime
  12
  13 LOG = logging.getLogger()
  14
  15 INITIAL_PROGRESS = 10
  16
  17
  18 class ProgressLogger:
  19     """ Tracks and prints progress for the indexing process.
  20         `name` is the name of the indexing step being tracked.
  21         `total` sets up the total number of items that need processing.
  22         `log_interval` denotes the interval in seconds at which progress
  23         should be reported.
  24     """
  25
  26     def __init__(self, name: str, total: int, log_interval: int = 1) -> None:
  27         self.name = name
  28         self.total_places = total
  29         self.done_places = 0
  30         self.rank_start_time = datetime.now()
  31         self.log_interval = log_interval
  32         self.next_info = INITIAL_PROGRESS if LOG.isEnabledFor(logging.WARNING) else total + 1
  33
  34     def add(self, num: int = 1) -> None:
  35         """ Mark `num` places as processed. Print a log message if the
  36             logging is at least info and the log interval has passed.
  37         """
  38         self.done_places += num
  39
  40         if self.done_places < self.next_info:
  41             return
  42
  43         now = datetime.now()
  44         done_time = (now - self.rank_start_time).total_seconds()
  45
  46         if done_time < 2:
  47             self.next_info = self.done_places + INITIAL_PROGRESS
  48             return
  49
  50         places_per_sec = self.done_places / done_time
  51         eta = (self.total_places - self.done_places) / places_per_sec
  52
  53         LOG.warning("Done %d in %.0f @ %.3f per second - %s ETA (seconds): %.2f",
  54                     self.done_places, done_time,
  55                     places_per_sec, self.name, eta)
  56
  57         self.next_info += int(places_per_sec) * self.log_interval
  58
  59     def done(self) -> int:
  60         """ Print final statistics about the progress.
  61         """
  62         rank_end_time = datetime.now()
  63
  64         if rank_end_time == self.rank_start_time:
  65             diff_seconds = 0.0
  66             places_per_sec = float(self.done_places)
  67         else:
  68             diff_seconds = (rank_end_time - self.rank_start_time).total_seconds()
  69             places_per_sec = self.done_places / diff_seconds
  70
  71         LOG.warning("Done %d/%d in %.0f @ %.3f per second - FINISHED %s\n",
  72                     self.done_places, self.total_places, diff_seconds,
  73                     places_per_sec, self.name)
  74
  75         return self.done_places