]> git.openstreetmap.org Git - nominatim.git/blob - nominatim/indexer/progress.py
Merge pull request #2562 from lonvia/copyright-headers
[nominatim.git] / nominatim / indexer / progress.py
1 # SPDX-License-Identifier: GPL-2.0-only
2 #
3 # This file is part of Nominatim. (https://nominatim.org)
4 #
5 # Copyright (C) 2022 by the Nominatim developer community.
6 # For a full list of authors see the git log.
7 """
8 Helpers for progress logging.
9 """
10 import logging
11 from datetime import datetime
12
13 LOG = logging.getLogger()
14
15 INITIAL_PROGRESS = 10
16
17 class ProgressLogger:
18     """ Tracks and prints progress for the indexing process.
19         `name` is the name of the indexing step being tracked.
20         `total` sets up the total number of items that need processing.
21         `log_interval` denotes the interval in seconds at which progres
22         should be reported.
23     """
24
25     def __init__(self, name, total, log_interval=1):
26         self.name = name
27         self.total_places = total
28         self.done_places = 0
29         self.rank_start_time = datetime.now()
30         self.log_interval = log_interval
31         self.next_info = INITIAL_PROGRESS if LOG.isEnabledFor(logging.WARNING) else total + 1
32
33     def add(self, num=1):
34         """ Mark `num` places as processed. Print a log message if the
35             logging is at least info and the log interval has passed.
36         """
37         self.done_places += num
38
39         if self.done_places < self.next_info:
40             return
41
42         now = datetime.now()
43         done_time = (now - self.rank_start_time).total_seconds()
44
45         if done_time < 2:
46             self.next_info = self.done_places + INITIAL_PROGRESS
47             return
48
49         places_per_sec = self.done_places / done_time
50         eta = (self.total_places - self.done_places) / places_per_sec
51
52         LOG.warning("Done %d in %d @ %.3f per second - %s ETA (seconds): %.2f",
53                     self.done_places, int(done_time),
54                     places_per_sec, self.name, eta)
55
56         self.next_info += int(places_per_sec) * self.log_interval
57
58     def done(self):
59         """ Print final statistics about the progress.
60         """
61         rank_end_time = datetime.now()
62
63         if rank_end_time == self.rank_start_time:
64             diff_seconds = 0
65             places_per_sec = self.done_places
66         else:
67             diff_seconds = (rank_end_time - self.rank_start_time).total_seconds()
68             places_per_sec = self.done_places / diff_seconds
69
70         LOG.warning("Done %d/%d in %d @ %.3f per second - FINISHED %s\n",
71                     self.done_places, self.total_places, int(diff_seconds),
72                     places_per_sec, self.name)