]> git.openstreetmap.org Git - nominatim.git/blob - nominatim/indexer/progress.py
indexer: allow batch processing of places
[nominatim.git] / nominatim / indexer / progress.py
1 # SPDX-License-Identifier: GPL-2.0-only
2 #
3 # This file is part of Nominatim.
4 # Copyright (C) 2020 Sarah Hoffmann
5
6 import logging
7 from datetime import datetime
8
9 log = logging.getLogger()
10
11 class ProgressLogger(object):
12     """ Tracks and prints progress for the indexing process.
13         `name` is the name of the indexing step being tracked.
14         `total` sets up the total number of items that need processing.
15         `log_interval` denotes the interval in seconds at which progres
16         should be reported.
17     """
18
19     def __init__(self, name, total, log_interval=1):
20         self.name = name
21         self.total_places = total
22         self.done_places = 0
23         self.rank_start_time = datetime.now()
24         self.next_info = 100 if log.isEnabledFor(logging.INFO) else total + 1
25
26     def add(self, num=1):
27         """ Mark `num` places as processed. Print a log message if the
28             logging is at least info and the log interval has past.
29         """
30         self.done_places += num
31
32         if self.done_places >= self.next_info:
33             now = datetime.now()
34             done_time = (now - self.rank_start_time).total_seconds()
35             places_per_sec = self.done_places / done_time
36             eta = (self.total_places - self.done_places)/places_per_sec
37
38             log.info("Done {} in {} @ {:.3f} per second - {} ETA (seconds): {:.2f}"
39                      .format(self.done_places, int(done_time),
40                              places_per_sec, self.name, eta))
41
42             self.next_info += int(places_per_sec)
43
44     def done(self):
45         """ Print final staticstics about the progress.
46         """
47         rank_end_time = datetime.now()
48         diff_seconds = (rank_end_time-self.rank_start_time).total_seconds()
49
50         log.warning("Done {}/{} in {} @ {:.3f} per second - FINISHED {}\n".format(
51                     self.done_places, self.total_places, int(diff_seconds),
52                     self.done_places/diff_seconds, self.name))