munin/nominatim_requests_querylog

   1 #!/usr/bin/python3
   2 #
   3 # Plugin to monitor the types of requsts made to the API
   4 #
   5 # Uses the query log.
   6 #
   7 # Parameters:
   8 #
   9 #       config   (required)
  10 #       autoconf (optional - used by munin-config)
  11 #
  12
  13 import re
  14 import os
  15 import sys
  16 from datetime import datetime, timedelta
  17
  18 CONFIG="""graph_title Requests by API call
  19 graph_args --base 1000 -l 0
  20 graph_vlabel requests per minute
  21 graph_category nominatim
  22 z1.label reverse
  23 z1.draw AREA
  24 z1.type GAUGE
  25 z2.label search (successful)
  26 z2.draw STACK
  27 z2.type GAUGE
  28 z3.label search (no result)
  29 z3.draw STACK
  30 z3.type GAUGE
  31 z4.label details
  32 z4.draw STACK
  33 z4.type GAUGE"""
  34
  35 ENTRY_REGEX = re.compile(r'\[[^]]+\] (?P<dur>[0-9.]+) (?P<numres>\d+) (?P<type>[a-z]+) ')
  36 TIME_REGEX = re.compile(r'\[(?P<t_year>\d\d\d\d)-(?P<t_month>\d\d)-(?P<t_day>\d\d) (?P<t_hour>\d\d):(?P<t_min>\d\d):(?P<t_sec>\d\d)[0-9.]*\] ')
  37
  38
  39 class LogFile:
  40     """ A query log file, unpacked. """
  41
  42     def __init__(self, filename):
  43         self.fd = open(filename, encoding='utf-8', errors='replace')
  44         self.len = os.path.getsize(filename)
  45
  46     def __del__(self):
  47         self.fd.close()
  48
  49     def seek_next(self, abstime):
  50         self.fd.seek(abstime)
  51         self.fd.readline()
  52         l = self.fd.readline()
  53         e = TIME_REGEX.match(l)
  54         if e is None:
  55             return None
  56         e = e.groupdict()
  57         return datetime(int(e['t_year']), int(e['t_month']), int(e['t_day']),
  58                              int(e['t_hour']), int(e['t_min']), int(e['t_sec']))
  59
  60     def seek_to_date(self, target):
  61         # start position for binary search
  62         fromseek = 0
  63         fromdate = self.seek_next(0)
  64         if fromdate > target:
  65             return True
  66         # end position for binary search
  67         toseek = -100
  68         while -toseek < self.len:
  69             todate = self.seek_next(self.len + toseek)
  70             if todate is not None:
  71                 break
  72             toseek -= 100
  73         if todate is None or todate < target:
  74             return False
  75         toseek = self.len + toseek
  76
  77
  78         while True:
  79             bps = (toseek - fromseek) / (todate - fromdate).total_seconds()
  80             newseek = fromseek + int((target - fromdate).total_seconds() * bps)
  81             newdate = self.seek_next(newseek)
  82             if newdate is None:
  83                 return False;
  84             error = abs((target - newdate).total_seconds())
  85             if error < 1:
  86                 return True
  87             if newdate > target:
  88                 toseek = newseek
  89                 todate = newdate
  90                 oldfromseek = fromseek
  91                 fromseek = toseek - error * bps
  92                 while True:
  93                     if fromseek <= oldfromseek:
  94                         fromseek = oldfromseek
  95                         fromdate = self.seek_next(fromseek)
  96                         break
  97                     fromdate = self.seek_next(fromseek)
  98                     if fromdate < target:
  99                         break;
 100                     bps *=2
 101                     fromseek -= error * bps
 102             else:
 103                 fromseek = newseek
 104                 fromdate = newdate
 105                 oldtoseek = toseek
 106                 toseek = fromseek + error * bps
 107                 while True:
 108                     if toseek > oldtoseek:
 109                         toseek = oldtoseek
 110                         todate = self.seek_next(toseek)
 111                         break
 112                     todate = self.seek_next(toseek)
 113                     if todate > target:
 114                         break
 115                     bps *=2
 116                     toseek += error * bps
 117             if toseek - fromseek < 500:
 118                 return True
 119
 120
 121     def loglines(self):
 122         for l in self.fd:
 123             e = ENTRY_REGEX.match(l)
 124             if e is None:
 125                 raise ValueError("Invalid log line:", l)
 126             yield e.groupdict()
 127
 128
 129 if __name__ == '__main__':
 130
 131     if len(sys.argv) > 1 and sys.argv[1] == 'config':
 132         print(CONFIG)
 133         sys.exit(0)
 134
 135     reverse = 0
 136     searchy = 0
 137     searchn = 0
 138     details = 0
 139     if 'NOMINATIM_QUERYLOG' in os.environ:
 140         lf = LogFile(os.environ['NOMINATIM_QUERYLOG'])
 141         if lf.seek_to_date(datetime.now() - timedelta(minutes=5)):
 142             for l in lf.loglines():
 143                 if l['type'] == 'reverse':
 144                     reverse += 1
 145                 elif  l['type'] == 'search':
 146                     if l['numres'] == '0':
 147                         searchn += 1
 148                     else:
 149                         searchy += 1
 150                 else:
 151                     details += 1
 152
 153
 154     print('z1.value', reverse/5)
 155     print('z2.value', searchy/5)
 156     print('z3.value', searchn/5)
 157     print('z4.value', details/5)