]> git.openstreetmap.org Git - chef.git/blob - cookbooks/prometheus/templates/default/alert_rules.yml.erb
Restrict taginfo alerts to active instances
[chef.git] / cookbooks / prometheus / templates / default / alert_rules.yml.erb
1 # DO NOT EDIT - This file is being maintained by Chef
2
3 groups:
4   - name: amsterdam
5     rules:
6       - alert: uplink
7         expr: junos_interface_up{site="amsterdam",name=~"ge-[01]/2/2"} != 1
8         for: 6m
9         labels:
10           alertgroup: "amsterdam"
11         annotations:
12           status: "{{ $value }}"
13       - alert: pdu current draw
14         expr: rPDU2PhaseStatusCurrent{site="amsterdam",rPDU2PhaseStatusIndex="1"} / 10 > 28
15         for: 6m
16         labels:
17           alertgroup: "amsterdam"
18         annotations:
19           current: "{{ $value | humanize }}A"
20       - alert: site power
21         expr: sum(avg_over_time(rPDU2PhaseStatusApparentPower{site="amsterdam",rPDU2PhaseStatusIndex="1"}[1h]) / 100) > 3.5
22         for: 6m
23         labels:
24           alertgroup: "amsterdam"
25         annotations:
26           current: "{{ $value | humanize }}kVA"
27       - alert: site temperature
28         expr: min(rPDU2SensorTempHumidityStatusTempC{site="amsterdam"}) / 10 < 18 or min(rPDU2SensorTempHumidityStatusTempC{site="amsterdam"}) / 10 > 26
29         for: 6m
30         labels:
31           alertgroup: "amsterdam"
32         annotations:
33           temperature: "{{ $value | humanize }}C"
34       - alert: site humidity
35         expr: max(rPDU2SensorTempHumidityStatusRelativeHumidity{site="amsterdam"}) / 100 < 0.25 or max(rPDU2SensorTempHumidityStatusRelativeHumidity{site="amsterdam"}) / 100 > 0.65
36         for: 6m
37         labels:
38           alertgroup: "amsterdam"
39         annotations:
40           humidity: "{{ $value | humanizePercentage }}"
41   - name: apache
42     rules:
43       - alert: apache down
44         expr: apache_up == 0
45         for: 5m
46         labels:
47           alertgroup: "{{ $labels.instance }}"
48       - alert: apache workers busy
49         expr: sum(apache_workers{state="busy"}) by (instance) / sum(apache_scoreboard) by (instance) > 0.8
50         for: 5m
51         labels:
52           alertgroup: "{{ $labels.instance }}"
53         annotations:
54           busy_workers: "{{ $value | humanizePercentage }}"
55   - name: chef
56     rules:
57       - alert: chef client not running
58         expr: time() - node_systemd_timer_last_trigger_seconds{name="chef-client.timer"} > 3600
59         for: 12h
60         labels:
61           alertgroup: "{{ $labels.instance }}"
62         annotations:
63           down_time: "{{ $value | humanizeDuration }}"
64   - name: cisco
65     rules:
66       - alert: cisco fan alarm
67         expr: rlPhdUnitEnvParamFan1Status{rlPhdUnitEnvParamFan1Status!="normal"} > 0 or rlPhdUnitEnvParamFan2Status{rlPhdUnitEnvParamFan2Status!="normal"} > 0
68         for: 5m
69         labels:
70           alertgroup: "{{ $labels.site }}"
71         annotations:
72           fan_rpm: "{{ with printf \"rlPhdUnitEnvParamFan1Speed{site='%s',instance='%s',rlPhdUnitEnvParamStackUnit='%s'}\" $labels.site $labels.instance $labels.rlPhdUnitEnvParamStackUnit | query }}{{ . | first | value | humanize }}rpm{{end}}"
73       - alert: cisco temperature alarm
74         expr: rlPhdUnitEnvParamTempSensorStatus{rlPhdUnitEnvParamTempSensorStatus!="ok"} > 0
75         for: 5m
76         labels:
77           alertgroup: "{{ $labels.site }}"
78         annotations:
79           temp_celsius: "{{ with printf \"rlPhdUnitEnvParamTempSensorValue{site='%s',instance='%s',rlPhdUnitEnvParamStackUnit='%s'}\" $labels.site $labels.instance $labels.rlPhdUnitEnvParamStackUnit | query }}{{ . | first | value | humanize }}C{{end}}"
80       - alert: cisco main power alarm
81         expr: rlPhdUnitEnvParamMainPSStatus{rlPhdUnitEnvParamMainPSStatus!="normal"} > 0
82         for: 5m
83         labels:
84           alertgroup: "{{ $labels.site }}"
85       - alert: cisco redundant power alarm
86         expr: rlPhdUnitEnvParamRedundantPSStatus{rlPhdUnitEnvParamRedundantPSStatus!="normal"} > 0
87         for: 5m
88         labels:
89           alertgroup: "{{ $labels.site }}"
90   - name: cpu
91     rules:
92       - alert: cpu pressure
93         expr: rate(node_pressure_cpu_waiting_seconds_total[5m]) > 0.75
94         for: 60m
95         labels:
96           alertgroup: "{{ $labels.instance }}"
97         annotations:
98           pressure: "{{ $value | humanizePercentage }}"
99   - name: database
100     rules:
101       - alert: postgres replication delay
102         expr: pg_replication_lag_seconds > 30
103         for: 15m
104         labels:
105           alertgroup: database
106         annotations:
107           delay: "{{ $value | humanizeDuration }}"
108   - name: discourse
109     rules:
110       - alert: discourse job failure rate
111         expr: rate(discourse_job_failures[5m]) > 0
112         for: 5m
113         labels:
114           alertgroup: discourse
115         annotations:
116           failure_rate: "{{ $value }} jobs/s"
117   - name: dublin
118     rules:
119       - alert: uplink
120         expr: junos_interface_up{site="dublin",name=~"ge-[01]/2/2"} != 1
121         for: 6m
122         labels:
123           alertgroup: "dublin"
124         annotations:
125           status: "{{ $value }}"
126       - alert: pdu current draw
127         expr: rPDU2PhaseStatusCurrent{site="dublin",rPDU2PhaseStatusIndex="1"} / 10 > 28
128         for: 6m
129         labels:
130           alertgroup: "dublin"
131         annotations:
132           current: "{{ $value | humanize }}A"
133       - alert: site power
134         expr: sum(avg_over_time(rPDU2PhaseStatusApparentPower{site="dublin",rPDU2PhaseStatusIndex="1"}[1h]) / 100) > 4
135         for: 6m
136         labels:
137           alertgroup: "dublin"
138         annotations:
139           current: "{{ $value | humanize }}kVA"
140       - alert: site temperature
141         expr: min(rPDU2SensorTempHumidityStatusTempC{site="dublin"}) / 10 < 18 or min(rPDU2SensorTempHumidityStatusTempC{site="dublin"}) / 10 > 26
142         for: 6m
143         labels:
144           alertgroup: "dublin"
145         annotations:
146           temperature: "{{ $value | humanize }}C"
147       - alert: site humidity
148         expr: max(rPDU2SensorTempHumidityStatusRelativeHumidity{site="dublin"}) / 100 < 0.25 or max(rPDU2SensorTempHumidityStatusRelativeHumidity{site="dublin"}) / 100 > 0.65
149         for: 6m
150         labels:
151           alertgroup: "dublin"
152         annotations:
153           humidity: "{{ $value | humanizePercentage }}"
154   - name: fastly
155     rules:
156       - alert: fastly error rate
157         expr: sum(rate(fastly_rt_status_group_total{status_group="5xx"}[5m])) by (service_name, datacenter) / sum(rate(fastly_rt_status_group_total[5m])) by (service_name, datacenter) > 0.005
158         for: 15m
159         labels:
160           alertgroup: fastly
161         annotations:
162           error_rate: "{{ $value | humanizePercentage }}"
163       - alert: fastly frontend healthcheck warning
164         expr: count(fastly_healthcheck_status == 0) by (service, datacenter) > 2
165         for: 15m
166         labels:
167           alertgroup: fastly
168       - alert: fastly frontend healthcheck critical
169         expr: count(fastly_healthcheck_status == 0) by (service, datacenter) == count(fastly_healthcheck_status) by (service, datacenter)
170         for: 5m
171         labels:
172           alertgroup: fastly
173       - alert: fastly backend healthcheck warning
174         expr: count(fastly_healthcheck_status == 0) by (service, backend) > 10
175         for: 15m
176         labels:
177           alertgroup: fastly
178       - alert: fastly backend healthcheck critical
179         expr: count(fastly_healthcheck_status == 0) by (service, backend) == count(fastly_healthcheck_status) by (service, backend)
180         for: 5m
181         labels:
182           alertgroup: fastly
183   - name: filesystem
184     rules:
185       - alert: readonly filesystem
186         expr: node_filesystem_readonly > min_over_time(node_filesystem_readonly[7d])
187         for: 0m
188         labels:
189           alertgroup: "{{ $labels.instance }}"
190       - alert: filesystem low on space
191         expr: node_filesystem_avail_bytes / node_filesystem_size_bytes < 0.05
192         for: 5m
193         labels:
194           alertgroup: "{{ $labels.instance }}"
195         annotations:
196           percentage_free: "{{ $value | humanizePercentage }}"
197           free_bytes: "{{ with printf \"node_filesystem_avail_bytes{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value | humanize1024 }}bytes{{end}}"
198           total_total: "{{ with printf \"node_filesystem_size_bytes{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value | humanize1024 }}bytes{{end}}"
199       - alert: filesystem low on inodes
200         expr: node_filesystem_files_free / node_filesystem_files < 0.1
201         for: 5m
202         labels:
203           alertgroup: "{{ $labels.instance }}"
204         annotations:
205           percentage_free: "{{ $value | humanizePercentage }}"
206           free_inodes: "{{ with printf \"node_filesystem_files_free{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value }}{{end}}"
207           total_inodes: "{{ with printf \"node_filesystem_files{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value }}{{end}}"
208   - name: hwmon
209     rules:
210       - alert: hwmon fan alarm
211         expr: node_hwmon_fan_alarm == 1
212         for: 5m
213         labels:
214           alertgroup: "{{ $labels.instance }}"
215         annotations:
216           fan_rpm: "{{ with printf \"node_hwmon_fan_rpm{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}rpm{{end}}"
217           fan_min_rpm: "{{ with printf \"node_hwmon_fan_min_rpm{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}rpm{{end}}"
218       - alert: hwmon temperature alarm
219         expr: node_hwmon_temp_alarm == 1
220         for: 5m
221         labels:
222           alertgroup: "{{ $labels.instance }}"
223         annotations:
224           temp_celsius: "{{ with printf \"node_hwmon_temp_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
225           temp_max_celsius: "{{ with printf \"node_hwmon_temp_max_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
226           temp_crit_celsius: "{{ with printf \"node_hwmon_temp_crit_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
227       - alert: hwmon voltage alarm
228         expr: node_hwmon_in_alarm == 1
229         for: 5m
230         labels:
231           alertgroup: "{{ $labels.instance }}"
232         annotations:
233           in_volts: "{{ with printf \"node_hwmon_in_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
234           in_min_volts: "{{ with printf \"node_hwmon_in_min_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
235           in_max_volts: "{{ with printf \"node_hwmon_in_max_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
236   - name: io
237     rules:
238       - alert: io pressure
239         expr: rate(node_pressure_io_waiting_seconds_total[5m]) > 0.6
240         for: 60m
241         labels:
242           alertgroup: "{{ $labels.instance }}"
243         annotations:
244           pressure: "{{ $value | humanizePercentage }}"
245   - name: ipmi
246     rules:
247       - alert: ipmi fan alarm
248         expr: ipmi_fan_speed_state > 0
249         for: 5m
250         labels:
251           alertgroup: "{{ $labels.instance }}"
252         annotations:
253           fan_speed_rpm: "{{ with printf \"ipmi_fan_speed_rpm{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}rpm{{end}}"
254       - alert: ipmi temperature alarm
255         expr: ipmi_temperature_state > 0
256         for: 5m
257         labels:
258           alertgroup: "{{ $labels.instance }}"
259         annotations:
260           temperature_celsius: "{{ with printf \"ipmi_temperature_celsius{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}C{{end}}"
261       - alert: ipmi voltage alarm
262         expr: ipmi_voltage_state > 0
263         for: 5m
264         labels:
265           alertgroup: "{{ $labels.instance }}"
266         annotations:
267           voltage_volts: "{{ with printf \"ipmi_voltage_volts{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}V{{end}}"
268       - alert: ipmi power alarm
269         expr: ipmi_power_state > 0 or ipmi_sensor_state{type=~"Power .*"} > 0
270         for: 5m
271         labels:
272           alertgroup: "{{ $labels.instance }}"
273   - name: juniper
274     rules:
275       - alert: juniper red alarms
276         expr: juniper_alarms_red_count > 0
277         for: 5m
278         labels:
279           alertgroup: "{{ $labels.site }}"
280         annotations:
281           alarm_count: "{{ $value }} alarms"
282       - alert: juniper yellow alarms
283         expr: juniper_alarms_yellow_count > 0
284         for: 5m
285         labels:
286           alertgroup: "{{ $labels.site }}"
287         annotations:
288           alarm_count: "{{ $value }} alarms"
289       - alert: juniper cpu alarm
290         expr: junos_route_engine_load_average_five / 2 > 0.5
291         for: 5m
292         labels:
293           alertgroup: "{{ $labels.site }}"
294         annotations:
295           load_average: "{{ $value | humanizePercentage }}"
296       - alert: juniper fan alarm
297         expr: junos_environment_fan_up != 1
298         for: 5m
299         labels:
300           alertgroup: "{{ $labels.site }}"
301       - alert: juniper power alarm
302         expr: junos_environment_power_up != 1
303         for: 5m
304         labels:
305           alertgroup: "{{ $labels.site }}"
306   - name: mail
307     rules:
308       - alert: exim down
309         expr: exim_up == 0
310         for: 5m
311         labels:
312           alertgroup: "{{ $labels.instance }}"
313       - alert: exim queue length
314         expr: exim_queue > ignoring(job) exim_queue_limit
315         for: 60m
316         labels:
317           alertgroup: mail
318         annotations:
319           queue_length: "{{ $value }}"
320       - alert: mailman queue length
321         expr: mailman_queue_length > 200
322         for: 60m
323         labels:
324           alertgroup: mail
325         annotations:
326           queue_length: "{{ $value }}"
327   - name: mdadm
328     rules:
329       - alert: mdadm array inactive
330         expr: node_md_state{state="inactive"} > 0
331         for: 0m
332         labels:
333           alertgroup: "{{ $labels.instance }}"
334         annotations:
335           required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
336           active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
337           failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
338           spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
339       - alert: mdadm array degraded
340         expr: sum (node_md_disks{state="active"}) without (state) < node_md_disks_required
341         for: 0m
342         labels:
343           alertgroup: "{{ $labels.instance }}"
344         annotations:
345           required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
346           active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
347           failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
348           spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
349       - alert: mdadm disk failed
350         expr: node_md_disks{state="failed"} > 0
351         for: 0m
352         labels:
353           alertgroup: "{{ $labels.instance }}"
354         annotations:
355           required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
356           active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
357           failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
358           spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
359   - name: memory
360     rules:
361       - alert: low memory
362         expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1
363         for: 15m
364         labels:
365           alertgroup: "{{ $labels.instance }}"
366         annotations:
367           memory_free: "{{ $value | humanizePercentage }}"
368       - alert: memory pressure
369         expr: rate(node_pressure_memory_waiting_seconds_total[5m]) > 0.6
370         for: 60m
371         labels:
372           alertgroup: "{{ $labels.instance }}"
373         annotations:
374           pressure: "{{ $value | humanizePercentage }}"
375       - alert: oom kill detected
376         expr: increase(node_vmstat_oom_kill[1m]) > 0
377         for: 0m
378         labels:
379           alertgroup: "{{ $labels.instance }}"
380         annotations:
381           new_oom_kills: "{{ $value }}"
382   - name: mysql
383     rules:
384       - alert: mysql down
385         expr: mysql_up == 0
386         for: 1m
387         labels:
388           alertgroup: "{{ $labels.instance }}"
389       - alert: mysql connection limit
390         expr: mysql_global_status_max_used_connections / mysql_global_variables_max_connections > 0.8
391         for: 1m
392         labels:
393           alertgroup: "{{ $labels.instance }}"
394         annotations:
395           connections_used: "{{ $value | humanizePercentage }}"
396   - name: network
397     rules:
398       - alert: interface transmit rate
399         expr: rate(node_network_transmit_bytes_total[1m]) / node_network_speed_bytes > 0.99
400         for: 5m
401         labels:
402           alertgroup: "{{ $labels.instance }}"
403         annotations:
404           bandwidth_used: "{{ $value | humanizePercentage }}"
405       - alert: interface receive rate
406         expr: rate(node_network_receive_bytes_total[1m]) / node_network_speed_bytes > 0.99
407         for: 5m
408         labels:
409           alertgroup: "{{ $labels.instance }}"
410         annotations:
411           bandwidth_used: "{{ $value | humanizePercentage }}"
412       - alert: interface transmit errors
413         expr: rate(node_network_transmit_errs_total{device!~"wg.*"}[1m]) / rate(node_network_transmit_packets_total{device!~"wg.*"}[1m]) > 0.01
414         for: 5m
415         labels:
416           alertgroup: "{{ $labels.instance }}"
417         annotations:
418           error_rate: "{{ $value | humanizePercentage }}"
419       - alert: wireguard interface transmit errors
420         expr: rate(node_network_transmit_errs_total{device=~"wg.*"}[1m]) / rate(node_network_transmit_packets_total{device=~"wg.*"}[1m]) > 0.05
421         for: 1h
422         labels:
423           alertgroup: "{{ $labels.instance }}"
424         annotations:
425           error_rate: "{{ $value | humanizePercentage }}"
426       - alert: interface receive errors
427         expr: rate(node_network_receive_errs_total[1m]) / rate(node_network_receive_packets_total[1m]) > 0.01
428         for: 5m
429         labels:
430           alertgroup: "{{ $labels.instance }}"
431         annotations:
432           error_rate: "{{ $value | humanizePercentage }}"
433       - alert: conntrack entries
434         expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
435         for: 5m
436         labels:
437           alertgroup: "{{ $labels.instance }}"
438         annotations:
439           entries_used: "{{ $value | humanizePercentage }}"
440   - name: nominatim
441     rules:
442       - alert: nominatim replication delay
443         expr: nominatim_replication_delay > 10800
444         for: 1h
445         labels:
446           alertgroup: nominatim
447         annotations:
448           delay: "{{ $value | humanizeDuration }}"
449   - name: overpass
450     rules:
451       - alert: overpass osm database age
452         expr: overpass_database_age_seconds{database="osm"} > 3600
453         for: 1h
454         labels:
455           alertgroup: overpass
456         annotations:
457           age: "{{ $value | humanizeDuration }}"
458       - alert: overpass area database age
459         expr: overpass_database_age_seconds{database="area"} > 86400
460         for: 1h
461         labels:
462           alertgroup: overpass
463         annotations:
464           age: "{{ $value | humanizeDuration }}"
465   - name: passenger
466     rules:
467       - alert: passenger down
468         expr: passenger_up == 0
469         for: 5m
470         labels:
471           alertgroup: "{{ $labels.instance }}"
472       - alert: passenger queuing
473         expr: passenger_top_level_request_queue > 0
474         for: 5m
475         labels:
476           alertgroup: "{{ $labels.instance }}"
477       - alert: passenger application queuing
478         expr: passenger_app_request_queue > 0
479         for: 5m
480         labels:
481           alertgroup: "{{ $labels.instance }}"
482   - name: planet
483     rules:
484       - alert: planet dump overdue
485         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/(pbf|planet)/.*"} > 7 * 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
486         for: 24h
487         labels:
488           alertgroup: planet
489         annotations:
490           overdue_by: "{{ $value | humanizeDuration }}"
491       - alert: notes dump overdue
492         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/notes/.*"} > 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
493         for: 6h
494         labels:
495           alertgroup: planet
496         annotations:
497           overdue_by: "{{ $value | humanizeDuration }}"
498       - alert: daily replication feed delayed
499         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/day/.*"} > 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
500         for: 3h
501         labels:
502           alertgroup: planet
503         annotations:
504           delayed_by: "{{ $value | humanizeDuration }}"
505       - alert: hourly replication feed delayed
506         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/hour/.*"} > 3600 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
507         for: 30m
508         labels:
509           alertgroup: planet
510         annotations:
511           delayed_by: "{{ $value | humanizeDuration }}"
512       - alert: minutely replication feed delayed
513         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/minute/.*"} > 60 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
514         for: 5m
515         labels:
516           alertgroup: planet
517         annotations:
518           delayed_by: "{{ $value | humanizeDuration }}"
519       - alert: changeset replication feed delayed
520         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/changesets/.*"} > 60 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
521         for: 5m
522         labels:
523           alertgroup: planet
524         annotations:
525           delayed_by: "{{ $value | humanizeDuration }}"
526   - name: postgresql
527     rules:
528       - alert: postgresql down
529         expr: pg_up == 0
530         for: 1m
531         labels:
532           alertgroup: "{{ $labels.instance }}"
533       - alert: postgresql replication delay
534         expr: pg_replication_lag_seconds > 30
535         for: 15m
536         labels:
537           alertgroup: "{{ $labels.instance }}"
538         annotations:
539           delay: "{{ $value | humanizeDuration }}"
540       - alert: postgresql connection limit
541         expr: sum (pg_stat_activity_count) by (instance, server) / sum (pg_settings_max_connections) by (instance, server) > 0.8
542         for: 1m
543         labels:
544           alertgroup: "{{ $labels.instance }}"
545         annotations:
546           connections_used: "{{ $value | humanizePercentage }}"
547       - alert: postgresql deadlocks
548         expr: increase(pg_stat_database_deadlocks{datname!="nominatim"}[1m]) > 5
549         for: 0m
550         labels:
551           alertgroup: "{{ $labels.instance }}"
552         annotations:
553           new_deadlocks: "{{ $value }}"
554       - alert: postgresql slow queries
555         expr: pg_slow_queries > 0
556         for: 5m
557         labels:
558           alertgroup: "{{ $labels.instance }}"
559         annotations:
560           queries: "{{ $value }}"
561       - alert: postgresql idle transactions
562         expr: sum(pg_process_idle_seconds_count{state="idle in transaction"}) by (instance, server) > sum(pg_process_idle_seconds_bucket{state="idle in transaction",le="300"}) by (instance, server)
563         for: 5m
564         labels:
565           alertgroup: "{{ $labels.instance }}"
566         annotations:
567           queries: "{{ $value }}"
568   - name: prometheus
569     rules:
570       - alert: prometheus configuration error
571         expr: prometheus_config_last_reload_successful == 0
572         for: 10m
573         labels:
574           alertgroup: "prometheus"
575       - alert: prometheus target missing
576         expr: up == 0
577         for: 10m
578         labels:
579           alertgroup: "prometheus"
580   - name: raid
581     rules:
582       - alert: raid controller battery failed
583         expr: ohai_controller_info{battery_status="failed"} > 0
584         for: 5m
585         labels:
586           alertgroup: "{{ $labels.instance }}"
587       - alert: raid controller battery recharging
588         expr: ohai_controller_info{battery_status="recharging"} > 0
589         for: 4h
590         labels:
591           alertgroup: "{{ $labels.instance }}"
592       - alert: raid array degraded
593         expr: ohai_array_info{status="degraded"} > 0
594         for: 5m
595         labels:
596           alertgroup: "{{ $labels.instance }}"
597       - alert: raid disk failed
598         expr: ohai_disk_info{status="failed"} > 0
599         for: 5m
600         labels:
601           alertgroup: "{{ $labels.instance }}"
602   - name: rasdaemon
603     rules:
604       - alert: memory controller errors
605         expr: increase(rasdaemon_mc_events_total[1m]) > 0
606         for: 0m
607         labels:
608           alertgroup: "{{ $labels.instance }}"
609         annotations:
610           new_errors: "{{ $value }}"
611       - alert: pcie aer errors
612         expr: increase(rasdaemon_aer_events_total[1m]) > 0
613         for: 0m
614         labels:
615           alertgroup: "{{ $labels.instance }}"
616         annotations:
617           new_errors: "{{ $value }}"
618   - name: smart
619     rules:
620       - alert: smart failure
621         expr: smart_health_status == 0
622         for: 60m
623         labels:
624           alertgroup: "{{ $labels.instance }}"
625       - alert: smart ssd wearout approaching
626         expr: smart_percentage_used / 100 >= 0.8
627         for: 60m
628         labels:
629           alertgroup: "{{ $labels.instance }}"
630         annotations:
631           percentage_used: "{{ $value | humanizePercentage }}"
632   - name: smokeping
633     rules:
634       - alert: packet loss
635         expr: 1 - (rate(smokeping_response_duration_seconds_count[5m]) / rate(smokeping_requests_total[5m])) > 0.02
636         for: 10m
637         labels:
638           alertgroup: smokeping
639         annotations:
640           loss_rate: "{{ $value | humanizePercentage }}"
641   - name: snmp
642     rules:
643       - alert: snmp pdus missing
644         expr: max_over_time(snmp_scrape_pdus_returned[1d]) - snmp_scrape_pdus_returned > 0
645         for: 15m
646         labels:
647           alertgroup: snmp
648         annotations:
649           missing_pdus: "{{ $value }}"
650   - name: ssl
651     rules:
652       - alert: ssl certificate probe failed
653         expr: ssl_probe_success == 0
654         for: 60m
655         labels:
656           alertgroup: ssl
657       - alert: ssl certificate expiry
658         expr: ssl_verified_cert_not_after{chain_no="0"} - time() < 86400 * 14
659         for: 0m
660         labels:
661           alertgroup: ssl
662         annotations:
663           expires_in: "{{ $value | humanizeDuration }}"
664       - alert: ssl certificate revoked
665         expr: ssl_ocsp_response_status == 1
666         for: 0m
667         labels:
668           alertgroup: ssl
669       - alert: ocsp status unknown
670         expr: ssl_ocsp_response_status == 1
671         for: 0m
672         labels:
673           alertgroup: ssl
674   - name: statuscake
675     rules:
676       - alert: statuscake uptime check failing
677         expr: statuscake_paused == 0 and statuscake_up == 0
678         for: 10m
679         labels:
680           alertgroup: statuscake
681   - name: systemd
682     rules:
683       - alert: systemd failed service
684         expr: node_systemd_unit_state{state="failed",name!="chef-client.service"} == 1
685         for: 5m
686         labels:
687           alertgroup: "{{ $labels.instance }}"
688       - alert: systemd failed chef client service
689         expr: sum_over_time(node_systemd_unit_state{state="inactive",name="chef-client.service"}[6h]) == 0
690         for: 0m
691         labels:
692           alertgroup: "{{ $labels.instance }}"
693   - name: taginfo
694     rules:
695       - alert: taginfo planet age
696         expr: time() - taginfo_data_from_seconds > 129600 and on (instance) chef_role{name="taginfo"}
697         for: 0m
698         labels:
699           alertgroup: taginfo
700         annotations:
701           age: "{{ $value | humanizeDuration }}"
702       - alert: taginfo database age
703         expr: time() - taginfo_database_update_finish_seconds > 129600 and on (instance) chef_role{name="taginfo"}
704         for: 0m
705         labels:
706           alertgroup: taginfo
707         annotations:
708           age: "{{ $value | humanizeDuration }}"
709       - alert: taginfo database size
710         expr: abs(delta(taginfo_database_size_bytes[30m])) / taginfo_database_size_bytes > 0.1
711         for: 30m
712         labels:
713           alertgroup: taginfo
714         annotations:
715           size_change: "{{ $value | humanizePercentage }}"
716   - name: tile
717     rules:
718       - alert: renderd replication delay
719         expr: renderd_replication_delay > 120
720         for: 15m
721         labels:
722           alertgroup: tile
723         annotations:
724           delay: "{{ $value | humanizeDuration }}"
725       - alert: missed tile rate
726         expr: sum(rate(modtile_http_response_total{code="404"}[5m])) by (instance) / sum(rate(modtile_http_response_total[5m])) by (instance) > 0.05
727         for: 5m
728         labels:
729           alertgroup: tile
730         annotations:
731           miss_rate: "{{ $value | humanizePercentage }}"
732       - alert: tile render rate
733         expr: sum(rate(renderd_zoom_metatiles_total[5m])) by (instance) == 0
734         for: 15m
735         labels:
736           alertgroup: tile
737         annotations:
738           render_rate: "{{ $value }} tiles/s"
739   - name: time
740     rules:
741       - alert: clock not synchronising
742         expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16
743         for: 5m
744         labels:
745           alertgroup: "{{ $labels.instance }}"
746       - alert: clock skew detected
747         expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)
748         for: 5m
749         labels:
750           alertgroup: "{{ $labels.instance }}"
751         annotations:
752           skew: "{{ with printf \"node_timex_offset_seconds{instance='%s'}\" $labels.instance | query }} {{ . | humanizeDuration }}{{ end }}"
753   - name: web
754     rules:
755       - alert: web error rate
756         expr: sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) / sum(rate(api_call_count_total[5m])) by (instance) > 0.002
757         for: 5m
758         labels:
759           alertgroup: web
760         annotations:
761           error_rate: "{{ $value | humanizePercentage }}"
762       - alert: job processing rate
763         expr: rate(pg_stat_user_tables_n_tup_del{datname="openstreetmap",relname="delayed_jobs"}[1h]) / rate(pg_stat_user_tables_n_tup_ins{datname="openstreetmap",relname="delayed_jobs"}[1h]) < 0.9 and ignoring(job, name, datname, relname, schemaname, server) chef_role{name="db-master"} == 1
764         for: 1h
765         labels:
766           alertgroup: web
767         annotations:
768           job_processing_rate: "{{ $value | humanizePercentage }}"