]> git.openstreetmap.org Git - chef.git/blob - cookbooks/prometheus/templates/default/alert_rules.yml.erb
Correct annotation for pressure alerts
[chef.git] / cookbooks / prometheus / templates / default / alert_rules.yml.erb
1 # DO NOT EDIT - This file is being maintained by Chef
2
3 groups:
4   - name: alertmanager
5     rules:
6       - alert: prometheus target missing
7         expr: up == 0
8         for: 5m
9         labels:
10           alertgroup: "prometheus"
11   - name: apache
12     rules:
13       - alert: apache down
14         expr: apache_up == 0
15         for: 5m
16         labels:
17           alertgroup: "{{ $labels.instance }}"
18       - alert: apache workers busy
19         expr: sum(apache_workers{state="busy"}) by (instance) / sum(apache_scoreboard) by (instance) > 0.8
20         for: 5m
21         labels:
22           alertgroup: "{{ $labels.instance }}"
23         annotations:
24           busy_workers: "{{ $value | humanizePercentage }}"
25   - name: cpu
26     rules:
27       - alert: cpu pressure
28         expr: rate(node_pressure_memory_waiting_seconds_total[5m]) > 0.3
29         for: 15m
30         labels:
31           alertgroup: "{{ $labels.instance }}"
32         annotations:
33           pressure: "{{ $value | humanizePercentage }}"
34   - name: database
35     rules:
36       - alert: postgres replication delay
37         expr: pg_replication_lag_seconds > 5
38         for: 5m
39         labels:
40           alertgroup: database
41         annotations:
42           delay: "{{ $value | humanizeDuration }}"
43   - name: filesystem
44     rules:
45       - alert: readonly filesystem
46         expr: node_filesystem_readonly == 1
47         for: 0m
48         labels:
49           alertgroup: "{{ $labels.instance }}"
50       - alert: filesystem low on space
51         expr: node_filesystem_avail_bytes / node_filesystem_size_bytes < 0.05
52         for: 5m
53         labels:
54           alertgroup: "{{ $labels.instance }}"
55         annotations:
56           percentage_free: "{{ $value | humanizePercentage }}"
57           free_bytes: "{{ with printf \"node_filesystem_avail_bytes{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value | humanize1024 }}bytes{{end}}"
58           total_total: "{{ with printf \"node_filesystem_size_bytes{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value | humanize1024 }}bytes{{end}}"
59       - alert: filesystem low on inodes
60         expr: node_filesystem_files_free / node_filesystem_files < 0.1
61         for: 5m
62         labels:
63           alertgroup: "{{ $labels.instance }}"
64         annotations:
65           percentage_free: "{{ $value | humanizePercentage }}"
66           free_inodes: "{{ with printf \"node_filesystem_files_free{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value }}{{end}}"
67           total_inodes: "{{ with printf \"node_filesystem_files{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value }}{{end}}"
68   - name: hwmon
69     rules:
70       - alert: hwmon fan alarm
71         expr: node_hwmon_fan_alarm == 1
72         for: 5m
73         labels:
74           alertgroup: "{{ $labels.instance }}"
75         annotations:
76           fan_rpm: "{{ with printf \"node_hwmon_fan_rpm{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}rpm{{end}}"
77           fan_min_rpm: "{{ with printf \"node_hwmon_fan_min_rpm{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}rpm{{end}}"
78       - alert: hwmon temperature alarm
79         expr: node_hwmon_temp_alarm == 1
80         for: 5m
81         labels:
82           alertgroup: "{{ $labels.instance }}"
83         annotations:
84           temp_celsius: "{{ with printf \"node_hwmon_temp_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
85           temp_max_celsius: "{{ with printf \"node_hwmon_temp_max_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
86           temp_crit_celsius: "{{ with printf \"node_hwmon_temp_crit_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
87       - alert: hwmon voltage alarm
88         expr: node_hwmon_in_alarm == 1
89         for: 5m
90         labels:
91           alertgroup: "{{ $labels.instance }}"
92         annotations:
93           in_volts: "{{ with printf \"node_hwmon_in_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
94           in_min_volts: "{{ with printf \"node_hwmon_in_min_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
95           in_max_volts: "{{ with printf \"node_hwmon_in_max_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
96   - name: io
97     rules:
98       - alert: cpu pressure
99         expr: rate(node_pressure_io_waiting_seconds_total[5m]) > 0.6
100         for: 60m
101         labels:
102           alertgroup: "{{ $labels.instance }}"
103         annotations:
104           pressure: "{{ $value | humanizePercentage }}"
105   - name: ipmi
106     rules:
107       - alert: ipmi fan alarm
108         expr: ipmi_fan_speed_state > 0
109         for: 5m
110         labels:
111           alertgroup: "{{ $labels.instance }}"
112         annotations:
113           fan_speed_rpm: "{{ with printf \"ipmi_fan_speed_rpm{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}rpm{{end}}"
114       - alert: ipmi temperature alarm
115         expr: ipmi_temperature_state > 0
116         for: 5m
117         labels:
118           alertgroup: "{{ $labels.instance }}"
119         annotations:
120           temperature_celsius: "{{ with printf \"ipmi_temperature_celsius{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}C{{end}}"
121       - alert: ipmi voltage alarm
122         expr: ipmi_voltage_state > 0
123         for: 5m
124         labels:
125           alertgroup: "{{ $labels.instance }}"
126         annotations:
127           voltage_volts: "{{ with printf \"ipmi_voltage_volts{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}V{{end}}"
128       - alert: ipmi power alarm
129         expr: ipmi_power_state > 0 or ipmi_sensor_state{type=~"Power .*"} > 0
130         for: 5m
131         labels:
132           alertgroup: "{{ $labels.instance }}"
133   - name: mdadm
134     rules:
135       - alert: mdadm array inactive
136         expr: node_md_state{state="inactive"} > 0
137         for: 0m
138         labels:
139           alertgroup: "{{ $labels.instance }}"
140         annotations:
141           required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
142           active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
143           failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
144           spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
145       - alert: mdadm array degraded
146         expr: sum (node_md_disks{state="active"}) without (state) < node_md_disks_required
147         for: 0m
148         labels:
149           alertgroup: "{{ $labels.instance }}"
150         annotations:
151           required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
152           active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
153           failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
154           spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
155       - alert: mdadm disk failed
156         expr: node_md_disks{state="failed"} > 0
157         for: 0m
158         labels:
159           alertgroup: "{{ $labels.instance }}"
160         annotations:
161           required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
162           active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
163           failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
164           spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
165   - name: memory
166     rules:
167       - alert: low memory
168         expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1
169         for: 15m
170         labels:
171           alertgroup: "{{ $labels.instance }}"
172         annotations:
173           memory_free: "{{ $value | humanizePercentage }}"
174       - alert: memory pressure
175         expr: rate(node_pressure_memory_waiting_seconds_total[5m]) > 0.6
176         for: 60m
177         labels:
178           alertgroup: "{{ $labels.instance }}"
179         annotations:
180           pressure: "{{ $value | humanizePercentage }}"
181       - alert: oom kill detected
182         expr: increase(node_vmstat_oom_kill[1m]) > 0
183         for: 0m
184         labels:
185           alertgroup: "{{ $labels.instance }}"
186         annotations:
187           new_oom_kills: "{{ $value }}"
188   - name: network
189     rules:
190       - alert: interface transmit rate
191         expr: rate(node_network_transmit_bytes_total[1m]) / node_network_speed_bytes > 0.98
192         for: 5m
193         labels:
194           alertgroup: "{{ $labels.instance }}"
195         annotations:
196           bandwidth_used: "{{ $value | humanizePercentage }}"
197       - alert: interface receive rate
198         expr: rate(node_network_receive_bytes_total[1m]) / node_network_speed_bytes > 0.98
199         for: 5m
200         labels:
201           alertgroup: "{{ $labels.instance }}"
202         annotations:
203           bandwidth_used: "{{ $value | humanizePercentage }}"
204       - alert: interface transmit errors
205         expr: rate(node_network_transmit_errs_total[1m]) / rate(node_network_transmit_packets_total[1m]) > 0.01
206         for: 5m
207         labels:
208           alertgroup: "{{ $labels.instance }}"
209         annotations:
210           error_rate: "{{ $value | humanizePercentage }}"
211       - alert: interface receive errors
212         expr: rate(node_network_receive_errs_total[1m]) / rate(node_network_receive_packets_total[1m]) > 0.01
213         for: 5m
214         labels:
215           alertgroup: "{{ $labels.instance }}"
216         annotations:
217           error_rate: "{{ $value | humanizePercentage }}"
218       - alert: conntrack entries
219         expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
220         for: 5m
221         labels:
222           alertgroup: "{{ $labels.instance }}"
223         annotations:
224           entries_used: "{{ $value | humanizePercentage }}"
225   - name: planet
226     rules:
227       - alert: planet dump overdue
228         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/(pbf|planet)/.*"} > 7 * 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
229         for: 24h
230         labels:
231           alertgroup: planet
232         annotations:
233           overdue_by: "{{ $value | humanizeDuration }}"
234       - alert: notes dump overdue
235         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/notes/.*"} > 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
236         for: 6h
237         labels:
238           alertgroup: planet
239         annotations:
240           overdue_by: "{{ $value | humanizeDuration }}"
241       - alert: daily replication feed delayed
242         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/day/.*"} > 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
243         for: 3h
244         labels:
245           alertgroup: planet
246         annotations:
247           delayed_by: "{{ $value | humanizeDuration }}"
248       - alert: hourly replication feed delayed
249         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/hour/.*"} > 3600 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
250         for: 30m
251         labels:
252           alertgroup: planet
253         annotations:
254           delayed_by: "{{ $value | humanizeDuration }}"
255       - alert: minutely replication feed delayed
256         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/minute/.*"} > 60 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
257         for: 5m
258         labels:
259           alertgroup: planet
260         annotations:
261           delayed_by: "{{ $value | humanizeDuration }}"
262       - alert: changeset replication feed delayed
263         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/changesets/.*"} > 60 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
264         for: 5m
265         labels:
266           alertgroup: planet
267         annotations:
268           delayed_by: "{{ $value | humanizeDuration }}"
269   - name: postgresql
270     rules:
271       - alert: postgresql down
272         expr: pg_up == 0
273         for: 1m
274         labels:
275           alertgroup: "{{ $labels.instance }}"
276       - alert: postgresql replication delay
277         expr: pg_replication_lag_seconds > 5
278         for: 1m
279         labels:
280           alertgroup: "{{ $labels.instance }}"
281         annotations:
282           delay: "{{ $value | humanizeDuration }}"
283       - alert: postgresql connection limit
284         expr: sum (pg_stat_activity_count) by (instance, server) / sum (pg_settings_max_connections) by (instance, server) > 0.8
285         for: 1m
286         labels:
287           alertgroup: "{{ $labels.instance }}"
288         annotations:
289           connections_used: "{{ $value | humanizePercentage }}"
290       - alert: postgresql deadlocks
291         expr: increase(pg_stat_database_deadlocks[1m]) > 5
292         for: 0m
293         labels:
294           alertgroup: "{{ $labels.instance }}"
295         annotations:
296           new_deadlocks: "{{ $value }}"
297       - alert: postgresql slow queries
298         expr: pg_slow_queries > 0
299         for: 5m
300         labels:
301           alertgroup: "{{ $labels.instance }}"
302         annotations:
303           queries: "{{ $value }}"
304   - name: smart
305     rules:
306       - alert: smart failure
307         expr: smart_health_status == 0
308         for: 60m
309         labels:
310           alertgroup: "{{ $labels.instance }}"
311       - alert: smart ssd wearout approaching
312         expr: smart_percentage_used >= 90
313         for: 60m
314         labels:
315           alertgroup: "{{ $labels.instance }}"
316         annotations:
317           percentage_used: "{{ $value | humanizePercentage }}"
318   - name: ssl
319     rules:
320       - alert: ssl certificate probe failed
321         expr: ssl_probe_success == 0
322         for: 60m
323         labels:
324           alertgroup: ssl
325       - alert: ssl certificate expiry
326         expr: ssl_verified_cert_not_after{chain_no="0"} - time() < 86400 * 14
327         for: 0m
328         labels:
329           alertgroup: ssl
330         annotations:
331           expires_in: "{{ $value | humanizeDuration }}"
332       - alert: ssl certificate revoked
333         expr: ssl_ocsp_response_status == 1
334         for: 0m
335         labels:
336           alertgroup: ssl
337       - alert: ocsp status unknown
338         expr: ssl_ocsp_response_status == 1
339         for: 0m
340         labels:
341           alertgroup: ssl
342   - name: systemd
343     rules:
344       - alert: systemd failed service
345         expr: node_systemd_unit_state{state="failed"} == 1
346         for: 5m
347         labels:
348           alertgroup: "{{ $labels.instance }}"
349   - name: tile
350     rules:
351       - alert: renderd replication delay
352         expr: renderd_replication_delay > 120
353         for: 5m
354         labels:
355           alertgroup: tile
356         annotations:
357           delay: "{{ $value | humanizeDuration }}"
358       - alert: missed tile rate
359         expr: sum(rate(modtile_http_response_total{code="404"}[5m])) by (instance) / sum(rate(modtile_http_response_total[5m])) by (instance) > 0.05
360         for: 5m
361         labels:
362           alertgroup: tile
363         annotations:
364           miss_rate: "{{ $value | humanizePercentage }}"
365   - name: time
366     rules:
367       - alert: clock not synchronising
368         expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16
369         for: 5m
370         labels:
371           alertgroup: "{{ $labels.instance }}"
372       - alert: clock skew detected
373         expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)
374         for: 5m
375         labels:
376           alertgroup: "{{ $labels.instance }}"
377         annotations:
378           skew: "{{ with printf \"node_timex_offset_seconds{instance='%s'}\" $labels.instance | query }} {{ . | humanizeDuration }}{{ end }}"
379   - name: web
380     rules:
381       - alert: web error rate
382         expr: sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) / sum(rate(api_call_count_total[5m])) by (instance) > 0.002
383         for: 5m
384         labels:
385           alertgroup: web
386         annotations:
387           error_rate: "{{ $value | humanizePercentage }}"