]> git.openstreetmap.org Git - chef.git/blob - cookbooks/prometheus/templates/default/alert_rules.yml.erb
Add an alert for chef not running for an extended time
[chef.git] / cookbooks / prometheus / templates / default / alert_rules.yml.erb
1 # DO NOT EDIT - This file is being maintained by Chef
2
3 groups:
4   - name: alertmanager
5     rules:
6       - alert: prometheus target missing
7         expr: up == 0
8         for: 5m
9         labels:
10           alertgroup: "prometheus"
11   - name: apache
12     rules:
13       - alert: apache down
14         expr: apache_up == 0
15         for: 5m
16         labels:
17           alertgroup: "{{ $labels.instance }}"
18       - alert: apache workers busy
19         expr: sum(apache_workers{state="busy"}) by (instance) / sum(apache_scoreboard) by (instance) > 0.8
20         for: 5m
21         labels:
22           alertgroup: "{{ $labels.instance }}"
23         annotations:
24           busy_workers: "{{ $value | humanizePercentage }}"
25   - name: chef
26     rules:
27       - alert: chef client not running
28         expr: time() - node_systemd_timer_last_trigger_seconds{name="chef-client.timer"} > 3600
29         for: 12h
30         labels:
31           alertgroup: "{{ $labels.instance }}"
32         annotations:
33           down_time: "{{ $value | humanizeDuration }}"
34   - name: cpu
35     rules:
36       - alert: cpu pressure
37         expr: rate(node_pressure_cpu_waiting_seconds_total[5m]) > 0.3
38         for: 15m
39         labels:
40           alertgroup: "{{ $labels.instance }}"
41         annotations:
42           pressure: "{{ $value | humanizePercentage }}"
43   - name: database
44     rules:
45       - alert: postgres replication delay
46         expr: pg_replication_lag_seconds > 5
47         for: 5m
48         labels:
49           alertgroup: database
50         annotations:
51           delay: "{{ $value | humanizeDuration }}"
52   - name: fastly
53     rules:
54       - alert: error rate
55         expr: sum(rate(fastly_rt_status_group_total{status_group="5xx"}[5m])) by (service_name, datacenter) / sum(rate(fastly_rt_status_group_total[5m])) by (service_name, datacenter) > 0.005
56         for: 15m
57         labels:
58           alertgroup: fastly
59         annotations:
60           error_rate: "{{ $value | humanizePercentage }}"
61   - name: filesystem
62     rules:
63       - alert: readonly filesystem
64         expr: node_filesystem_readonly == 1
65         for: 0m
66         labels:
67           alertgroup: "{{ $labels.instance }}"
68       - alert: filesystem low on space
69         expr: node_filesystem_avail_bytes / node_filesystem_size_bytes < 0.05
70         for: 5m
71         labels:
72           alertgroup: "{{ $labels.instance }}"
73         annotations:
74           percentage_free: "{{ $value | humanizePercentage }}"
75           free_bytes: "{{ with printf \"node_filesystem_avail_bytes{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value | humanize1024 }}bytes{{end}}"
76           total_total: "{{ with printf \"node_filesystem_size_bytes{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value | humanize1024 }}bytes{{end}}"
77       - alert: filesystem low on inodes
78         expr: node_filesystem_files_free / node_filesystem_files < 0.1
79         for: 5m
80         labels:
81           alertgroup: "{{ $labels.instance }}"
82         annotations:
83           percentage_free: "{{ $value | humanizePercentage }}"
84           free_inodes: "{{ with printf \"node_filesystem_files_free{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value }}{{end}}"
85           total_inodes: "{{ with printf \"node_filesystem_files{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value }}{{end}}"
86   - name: hwmon
87     rules:
88       - alert: hwmon fan alarm
89         expr: node_hwmon_fan_alarm == 1
90         for: 5m
91         labels:
92           alertgroup: "{{ $labels.instance }}"
93         annotations:
94           fan_rpm: "{{ with printf \"node_hwmon_fan_rpm{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}rpm{{end}}"
95           fan_min_rpm: "{{ with printf \"node_hwmon_fan_min_rpm{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}rpm{{end}}"
96       - alert: hwmon temperature alarm
97         expr: node_hwmon_temp_alarm == 1
98         for: 5m
99         labels:
100           alertgroup: "{{ $labels.instance }}"
101         annotations:
102           temp_celsius: "{{ with printf \"node_hwmon_temp_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
103           temp_max_celsius: "{{ with printf \"node_hwmon_temp_max_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
104           temp_crit_celsius: "{{ with printf \"node_hwmon_temp_crit_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
105       - alert: hwmon voltage alarm
106         expr: node_hwmon_in_alarm == 1
107         for: 5m
108         labels:
109           alertgroup: "{{ $labels.instance }}"
110         annotations:
111           in_volts: "{{ with printf \"node_hwmon_in_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
112           in_min_volts: "{{ with printf \"node_hwmon_in_min_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
113           in_max_volts: "{{ with printf \"node_hwmon_in_max_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
114   - name: io
115     rules:
116       - alert: io pressure
117         expr: rate(node_pressure_io_waiting_seconds_total[5m]) > 0.6
118         for: 60m
119         labels:
120           alertgroup: "{{ $labels.instance }}"
121         annotations:
122           pressure: "{{ $value | humanizePercentage }}"
123   - name: ipmi
124     rules:
125       - alert: ipmi fan alarm
126         expr: ipmi_fan_speed_state > 0
127         for: 5m
128         labels:
129           alertgroup: "{{ $labels.instance }}"
130         annotations:
131           fan_speed_rpm: "{{ with printf \"ipmi_fan_speed_rpm{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}rpm{{end}}"
132       - alert: ipmi temperature alarm
133         expr: ipmi_temperature_state > 0
134         for: 5m
135         labels:
136           alertgroup: "{{ $labels.instance }}"
137         annotations:
138           temperature_celsius: "{{ with printf \"ipmi_temperature_celsius{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}C{{end}}"
139       - alert: ipmi voltage alarm
140         expr: ipmi_voltage_state > 0
141         for: 5m
142         labels:
143           alertgroup: "{{ $labels.instance }}"
144         annotations:
145           voltage_volts: "{{ with printf \"ipmi_voltage_volts{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}V{{end}}"
146       - alert: ipmi power alarm
147         expr: ipmi_power_state > 0 or ipmi_sensor_state{type=~"Power .*"} > 0
148         for: 5m
149         labels:
150           alertgroup: "{{ $labels.instance }}"
151   - name: mail
152     rules:
153       - alert: exim queue length
154         expr: exim_queue > exim_queue_limit
155         for: 60m
156         labels:
157           alertgroup: mail
158         annotations:
159           queue_length: "{{ $value }}"
160       - alert: mailman queue length
161         expr: mailman_queue_length > 200
162         for: 60m
163         labels:
164           alertgroup: mail
165         annotations:
166           queue_length: "{{ $value }}"
167   - name: mdadm
168     rules:
169       - alert: mdadm array inactive
170         expr: node_md_state{state="inactive"} > 0
171         for: 0m
172         labels:
173           alertgroup: "{{ $labels.instance }}"
174         annotations:
175           required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
176           active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
177           failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
178           spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
179       - alert: mdadm array degraded
180         expr: sum (node_md_disks{state="active"}) without (state) < node_md_disks_required
181         for: 0m
182         labels:
183           alertgroup: "{{ $labels.instance }}"
184         annotations:
185           required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
186           active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
187           failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
188           spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
189       - alert: mdadm disk failed
190         expr: node_md_disks{state="failed"} > 0
191         for: 0m
192         labels:
193           alertgroup: "{{ $labels.instance }}"
194         annotations:
195           required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
196           active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
197           failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
198           spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
199   - name: memory
200     rules:
201       - alert: low memory
202         expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1
203         for: 15m
204         labels:
205           alertgroup: "{{ $labels.instance }}"
206         annotations:
207           memory_free: "{{ $value | humanizePercentage }}"
208       - alert: memory pressure
209         expr: rate(node_pressure_memory_waiting_seconds_total[5m]) > 0.6
210         for: 60m
211         labels:
212           alertgroup: "{{ $labels.instance }}"
213         annotations:
214           pressure: "{{ $value | humanizePercentage }}"
215       - alert: oom kill detected
216         expr: increase(node_vmstat_oom_kill[1m]) > 0
217         for: 0m
218         labels:
219           alertgroup: "{{ $labels.instance }}"
220         annotations:
221           new_oom_kills: "{{ $value }}"
222   - name: network
223     rules:
224       - alert: interface transmit rate
225         expr: rate(node_network_transmit_bytes_total[1m]) / node_network_speed_bytes > 0.98
226         for: 5m
227         labels:
228           alertgroup: "{{ $labels.instance }}"
229         annotations:
230           bandwidth_used: "{{ $value | humanizePercentage }}"
231       - alert: interface receive rate
232         expr: rate(node_network_receive_bytes_total[1m]) / node_network_speed_bytes > 0.98
233         for: 5m
234         labels:
235           alertgroup: "{{ $labels.instance }}"
236         annotations:
237           bandwidth_used: "{{ $value | humanizePercentage }}"
238       - alert: interface transmit errors
239         expr: rate(node_network_transmit_errs_total[1m]) / rate(node_network_transmit_packets_total[1m]) > 0.01
240         for: 5m
241         labels:
242           alertgroup: "{{ $labels.instance }}"
243         annotations:
244           error_rate: "{{ $value | humanizePercentage }}"
245       - alert: interface receive errors
246         expr: rate(node_network_receive_errs_total[1m]) / rate(node_network_receive_packets_total[1m]) > 0.01
247         for: 5m
248         labels:
249           alertgroup: "{{ $labels.instance }}"
250         annotations:
251           error_rate: "{{ $value | humanizePercentage }}"
252       - alert: conntrack entries
253         expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
254         for: 5m
255         labels:
256           alertgroup: "{{ $labels.instance }}"
257         annotations:
258           entries_used: "{{ $value | humanizePercentage }}"
259   - name: planet
260     rules:
261       - alert: planet dump overdue
262         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/(pbf|planet)/.*"} > 7 * 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
263         for: 24h
264         labels:
265           alertgroup: planet
266         annotations:
267           overdue_by: "{{ $value | humanizeDuration }}"
268       - alert: notes dump overdue
269         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/notes/.*"} > 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
270         for: 6h
271         labels:
272           alertgroup: planet
273         annotations:
274           overdue_by: "{{ $value | humanizeDuration }}"
275       - alert: daily replication feed delayed
276         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/day/.*"} > 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
277         for: 3h
278         labels:
279           alertgroup: planet
280         annotations:
281           delayed_by: "{{ $value | humanizeDuration }}"
282       - alert: hourly replication feed delayed
283         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/hour/.*"} > 3600 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
284         for: 30m
285         labels:
286           alertgroup: planet
287         annotations:
288           delayed_by: "{{ $value | humanizeDuration }}"
289       - alert: minutely replication feed delayed
290         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/minute/.*"} > 60 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
291         for: 5m
292         labels:
293           alertgroup: planet
294         annotations:
295           delayed_by: "{{ $value | humanizeDuration }}"
296       - alert: changeset replication feed delayed
297         expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/changesets/.*"} > 60 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
298         for: 5m
299         labels:
300           alertgroup: planet
301         annotations:
302           delayed_by: "{{ $value | humanizeDuration }}"
303   - name: postgresql
304     rules:
305       - alert: postgresql down
306         expr: pg_up == 0
307         for: 1m
308         labels:
309           alertgroup: "{{ $labels.instance }}"
310       - alert: postgresql replication delay
311         expr: pg_replication_lag_seconds > 5
312         for: 1m
313         labels:
314           alertgroup: "{{ $labels.instance }}"
315         annotations:
316           delay: "{{ $value | humanizeDuration }}"
317       - alert: postgresql connection limit
318         expr: sum (pg_stat_activity_count) by (instance, server) / sum (pg_settings_max_connections) by (instance, server) > 0.8
319         for: 1m
320         labels:
321           alertgroup: "{{ $labels.instance }}"
322         annotations:
323           connections_used: "{{ $value | humanizePercentage }}"
324       - alert: postgresql deadlocks
325         expr: increase(pg_stat_database_deadlocks[1m]) > 5
326         for: 0m
327         labels:
328           alertgroup: "{{ $labels.instance }}"
329         annotations:
330           new_deadlocks: "{{ $value }}"
331       - alert: postgresql slow queries
332         expr: pg_slow_queries > 0
333         for: 5m
334         labels:
335           alertgroup: "{{ $labels.instance }}"
336         annotations:
337           queries: "{{ $value }}"
338   - name: smart
339     rules:
340       - alert: smart failure
341         expr: smart_health_status == 0
342         for: 60m
343         labels:
344           alertgroup: "{{ $labels.instance }}"
345       - alert: smart ssd wearout approaching
346         expr: smart_percentage_used >= 90
347         for: 60m
348         labels:
349           alertgroup: "{{ $labels.instance }}"
350         annotations:
351           percentage_used: "{{ $value | humanizePercentage }}"
352   - name: ssl
353     rules:
354       - alert: ssl certificate probe failed
355         expr: ssl_probe_success == 0
356         for: 60m
357         labels:
358           alertgroup: ssl
359       - alert: ssl certificate expiry
360         expr: ssl_verified_cert_not_after{chain_no="0"} - time() < 86400 * 14
361         for: 0m
362         labels:
363           alertgroup: ssl
364         annotations:
365           expires_in: "{{ $value | humanizeDuration }}"
366       - alert: ssl certificate revoked
367         expr: ssl_ocsp_response_status == 1
368         for: 0m
369         labels:
370           alertgroup: ssl
371       - alert: ocsp status unknown
372         expr: ssl_ocsp_response_status == 1
373         for: 0m
374         labels:
375           alertgroup: ssl
376   - name: systemd
377     rules:
378       - alert: systemd failed service
379         expr: node_systemd_unit_state{state="failed",name!="chef-client.service"} == 1
380         for: 5m
381         labels:
382           alertgroup: "{{ $labels.instance }}"
383       - alert: systemd failed service
384         expr: node_systemd_unit_state{state="failed",name="chef-client.service"} == 1
385         for: 6h
386         labels:
387           alertgroup: "{{ $labels.instance }}"
388   - name: tile
389     rules:
390       - alert: renderd replication delay
391         expr: renderd_replication_delay > 120
392         for: 5m
393         labels:
394           alertgroup: tile
395         annotations:
396           delay: "{{ $value | humanizeDuration }}"
397       - alert: missed tile rate
398         expr: sum(rate(modtile_http_response_total{code="404"}[5m])) by (instance) / sum(rate(modtile_http_response_total[5m])) by (instance) > 0.05
399         for: 5m
400         labels:
401           alertgroup: tile
402         annotations:
403           miss_rate: "{{ $value | humanizePercentage }}"
404   - name: time
405     rules:
406       - alert: clock not synchronising
407         expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16
408         for: 5m
409         labels:
410           alertgroup: "{{ $labels.instance }}"
411       - alert: clock skew detected
412         expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)
413         for: 5m
414         labels:
415           alertgroup: "{{ $labels.instance }}"
416         annotations:
417           skew: "{{ with printf \"node_timex_offset_seconds{instance='%s'}\" $labels.instance | query }} {{ . | humanizeDuration }}{{ end }}"
418   - name: web
419     rules:
420       - alert: web error rate
421         expr: sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) / sum(rate(api_call_count_total[5m])) by (instance) > 0.002
422         for: 5m
423         labels:
424           alertgroup: web
425         annotations:
426           error_rate: "{{ $value | humanizePercentage }}"
427       - alert: job processing rate
428         expr: rate(pg_stat_user_tables_n_tup_del{datname="openstreetmap",relname="delayed_jobs"}[5m]) / rate(pg_stat_user_tables_n_tup_ins{datname="openstreetmap",relname="delayed_jobs"}[5m]) < 0.9 and ignoring(job, name, datname, relname, schemaname, server) chef_role{name="db-master"} == 1
429         for: 5m
430         labels:
431           alertgroup: web
432         annotations:
433           job_processing_rate: "{{ $value | humanizePercentage }}"