1 # DO NOT EDIT - This file is being maintained by Chef
6 - alert: prometheus target missing
10 alertgroup: "prometheus"
17 alertgroup: "{{ $labels.instance }}"
18 - alert: apache workers busy
19 expr: sum(apache_workers{state="busy"}) by (instance) / sum(apache_scoreboard) by (instance) > 0.8
22 alertgroup: "{{ $labels.instance }}"
24 busy_workers: "{{ $value | humanizePercentage }}"
27 - alert: postgres replication delay
28 expr: pg_replication_lag_seconds > 5
33 delay: "{{ $value | humanizeDuration }}"
36 - alert: hwmon fan alarm
37 expr: node_hwmon_fan_alarm == 1
40 alertgroup: "{{ $labels.instance }}"
42 fan_rpm: "{{ with printf \"node_hwmon_fan_rpm{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}rpm{{end}}"
43 fan_min_rpm: "{{ with printf \"node_hwmon_fan_min_rpm{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}rpm{{end}}"
44 - alert: hwmon temperature alarm
45 expr: node_hwmon_temp_alarm == 1
48 alertgroup: "{{ $labels.instance }}"
50 temp_celsius: "{{ with printf \"node_hwmon_temp_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
51 temp_max_celsius: "{{ with printf \"node_hwmon_temp_max_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
52 temp_crit_celsius: "{{ with printf \"node_hwmon_temp_crit_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
53 - alert: hwmon voltage alarm
54 expr: node_hwmon_in_alarm == 1
57 alertgroup: "{{ $labels.instance }}"
59 in_volts: "{{ with printf \"node_hwmon_in_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
60 in_min_volts: "{{ with printf \"node_hwmon_in_min_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
61 in_max_volts: "{{ with printf \"node_hwmon_in_max_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
64 - alert: ipmi fan alarm
65 expr: ipmi_fan_speed_state > 0
68 alertgroup: "{{ $labels.instance }}"
70 fan_speed_rpm: "{{ with printf \"ipmi_fan_speed_rpm{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}rpm{{end}}"
71 - alert: ipmi temperature alarm
72 expr: ipmi_temperature_state > 0
75 alertgroup: "{{ $labels.instance }}"
77 temperature_celsius: "{{ with printf \"ipmi_temperature_celsius{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}C{{end}}"
78 - alert: ipmi voltage alarm
79 expr: ipmi_voltage_state > 0
82 alertgroup: "{{ $labels.instance }}"
84 voltage_volts: "{{ with printf \"ipmi_voltage_volts{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}V{{end}}"
87 - alert: mdadm array inactive
88 expr: node_md_state{state="inactive"} > 0
91 alertgroup: "{{ $labels.instance }}"
93 required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
94 active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
95 failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
96 spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
97 - alert: mdadm disk failed
98 expr: node_md_disks{state="failed"} > 0
101 alertgroup: "{{ $labels.instance }}"
103 required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
104 active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
105 failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
106 spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
110 expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1
113 alertgroup: "{{ $labels.instance }}"
115 memory_free: "{{ $value | humanizePercentage }}"
116 - alert: memory pressure
117 expr: rate(node_vmstat_pgmajfault[1m]) > 1000
120 alertgroup: "{{ $labels.instance }}"
122 major_page_faults: "{{ $value }} faults/s"
123 - alert: oom kill detected
124 expr: increase(node_vmstat_oom_kill[1m]) > 0
127 alertgroup: "{{ $labels.instance }}"
129 new_oom_kills: "{{ $value }}"
132 - alert: interface transmit rate
133 expr: rate(node_network_transmit_bytes_total[1m]) / node_network_speed_bytes > 0.98
136 alertgroup: "{{ $labels.instance }}"
138 bandwidth_used: "{{ $value | humanizePercentage }}"
139 - alert: interface receive rate
140 expr: rate(node_network_receive_bytes_total[1m]) / node_network_speed_bytes > 0.98
143 alertgroup: "{{ $labels.instance }}"
145 bandwidth_used: "{{ $value | humanizePercentage }}"
146 - alert: interface transmit errors
147 expr: rate(node_network_transmit_errs_total[1m]) / rate(node_network_transmit_packets_total[1m]) > 0.01
150 alertgroup: "{{ $labels.instance }}"
152 error_rate: "{{ $value | humanizePercentage }}"
153 - alert: interface receive errors
154 expr: rate(node_network_receive_errs_total[1m]) / rate(node_network_receive_packets_total[1m]) > 0.01
157 alertgroup: "{{ $labels.instance }}"
159 error_rate: "{{ $value | humanizePercentage }}"
160 - alert: conntrack entries
161 expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
164 alertgroup: "{{ $labels.instance }}"
166 entries_used: "{{ $value | humanizePercentage }}"
169 - alert: postgresql down
173 alertgroup: "{{ $labels.instance }}"
174 - alert: postgresql replication delay
175 expr: pg_replication_lag_seconds > 5
178 alertgroup: "{{ $labels.instance }}"
180 delay: "{{ $value | humanizeDuration }}"
181 - alert: postgresql connection limit
182 expr: sum (pg_stat_activity_count) by (instance, server) / sum (pg_settings_max_connections) by (instance, server) > 0.8
185 alertgroup: "{{ $labels.instance }}"
187 connections_used: "{{ $value | humanizePercentage }}"
188 - alert: postgresql deadlocks
189 expr: increase(pg_stat_database_deadlocks[1m]) > 5
192 alertgroup: "{{ $labels.instance }}"
194 new_deadlocks: "{{ $value }}"
195 - alert: postgresql slow queries
196 expr: pg_slow_queries > 0
199 alertgroup: "{{ $labels.instance }}"
201 queries: "{{ $value }}"
204 - alert: ssl certificate probe failed
205 expr: ssl_probe_success == 0
209 - alert: ssl certificate expiry
210 expr: ssl_verified_cert_not_after{chain_no="0"} - time() < 86400 * 14
215 expires_in: "{{ $value | humanizeDuration }}"
216 - alert: ssl certificate revoked
217 expr: ssl_ocsp_response_status == 1
221 - alert: ocsp status unknown
222 expr: ssl_ocsp_response_status == 1
228 - alert: renderd replication delay
229 expr: renderd_replication_delay > 120
234 delay: "{{ $value | humanizeDuration }}"
235 - alert: missed tile rate
236 expr: sum(rate(modtile_http_response_total{code="404"}[5m])) by (instance) / sum(rate(modtile_http_response_total[5m])) by (instance) > 0.05
241 miss_rate: "{{ $value | humanizePercentage }}"
244 - alert: clock not synchronising
245 expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16
248 alertgroup: "{{ $labels.instance }}"
249 - alert: clock skew detected
250 expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)
253 alertgroup: "{{ $labels.instance }}"
255 skew: "{{ with printf \"node_timex_offset_seconds{instance='%s'}\" $labels.instance | query }} {{ . | humanizeDuration }}{{ end }}"