1 # DO NOT EDIT - This file is being maintained by Chef
6 - alert: pdu current draw
7 expr: rPDU2PhaseStatusCurrent{site="amsterdam",rPDU2PhaseStatusIndex="1"} / 10 > 10
10 alertgroup: "amsterdam"
12 current: "{{ $value | humanize }}A"
13 - alert: site current draw
14 expr: sum(rPDU2PhaseStatusCurrent{site="amsterdam",rPDU2PhaseStatusIndex="1"} / 10) > 13
17 alertgroup: "amsterdam"
19 current: "{{ $value | humanize }}A"
20 - alert: site temperature
21 expr: min(rPDU2SensorTempHumidityStatusTempC{site="amsterdam"}) / 10 < 18 or min(rPDU2SensorTempHumidityStatusTempC{site="amsterdam"}) / 10 > 25
24 alertgroup: "amsterdam"
26 temperature: "{{ $value | humanize }}C"
27 - alert: site humidity
28 expr: max(rPDU2SensorTempHumidityStatusRelativeHumidity{site="amsterdam"}) / 100 < 0.25 or max(rPDU2SensorTempHumidityStatusRelativeHumidity{site="amsterdam"}) / 100 > 0.65
31 alertgroup: "amsterdam"
33 humidity: "{{ $value | humanizePercentage }}"
40 alertgroup: "{{ $labels.instance }}"
41 - alert: apache workers busy
42 expr: sum(apache_workers{state="busy"}) by (instance) / sum(apache_scoreboard) by (instance) > 0.8
45 alertgroup: "{{ $labels.instance }}"
47 busy_workers: "{{ $value | humanizePercentage }}"
48 - alert: apache low request rate
49 expr: rate(apache_accesses_total[5m]) / rate(apache_accesses_total[1h] offset 1w) < 0.25 and rate(apache_accesses_total[1h] offset 1w) > 2
52 alertgroup: "{{ $labels.instance }}"
54 request_rate: "{{ $value | humanizePercentage }}"
57 - alert: chef client not running
58 expr: time() - node_systemd_timer_last_trigger_seconds{name="chef-client.timer"} > 3600
61 alertgroup: "{{ $labels.instance }}"
63 down_time: "{{ $value | humanizeDuration }}"
66 - alert: cisco fan alarm
67 expr: rlPhdUnitEnvParamFan1Status{rlPhdUnitEnvParamFan1Status!="normal"} > 0 or rlPhdUnitEnvParamFan2Status{rlPhdUnitEnvParamFan2Status!="normal"} > 0
70 alertgroup: "{{ $labels.site }}"
72 fan_rpm: "{{ with printf \"rlPhdUnitEnvParamFan1Speed{site='%s',instance='%s',rlPhdUnitEnvParamStackUnit='%s'}\" $labels.site $labels.instance $labels.rlPhdUnitEnvParamStackUnit | query }}{{ . | first | value | humanize }}rpm{{end}}"
73 - alert: cisco temperature alarm
74 expr: rlPhdUnitEnvParamTempSensorStatus{rlPhdUnitEnvParamTempSensorStatus!="ok"} > 0
77 alertgroup: "{{ $labels.site }}"
79 temp_celsius: "{{ with printf \"rlPhdUnitEnvParamTempSensorValue{site='%s',instance='%s',rlPhdUnitEnvParamStackUnit='%s'}\" $labels.site $labels.instance $labels.rlPhdUnitEnvParamStackUnit | query }}{{ . | first | value | humanize }}C{{end}}"
80 - alert: cisco main power alarm
81 expr: rlPhdUnitEnvParamMainPSStatus{rlPhdUnitEnvParamMainPSStatus!="normal"} > 0
84 alertgroup: "{{ $labels.site }}"
85 - alert: cisco redundant power alarm
86 expr: rlPhdUnitEnvParamRedundantPSStatus{rlPhdUnitEnvParamRedundantPSStatus!="normal"} > 0
89 alertgroup: "{{ $labels.site }}"
93 expr: rate(node_pressure_cpu_waiting_seconds_total[5m]) > 0.6
96 alertgroup: "{{ $labels.instance }}"
98 pressure: "{{ $value | humanizePercentage }}"
101 - alert: postgres replication delay
102 expr: pg_replication_lag_seconds > 5
107 delay: "{{ $value | humanizeDuration }}"
110 - alert: fastly error rate
111 expr: sum(rate(fastly_rt_status_group_total{status_group="5xx"}[5m])) by (service_name, datacenter) / sum(rate(fastly_rt_status_group_total[5m])) by (service_name, datacenter) > 0.005
116 error_rate: "{{ $value | humanizePercentage }}"
117 - alert: fastly healthcheck failing
118 expr: fastly_healthcheck_status == 0
124 - alert: readonly filesystem
125 expr: node_filesystem_readonly == 1
128 alertgroup: "{{ $labels.instance }}"
129 - alert: filesystem low on space
130 expr: node_filesystem_avail_bytes / node_filesystem_size_bytes < 0.05
133 alertgroup: "{{ $labels.instance }}"
135 percentage_free: "{{ $value | humanizePercentage }}"
136 free_bytes: "{{ with printf \"node_filesystem_avail_bytes{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value | humanize1024 }}bytes{{end}}"
137 total_total: "{{ with printf \"node_filesystem_size_bytes{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value | humanize1024 }}bytes{{end}}"
138 - alert: filesystem low on inodes
139 expr: node_filesystem_files_free / node_filesystem_files < 0.1
142 alertgroup: "{{ $labels.instance }}"
144 percentage_free: "{{ $value | humanizePercentage }}"
145 free_inodes: "{{ with printf \"node_filesystem_files_free{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value }}{{end}}"
146 total_inodes: "{{ with printf \"node_filesystem_files{instance='%s',mountpoint='%s'}\" $labels.instance $labels.mountpoint | query }}{{ . | first | value }}{{end}}"
149 - alert: hwmon fan alarm
150 expr: node_hwmon_fan_alarm == 1
153 alertgroup: "{{ $labels.instance }}"
155 fan_rpm: "{{ with printf \"node_hwmon_fan_rpm{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}rpm{{end}}"
156 fan_min_rpm: "{{ with printf \"node_hwmon_fan_min_rpm{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}rpm{{end}}"
157 - alert: hwmon temperature alarm
158 expr: node_hwmon_temp_alarm == 1
161 alertgroup: "{{ $labels.instance }}"
163 temp_celsius: "{{ with printf \"node_hwmon_temp_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
164 temp_max_celsius: "{{ with printf \"node_hwmon_temp_max_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
165 temp_crit_celsius: "{{ with printf \"node_hwmon_temp_crit_celsius{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}C{{end}}"
166 - alert: hwmon voltage alarm
167 expr: node_hwmon_in_alarm == 1
170 alertgroup: "{{ $labels.instance }}"
172 in_volts: "{{ with printf \"node_hwmon_in_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
173 in_min_volts: "{{ with printf \"node_hwmon_in_min_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
174 in_max_volts: "{{ with printf \"node_hwmon_in_max_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}"
178 expr: rate(node_pressure_io_waiting_seconds_total[5m]) > 0.6
181 alertgroup: "{{ $labels.instance }}"
183 pressure: "{{ $value | humanizePercentage }}"
186 - alert: ipmi fan alarm
187 expr: ipmi_fan_speed_state > 0
190 alertgroup: "{{ $labels.instance }}"
192 fan_speed_rpm: "{{ with printf \"ipmi_fan_speed_rpm{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}rpm{{end}}"
193 - alert: ipmi temperature alarm
194 expr: ipmi_temperature_state > 0
197 alertgroup: "{{ $labels.instance }}"
199 temperature_celsius: "{{ with printf \"ipmi_temperature_celsius{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}C{{end}}"
200 - alert: ipmi voltage alarm
201 expr: ipmi_voltage_state > 0
204 alertgroup: "{{ $labels.instance }}"
206 voltage_volts: "{{ with printf \"ipmi_voltage_volts{instance='%s',id='%s'}\" $labels.instance $labels.id | query }}{{ . | first | value | humanize }}V{{end}}"
207 - alert: ipmi power alarm
208 expr: ipmi_power_state > 0 or ipmi_sensor_state{type=~"Power .*"} > 0
211 alertgroup: "{{ $labels.instance }}"
214 - alert: juniper fan alarm
215 expr: jnxOperatingState{jnxOperatingContentsIndex="4",jnxOperatingState!="running"} > 0
218 alertgroup: "{{ $labels.site }}"
219 - alert: juniper power alarm
220 expr: jnxOperatingState{jnxOperatingContentsIndex="2",jnxOperatingState!="running"} > 0
223 alertgroup: "{{ $labels.site }}"
226 - alert: exim queue length
227 expr: exim_queue > exim_queue_limit
232 queue_length: "{{ $value }}"
233 - alert: mailman queue length
234 expr: mailman_queue_length > 200
239 queue_length: "{{ $value }}"
242 - alert: mdadm array inactive
243 expr: node_md_state{state="inactive"} > 0
246 alertgroup: "{{ $labels.instance }}"
248 required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
249 active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
250 failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
251 spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
252 - alert: mdadm array degraded
253 expr: sum (node_md_disks{state="active"}) without (state) < node_md_disks_required
256 alertgroup: "{{ $labels.instance }}"
258 required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
259 active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
260 failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
261 spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
262 - alert: mdadm disk failed
263 expr: node_md_disks{state="failed"} > 0
266 alertgroup: "{{ $labels.instance }}"
268 required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
269 active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
270 failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
271 spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}"
275 expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1
278 alertgroup: "{{ $labels.instance }}"
280 memory_free: "{{ $value | humanizePercentage }}"
281 - alert: memory pressure
282 expr: rate(node_pressure_memory_waiting_seconds_total[5m]) > 0.6
285 alertgroup: "{{ $labels.instance }}"
287 pressure: "{{ $value | humanizePercentage }}"
288 - alert: oom kill detected
289 expr: increase(node_vmstat_oom_kill[1m]) > 0
292 alertgroup: "{{ $labels.instance }}"
294 new_oom_kills: "{{ $value }}"
297 - alert: interface transmit rate
298 expr: rate(node_network_transmit_bytes_total[1m]) / node_network_speed_bytes > 0.98
301 alertgroup: "{{ $labels.instance }}"
303 bandwidth_used: "{{ $value | humanizePercentage }}"
304 - alert: interface receive rate
305 expr: rate(node_network_receive_bytes_total[1m]) / node_network_speed_bytes > 0.98
308 alertgroup: "{{ $labels.instance }}"
310 bandwidth_used: "{{ $value | humanizePercentage }}"
311 - alert: interface transmit errors
312 expr: rate(node_network_transmit_errs_total{device!~"wg.*"}[1m]) / rate(node_network_transmit_packets_total{device!~"wg.*"}[1m]) > 0.01
315 alertgroup: "{{ $labels.instance }}"
317 error_rate: "{{ $value | humanizePercentage }}"
318 - alert: interface transmit errors
319 expr: rate(node_network_transmit_errs_total{device=~"wg.*"}[1m]) / rate(node_network_transmit_packets_total{device=~"wg.*"}[1m]) > 0.05
322 alertgroup: "{{ $labels.instance }}"
324 error_rate: "{{ $value | humanizePercentage }}"
325 - alert: interface receive errors
326 expr: rate(node_network_receive_errs_total[1m]) / rate(node_network_receive_packets_total[1m]) > 0.01
329 alertgroup: "{{ $labels.instance }}"
331 error_rate: "{{ $value | humanizePercentage }}"
332 - alert: conntrack entries
333 expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
336 alertgroup: "{{ $labels.instance }}"
338 entries_used: "{{ $value | humanizePercentage }}"
341 - alert: planet dump overdue
342 expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/(pbf|planet)/.*"} > 7 * 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
347 overdue_by: "{{ $value | humanizeDuration }}"
348 - alert: notes dump overdue
349 expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/notes/.*"} > 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
354 overdue_by: "{{ $value | humanizeDuration }}"
355 - alert: daily replication feed delayed
356 expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/day/.*"} > 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
361 delayed_by: "{{ $value | humanizeDuration }}"
362 - alert: hourly replication feed delayed
363 expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/hour/.*"} > 3600 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
368 delayed_by: "{{ $value | humanizeDuration }}"
369 - alert: minutely replication feed delayed
370 expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/minute/.*"} > 60 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
375 delayed_by: "{{ $value | humanizeDuration }}"
376 - alert: changeset replication feed delayed
377 expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/changesets/.*"} > 60 and ignoring (job, name, path) chef_role{name="planetdump"} == 1
382 delayed_by: "{{ $value | humanizeDuration }}"
385 - alert: postgresql down
389 alertgroup: "{{ $labels.instance }}"
390 - alert: postgresql replication delay
391 expr: pg_replication_lag_seconds > 5
394 alertgroup: "{{ $labels.instance }}"
396 delay: "{{ $value | humanizeDuration }}"
397 - alert: postgresql connection limit
398 expr: sum (pg_stat_activity_count) by (instance, server) / sum (pg_settings_max_connections) by (instance, server) > 0.8
401 alertgroup: "{{ $labels.instance }}"
403 connections_used: "{{ $value | humanizePercentage }}"
404 - alert: postgresql deadlocks
405 expr: increase(pg_stat_database_deadlocks[1m]) > 5
408 alertgroup: "{{ $labels.instance }}"
410 new_deadlocks: "{{ $value }}"
411 - alert: postgresql slow queries
412 expr: pg_slow_queries > 0
415 alertgroup: "{{ $labels.instance }}"
417 queries: "{{ $value }}"
420 - alert: prometheus configuration error
421 expr: prometheus_config_last_reload_successful == 0
424 alertgroup: "prometheus"
425 - alert: prometheus target missing
429 alertgroup: "prometheus"
432 - alert: raid array degraded
433 expr: ohai_array_info{status="degraded"} > 0
436 alertgroup: "{{ $labels.instance }}"
437 - alert: raid disk failed
438 expr: ohai_disk_info{status="failed"} > 0
441 alertgroup: "{{ $labels.instance }}"
444 - alert: smart failure
445 expr: smart_health_status == 0
448 alertgroup: "{{ $labels.instance }}"
449 - alert: smart ssd wearout approaching
450 expr: smart_percentage_used >= 90
453 alertgroup: "{{ $labels.instance }}"
455 percentage_used: "{{ $value | humanizePercentage }}"
458 - alert: ssl certificate probe failed
459 expr: ssl_probe_success == 0
463 - alert: ssl certificate expiry
464 expr: ssl_verified_cert_not_after{chain_no="0"} - time() < 86400 * 14
469 expires_in: "{{ $value | humanizeDuration }}"
470 - alert: ssl certificate revoked
471 expr: ssl_ocsp_response_status == 1
475 - alert: ocsp status unknown
476 expr: ssl_ocsp_response_status == 1
482 - alert: statuscake uptime check failing
483 expr: statuscake_uptime{status="down",paused="false"} > 0
486 alertgroup: statuscake
489 - alert: systemd failed service
490 expr: node_systemd_unit_state{state="failed",name!="chef-client.service"} == 1
493 alertgroup: "{{ $labels.instance }}"
494 - alert: systemd failed service
495 expr: node_systemd_unit_state{state="failed",name="chef-client.service"} == 1
498 alertgroup: "{{ $labels.instance }}"
501 - alert: renderd replication delay
502 expr: renderd_replication_delay > 120
507 delay: "{{ $value | humanizeDuration }}"
508 - alert: missed tile rate
509 expr: sum(rate(modtile_http_response_total{code="404"}[5m])) by (instance) / sum(rate(modtile_http_response_total[5m])) by (instance) > 0.05
514 miss_rate: "{{ $value | humanizePercentage }}"
517 - alert: clock not synchronising
518 expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16
521 alertgroup: "{{ $labels.instance }}"
522 - alert: clock skew detected
523 expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)
526 alertgroup: "{{ $labels.instance }}"
528 skew: "{{ with printf \"node_timex_offset_seconds{instance='%s'}\" $labels.instance | query }} {{ . | humanizeDuration }}{{ end }}"
531 - alert: web error rate
532 expr: sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) / sum(rate(api_call_count_total[5m])) by (instance) > 0.002
537 error_rate: "{{ $value | humanizePercentage }}"
538 - alert: job processing rate
539 expr: rate(pg_stat_user_tables_n_tup_del{datname="openstreetmap",relname="delayed_jobs"}[5m]) / rate(pg_stat_user_tables_n_tup_ins{datname="openstreetmap",relname="delayed_jobs"}[5m]) < 0.9 and ignoring(job, name, datname, relname, schemaname, server) chef_role{name="db-master"} == 1
544 job_processing_rate: "{{ $value | humanizePercentage }}"