X-Git-Url: https://git.openstreetmap.org/chef.git/blobdiff_plain/3b212e01c17597264ff6303498329ffa52157f38..0d12afe2ac536a6e7696800798b2e553f36e1a44:/cookbooks/prometheus/templates/default/alert_rules.yml.erb diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb index b9ffa9f44..0aa0b64cb 100644 --- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb +++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb @@ -22,6 +22,24 @@ groups: alertgroup: "{{ $labels.instance }}" annotations: busy_workers: "{{ $value | humanizePercentage }}" + - name: chef + rules: + - alert: chef client not running + expr: time() - node_systemd_timer_last_trigger_seconds{name="chef-client.timer"} > 3600 + for: 12h + labels: + alertgroup: "{{ $labels.instance }}" + annotations: + down_time: "{{ $value | humanizeDuration }}" + - name: cpu + rules: + - alert: cpu pressure + expr: rate(node_pressure_cpu_waiting_seconds_total[5m]) > 0.6 + for: 15m + labels: + alertgroup: "{{ $labels.instance }}" + annotations: + pressure: "{{ $value | humanizePercentage }}" - name: database rules: - alert: postgres replication delay @@ -31,8 +49,22 @@ groups: alertgroup: database annotations: delay: "{{ $value | humanizeDuration }}" + - name: fastly + rules: + - alert: error rate + expr: sum(rate(fastly_rt_status_group_total{status_group="5xx"}[5m])) by (service_name, datacenter) / sum(rate(fastly_rt_status_group_total[5m])) by (service_name, datacenter) > 0.005 + for: 15m + labels: + alertgroup: fastly + annotations: + error_rate: "{{ $value | humanizePercentage }}" - name: filesystem rules: + - alert: readonly filesystem + expr: node_filesystem_readonly == 1 + for: 0m + labels: + alertgroup: "{{ $labels.instance }}" - alert: filesystem low on space expr: node_filesystem_avail_bytes / node_filesystem_size_bytes < 0.05 for: 5m @@ -79,6 +111,15 @@ groups: in_volts: "{{ with printf \"node_hwmon_in_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}" in_min_volts: "{{ with printf \"node_hwmon_in_min_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}" in_max_volts: "{{ with printf \"node_hwmon_in_max_volts{instance='%s',chip='%s',sensor='%s'}\" $labels.instance $labels.chip $labels.sensor | query }}{{ . | first | value | humanize }}V{{end}}" + - name: io + rules: + - alert: io pressure + expr: rate(node_pressure_io_waiting_seconds_total[5m]) > 0.6 + for: 60m + labels: + alertgroup: "{{ $labels.instance }}" + annotations: + pressure: "{{ $value | humanizePercentage }}" - name: ipmi rules: - alert: ipmi fan alarm @@ -107,6 +148,22 @@ groups: for: 5m labels: alertgroup: "{{ $labels.instance }}" + - name: mail + rules: + - alert: exim queue length + expr: exim_queue > exim_queue_limit + for: 60m + labels: + alertgroup: mail + annotations: + queue_length: "{{ $value }}" + - alert: mailman queue length + expr: mailman_queue_length > 200 + for: 60m + labels: + alertgroup: mail + annotations: + queue_length: "{{ $value }}" - name: mdadm rules: - alert: mdadm array inactive @@ -119,6 +176,16 @@ groups: active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}" failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}" spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}" + - alert: mdadm array degraded + expr: sum (node_md_disks{state="active"}) without (state) < node_md_disks_required + for: 0m + labels: + alertgroup: "{{ $labels.instance }}" + annotations: + required: "{{ with printf \"node_md_disks_required{instance='%s',device='%s'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}" + active: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='active'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}" + failed: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='failed'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}" + spare: "{{ with printf \"node_md_disks{instance='%s',device='%s',state='spare'}\" $labels.instance $labels.device | query }}{{ . | first | value | humanize }} disks{{end}}" - alert: mdadm disk failed expr: node_md_disks{state="failed"} > 0 for: 0m @@ -133,18 +200,18 @@ groups: rules: - alert: low memory expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1 - for: 5m + for: 15m labels: alertgroup: "{{ $labels.instance }}" annotations: memory_free: "{{ $value | humanizePercentage }}" - alert: memory pressure - expr: rate(node_vmstat_pgmajfault[1m]) > 1000 - for: 5m + expr: rate(node_pressure_memory_waiting_seconds_total[5m]) > 0.6 + for: 60m labels: alertgroup: "{{ $labels.instance }}" annotations: - major_page_faults: "{{ $value }} faults/s" + pressure: "{{ $value | humanizePercentage }}" - alert: oom kill detected expr: increase(node_vmstat_oom_kill[1m]) > 0 for: 0m @@ -189,6 +256,50 @@ groups: alertgroup: "{{ $labels.instance }}" annotations: entries_used: "{{ $value | humanizePercentage }}" + - name: planet + rules: + - alert: planet dump overdue + expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/(pbf|planet)/.*"} > 7 * 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1 + for: 24h + labels: + alertgroup: planet + annotations: + overdue_by: "{{ $value | humanizeDuration }}" + - alert: notes dump overdue + expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/notes/.*"} > 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1 + for: 6h + labels: + alertgroup: planet + annotations: + overdue_by: "{{ $value | humanizeDuration }}" + - alert: daily replication feed delayed + expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/day/.*"} > 86400 and ignoring (job, name, path) chef_role{name="planetdump"} == 1 + for: 3h + labels: + alertgroup: planet + annotations: + delayed_by: "{{ $value | humanizeDuration }}" + - alert: hourly replication feed delayed + expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/hour/.*"} > 3600 and ignoring (job, name, path) chef_role{name="planetdump"} == 1 + for: 30m + labels: + alertgroup: planet + annotations: + delayed_by: "{{ $value | humanizeDuration }}" + - alert: minutely replication feed delayed + expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/minute/.*"} > 60 and ignoring (job, name, path) chef_role{name="planetdump"} == 1 + for: 5m + labels: + alertgroup: planet + annotations: + delayed_by: "{{ $value | humanizeDuration }}" + - alert: changeset replication feed delayed + expr: time() - file_stat_modif_time_seconds{path=~"/store/planet/replication/changesets/.*"} > 60 and ignoring (job, name, path) chef_role{name="planetdump"} == 1 + for: 5m + labels: + alertgroup: planet + annotations: + delayed_by: "{{ $value | humanizeDuration }}" - name: postgresql rules: - alert: postgresql down @@ -265,10 +376,15 @@ groups: - name: systemd rules: - alert: systemd failed service - expr: node_systemd_unit_state{state="failed"} == 1 + expr: node_systemd_unit_state{state="failed",name!="chef-client.service"} == 1 for: 5m labels: alertgroup: "{{ $labels.instance }}" + - alert: systemd failed service + expr: node_systemd_unit_state{state="failed",name="chef-client.service"} == 1 + for: 6h + labels: + alertgroup: "{{ $labels.instance }}" - name: tile rules: - alert: renderd replication delay @@ -299,3 +415,19 @@ groups: alertgroup: "{{ $labels.instance }}" annotations: skew: "{{ with printf \"node_timex_offset_seconds{instance='%s'}\" $labels.instance | query }} {{ . | humanizeDuration }}{{ end }}" + - name: web + rules: + - alert: web error rate + expr: sum(rate(api_call_count_total{status=~"50[0-8]|5[1-9][0-9]"}[5m])) by (instance) / sum(rate(api_call_count_total[5m])) by (instance) > 0.002 + for: 5m + labels: + alertgroup: web + annotations: + error_rate: "{{ $value | humanizePercentage }}" + - alert: job processing rate + expr: rate(pg_stat_user_tables_n_tup_del{datname="openstreetmap",relname="delayed_jobs"}[5m]) / rate(pg_stat_user_tables_n_tup_ins{datname="openstreetmap",relname="delayed_jobs"}[5m]) < 0.9 and ignoring(job, name, datname, relname, schemaname, server) chef_role{name="db-master"} == 1 + for: 5m + labels: + alertgroup: web + annotations: + job_processing_rate: "{{ $value | humanizePercentage }}"