From: Tom Hughes <tom@compton.nu>
Date: Mon, 15 Nov 2021 15:39:26 +0000 (+0000)
Subject: Add alert for failed services
X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/3b212e01c17597264ff6303498329ffa52157f38

Add alert for failed services
---

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 1b3afcba5..b9ffa9f44 100644
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -262,6 +262,13 @@ groups:
         for: 0m
         labels:
           alertgroup: ssl
+  - name: systemd
+    rules:
+      - alert: systemd failed service
+        expr: node_systemd_unit_state{state="failed"} == 1
+        for: 5m
+        labels:
+          alertgroup: "{{ $labels.instance }}"
   - name: tile
     rules:
       - alert: renderd replication delay