From: Tom Hughes <tom@compton.nu>
Date: Mon, 10 Jul 2023 13:20:15 +0000 (+0100)
Subject: Improve fastly alerts to make them more useful
X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/ae501e32e6b6b901122e511470c2f838cd1b9e0a

Improve fastly alerts to make them more useful
---

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 396de8de4..899c765e4 100644
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -160,13 +160,23 @@ groups:
           alertgroup: fastly
         annotations:
           error_rate: "{{ $value | humanizePercentage }}"
-      - alert: fastly healthcheck failing
-        expr: count(fastly_healthcheck_status == 0) by (service) > 0
+      - alert: fastly frontend healthcheck warning
+        expr: count(fastly_healthcheck_status == 0) by (service, datacenter) > 2
         for: 15m
         labels:
           alertgroup: fastly
-      - alert: multiple fastly healthchecks failing
-        expr: count(fastly_healthcheck_status == 0) by (service) > 4
+      - alert: fastly frontend healthcheck critical
+        expr: count(fastly_healthcheck_status == 0) by (service, datacenter) == count(fastly_healthcheck_status) by (service, datacenter)
+        for: 5m
+        labels:
+          alertgroup: fastly
+      - alert: fastly backend healthcheck warning
+        expr: count(fastly_healthcheck_status == 0) by (service, backend) > 10
+        for: 15m
+        labels:
+          alertgroup: fastly
+      - alert: fastly backend healthcheck critical
+        expr: count(fastly_healthcheck_status == 0) by (service, backend) == count(fastly_healthcheck_status) by (service, backend)
         for: 5m
         labels:
           alertgroup: fastly