From: Tom Hughes <tom@compton.nu>
Date: Thu, 28 Jul 2022 07:28:32 +0000 (+0100)
Subject: Reduce sensitivity of fastly healthcheck alerts
X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/210ef98310a7e983a28e6136c8835834fe531ddf

Reduce sensitivity of fastly healthcheck alerts
---

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 30a14a471..32e536736 100644
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -115,7 +115,12 @@ groups:
         annotations:
           error_rate: "{{ $value | humanizePercentage }}"
       - alert: fastly healthcheck failing
-        expr: fastly_healthcheck_status == 0
+        expr: count(fastly_healthcheck_status == 0) > 0
+        for: 15m
+        labels:
+          alertgroup: fastly
+      - alert: fastly healthcheck failing
+        expr: count(fastly_healthcheck_status == 0) > 4
         for: 5m
         labels:
           alertgroup: fastly