From: Tom Hughes <tom@compton.nu>
Date: Tue, 23 Jan 2024 09:09:38 +0000 (+0000)
Subject: Add alerts for high numbers of active queries on the main database
X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/b8d8c5b86947c933602e98057c106a2f97c040f7

Add alerts for high numbers of active queries on the main database
---

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 598179d5c..f64b7fc08 100644
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -112,6 +112,20 @@ groups:
           alertgroup: database
         annotations:
           delay: "{{ $value | humanizeDuration }}"
+      - alert: active rails queries
+        expr: sum(pg_stat_activity_count{instance="snap-01",datname="openstreetmap",usename="rails",state="active"}) > 50 and on (instance) chef_role{name="db-master"}
+        for: 5m
+        labels:
+          alertgroup: database
+        annotations:
+          queries: "{{ $value }}"
+      - alert: active cgimap queries
+        expr: sum(pg_stat_activity_count{instance="snap-01",datname="openstreetmap",usename="cgimap",state="active"}) > 30 and on (instance) chef_role{name="db-master"}
+        for: 5m
+        labels:
+          alertgroup: database
+        annotations:
+          delay: "{{ $value }}"
   - name: discourse
     rules:
       - alert: discourse job failure rate