Add alerts for high numbers of active queries on the main database

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 598179d5cb4a6f5303d64f601766a61b5f65e61f..f64b7fc08f17f59d075c231baf474e2c44c7dba4 100644 (file)
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -112,6 +112,20 @@ groups:
           alertgroup: database
         annotations:
           delay: "{{ $value | humanizeDuration }}"
+      - alert: active rails queries
+        expr: sum(pg_stat_activity_count{instance="snap-01",datname="openstreetmap",usename="rails",state="active"}) > 50 and on (instance) chef_role{name="db-master"}
+        for: 5m
+        labels:
+          alertgroup: database
+        annotations:
+          queries: "{{ $value }}"
+      - alert: active cgimap queries
+        expr: sum(pg_stat_activity_count{instance="snap-01",datname="openstreetmap",usename="cgimap",state="active"}) > 30 and on (instance) chef_role{name="db-master"}
+        for: 5m
+        labels:
+          alertgroup: database
+        annotations:
+          delay: "{{ $value }}"
   - name: discourse
     rules:
       - alert: discourse job failure rate