From aad368fcb42c6955ccadbabe1f22aca5d31730dd Mon Sep 17 00:00:00 2001
From: Tom Hughes <tom@compton.nu>
Date: Thu, 24 Feb 2022 22:52:42 +0000
Subject: [PATCH] Add alert for job processing rate

---
 cookbooks/prometheus/templates/default/alert_rules.yml.erb | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 92c38d5c5..be6a89b43 100644
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -410,3 +410,10 @@ groups:
           alertgroup: web
         annotations:
           error_rate: "{{ $value | humanizePercentage }}"
+      - alert: job processing rate
+        expr: rate(pg_stat_user_tables_n_tup_del{datname="openstreetmap",relname="delayed_jobs"}[5m]) / rate(pg_stat_user_tables_n_tup_ins{datname="openstreetmap",relname="delayed_jobs"}[5m]) < 0.9 and ignoring(job, name, datname, relname, schemaname, server) chef_role{name="db-master"} == 1
+        for: 5m
+        labels:
+          alertgroup: web
+        annotations:
+          job_processing_rate: "{{ $value | humanizePercentage }}"
-- 
2.39.5