Add an alert for the mail queue

diff --git a/cookbooks/prometheus/templates/default/alert_rules.yml.erb b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
index 620b44f545e720df6df7a0dcf17c36beb032a528..9a3609e03a1876f10a99f4bcfac63033e100b645 100644 (file)
--- a/cookbooks/prometheus/templates/default/alert_rules.yml.erb
+++ b/cookbooks/prometheus/templates/default/alert_rules.yml.erb
@@ -130,6 +130,15 @@ groups:
         for: 5m
         labels:
           alertgroup: "{{ $labels.instance }}"
+  - name: mail
+    rules:
+      - alert: mail queue length
+        expr: exim_queue > exim_queue_limit
+        for: 60m
+        labels:
+          alertgroup: mail
+        annotations:
+          queue_length: "{{ $value }}"
   - name: mdadm
     rules:
       - alert: mdadm array inactive

diff --git a/cookbooks/prometheus/templates/default/chef.prom.erb b/cookbooks/prometheus/templates/default/chef.prom.erb
index f9be619f095f06b72a5c85faa28a7dd3cfc7013c..fd8089f4b195c932db57e349693bface5d00d5f8 100644 (file)
--- a/cookbooks/prometheus/templates/default/chef.prom.erb
+++ b/cookbooks/prometheus/templates/default/chef.prom.erb
@@ -11,5 +11,5 @@ chef_role{name="<%= role %>"} 1
 <% node[:prometheus][:metrics].sort.each do |name, details| -%>
 # HELP <%= name %> <%= details[:help] %>
 # TYPE <%= name %> gauge
-<%= name %>{<%= details[:labels].map { |k,v| "#{k}=\"#{v}\"" }.join(",") %>} 1
+<%= name %>{<%= Hash(details[:labels]).map { |k,v| "#{k}=\"#{v}\"" }.join(",") %>} <%= details[:metric] || 1 %>
 <% end -%>

diff --git a/roles/base.rb b/roles/base.rb
index 4d453e45ce32d779f0dc67fdfd83ecca58387d3c..d5230dc602c20b2a8d33726b43e21fbc077cf3ea 100644 (file)
--- a/roles/base.rb
+++ b/roles/base.rb
@@ -24,6 +24,14 @@ default_attributes(
     },
     :search => ["openstreetmap.org"]
   },
+  :prometheus => {
+    :metrics => {
+      :exim_queue_limit => {
+        :help => "Mail queue alert level",
+        :metric => 50
+      }
+    }
+  },
   :sysctl => {
     :panic => {
       :comment => "Reboot automatically after a panic",

diff --git a/roles/mail.rb b/roles/mail.rb
index 50382b03bef7562cfd9b57841612aa175a2f7491..4184b47f30d41a6c7040383242ee5413e2ebb9b6 100644 (file)
--- a/roles/mail.rb
+++ b/roles/mail.rb
@@ -90,6 +90,11 @@ default_attributes(
         }
       }
     }
+  },
+  :prometheus => {
+    :metrics => {
+      :exim_queue_limit => { :metric => 500 }
+    }
   }
 )