X-Git-Url: https://git.openstreetmap.org/chef.git/blobdiff_plain/65d913a7f1b116d8e53cfcc70973ec8bb467ab76..cf017d3cfcfb0916a489ed83471f833494ef3f78:/cookbooks/prometheus/recipes/server.rb

diff --git a/cookbooks/prometheus/recipes/server.rb b/cookbooks/prometheus/recipes/server.rb
index 1a6639abe..4b9058c8e 100644
--- a/cookbooks/prometheus/recipes/server.rb
+++ b/cookbooks/prometheus/recipes/server.rb
@@ -19,6 +19,7 @@
 
 include_recipe "apache"
 include_recipe "apt::grafana"
+include_recipe "awscli"
 include_recipe "networking"
 
 passwords = data_bag_item("prometheus", "passwords")
@@ -55,9 +56,14 @@ prometheus_exporter "cloudwatch" do
   address "127.0.0.1"
   port 5000
   listen_switch "listen-address"
-  options "--config.file=/etc/prometheus/cloudwatch.yml"
+  options %w[
+    --config.file=/etc/prometheus/cloudwatch.yml
+    --enable-feature=aws-sdk-v2
+    --enable-feature=always-return-info-metrics
+  ]
   environment "AWS_ACCESS_KEY_ID" => "AKIASQUXHPE7JHG37EA6",
               "AWS_SECRET_ACCESS_KEY" => tokens["cloudwatch"]
+  subscribes :restart, "template[/etc/prometheus/cloudwatch.yml]"
 end
 
 cache_dir = Chef::Config[:file_cache_path]
@@ -236,7 +242,7 @@ package "prometheus"
 systemd_service "prometheus-executable" do
   service "prometheus"
   dropin "executable"
-  exec_start "/opt/prometheus-server/prometheus/prometheus --config.file=/etc/prometheus/prometheus.yml --web.external-url=https://prometheus.openstreetmap.org/prometheus --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=540d"
+  exec_start "/opt/prometheus-server/prometheus/prometheus --config.file=/etc/prometheus/prometheus.yml --web.enable-admin-api --web.external-url=https://prometheus.openstreetmap.org/prometheus --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=540d"
   timeout_stop_sec 300
   notifies :restart, "service[prometheus]"
 end
@@ -357,7 +363,7 @@ apache_module "proxy_http"
 apache_module "proxy_wstunnel"
 
 ssl_certificate "prometheus.openstreetmap.org" do
-  domains ["prometheus.openstreetmap.org", "prometheus.osm.org"]
+  domains ["prometheus.openstreetmap.org", "prometheus.osm.org", "munin.openstreetmap.org", "munin.osm.org"]
   notifies :reload, "service[apache2]"
 end
 
@@ -372,3 +378,49 @@ template "/etc/cron.daily/prometheus-backup" do
   group "root"
   mode "750"
 end
+
+package %w[
+  curl
+  jq
+]
+
+directory "/var/lib/prometheus/.aws" do
+  user "prometheus"
+  group "prometheus"
+  mode "755"
+end
+
+template "/var/lib/prometheus/.aws/credentials" do
+  source "aws-credentials.erb"
+  user "prometheus"
+  group "prometheus"
+  mode "600"
+  variables :passwords => passwords
+end
+
+template "/usr/local/bin/prometheus-backup-data" do
+  source "backup-data.erb"
+  owner "root"
+  group "root"
+  mode "755"
+end
+
+systemd_service "prometheus-backup-data" do
+  description "Backup prometheus data to S3"
+  user "prometheus"
+  exec_start "/usr/local/bin/prometheus-backup-data"
+  read_write_paths %w[
+    /var/lib/prometheus/.aws
+    /var/lib/prometheus/metrics2/snapshots
+  ]
+  sandbox :enable_network => true
+end
+
+systemd_timer "prometheus-backup-data" do
+  description "Backup prometheus data to S3"
+  on_calendar "03:11"
+end
+
+service "prometheus-backup-data.timer" do
+  action [:enable, :start]
+end