2 # Cookbook:: prometheus
5 # Copyright:: 2020, OpenStreetMap Foundation
7 # Licensed under the Apache License, Version 2.0 (the "License");
8 # you may not use this file except in compliance with the License.
9 # You may obtain a copy of the License at
11 # https://www.apache.org/licenses/LICENSE-2.0
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS,
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
20 include_recipe "apache"
21 include_recipe "apt::grafana"
22 include_recipe "networking"
24 passwords = data_bag_item("prometheus", "passwords")
25 tokens = data_bag_item("prometheus", "tokens")
26 admins = data_bag_item("apache", "admins")
28 prometheus_exporter "fastly" do
30 listen_switch "listen"
31 environment "FASTLY_API_TOKEN" => tokens["fastly"]
34 prometheus_exporter "fastly_healthcheck" do
37 environment "FASTLY_API_TOKEN" => tokens["fastly"]
40 prometheus_exporter "statuscake" do
44 environment "STATUSCAKE_APIKEY" => tokens["statuscake"]
47 cache_dir = Chef::Config[:file_cache_path]
49 prometheus_version = "2.41.0"
50 alertmanager_version = "0.24.0"
51 karma_version = "0.111"
53 directory "/opt/prometheus-server" do
59 remote_file "#{cache_dir}/prometheus.linux-amd64.tar.gz" do
60 source "https://github.com/prometheus/prometheus/releases/download/v#{prometheus_version}/prometheus-#{prometheus_version}.linux-amd64.tar.gz"
67 archive_file "#{cache_dir}/prometheus.linux-amd64.tar.gz" do
69 destination "/opt/prometheus-server/prometheus"
74 subscribes :extract, "remote_file[#{cache_dir}/prometheus.linux-amd64.tar.gz]", :immediately
77 remote_file "#{cache_dir}/alertmanager.linux-amd64.tar.gz" do
78 source "https://github.com/prometheus/alertmanager/releases/download/v#{alertmanager_version}/alertmanager-#{alertmanager_version}.linux-amd64.tar.gz"
85 archive_file "#{cache_dir}/alertmanager.linux-amd64.tar.gz" do
87 destination "/opt/prometheus-server/alertmanager"
92 subscribes :extract, "remote_file[#{cache_dir}/alertmanager.linux-amd64.tar.gz]", :immediately
95 remote_file "#{cache_dir}/karma-linux-amd64.tar.gz" do
96 source "https://github.com/prymitive/karma/releases/download/v#{karma_version}/karma-linux-amd64.tar.gz"
103 archive_file "#{cache_dir}/karma-linux-amd64.tar.gz" do
105 destination "/opt/prometheus-server/karma"
109 subscribes :extract, "remote_file[#{cache_dir}/karma-linux-amd64.tar.gz]", :immediately
112 search(:node, "roles:gateway") do |gateway|
113 allowed_ips = gateway.ipaddresses(:role => :internal).map(&:subnet)
115 node.default[:networking][:wireguard][:peers] << {
116 :public_key => gateway[:networking][:wireguard][:public_key],
117 :allowed_ips => allowed_ips,
118 :endpoint => "#{gateway.name}:51820"
125 search(:node, "recipes:prometheus\\:\\:default").sort_by(&:name).each do |client|
126 if client[:prometheus][:mode] == "wireguard"
127 node.default[:networking][:wireguard][:peers] << {
128 :public_key => client[:networking][:wireguard][:public_key],
129 :allowed_ips => client[:networking][:wireguard][:address],
130 :endpoint => "#{client.name}:51820"
134 client[:prometheus][:exporters].each do |key, exporter|
135 if exporter.is_a?(Hash)
136 name = exporter[:name]
137 address = exporter[:address]
139 scrape_interval = exporter[:scrape_interval]
140 scrape_timeout = exporter[:scrape_timeout]
141 metric_relabel = exporter[:metric_relabel] || []
146 scrape_interval = nil
155 :instance => client.name.split(".").first,
156 :scrape_interval => scrape_interval,
157 :scrape_timeout => scrape_timeout,
158 :metric_relabel => metric_relabel
162 Hash(client[:prometheus][:snmp]).each do |instance, details|
164 :instance => instance,
165 :target => details[:address],
166 :modules => details[:modules],
167 :address => client[:prometheus][:addresses]["snmp"],
168 :labels => Array(details[:labels])
173 certificates = search(:node, "letsencrypt:certificates").each_with_object({}) do |n, c|
174 n[:letsencrypt][:certificates].each do |name, details|
175 c[name] ||= details.merge(:nodes => [])
179 :address => n.external_ipaddress || n.internal_ipaddress
184 template "/etc/prometheus/ssl.yml" do
189 variables :certificates => certificates
192 prometheus_exporter "ssl" do
195 options "--config.file=/etc/prometheus/ssl.yml"
196 register_target false
199 systemd_service "prometheus-executable" do
202 exec_start "/opt/prometheus-server/prometheus/prometheus --config.file=/etc/prometheus/prometheus.yml --web.external-url=https://prometheus.openstreetmap.org/prometheus --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=540d"
204 notifies :restart, "service[prometheus]"
207 template "/etc/prometheus/prometheus.yml" do
208 source "prometheus.yml.erb"
212 variables :jobs => jobs, :snmp_targets => snmp_targets, :certificates => certificates
215 template "/etc/prometheus/alert_rules.yml" do
216 source "alert_rules.yml.erb"
222 service "prometheus" do
223 action [:enable, :start]
224 subscribes :reload, "template[/etc/prometheus/prometheus.yml]"
225 subscribes :reload, "template[/etc/prometheus/alert_rules.yml]"
226 subscribes :restart, "archive_file[#{cache_dir}/prometheus.linux-amd64.tar.gz]"
229 systemd_service "prometheus-alertmanager" do
230 description "Prometheus alert manager"
233 exec_start "/opt/prometheus-server/alertmanager/alertmanager --config.file=/etc/prometheus/alertmanager.yml --storage.path=/var/lib/prometheus/alertmanager --web.external-url=https://prometheus.openstreetmap.org/alertmanager"
234 exec_reload "/bin/kill -HUP $MAINPID"
237 notifies :restart, "service[prometheus-alertmanager]"
240 link "/usr/local/bin/promtool" do
241 to "/opt/prometheus-server/prometheus/promtool"
244 template "/etc/prometheus/alertmanager.yml" do
245 source "alertmanager.yml.erb"
251 directory "/var/lib/prometheus/alertmanager" do
257 service "prometheus-alertmanager" do
258 action [:enable, :start]
259 subscribes :reload, "template[/etc/prometheus/alertmanager.yml]"
260 subscribes :restart, "systemd_service[prometheus-alertmanager]"
261 subscribes :restart, "archive_file[#{cache_dir}/alertmanager.linux-amd64.tar.gz]"
264 directory "/etc/amtool" do
270 template "/etc/amtool/config.yml" do
271 source "amtool.yml.erb"
277 link "/usr/local/bin/amtool" do
278 to "/opt/prometheus-server/alertmanager/amtool"
281 template "/etc/prometheus/karma.yml" do
282 source "karma.yml.erb"
288 systemd_service "prometheus-karma" do
289 description "Alert dashboard for Prometheus Alertmanager"
291 exec_start "/opt/prometheus-server/karma/karma-linux-amd64 --config.file=/etc/prometheus/karma.yml"
292 sandbox :enable_network => true
296 service "prometheus-karma" do
297 action [:enable, :start]
298 subscribes :restart, "template[/etc/prometheus/karma.yml]"
299 subscribes :restart, "archive_file[#{cache_dir}/karma-linux-amd64.tar.gz]"
300 subscribes :restart, "systemd_service[prometheus-karma]"
303 package "grafana-enterprise"
305 template "/etc/grafana/grafana.ini" do
306 source "grafana.ini.erb"
310 variables :passwords => passwords
313 service "grafana-server" do
314 action [:enable, :start]
315 subscribes :restart, "template[/etc/grafana/grafana.ini]"
318 apache_module "alias"
319 apache_module "proxy_http"
320 apache_module "proxy_wstunnel"
322 ssl_certificate "prometheus.openstreetmap.org" do
323 domains ["prometheus.openstreetmap.org", "prometheus.osm.org"]
324 notifies :reload, "service[apache2]"
327 apache_site "prometheus.openstreetmap.org" do
328 template "apache.erb"
329 variables :admin_hosts => admins["hosts"]
332 template "/etc/cron.daily/prometheus-backup" do
333 source "backup.cron.erb"