From 2b742aa96a7c7e1256ad9a7c8e9ee698130b5226 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Sun, 26 Jul 2015 18:50:08 +0100 Subject: [PATCH] Add a kibana frontend for logstash --- cookbooks/kibana/.foodcritic | 2 + cookbooks/kibana/README.md | 34 +++++++ cookbooks/kibana/attributes/default.rb | 2 + cookbooks/kibana/metadata.rb | 8 ++ cookbooks/kibana/recipes/default.rb | 99 +++++++++++++++++++ cookbooks/kibana/templates/default/apache.erb | 32 ++++++ .../kibana/templates/default/kibana.conf.erb | 18 ++++ roles/kibana.rb | 14 +++ roles/logstash.rb | 10 ++ 9 files changed, 219 insertions(+) create mode 100644 cookbooks/kibana/.foodcritic create mode 100644 cookbooks/kibana/README.md create mode 100644 cookbooks/kibana/attributes/default.rb create mode 100644 cookbooks/kibana/metadata.rb create mode 100644 cookbooks/kibana/recipes/default.rb create mode 100644 cookbooks/kibana/templates/default/apache.erb create mode 100644 cookbooks/kibana/templates/default/kibana.conf.erb create mode 100644 roles/kibana.rb diff --git a/cookbooks/kibana/.foodcritic b/cookbooks/kibana/.foodcritic new file mode 100644 index 000000000..37c7131e2 --- /dev/null +++ b/cookbooks/kibana/.foodcritic @@ -0,0 +1,2 @@ +~FC001 +~FC003 diff --git a/cookbooks/kibana/README.md b/cookbooks/kibana/README.md new file mode 100644 index 000000000..205739199 --- /dev/null +++ b/cookbooks/kibana/README.md @@ -0,0 +1,34 @@ +Cookbook +======== +TODO: Enter the cookbook description here. + +e.g. +This cookbook makes your favorite breakfast sandwich. + +Requirements +------------ +TODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc. + +Attributes +---------- +TODO: List you cookbook attributes here. + +Usage +----- +TODO: Write usage instructions for each cookbook. + +Contributing +------------ +TODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section. + +e.g. +1. Fork the repository on Github +2. Create a named feature branch (like `add_component_x`) +3. Write your change +4. Write tests for your change (if applicable) +5. Run the tests, ensuring they all pass +6. Submit a Pull Request using Github + +License and Authors +------------------- +Authors: TODO: List authors diff --git a/cookbooks/kibana/attributes/default.rb b/cookbooks/kibana/attributes/default.rb new file mode 100644 index 000000000..46dda49ce --- /dev/null +++ b/cookbooks/kibana/attributes/default.rb @@ -0,0 +1,2 @@ +default[:kibana][:version] = "4.1.1" +default[:kibana][:sites] = {} diff --git a/cookbooks/kibana/metadata.rb b/cookbooks/kibana/metadata.rb new file mode 100644 index 000000000..a19cdfedb --- /dev/null +++ b/cookbooks/kibana/metadata.rb @@ -0,0 +1,8 @@ +name "kibana" +maintainer "OpenStreetMap Administrators" +maintainer_email "admins@openstreetmap.org" +license "Apache 2.0" +description "Installs and configures a kibana server" +long_description IO.read(File.join(File.dirname(__FILE__), "README.md")) +version "1.0.0" +depends "apache" diff --git a/cookbooks/kibana/recipes/default.rb b/cookbooks/kibana/recipes/default.rb new file mode 100644 index 000000000..30ee757aa --- /dev/null +++ b/cookbooks/kibana/recipes/default.rb @@ -0,0 +1,99 @@ +# coding: utf-8 +# +# Cookbook Name:: kibana +# Recipe:: default +# +# Copyright 2015, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "yaml" + +include_recipe "apache::ssl" + +apache_module "proxy_http" + +version = node[:kibana][:version] + +remote_file "#{Chef::Config[:file_cache_path]}/kibana-#{version}.tar.gz" do + source "https://download.elastic.co/kibana/kibana/kibana-4.1.1-linux-x64.tar.gz" + not_if { File.exist?("/opt/kibana-#{version}/bin/kibana") } +end + +directory "/opt/kibana-#{version}" do + owner "root" + group "root" + mode 0755 +end + +execute "unzip-kibana-#{version}" do + command "tar --gunzip --extract --strip-components=1 --file=#{Chef::Config[:file_cache_path]}/kibana-#{version}.tar.gz" + cwd "/opt/kibana-#{version}" + user "root" + group "root" + not_if { File.exist?("/opt/kibana-#{version}/bin/kibana") } +end + +directory "/etc/kibana" do + owner "root" + group "root" + mode 0755 +end + +directory "/var/run/kibana" do + owner "kibana" + group "kibana" + mode 0755 +end + +directory "/var/log/kibana" do + owner "kibana" + group "kibana" + mode 0755 +end + +node[:kibana][:sites].each do |name, details| + file "/etc/kibana/#{name}.yml" do + content YAML.dump(YAML.load(File.read("/opt/kibana-#{version}/config/kibana.yml")).merge( + "port" => details[:port], + "host" => "127.0.0.1", + "elasticsearch_url" => details[:elasticsearch_url], + "pid_file" => "/var/run/kibana/#{name}.pid", + "log_file" => "/var/log/kibana/#{name}.log" + )) + owner "root" + group "root" + mode 0644 + notifies :restart, "service[kibana-#{name}]" + end + + template "/etc/init/kibana-#{name}.conf" do + source "kibana.conf.erb" + owner "root" + group "root" + mode 0644 + variables :config => "/etc/kibana/#{name}.yml" + notifies :restart, "service[kibana-#{name}]" + end + + service "kibana-#{name}" do + action [:enable, :start] + supports :status => true, :restart => true, :reload => false + end + + apache_site details[:site] do + template "apache.erb" + variables details.merge(:passwd => "/etc/kibana/#{name}.passwd") + end +end diff --git a/cookbooks/kibana/templates/default/apache.erb b/cookbooks/kibana/templates/default/apache.erb new file mode 100644 index 000000000..b9f50de53 --- /dev/null +++ b/cookbooks/kibana/templates/default/apache.erb @@ -0,0 +1,32 @@ +# DO NOT EDIT - This file is being maintained by Chef + + + ServerName <%= @site %> + ServerAdmin webmaster@openstreetmap.org + + CustomLog /var/log/apache2/<%= @site %>-access.log combined + ErrorLog /var/log/apache2/<%= @site %>-error.log + + Redirect permanent / https://<%= @site %>/ + + + + ServerName <%= @site %> + ServerAdmin webmaster@openstreetmap.org + + CustomLog /var/log/apache2/<%= @site %>-access.log combined + ErrorLog /var/log/apache2/<%= @site %>-error.log + + SSLEngine on + SSLProxyEngine on + + ProxyPass / http://127.0.0.1:<%= @port %>/ + + + AuthType basic + AuthName <%= @site %> + AuthBasicProvider file + AuthUserFile "<%= @passwd %>" + Require valid-user + + diff --git a/cookbooks/kibana/templates/default/kibana.conf.erb b/cookbooks/kibana/templates/default/kibana.conf.erb new file mode 100644 index 000000000..c81b9fa0a --- /dev/null +++ b/cookbooks/kibana/templates/default/kibana.conf.erb @@ -0,0 +1,18 @@ +# kibana +# +# Startup script for kibana + +description "starts up kibana" + +start on (net-device-up + and local-filesystems + and runlevel [345]) +stop on runlevel [!345] + +setuid kibana + +script + exec /opt/kibana-<%= node[:kibana][:version] %>/bin/kibana -c <%= @config %> +end script + +respawn diff --git a/roles/kibana.rb b/roles/kibana.rb new file mode 100644 index 000000000..6198120cf --- /dev/null +++ b/roles/kibana.rb @@ -0,0 +1,14 @@ +name "kibana" +description "Role applied to all kibana servers" + +default_attributes( + :accounts => { + :users => { + :kibana => { :status => :role } + } + } +) + +run_list( + "recipe[kibana]" +) diff --git a/roles/logstash.rb b/roles/logstash.rb index 0c3d9ce10..860e015ff 100644 --- a/roles/logstash.rb +++ b/roles/logstash.rb @@ -9,10 +9,20 @@ default_attributes( :cluster => { :name => "logstash" } + }, + :kibana => { + :sites => { + :logstash => { + :site => "logstash.openstreetmap.org", + :port => 5601, + :elasticsearch_url => "http://127.0.0.1:9200/" + } + } } ) run_list( "role[elasticsearch]", + "role[kibana]", "recipe[logstash]" ) -- 2.43.2