From: Tom Hughes <tom@compton.nu>
Date: Fri, 24 Jul 2015 15:27:06 +0000 (+0100)
Subject: Add support for storing web logs in logstash
X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/a92c80c41d88faa20123f2841cc335c37905f638?hp=a9bee8489dc2cfe75b26bfbc567c13f90143f47e

Add support for storing web logs in logstash
---

diff --git a/cookbooks/apt/recipes/default.rb b/cookbooks/apt/recipes/default.rb
index 050f806f6..0d793d4b7 100644
--- a/cookbooks/apt/recipes/default.rb
+++ b/cookbooks/apt/recipes/default.rb
@@ -87,6 +87,18 @@ apt_source "elasticsearch" do
   key "D88E42B4"
 end
 
+apt_source "logstash" do
+  template "elasticsearch.list.erb"
+  url "http://packages.elasticsearch.org/logstash/1.5/debian"
+  key "D88E42B4"
+end
+
+apt_source "logstash-forwarder" do
+  template "elasticsearch.list.erb"
+  url "http://packages.elasticsearch.org/logstashforwarder/debian"
+  key "D88E42B4"
+end
+
 apt_source "passenger" do
   url "https://oss-binaries.phusionpassenger.com/apt/passenger"
   key "AC40B2F7"
diff --git a/cookbooks/elasticsearch/attributes/default.rb b/cookbooks/elasticsearch/attributes/default.rb
deleted file mode 100644
index e5e5c3553..000000000
--- a/cookbooks/elasticsearch/attributes/default.rb
+++ /dev/null
@@ -1 +0,0 @@
-default[:apt][:sources] |= ["elasticsearch"]
diff --git a/cookbooks/logstash/.foodcritic b/cookbooks/logstash/.foodcritic
new file mode 100644
index 000000000..37c7131e2
--- /dev/null
+++ b/cookbooks/logstash/.foodcritic
@@ -0,0 +1,2 @@
+~FC001
+~FC003
diff --git a/cookbooks/logstash/README.md b/cookbooks/logstash/README.md
new file mode 100644
index 000000000..205739199
--- /dev/null
+++ b/cookbooks/logstash/README.md
@@ -0,0 +1,34 @@
+Cookbook
+========
+TODO: Enter the cookbook description here.
+
+e.g.
+This cookbook makes your favorite breakfast sandwich.
+
+Requirements
+------------
+TODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc.
+
+Attributes
+----------
+TODO: List you cookbook attributes here.
+
+Usage
+-----
+TODO: Write usage instructions for each cookbook.
+
+Contributing
+------------
+TODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section.
+
+e.g.
+1. Fork the repository on Github
+2. Create a named feature branch (like `add_component_x`)
+3. Write your change
+4. Write tests for your change (if applicable)
+5. Run the tests, ensuring they all pass
+6. Submit a Pull Request using Github
+
+License and Authors
+-------------------
+Authors: TODO: List authors
diff --git a/cookbooks/logstash/attributes/default.rb b/cookbooks/logstash/attributes/default.rb
new file mode 100644
index 000000000..9a56f3836
--- /dev/null
+++ b/cookbooks/logstash/attributes/default.rb
@@ -0,0 +1,3 @@
+default[:logstash][:forwarder][:network][:servers] = ["logstash.openstreetmap.org:5043"]
+default[:logstash][:forwarder][:network][:"ssl ca"] = "/var/lib/logstash-forwarder/lumberjack.crt"
+default[:logstash][:forwarder][:files] = []
diff --git a/cookbooks/logstash/files/default/lumberjack.crt b/cookbooks/logstash/files/default/lumberjack.crt
new file mode 100644
index 000000000..d4d0ef4f7
--- /dev/null
+++ b/cookbooks/logstash/files/default/lumberjack.crt
@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDHTCCAgWgAwIBAgIJAKZiPjhmhjctMA0GCSqGSIb3DQEBCwUAMCUxIzAhBgNV
+BAMMGmxvZ3N0YXNoLm9wZW5zdHJlZXRtYXAub3JnMB4XDTE1MDcyMzIzMDkxNVoX
+DTE1MDgyMjIzMDkxNVowJTEjMCEGA1UEAwwabG9nc3Rhc2gub3BlbnN0cmVldG1h
+cC5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQClqGmqXzf4Woj3
+Y3JEVaOdSWCKODABCMsbCE5Wca3+72xQwtwI/jhH+IMdXk2+4Puw+5j01Ko3dGbu
+ZWPZ8vWfVg0QXMygZTLNj/Vd+vKZ77DDiaeJHtYudQ0Q5X1sVDScNHeE20IwGnot
+5QcnvzxRhFPaP/UFP4x6v0UUMWVbI211mD+Ebdx6o0829hI+NYNmbnCmk8gBIBQ7
+YaddCF8MDAJDjFIcC/+DrxPF8iFa2lEcpGcsldVBVcgFaNVA/cXI/ysncZjJGigv
+mnk8Oq6xahfdBZyae53XbaO3AghHtJ9kMOYPLLmcMDWJcuFHQOS3NH4HeZ5O5chk
+xsLahqm3AgMBAAGjUDBOMB0GA1UdDgQWBBTiloMwMnJ4n+8f4qLqKW2Ee4FyfjAf
+BgNVHSMEGDAWgBTiloMwMnJ4n+8f4qLqKW2Ee4FyfjAMBgNVHRMEBTADAQH/MA0G
+CSqGSIb3DQEBCwUAA4IBAQB/KFOEA9VvqA/85C0VDAu/3kPXUnCkSNW8UU7xGJTY
+ac5tN8EP9hA+sUmqxTyqt5HlFVPFnM+d/qMbhS5cKttt60F0deDMEwwncQoT+tcC
+oRQPLQshkCUCWA/khAAeYvD9BsFM1jVan8HultTksLZ3U0DwDkn47K4CCVhssv0W
+z2aJctJQbvkADjeQ5eVgatrHpHv8GcXyN8olPbDHo9IH+7nw1lx1oaAIYuc9oHhW
+4ZMF5hBCTgOjq48O7604V731hxVC3lzYaT3qURPW9fOu9qzDSza4SUaJvATpfLrj
+sHJ7QDlOSt11AjHDD6C01L5SeMturwRe4lWYzCiNuW4D
+-----END CERTIFICATE-----
diff --git a/cookbooks/logstash/metadata.rb b/cookbooks/logstash/metadata.rb
new file mode 100644
index 000000000..5e4405a21
--- /dev/null
+++ b/cookbooks/logstash/metadata.rb
@@ -0,0 +1,8 @@
+name              "logstash"
+maintainer        "OpenStreetMap Administrators"
+maintainer_email  "admins@openstreetmap.org"
+license           "Apache 2.0"
+description       "Installs and configures a elasticsearch server"
+long_description  IO.read(File.join(File.dirname(__FILE__), "README.md"))
+version           "1.0.0"
+depends           "networking"
diff --git a/cookbooks/logstash/recipes/default.rb b/cookbooks/logstash/recipes/default.rb
new file mode 100644
index 000000000..b2932aa97
--- /dev/null
+++ b/cookbooks/logstash/recipes/default.rb
@@ -0,0 +1,70 @@
+#
+# Cookbook Name:: logstash
+# Recipe:: default
+#
+# Copyright 2015, OpenStreetMap Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "networking"
+
+keys = data_bag_item("logstash", "keys")
+
+package "openjdk-7-jre-headless"
+package "logstash"
+
+cookbook_file "/var/lib/logstash/lumberjack.crt" do
+  source "lumberjack.crt"
+  user "root"
+  group "logstash"
+  mode 0644
+  notifies :restart, "service[logstash]"
+end
+
+file "/var/lib/logstash/lumberjack.key" do
+  content keys["lumberjack"].join("\n")
+  user "root"
+  group "logstash"
+  mode 0640
+  notifies :restart, "service[logstash]"
+end
+
+template "/etc/logstash/conf.d/chef.conf" do
+  source "logstash.conf.erb"
+  user "root"
+  group "root"
+  mode 0644
+  notifies :restart, "service[logstash]"
+end
+
+service "logstash" do
+  action [:enable, :start]
+  supports :status => true, :restart => true
+end
+
+forwarders = search(:node, "recipes:logstash\\:\\:forwarder")
+
+forwarders.each do |forwarder|
+  forwarder.interfaces(:role => :external) do |interface|
+    firewall_rule "accept-lumberjack-#{forwarder}" do
+      action :accept
+      family interface[:family]
+      source "#{interface[:zone]}:#{interface[:address]}"
+      dest "fw"
+      proto "tcp:syn"
+      dest_ports "5043"
+      source_ports "1024:"
+    end
+  end
+end
diff --git a/cookbooks/logstash/recipes/forwarder.rb b/cookbooks/logstash/recipes/forwarder.rb
new file mode 100644
index 000000000..093912d7c
--- /dev/null
+++ b/cookbooks/logstash/recipes/forwarder.rb
@@ -0,0 +1,43 @@
+#
+# Cookbook Name:: logstash
+# Recipe:: default
+#
+# Copyright 2015, OpenStreetMap Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "json"
+
+package "logstash-forwarder"
+
+cookbook_file "/var/lib/logstash-forwarder/lumberjack.crt" do
+  source "lumberjack.crt"
+  user "root"
+  group "root"
+  mode 0644
+  notifies :restart, "service[logstash-forwarder]"
+end
+
+file "/etc/logstash-forwarder.conf" do
+  content JSON.pretty_generate(node[:logstash][:forwarder])
+  user "root"
+  group "root"
+  mode 0644
+  notifies :restart, "service[logstash-forwarder]"
+end
+
+service "logstash-forwarder" do
+  action [:enable, :start]
+  supports :status => true, :restart => true
+end
diff --git a/cookbooks/logstash/templates/default/logstash.conf.erb b/cookbooks/logstash/templates/default/logstash.conf.erb
new file mode 100644
index 000000000..f6c207a61
--- /dev/null
+++ b/cookbooks/logstash/templates/default/logstash.conf.erb
@@ -0,0 +1,28 @@
+input {
+  lumberjack {
+    port => 5043
+    ssl_certificate => "/var/lib/logstash/lumberjack.crt"
+    ssl_key => "/var/lib/logstash/lumberjack.key"
+  }
+}
+
+filter {
+  if [type] == "apache" {
+    grok {
+      match => [ "message", "%{COMBINEDAPACHELOG} %{NUMBER:duration:int}us %{WORD:request_id} %{NOTSPACE:ssl_protocol} %{NOTSPACE:ssl_cipher}" ]
+    }
+    date {
+      match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
+    }
+  } else if [type] == "rails" {
+    json {
+      source => "message"
+    }
+  }
+}
+
+output {
+  elasticsearch {
+    host => [ "127.0.0.1" ]
+  }
+}
diff --git a/cookbooks/mediawiki/metadata.rb b/cookbooks/mediawiki/metadata.rb
index 2551b291e..46daf9a65 100644
--- a/cookbooks/mediawiki/metadata.rb
+++ b/cookbooks/mediawiki/metadata.rb
@@ -9,4 +9,3 @@ depends           "memcached"
 depends           "apache"
 depends           "mysql"
 depends           "git"
-depends           "elasticsearch"
diff --git a/cookbooks/mediawiki/recipes/default.rb b/cookbooks/mediawiki/recipes/default.rb
index 61c761725..267c4f4b1 100644
--- a/cookbooks/mediawiki/recipes/default.rb
+++ b/cookbooks/mediawiki/recipes/default.rb
@@ -21,7 +21,6 @@ include_recipe "memcached"
 include_recipe "apache::ssl"
 include_recipe "mysql"
 include_recipe "git"
-include_recipe "elasticsearch"
 
 # Mediawiki Base Requirements
 package "php5"
diff --git a/roles/elasticsearch.rb b/roles/elasticsearch.rb
new file mode 100644
index 000000000..ae0e2b6cc
--- /dev/null
+++ b/roles/elasticsearch.rb
@@ -0,0 +1,12 @@
+name "elasticsearch"
+description "Role applied to all elasticsearch servers"
+
+default_attributes(
+  :apt => {
+    :sources => ["elasticsearch"]
+  }
+)
+
+run_list(
+  "recipe[elasticsearch]"
+)
diff --git a/roles/foundation.rb b/roles/foundation.rb
index fa96196f5..031611775 100644
--- a/roles/foundation.rb
+++ b/roles/foundation.rb
@@ -19,6 +19,7 @@ default_attributes(
 
 run_list(
   "role[crm]",
+  "role[elasticsearch]",
   "recipe[foundation::wiki]",
   "recipe[foundation::board]"
 )
diff --git a/roles/ironbelly.rb b/roles/ironbelly.rb
index 4bcb8fb26..fb86c78f0 100644
--- a/roles/ironbelly.rb
+++ b/roles/ironbelly.rb
@@ -97,6 +97,7 @@ run_list(
   "role[stats]",
   "role[planet]",
   "role[planetdump]",
+  "role[logstash]",
   "recipe[rsyncd]",
   "recipe[openvpn]",
   "recipe[git::server]",
diff --git a/roles/logstash-forwarder.rb b/roles/logstash-forwarder.rb
new file mode 100644
index 000000000..2857223b1
--- /dev/null
+++ b/roles/logstash-forwarder.rb
@@ -0,0 +1,12 @@
+name "logstash-forwarder"
+description "Role applied to all logstash forwarders"
+
+default_attributes(
+  :apt => {
+    :sources => ["logstash-forwarder"]
+  }
+)
+
+run_list(
+  "recipe[logstash::forwarder]"
+)
diff --git a/roles/logstash.rb b/roles/logstash.rb
new file mode 100644
index 000000000..dabe91379
--- /dev/null
+++ b/roles/logstash.rb
@@ -0,0 +1,13 @@
+name "logstash"
+description "Role applied to all logstash servers"
+
+default_attributes(
+  :apt => {
+    :sources => ["logstash"]
+  }
+)
+
+run_list(
+  "role[elasticsearch]",
+  "recipe[logstash]"
+)
diff --git a/roles/web-backend.rb b/roles/web-backend.rb
index d77ea1b95..51d7c477e 100644
--- a/roles/web-backend.rb
+++ b/roles/web-backend.rb
@@ -8,6 +8,14 @@ default_attributes(
       :max_requests_per_child => 10000
     }
   },
+  :logstash => {
+    :forwarder => {
+      :files => [
+        { :paths => ["/var/log/apache2/access.log"], :fields => { :type => "apache" } },
+        { :paths => ["/var/log/web/rails-logstash.log"], :fields => { :type => "rails" } }
+      ]
+    }
+  },
   :memcached  => {
     :memory_limit => 4096
   },
@@ -18,5 +26,6 @@ default_attributes(
 
 run_list(
   "role[web]",
+  "role[logstash-forwarder]",
   "recipe[web::backend]"
 )
diff --git a/roles/web-frontend.rb b/roles/web-frontend.rb
index faa1017de..e228e0efc 100644
--- a/roles/web-frontend.rb
+++ b/roles/web-frontend.rb
@@ -13,6 +13,14 @@ default_attributes(
       :max_requests_per_child => 10000
     }
   },
+  :logstash => {
+    :forwarder => {
+      :files => [
+        { :paths => ["/var/log/apache2/access.log"], :fields => { :type => "apache" } },
+        { :paths => ["/var/log/web/rails-logstash.log"], :fields => { :type => "rails" } }
+      ]
+    }
+  },
   :passenger => {
     :max_pool_size => 50
   },
@@ -34,5 +42,6 @@ default_attributes(
 
 run_list(
   "role[web]",
+  "role[logstash-forwarder]",
   "recipe[web::frontend]"
 )
diff --git a/roles/wiki.rb b/roles/wiki.rb
index e79639928..4357ee17a 100644
--- a/roles/wiki.rb
+++ b/roles/wiki.rb
@@ -39,5 +39,6 @@ default_attributes(
 )
 
 run_list(
+  "role[elasticsearch]",
   "recipe[wiki]"
 )