key "D88E42B4"
end
+apt_source "logstash" do
+ template "elasticsearch.list.erb"
+ url "http://packages.elasticsearch.org/logstash/1.5/debian"
+ key "D88E42B4"
+end
+
+apt_source "logstash-forwarder" do
+ template "elasticsearch.list.erb"
+ url "http://packages.elasticsearch.org/logstashforwarder/debian"
+ key "D88E42B4"
+end
+
apt_source "passenger" do
url "https://oss-binaries.phusionpassenger.com/apt/passenger"
key "AC40B2F7"
+++ /dev/null
-default[:apt][:sources] |= ["elasticsearch"]
--- /dev/null
+~FC001
+~FC003
--- /dev/null
+Cookbook
+========
+TODO: Enter the cookbook description here.
+
+e.g.
+This cookbook makes your favorite breakfast sandwich.
+
+Requirements
+------------
+TODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc.
+
+Attributes
+----------
+TODO: List you cookbook attributes here.
+
+Usage
+-----
+TODO: Write usage instructions for each cookbook.
+
+Contributing
+------------
+TODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section.
+
+e.g.
+1. Fork the repository on Github
+2. Create a named feature branch (like `add_component_x`)
+3. Write your change
+4. Write tests for your change (if applicable)
+5. Run the tests, ensuring they all pass
+6. Submit a Pull Request using Github
+
+License and Authors
+-------------------
+Authors: TODO: List authors
--- /dev/null
+default[:logstash][:forwarder][:network][:servers] = ["logstash.openstreetmap.org:5043"]
+default[:logstash][:forwarder][:network][:"ssl ca"] = "/var/lib/logstash-forwarder/lumberjack.crt"
+default[:logstash][:forwarder][:files] = []
--- /dev/null
+-----BEGIN CERTIFICATE-----
+MIIDHTCCAgWgAwIBAgIJAKZiPjhmhjctMA0GCSqGSIb3DQEBCwUAMCUxIzAhBgNV
+BAMMGmxvZ3N0YXNoLm9wZW5zdHJlZXRtYXAub3JnMB4XDTE1MDcyMzIzMDkxNVoX
+DTE1MDgyMjIzMDkxNVowJTEjMCEGA1UEAwwabG9nc3Rhc2gub3BlbnN0cmVldG1h
+cC5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQClqGmqXzf4Woj3
+Y3JEVaOdSWCKODABCMsbCE5Wca3+72xQwtwI/jhH+IMdXk2+4Puw+5j01Ko3dGbu
+ZWPZ8vWfVg0QXMygZTLNj/Vd+vKZ77DDiaeJHtYudQ0Q5X1sVDScNHeE20IwGnot
+5QcnvzxRhFPaP/UFP4x6v0UUMWVbI211mD+Ebdx6o0829hI+NYNmbnCmk8gBIBQ7
+YaddCF8MDAJDjFIcC/+DrxPF8iFa2lEcpGcsldVBVcgFaNVA/cXI/ysncZjJGigv
+mnk8Oq6xahfdBZyae53XbaO3AghHtJ9kMOYPLLmcMDWJcuFHQOS3NH4HeZ5O5chk
+xsLahqm3AgMBAAGjUDBOMB0GA1UdDgQWBBTiloMwMnJ4n+8f4qLqKW2Ee4FyfjAf
+BgNVHSMEGDAWgBTiloMwMnJ4n+8f4qLqKW2Ee4FyfjAMBgNVHRMEBTADAQH/MA0G
+CSqGSIb3DQEBCwUAA4IBAQB/KFOEA9VvqA/85C0VDAu/3kPXUnCkSNW8UU7xGJTY
+ac5tN8EP9hA+sUmqxTyqt5HlFVPFnM+d/qMbhS5cKttt60F0deDMEwwncQoT+tcC
+oRQPLQshkCUCWA/khAAeYvD9BsFM1jVan8HultTksLZ3U0DwDkn47K4CCVhssv0W
+z2aJctJQbvkADjeQ5eVgatrHpHv8GcXyN8olPbDHo9IH+7nw1lx1oaAIYuc9oHhW
+4ZMF5hBCTgOjq48O7604V731hxVC3lzYaT3qURPW9fOu9qzDSza4SUaJvATpfLrj
+sHJ7QDlOSt11AjHDD6C01L5SeMturwRe4lWYzCiNuW4D
+-----END CERTIFICATE-----
--- /dev/null
+name "logstash"
+maintainer "OpenStreetMap Administrators"
+maintainer_email "admins@openstreetmap.org"
+license "Apache 2.0"
+description "Installs and configures a elasticsearch server"
+long_description IO.read(File.join(File.dirname(__FILE__), "README.md"))
+version "1.0.0"
+depends "networking"
--- /dev/null
+#
+# Cookbook Name:: logstash
+# Recipe:: default
+#
+# Copyright 2015, OpenStreetMap Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "networking"
+
+keys = data_bag_item("logstash", "keys")
+
+package "openjdk-7-jre-headless"
+package "logstash"
+
+cookbook_file "/var/lib/logstash/lumberjack.crt" do
+ source "lumberjack.crt"
+ user "root"
+ group "logstash"
+ mode 0644
+ notifies :restart, "service[logstash]"
+end
+
+file "/var/lib/logstash/lumberjack.key" do
+ content keys["lumberjack"].join("\n")
+ user "root"
+ group "logstash"
+ mode 0640
+ notifies :restart, "service[logstash]"
+end
+
+template "/etc/logstash/conf.d/chef.conf" do
+ source "logstash.conf.erb"
+ user "root"
+ group "root"
+ mode 0644
+ notifies :restart, "service[logstash]"
+end
+
+service "logstash" do
+ action [:enable, :start]
+ supports :status => true, :restart => true
+end
+
+forwarders = search(:node, "recipes:logstash\\:\\:forwarder")
+
+forwarders.each do |forwarder|
+ forwarder.interfaces(:role => :external) do |interface|
+ firewall_rule "accept-lumberjack-#{forwarder}" do
+ action :accept
+ family interface[:family]
+ source "#{interface[:zone]}:#{interface[:address]}"
+ dest "fw"
+ proto "tcp:syn"
+ dest_ports "5043"
+ source_ports "1024:"
+ end
+ end
+end
--- /dev/null
+#
+# Cookbook Name:: logstash
+# Recipe:: default
+#
+# Copyright 2015, OpenStreetMap Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "json"
+
+package "logstash-forwarder"
+
+cookbook_file "/var/lib/logstash-forwarder/lumberjack.crt" do
+ source "lumberjack.crt"
+ user "root"
+ group "root"
+ mode 0644
+ notifies :restart, "service[logstash-forwarder]"
+end
+
+file "/etc/logstash-forwarder.conf" do
+ content JSON.pretty_generate(node[:logstash][:forwarder])
+ user "root"
+ group "root"
+ mode 0644
+ notifies :restart, "service[logstash-forwarder]"
+end
+
+service "logstash-forwarder" do
+ action [:enable, :start]
+ supports :status => true, :restart => true
+end
--- /dev/null
+input {
+ lumberjack {
+ port => 5043
+ ssl_certificate => "/var/lib/logstash/lumberjack.crt"
+ ssl_key => "/var/lib/logstash/lumberjack.key"
+ }
+}
+
+filter {
+ if [type] == "apache" {
+ grok {
+ match => [ "message", "%{COMBINEDAPACHELOG} %{NUMBER:duration:int}us %{WORD:request_id} %{NOTSPACE:ssl_protocol} %{NOTSPACE:ssl_cipher}" ]
+ }
+ date {
+ match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
+ }
+ } else if [type] == "rails" {
+ json {
+ source => "message"
+ }
+ }
+}
+
+output {
+ elasticsearch {
+ host => [ "127.0.0.1" ]
+ }
+}
depends "apache"
depends "mysql"
depends "git"
-depends "elasticsearch"
include_recipe "apache::ssl"
include_recipe "mysql"
include_recipe "git"
-include_recipe "elasticsearch"
# Mediawiki Base Requirements
package "php5"
--- /dev/null
+name "elasticsearch"
+description "Role applied to all elasticsearch servers"
+
+default_attributes(
+ :apt => {
+ :sources => ["elasticsearch"]
+ }
+)
+
+run_list(
+ "recipe[elasticsearch]"
+)
run_list(
"role[crm]",
+ "role[elasticsearch]",
"recipe[foundation::wiki]",
"recipe[foundation::board]"
)
"role[stats]",
"role[planet]",
"role[planetdump]",
+ "role[logstash]",
"recipe[rsyncd]",
"recipe[openvpn]",
"recipe[git::server]",
--- /dev/null
+name "logstash-forwarder"
+description "Role applied to all logstash forwarders"
+
+default_attributes(
+ :apt => {
+ :sources => ["logstash-forwarder"]
+ }
+)
+
+run_list(
+ "recipe[logstash::forwarder]"
+)
--- /dev/null
+name "logstash"
+description "Role applied to all logstash servers"
+
+default_attributes(
+ :apt => {
+ :sources => ["logstash"]
+ }
+)
+
+run_list(
+ "role[elasticsearch]",
+ "recipe[logstash]"
+)
:max_requests_per_child => 10000
}
},
+ :logstash => {
+ :forwarder => {
+ :files => [
+ { :paths => ["/var/log/apache2/access.log"], :fields => { :type => "apache" } },
+ { :paths => ["/var/log/web/rails-logstash.log"], :fields => { :type => "rails" } }
+ ]
+ }
+ },
:memcached => {
:memory_limit => 4096
},
run_list(
"role[web]",
+ "role[logstash-forwarder]",
"recipe[web::backend]"
)
:max_requests_per_child => 10000
}
},
+ :logstash => {
+ :forwarder => {
+ :files => [
+ { :paths => ["/var/log/apache2/access.log"], :fields => { :type => "apache" } },
+ { :paths => ["/var/log/web/rails-logstash.log"], :fields => { :type => "rails" } }
+ ]
+ }
+ },
:passenger => {
:max_pool_size => 50
},
run_list(
"role[web]",
+ "role[logstash-forwarder]",
"recipe[web::frontend]"
)
)
run_list(
+ "role[elasticsearch]",
"recipe[wiki]"
)
projects / chef.git / commitdiff