Switch to use filebeat for forwarding to logstash

diff --git a/cookbooks/apt/recipes/default.rb b/cookbooks/apt/recipes/default.rb
index 8f57b88c9283508ab7e4bb8032e09b7ad57ed949..51586d61b460dbff4f7fffa9a692191fe4c600b9 100644 (file)
--- a/cookbooks/apt/recipes/default.rb
+++ b/cookbooks/apt/recipes/default.rb
@@ -138,12 +138,6 @@ apt_source "logstash" do
   key "D27D666CD88E42B4"
 end
 
   key "D27D666CD88E42B4"
 end
 

-apt_source "logstash-forwarder" do
-  source_template "elasticsearch.list.erb"
-  url "https://packages.elasticsearch.org/logstashforwarder/debian"
-  key "D27D666CD88E42B4"
-end
-

 apt_source "passenger" do
   url "https://oss-binaries.phusionpassenger.com/apt/passenger"
   key "561F9B9CAC40B2F7"
 apt_source "passenger" do
   url "https://oss-binaries.phusionpassenger.com/apt/passenger"
   key "561F9B9CAC40B2F7"

diff --git a/cookbooks/logstash/.foodcritic b/cookbooks/logstash/.foodcritic
index 53c271a0fbce34c5a8eaf13bf256b108ec32abc0..56333367428e5d7b5c49453ba50d1624dcadd7ef 100644 (file)
--- a/cookbooks/logstash/.foodcritic
+++ b/cookbooks/logstash/.foodcritic
@@ -1,5 +1,6 @@
 ~FC001
 ~FC003
 ~FC001
 ~FC003

+~FC019

 ~FC064
 ~FC065
 ~FC066
 ~FC064
 ~FC065
 ~FC066

diff --git a/cookbooks/logstash/attributes/default.rb b/cookbooks/logstash/attributes/default.rb
index 9a56f3836440f6ca1aa43b665c743c77962b145d..4014b7207bc99a5e8e53f917901b44b3ffe5befa 100644 (file)
--- a/cookbooks/logstash/attributes/default.rb
+++ b/cookbooks/logstash/attributes/default.rb
@@ -1,3 +1,4 @@
-default[:logstash][:forwarder][:network][:servers] = ["logstash.openstreetmap.org:5043"]
-default[:logstash][:forwarder][:network][:"ssl ca"] = "/var/lib/logstash-forwarder/lumberjack.crt"
-default[:logstash][:forwarder][:files] = []
+default[:logstash][:forwarder]["output.logstash"]["hosts"] = ["logstash.openstreetmap.org:5044"]
+default[:logstash][:forwarder]["output.logstash"]["ssl.certificate_authorities"] = "/etc/filebeat/filebeat.crt"
+default[:logstash][:forwarder]["output.logstash"]["ssl.verification_mode"] = "none"
+default[:logstash][:forwarder]["filebeat.prospectors"] = []

diff --git a/cookbooks/logstash/recipes/forwarder.rb b/cookbooks/logstash/recipes/forwarder.rb
index d6956a8f7180ee9af39bf97f0f7a13aa59ce1d23..ea653c8ef5b02efd4c362509a14f3062a934d956 100644 (file)
--- a/cookbooks/logstash/recipes/forwarder.rb
+++ b/cookbooks/logstash/recipes/forwarder.rb
@@ -17,27 +17,39 @@
 # limitations under the License.
 #
 
 # limitations under the License.
 #
 

-require "json"
+service "logstash-forwarder" do
+  action [:disable, :stop]
+end
+
+file "/var/lib/logstash-forwarder/lumberjack.crt" do
+  action :delete
+end

 
 

-package "logstash-forwarder"
+package "logstash-forwarder" do
+  action :purge
+end
+
+require "yaml"

 
 

-cookbook_file "/var/lib/logstash-forwarder/lumberjack.crt" do
-  source "lumberjack.crt"
+package "filebeat"
+
+cookbook_file "/etc/filebeat/filebeat.crt" do
+  source "beats.crt"

   user "root"
   group "root"
   user "root"
   group "root"

-  mode 0o644
-  notifies :restart, "service[logstash-forwarder]"
+  mode 0o600
+  notifies :restart, "service[filebeat]"

 end
 
 end
 

-file "/etc/logstash-forwarder.conf" do
-  content JSON.pretty_generate(node[:logstash][:forwarder])
+file "/etc/filebeat/filebeat.yml" do
+  content YAML.dump(node[:logstash][:forwarder].to_hash)

   user "root"
   group "root"
   user "root"
   group "root"

-  mode 0o644
-  notifies :restart, "service[logstash-forwarder]"
+  mode 0o600
+  notifies :restart, "service[filebeat]"

 end
 
 end
 

-service "logstash-forwarder" do
+service "filebeat" do

   action [:enable, :start]
   supports :status => true, :restart => true
 end
   action [:enable, :start]
   supports :status => true, :restart => true
 end

diff --git a/roles/logstash-forwarder.rb b/roles/logstash-forwarder.rb
index 3d8ba3abe5d9e238a3bada312a55f68f153047a1..60b34a31189107fe906b3326d28ee3dad47d08b0 100644 (file)
--- a/roles/logstash-forwarder.rb
+++ b/roles/logstash-forwarder.rb
@@ -3,7 +3,7 @@ description "Role applied to all logstash forwarders"
 
 default_attributes(
   :apt => {
 
 default_attributes(
   :apt => {

-    # :sources => ["logstash-forwarder"]
+    :sources => ["elasticsearch5.x"]

   }
 )
 
   }
 )
 

diff --git a/roles/web-backend.rb b/roles/web-backend.rb
index 4c6dff43f312116773f55df48343e7fe6b78989d..d59ce812e00c9608aa0301b520725e70f95477b2 100644 (file)
--- a/roles/web-backend.rb
+++ b/roles/web-backend.rb
@@ -10,9 +10,9 @@ default_attributes(
   },
   :logstash => {
     :forwarder => {
   },
   :logstash => {
     :forwarder => {

-      :files => [
-        { :paths => ["/var/log/apache2/access.log"], :fields => { :type => "apache" } },
-        { :paths => ["/var/log/web/rails-logstash.log"], :fields => { :type => "rails" } }
+      "filebeat.prospectors" => [
+        { "input_type" => "log", "paths" => ["/var/log/apache2/access.log"], "fields" => { "type" => "apache" } },
+        { "input_type" => "log", "paths" => ["/var/log/web/rails-logstash.log"], "fields" => { "type" => "rails" } }

       ]
     }
   },
       ]
     }
   },

diff --git a/roles/web-frontend.rb b/roles/web-frontend.rb
index 4d82bd4f65c4fafa482a7cd50a90441336429ccd..5f23e0618361d3e7ee76c300cc3821b42b808eb3 100644 (file)
--- a/roles/web-frontend.rb
+++ b/roles/web-frontend.rb
@@ -15,9 +15,9 @@ default_attributes(
   },
   :logstash => {
     :forwarder => {
   },
   :logstash => {
     :forwarder => {

-      :files => [
-        { :paths => ["/var/log/apache2/access.log"], :fields => { :type => "apache" } },
-        { :paths => ["/var/log/web/rails-logstash.log"], :fields => { :type => "rails" } }
+      "filebeat.prospectors" => [
+        { "input_type" => "log", "paths" => ["/var/log/apache2/access.log"], "fields" => { "type" => "apache" } },
+        { "input_type" => "log", "paths" => ["/var/log/web/rails-logstash.log"], "fields" => { "type" => "rails" } }

       ]
     }
   },
       ]
     }
   },