From cbf4196ce80c8b57a04aff7cd6b881671271b2c5 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Mon, 24 Feb 2020 21:34:03 +0000 Subject: [PATCH] Add tests for elasticsearch, kibana and logstash cookbooks --- .github/workflows/test-kitchen.yml | 4 ++++ .kitchen.yml | 20 ++++++++++++++++++ cookbooks/elasticsearch/attributes/default.rb | 3 ++- cookbooks/elasticsearch/metadata.rb | 1 + cookbooks/elasticsearch/recipes/default.rb | 8 +++---- cookbooks/kibana/attributes/default.rb | 2 ++ cookbooks/kibana/metadata.rb | 1 + cookbooks/kibana/recipes/default.rb | 3 ++- cookbooks/logstash/attributes/default.rb | 2 ++ cookbooks/logstash/metadata.rb | 2 ++ cookbooks/logstash/recipes/default.rb | 4 ++-- cookbooks/logstash/recipes/forwarder.rb | 2 ++ roles/kibana.rb | 8 ------- roles/logstash.rb | 6 ------ test/data_bags/accounts/kibana.json | 6 ++++++ test/data_bags/logstash/keys.json | 11 ++++++++++ .../serverspec/elasticsearch_spec.rb | 17 +++++++++++++++ .../kibana/serverspec/apache_spec.rb | 21 +++++++++++++++++++ .../serverspec/filebeat_spec.rb | 13 ++++++++++++ .../logstash/serverspec/elasticsearch_spec.rb | 17 +++++++++++++++ .../logstash/serverspec/logstash_spec.rb | 17 +++++++++++++++ 21 files changed, 146 insertions(+), 22 deletions(-) create mode 100644 test/data_bags/accounts/kibana.json create mode 100644 test/data_bags/logstash/keys.json create mode 100644 test/integration/elasticsearch/serverspec/elasticsearch_spec.rb create mode 100644 test/integration/kibana/serverspec/apache_spec.rb create mode 100644 test/integration/logstash-forwarder/serverspec/filebeat_spec.rb create mode 100644 test/integration/logstash/serverspec/elasticsearch_spec.rb create mode 100644 test/integration/logstash/serverspec/logstash_spec.rb diff --git a/.github/workflows/test-kitchen.yml b/.github/workflows/test-kitchen.yml index 5345c0ed3..7cb3e874a 100644 --- a/.github/workflows/test-kitchen.yml +++ b/.github/workflows/test-kitchen.yml @@ -26,6 +26,7 @@ jobs: - dhcpd - dmca - donate + - elasticsearch - exim - fail2ban - forum @@ -39,7 +40,10 @@ jobs: - hardware - hot - incron + - kibana - letsencrypt + - logstash + - logstash-forwarder - mailman - memcached - munin diff --git a/.kitchen.yml b/.kitchen.yml index 4195d7e0d..c21b1a177 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -93,6 +93,9 @@ suites: - name: donate run_list: - recipe[donate::default] + - name: elasticsearch + run_list: + - recipe[elasticsearch::default] - name: exim run_list: - recipe[exim::default] @@ -138,6 +141,9 @@ suites: - name: incron run_list: - recipe[incron::default] + - name: kibana + run_list: + - recipe[kibana::default] - name: letsencrypt run_list: - recipe[letsencrypt::default] @@ -145,6 +151,20 @@ suites: apt: sources: - openstreetmap + - name: logstash + run_list: + - recipe[logstash::default] + - name: logstash-forwarder + run_list: + - recipe[logstash::forwarder] + attributes: + logstash: + forwarder: + filebeat.prospectors: + - input_type: log + paths: + - /var/log/apache2/access.log + type: apache - name: mailman run_list: - recipe[mailman::default] diff --git a/cookbooks/elasticsearch/attributes/default.rb b/cookbooks/elasticsearch/attributes/default.rb index a2a643412..1de57c968 100644 --- a/cookbooks/elasticsearch/attributes/default.rb +++ b/cookbooks/elasticsearch/attributes/default.rb @@ -1,4 +1,5 @@ -default[:elasticsearch][:version] = "1.7" +default[:elasticsearch][:version] = "6.x" +default[:elasticsearch][:cluster][:name] = "default" default[:elasticsearch][:cluster][:routing][:allocation][:disk][:watermark][:low] = "85%" default[:elasticsearch][:cluster][:routing][:allocation][:disk][:watermark][:high] = "90%" default[:elasticsearch][:cluster][:routing][:allocation][:disk][:watermark][:flood_stage] = "95%" diff --git a/cookbooks/elasticsearch/metadata.rb b/cookbooks/elasticsearch/metadata.rb index 9bab57ae2..37f38ee1a 100644 --- a/cookbooks/elasticsearch/metadata.rb +++ b/cookbooks/elasticsearch/metadata.rb @@ -6,3 +6,4 @@ description "Installs and configures a elasticsearch server" version "1.0.0" supports "ubuntu" +depends "apt" diff --git a/cookbooks/elasticsearch/recipes/default.rb b/cookbooks/elasticsearch/recipes/default.rb index d2ba1d31b..7feaa4a4c 100644 --- a/cookbooks/elasticsearch/recipes/default.rb +++ b/cookbooks/elasticsearch/recipes/default.rb @@ -17,10 +17,10 @@ # limitations under the License. # -package %w[ - default-jre-headless - elasticsearch -] +include_recipe "apt" + +package "default-jre-headless" +package "elasticsearch" template "/etc/elasticsearch/elasticsearch.yml" do source "elasticsearch.yml.erb" diff --git a/cookbooks/kibana/attributes/default.rb b/cookbooks/kibana/attributes/default.rb index 46dda49ce..bb431975a 100644 --- a/cookbooks/kibana/attributes/default.rb +++ b/cookbooks/kibana/attributes/default.rb @@ -1,2 +1,4 @@ default[:kibana][:version] = "4.1.1" default[:kibana][:sites] = {} + +default[:accounts][:users][:kibana][:status] = :role diff --git a/cookbooks/kibana/metadata.rb b/cookbooks/kibana/metadata.rb index 83c575e9b..ec3e6f4c6 100644 --- a/cookbooks/kibana/metadata.rb +++ b/cookbooks/kibana/metadata.rb @@ -6,5 +6,6 @@ description "Installs and configures a kibana server" version "1.0.0" supports "ubuntu" +depends "accounts" depends "apache" depends "systemd" diff --git a/cookbooks/kibana/recipes/default.rb b/cookbooks/kibana/recipes/default.rb index 8f20eb523..ceeaf13df 100644 --- a/cookbooks/kibana/recipes/default.rb +++ b/cookbooks/kibana/recipes/default.rb @@ -19,6 +19,7 @@ require "yaml" +include_recipe "accounts" include_recipe "apache" apache_module "proxy_http" @@ -26,7 +27,7 @@ apache_module "proxy_http" version = node[:kibana][:version] remote_file "#{Chef::Config[:file_cache_path]}/kibana-#{version}.tar.gz" do - source "https://download.elastic.co/kibana/kibana/kibana-4.1.1-linux-x64.tar.gz" + source "https://download.elastic.co/kibana/kibana/kibana-#{version}-linux-x64.tar.gz" not_if { ::File.exist?("/opt/kibana-#{version}/bin/kibana") } end diff --git a/cookbooks/logstash/attributes/default.rb b/cookbooks/logstash/attributes/default.rb index 4014b7207..ff9cace3e 100644 --- a/cookbooks/logstash/attributes/default.rb +++ b/cookbooks/logstash/attributes/default.rb @@ -2,3 +2,5 @@ default[:logstash][:forwarder]["output.logstash"]["hosts"] = ["logstash.openstre default[:logstash][:forwarder]["output.logstash"]["ssl.certificate_authorities"] = "/etc/filebeat/filebeat.crt" default[:logstash][:forwarder]["output.logstash"]["ssl.verification_mode"] = "none" default[:logstash][:forwarder]["filebeat.prospectors"] = [] + +default[:elasticsearch][:cluster][:name] = "logstash" diff --git a/cookbooks/logstash/metadata.rb b/cookbooks/logstash/metadata.rb index e13ece1b0..a1a7d387b 100644 --- a/cookbooks/logstash/metadata.rb +++ b/cookbooks/logstash/metadata.rb @@ -6,4 +6,6 @@ description "Installs and configures a elasticsearch server" version "1.0.0" supports "ubuntu" +depends "apt" +depends "elasticsearch" depends "networking" diff --git a/cookbooks/logstash/recipes/default.rb b/cookbooks/logstash/recipes/default.rb index babff56df..e58df226b 100644 --- a/cookbooks/logstash/recipes/default.rb +++ b/cookbooks/logstash/recipes/default.rb @@ -17,6 +17,7 @@ # limitations under the License. # +include_recipe "elasticsearch" include_recipe "networking" keys = data_bag_item("logstash", "keys") @@ -47,7 +48,7 @@ template "/etc/logstash/conf.d/chef.conf" do user "root" group "root" mode 0o644 - notifies :reload, "service[logstash]" + notifies :start, "service[logstash]" end file "/etc/logrotate.d/logstash" do @@ -64,7 +65,6 @@ end service "logstash" do action [:enable, :start] - supports :status => true, :restart => true, :reload => true end template "/etc/cron.daily/expire-logstash" do diff --git a/cookbooks/logstash/recipes/forwarder.rb b/cookbooks/logstash/recipes/forwarder.rb index 14866558b..c87b9edf1 100644 --- a/cookbooks/logstash/recipes/forwarder.rb +++ b/cookbooks/logstash/recipes/forwarder.rb @@ -19,6 +19,8 @@ require "yaml" +include_recipe "apt" + package "filebeat" cookbook_file "/etc/filebeat/filebeat.crt" do diff --git a/roles/kibana.rb b/roles/kibana.rb index 6198120cf..d45d4e53f 100644 --- a/roles/kibana.rb +++ b/roles/kibana.rb @@ -1,14 +1,6 @@ name "kibana" description "Role applied to all kibana servers" -default_attributes( - :accounts => { - :users => { - :kibana => { :status => :role } - } - } -) - run_list( "recipe[kibana]" ) diff --git a/roles/logstash.rb b/roles/logstash.rb index 9e421fc8d..b5cbac836 100644 --- a/roles/logstash.rb +++ b/roles/logstash.rb @@ -2,12 +2,6 @@ name "logstash" description "Role applied to all logstash servers" default_attributes( - :elasticsearch => { - :cluster => { - :name => "logstash" - }, - :version => "6.x" - }, :kibana => { :sites => { :logstash => { diff --git a/test/data_bags/accounts/kibana.json b/test/data_bags/accounts/kibana.json new file mode 100644 index 000000000..1d2fc538b --- /dev/null +++ b/test/data_bags/accounts/kibana.json @@ -0,0 +1,6 @@ +{ + "id": "kibana", + "uid": "521", + "comment": "Kibana", + "manage_home": false +} diff --git a/test/data_bags/logstash/keys.json b/test/data_bags/logstash/keys.json new file mode 100644 index 000000000..69f7fe54c --- /dev/null +++ b/test/data_bags/logstash/keys.json @@ -0,0 +1,11 @@ +{ + "id": "keys", + "lumberjack": [ + "-----BEGIN PRIVATE KEY-----", + "-----END PRIVATE KEY-----" + ], + "beats": [ + "-----BEGIN PRIVATE KEY-----", + "-----END PRIVATE KEY-----" + ] +} diff --git a/test/integration/elasticsearch/serverspec/elasticsearch_spec.rb b/test/integration/elasticsearch/serverspec/elasticsearch_spec.rb new file mode 100644 index 000000000..b4e9fa77b --- /dev/null +++ b/test/integration/elasticsearch/serverspec/elasticsearch_spec.rb @@ -0,0 +1,17 @@ +require "serverspec" + +# Required by serverspec +set :backend, :exec + +describe package("elasticsearch") do + it { should be_installed } +end + +describe service("elasticsearch") do + it { should be_enabled } + it { should be_running } +end + +# describe port(9200) do +# it { should be_listening.with("tcp") } +# end diff --git a/test/integration/kibana/serverspec/apache_spec.rb b/test/integration/kibana/serverspec/apache_spec.rb new file mode 100644 index 000000000..446d3b915 --- /dev/null +++ b/test/integration/kibana/serverspec/apache_spec.rb @@ -0,0 +1,21 @@ +require "serverspec" + +# Required by serverspec +set :backend, :exec + +describe package("apache2") do + it { should be_installed } +end + +describe service("apache2") do + it { should be_enabled } + it { should be_running } +end + +describe port(80) do + it { should be_listening.with("tcp") } +end + +describe port(443) do + it { should be_listening.with("tcp") } +end diff --git a/test/integration/logstash-forwarder/serverspec/filebeat_spec.rb b/test/integration/logstash-forwarder/serverspec/filebeat_spec.rb new file mode 100644 index 000000000..6ac089acf --- /dev/null +++ b/test/integration/logstash-forwarder/serverspec/filebeat_spec.rb @@ -0,0 +1,13 @@ +require "serverspec" + +# Required by serverspec +set :backend, :exec + +describe package("filebeat") do + it { should be_installed } +end + +describe service("filebeat") do + it { should be_enabled } + it { should be_running } +end diff --git a/test/integration/logstash/serverspec/elasticsearch_spec.rb b/test/integration/logstash/serverspec/elasticsearch_spec.rb new file mode 100644 index 000000000..b4e9fa77b --- /dev/null +++ b/test/integration/logstash/serverspec/elasticsearch_spec.rb @@ -0,0 +1,17 @@ +require "serverspec" + +# Required by serverspec +set :backend, :exec + +describe package("elasticsearch") do + it { should be_installed } +end + +describe service("elasticsearch") do + it { should be_enabled } + it { should be_running } +end + +# describe port(9200) do +# it { should be_listening.with("tcp") } +# end diff --git a/test/integration/logstash/serverspec/logstash_spec.rb b/test/integration/logstash/serverspec/logstash_spec.rb new file mode 100644 index 000000000..c339de542 --- /dev/null +++ b/test/integration/logstash/serverspec/logstash_spec.rb @@ -0,0 +1,17 @@ +require "serverspec" + +# Required by serverspec +set :backend, :exec + +describe package("logstash") do + it { should be_installed } +end + +describe service("logstash") do + it { should be_enabled } + it { should be_running } +end + +# describe port(5044) do +# it { should be_listening.with("tcp") } +# end -- 2.43.2