From ee3840f506ead7827de2dcbf0746ef8c06da2058 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Sun, 31 Aug 2025 22:25:26 +0100 Subject: [PATCH] Remove unused logstash support --- .github/workflows/test-kitchen.yml | 2 - .kitchen.yml | 17 ---- cookbooks/logstash/README.md | 3 - cookbooks/logstash/attributes/default.rb | 6 -- cookbooks/logstash/files/default/beats.crt | 22 ----- .../logstash/files/default/lumberjack.crt | 22 ----- cookbooks/logstash/metadata.rb | 11 --- cookbooks/logstash/recipes/default.rb | 94 ------------------- cookbooks/logstash/recipes/forwarder.rb | 45 --------- .../logstash/templates/default/expire.erb | 7 -- .../templates/default/logstash.conf.erb | 79 ---------------- .../templates/default/logstash.default.erb | 13 --- cookbooks/web/recipes/rails.rb | 1 - cookbooks/web/resources/rails_port.rb | 6 -- roles/logstash-forwarder.rb | 6 -- roles/logstash.rb | 20 ---- .../logstash/inspec/elasticsearch_spec.rb | 13 --- .../logstash/inspec/logstash_spec.rb | 13 --- 18 files changed, 380 deletions(-) delete mode 100644 cookbooks/logstash/README.md delete mode 100644 cookbooks/logstash/attributes/default.rb delete mode 100644 cookbooks/logstash/files/default/beats.crt delete mode 100644 cookbooks/logstash/files/default/lumberjack.crt delete mode 100644 cookbooks/logstash/metadata.rb delete mode 100644 cookbooks/logstash/recipes/default.rb delete mode 100644 cookbooks/logstash/recipes/forwarder.rb delete mode 100644 cookbooks/logstash/templates/default/expire.erb delete mode 100644 cookbooks/logstash/templates/default/logstash.conf.erb delete mode 100644 cookbooks/logstash/templates/default/logstash.default.erb delete mode 100644 roles/logstash-forwarder.rb delete mode 100644 roles/logstash.rb delete mode 100644 test/integration/logstash/inspec/elasticsearch_spec.rb delete mode 100644 test/integration/logstash/inspec/logstash_spec.rb diff --git a/.github/workflows/test-kitchen.yml b/.github/workflows/test-kitchen.yml index e332c9260..c56c9bee7 100644 --- a/.github/workflows/test-kitchen.yml +++ b/.github/workflows/test-kitchen.yml @@ -64,8 +64,6 @@ jobs: - irc - kibana - letsencrypt - - logstash - - logstash-forwarder - mail - mailman - matomo diff --git a/.kitchen.yml b/.kitchen.yml index eb3629b8f..22d350799 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -253,23 +253,6 @@ suites: - name: letsencrypt run_list: - recipe[letsencrypt::default] - - name: logstash - run_list: - - recipe[logstash::default] - - name: logstash-forwarder - run_list: - - recipe[logstash::forwarder] - attributes: - logstash: - forwarder: - filebeat.inputs: - - type: filestream - id: apache - paths: - - /var/log/apache2/access.log - fields: - type: apache - fields_under_root: true - name: mail run_list: - role[mail] diff --git a/cookbooks/logstash/README.md b/cookbooks/logstash/README.md deleted file mode 100644 index 7bcc94a9d..000000000 --- a/cookbooks/logstash/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Logstash Cookbook - -This cookbook installs Logstash, which takes events and stores them in elasticsearch. diff --git a/cookbooks/logstash/attributes/default.rb b/cookbooks/logstash/attributes/default.rb deleted file mode 100644 index 1b97ce335..000000000 --- a/cookbooks/logstash/attributes/default.rb +++ /dev/null @@ -1,6 +0,0 @@ -default[:logstash][:forwarder]["output.logstash"]["hosts"] = ["logstash.openstreetmap.org:5044"] -default[:logstash][:forwarder]["output.logstash"]["ssl.certificate_authorities"] = "/etc/filebeat/filebeat.crt" -default[:logstash][:forwarder]["output.logstash"]["ssl.verification_mode"] = "none" -default[:logstash][:forwarder]["filebeat.inputs"] = [] - -default[:elasticsearch][:cluster][:name] = "logstash" diff --git a/cookbooks/logstash/files/default/beats.crt b/cookbooks/logstash/files/default/beats.crt deleted file mode 100644 index a1db9f8e8..000000000 --- a/cookbooks/logstash/files/default/beats.crt +++ /dev/null @@ -1,22 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDoTCCAomgAwIBAgIJAOWS1wQ8fa1RMA0GCSqGSIb3DQEBCwUAMGcxCzAJBgNV -BAYTAkdCMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg -Q29tcGFueSBMdGQxIzAhBgNVBAMMGmxvZ3N0YXNoLm9wZW5zdHJlZXRtYXAub3Jn -MB4XDTE1MDgyODA3NDYzNloXDTI1MDgyNTA3NDYzNlowZzELMAkGA1UEBhMCR0Ix -FTATBgNVBAcMDERlZmF1bHQgQ2l0eTEcMBoGA1UECgwTRGVmYXVsdCBDb21wYW55 -IEx0ZDEjMCEGA1UEAwwabG9nc3Rhc2gub3BlbnN0cmVldG1hcC5vcmcwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCxU5CoykubpFaYdJMJOGFmq8a3bMLx -KG9wlkDfYH65fxh8W+XAHR0oyi3kwqy9P1OmicdMJkFRpFsIDmg0EuirZCX2A4nw -ADJxXa/mqbFTNCHmVjhFqMdAaMW/O0WkcXxLc/9D/WLFEqTNMkZwZ1wo6JwAo26f -Dxn75TggFy0rTBYsOp87nK3fYEp8cY43tGWkqMbTdU2GB511+FeUbm7NTyQakHq8 -9/LtmJPzmsO30wJzF++NOJkis/xNTvPPybkljwSshoo53ed/kmxy/KiVgPqD/fR7 -8bkOfqknycyqV5zskMUtrN9PsWQx3bzY7dhYo1nNMd8oNLVKpSluvga5AgMBAAGj -UDBOMB0GA1UdDgQWBBS5qcKaMediosEUc6SHDGgTfGnpRzAfBgNVHSMEGDAWgBS5 -qcKaMediosEUc6SHDGgTfGnpRzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUA -A4IBAQA1G0BJ6rxyyuvY4iLtyah0p7hawt7jkQ35Y6gNoGGwHgMGslQ8URhx29zr -HpJPtCdrHagC58FmL74/SS8czNdt+V3xcYAAZxm3ZwlJr/GyxSlCD/3zpD/eWHTV -lMFqvGqLGxJfo9Q1iuyV35jEfi84toXT+zps/4voX6fYutNaouQDk5V/up9lPPCp -x2xq0vVDV+BhZ3Y7QxsZHEcAqPg9wGMT4UiJFSsatwnmSwSv429tdZeETEb3yCZv -kautRvtRRMB2QqNQRFGXmJLt4sMFuAjo5jcz4GvBWZTjPOoFrgnhlmWkCvqWrxJU -/CA3EQf+gGw2loeKZrxbKXdSuIPs ------END CERTIFICATE----- diff --git a/cookbooks/logstash/files/default/lumberjack.crt b/cookbooks/logstash/files/default/lumberjack.crt deleted file mode 100644 index a1db9f8e8..000000000 --- a/cookbooks/logstash/files/default/lumberjack.crt +++ /dev/null @@ -1,22 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDoTCCAomgAwIBAgIJAOWS1wQ8fa1RMA0GCSqGSIb3DQEBCwUAMGcxCzAJBgNV -BAYTAkdCMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg -Q29tcGFueSBMdGQxIzAhBgNVBAMMGmxvZ3N0YXNoLm9wZW5zdHJlZXRtYXAub3Jn -MB4XDTE1MDgyODA3NDYzNloXDTI1MDgyNTA3NDYzNlowZzELMAkGA1UEBhMCR0Ix -FTATBgNVBAcMDERlZmF1bHQgQ2l0eTEcMBoGA1UECgwTRGVmYXVsdCBDb21wYW55 -IEx0ZDEjMCEGA1UEAwwabG9nc3Rhc2gub3BlbnN0cmVldG1hcC5vcmcwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCxU5CoykubpFaYdJMJOGFmq8a3bMLx -KG9wlkDfYH65fxh8W+XAHR0oyi3kwqy9P1OmicdMJkFRpFsIDmg0EuirZCX2A4nw -ADJxXa/mqbFTNCHmVjhFqMdAaMW/O0WkcXxLc/9D/WLFEqTNMkZwZ1wo6JwAo26f -Dxn75TggFy0rTBYsOp87nK3fYEp8cY43tGWkqMbTdU2GB511+FeUbm7NTyQakHq8 -9/LtmJPzmsO30wJzF++NOJkis/xNTvPPybkljwSshoo53ed/kmxy/KiVgPqD/fR7 -8bkOfqknycyqV5zskMUtrN9PsWQx3bzY7dhYo1nNMd8oNLVKpSluvga5AgMBAAGj -UDBOMB0GA1UdDgQWBBS5qcKaMediosEUc6SHDGgTfGnpRzAfBgNVHSMEGDAWgBS5 -qcKaMediosEUc6SHDGgTfGnpRzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUA -A4IBAQA1G0BJ6rxyyuvY4iLtyah0p7hawt7jkQ35Y6gNoGGwHgMGslQ8URhx29zr -HpJPtCdrHagC58FmL74/SS8czNdt+V3xcYAAZxm3ZwlJr/GyxSlCD/3zpD/eWHTV -lMFqvGqLGxJfo9Q1iuyV35jEfi84toXT+zps/4voX6fYutNaouQDk5V/up9lPPCp -x2xq0vVDV+BhZ3Y7QxsZHEcAqPg9wGMT4UiJFSsatwnmSwSv429tdZeETEb3yCZv -kautRvtRRMB2QqNQRFGXmJLt4sMFuAjo5jcz4GvBWZTjPOoFrgnhlmWkCvqWrxJU -/CA3EQf+gGw2loeKZrxbKXdSuIPs ------END CERTIFICATE----- diff --git a/cookbooks/logstash/metadata.rb b/cookbooks/logstash/metadata.rb deleted file mode 100644 index a1a7d387b..000000000 --- a/cookbooks/logstash/metadata.rb +++ /dev/null @@ -1,11 +0,0 @@ -name "logstash" -maintainer "OpenStreetMap Administrators" -maintainer_email "admins@openstreetmap.org" -license "Apache-2.0" -description "Installs and configures a elasticsearch server" - -version "1.0.0" -supports "ubuntu" -depends "apt" -depends "elasticsearch" -depends "networking" diff --git a/cookbooks/logstash/recipes/default.rb b/cookbooks/logstash/recipes/default.rb deleted file mode 100644 index a4b237c00..000000000 --- a/cookbooks/logstash/recipes/default.rb +++ /dev/null @@ -1,94 +0,0 @@ -# -# Cookbook:: logstash -# Recipe:: default -# -# Copyright:: 2015, OpenStreetMap Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include_recipe "elasticsearch" -include_recipe "networking" - -keys = data_bag_item("logstash", "keys") - -package %w[ - openjdk-11-jre-headless - logstash - logrotate -] - -cookbook_file "/var/lib/logstash/beats.crt" do - source "beats.crt" - user "root" - group "logstash" - mode "644" - notifies :restart, "service[logstash]" -end - -file "/var/lib/logstash/beats.key" do - content keys["beats"].join("\n") - user "root" - group "logstash" - mode "640" - notifies :restart, "service[logstash]" -end - -template "/etc/logstash/conf.d/chef.conf" do - source "logstash.conf.erb" - user "root" - group "root" - mode "644" - notifies :start, "service[logstash]" -end - -file "/etc/logrotate.d/logstash" do - mode "644" -end - -template "/etc/default/logstash" do - source "logstash.default.erb" - user "root" - group "root" - mode "644" - notifies :restart, "service[logstash]" -end - -service "logstash" do - action [:enable, :start] -end - -template "/etc/cron.daily/expire-logstash" do - source "expire.erb" - owner "root" - group "root" - mode "755" -end - -forwarders = search(:node, "recipes:logstash\\:\\:forwarder").map do |forwarder| - forwarder.ipaddresses(:role => :external) -end - -search(:node, "roles:gateway").each do |forwarder| - forwarders.append(forwarder.ipaddresses(:role => :external)) -end - -firewall_rule "accept-logstash" do - action :accept - context :incoming - protocol :tcp - source forwarders - dest_ports %w[5043 5044] - source_ports "1024-65535" - not_if { forwarders.empty? } -end diff --git a/cookbooks/logstash/recipes/forwarder.rb b/cookbooks/logstash/recipes/forwarder.rb deleted file mode 100644 index 2184da048..000000000 --- a/cookbooks/logstash/recipes/forwarder.rb +++ /dev/null @@ -1,45 +0,0 @@ -# -# Cookbook:: logstash -# Recipe:: default -# -# Copyright:: 2015, OpenStreetMap Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require "yaml" - -include_recipe "apt::elasticsearch8" - -package "filebeat" - -cookbook_file "/etc/filebeat/filebeat.crt" do - source "beats.crt" - user "root" - group "root" - mode "600" - notifies :restart, "service[filebeat]" -end - -file "/etc/filebeat/filebeat.yml" do - content YAML.dump(node[:logstash][:forwarder].to_hash) - user "root" - group "root" - mode "600" - notifies :restart, "service[filebeat]" -end - -service "filebeat" do - action [:enable, :start] - supports :status => true, :restart => true -end diff --git a/cookbooks/logstash/templates/default/expire.erb b/cookbooks/logstash/templates/default/expire.erb deleted file mode 100644 index 582351933..000000000 --- a/cookbooks/logstash/templates/default/expire.erb +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -# DO NOT EDIT - This file is being maintained by Chef - -D=$(date --date="3 days ago" +"%Y.%m.%d") - -curl -s -XDELETE "http://localhost:9200/logstash-${D}" diff --git a/cookbooks/logstash/templates/default/logstash.conf.erb b/cookbooks/logstash/templates/default/logstash.conf.erb deleted file mode 100644 index cbc8c909b..000000000 --- a/cookbooks/logstash/templates/default/logstash.conf.erb +++ /dev/null @@ -1,79 +0,0 @@ -input { - beats { - port => 5044 - ssl => true - ssl_certificate => "/var/lib/logstash/beats.crt" - ssl_key => "/var/lib/logstash/beats.key" - } -} - -filter { - if [type] == "apache" { - grok { - match => [ "message", "%{COMBINEDAPACHELOG} %{NUMBER:duration:int}us %{NOTSPACE:request_id} %{NOTSPACE:ssl_protocol} %{NOTSPACE:ssl_cipher}" ] - } - date { - match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ] - } - if [agent] == "-" { - mutate { - remove_field => [ "agent" ] - } - } else { - useragent { - source => "agent" - target => "useragent" - } - grok { - match => { "agent" => "%{JOSM:[useragent][name]=JOSM}/%{POSINT:[useragent][major]}\.%{POSINT:[useragent][minor]} \(%{POSINT:[useragent][patch]} \w+\) " } - overwrite => [ "[useragent][name]", "[useragent][major]", "[useragent][minor]", "[useragent][patch]" ] - tag_on_failure => [] - } - mutate { - rename => { "agent" => "[useragent][raw]" } - } - } - } else if [type] == "rails" { - json { - source => "message" - remove_field => [ - "message", - "[parameters][authenticity_token]", - "[parameters][pass_crypt]", - "[parameters][pass_crypt_confirmation]", - "[parameters][utf8]" - ] - } - if [duration] { - ruby { - code => "event['duration'] = Integer(event['duration'] * 1000000)" - } - } - if [db] { - ruby { - code => "event['db'] = Integer(event['db'] * 1000000)" - } - } - if [view] { - ruby { - code => "event['view'] = Integer(event['view'] * 1000000)" - } - } - } - - if [host] =~ /^spike-/ { - mutate { - add_tag => [ "frontend" ] - } - } else if [host] =~ /^thorn-/ { - mutate { - add_tag => [ "backend" ] - } - } -} - -output { - elasticsearch { - hosts => [ "127.0.0.1" ] - } -} diff --git a/cookbooks/logstash/templates/default/logstash.default.erb b/cookbooks/logstash/templates/default/logstash.default.erb deleted file mode 100644 index ce376aff5..000000000 --- a/cookbooks/logstash/templates/default/logstash.default.erb +++ /dev/null @@ -1,13 +0,0 @@ -# DO NOT EDIT - This file is being maintained by Chef - -JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64" -LS_HOME="/usr/share/logstash" -LS_SETTINGS_DIR="/etc/logstash" -LS_PIDFILE="/var/run/logstash.pid" -LS_USER="logstash" -LS_GROUP="logstash" -LS_GC_LOG_FILE="/var/log/logstash/gc.log" -LS_OPEN_FILES="16384" -LS_NICE="19" -SERVICE_NAME="logstash" -SERVICE_DESCRIPTION="logstash" diff --git a/cookbooks/web/recipes/rails.rb b/cookbooks/web/recipes/rails.rb index fc9db249b..e21a988e2 100644 --- a/cookbooks/web/recipes/rails.rb +++ b/cookbooks/web/recipes/rails.rb @@ -107,7 +107,6 @@ rails_port "www.openstreetmap.org" do status node[:web][:status] messages_domain "messages.openstreetmap.org" log_path "#{node[:web][:log_directory]}/rails.log" - logstash_path "#{node[:web][:log_directory]}/rails-logstash.log" memcache_servers node[:web][:memcached_servers] potlatch2_key web_passwords["potlatch2_key"] id_key web_passwords["id_key"] diff --git a/cookbooks/web/resources/rails_port.rb b/cookbooks/web/resources/rails_port.rb index 3050c44de..b359711ff 100644 --- a/cookbooks/web/resources/rails_port.rb +++ b/cookbooks/web/resources/rails_port.rb @@ -46,7 +46,6 @@ property :messages_domain, String property :gpx_dir, String property :attachments_dir, String property :log_path, String -property :logstash_path, String property :memcache_servers, Array property :potlatch2_key, String property :id_key, String @@ -207,10 +206,6 @@ action :create do line.gsub!(/^( *)#log_path:.*$/, "\\1log_path: \"#{new_resource.log_path}\"") end - if new_resource.logstash_path - line.gsub!(/^( *)#logstash_path:.*$/, "\\1logstash_path: \"#{new_resource.logstash_path}\"") - end - if new_resource.memcache_servers line.gsub!(/^( *)#memcache_servers:.*$/, "\\1memcache_servers: [ \"#{new_resource.memcache_servers.join('", "')}\" ]") end @@ -313,7 +308,6 @@ action :create do "messages_domain", "attachments_dir", "log_path", - "logstash_path", "potlatch2_key", "id_key", "id_application", diff --git a/roles/logstash-forwarder.rb b/roles/logstash-forwarder.rb deleted file mode 100644 index bb0b777f3..000000000 --- a/roles/logstash-forwarder.rb +++ /dev/null @@ -1,6 +0,0 @@ -name "logstash-forwarder" -description "Role applied to all logstash forwarders" - -run_list( - "recipe[logstash::forwarder]" -) diff --git a/roles/logstash.rb b/roles/logstash.rb deleted file mode 100644 index b5cbac836..000000000 --- a/roles/logstash.rb +++ /dev/null @@ -1,20 +0,0 @@ -name "logstash" -description "Role applied to all logstash servers" - -default_attributes( - :kibana => { - :sites => { - :logstash => { - :site => "logstash.openstreetmap.org", - :port => 5601, - :elasticsearch_url => "http://127.0.0.1:9200/" - } - } - } -) - -run_list( - "role[elasticsearch]", - "role[kibana]", - "recipe[logstash]" -) diff --git a/test/integration/logstash/inspec/elasticsearch_spec.rb b/test/integration/logstash/inspec/elasticsearch_spec.rb deleted file mode 100644 index 5977b2393..000000000 --- a/test/integration/logstash/inspec/elasticsearch_spec.rb +++ /dev/null @@ -1,13 +0,0 @@ -describe package("elasticsearch") do - it { should be_installed } -end - -describe service("elasticsearch") do - it { should be_enabled } - it { should be_running } -end - -# describe port(9200) do -# it { should be_listening } -# its("protocols") { should cmp "tcp" } -# end diff --git a/test/integration/logstash/inspec/logstash_spec.rb b/test/integration/logstash/inspec/logstash_spec.rb deleted file mode 100644 index 0ebfe55d2..000000000 --- a/test/integration/logstash/inspec/logstash_spec.rb +++ /dev/null @@ -1,13 +0,0 @@ -describe package("logstash") do - it { should be_installed } -end - -describe service("logstash") do - it { should be_enabled } - it { should be_running } -end - -# describe port(5044) do -# it { should be_listening } -# its("protocols") { should cmp "tcp" } -# end -- 2.39.5