From be5f5bf83260f6fe16c253bb2069b34bd7421549 Mon Sep 17 00:00:00 2001
From: Tom Hughes <tom@compton.nu>
Date: Wed, 1 Apr 2015 00:10:34 +0100
Subject: [PATCH] Add an api-statistics daemon to gather data from apache logs

---
 .rubocop.yml                                  |  1 +
 cookbooks/web/recipes/rails.rb                | 24 +++++++++++
 .../web/templates/default/api-statistics.erb  | 42 +++++++++++++++++++
 .../templates/default/api-statistics.init.erb | 24 +++++++++++
 4 files changed, 91 insertions(+)
 create mode 100644 cookbooks/web/templates/default/api-statistics.erb
 create mode 100644 cookbooks/web/templates/default/api-statistics.init.erb

diff --git a/.rubocop.yml b/.rubocop.yml
index 1f5c531bf..b6fd6fcc3 100644
--- a/.rubocop.yml
+++ b/.rubocop.yml
@@ -8,6 +8,7 @@ Style/FileName:
   Exclude:
     - 'cookbooks/trac/files/default/trac-authenticate'
     - 'cookbooks/planet/files/default/replication-bin/replicate-changesets'
+    - 'cookbooks/*/templates/*/*.erb'
     - 'hooks/*'
     - 'roles/*.rb'
 
diff --git a/cookbooks/web/recipes/rails.rb b/cookbooks/web/recipes/rails.rb
index 926ff38ce..52a49aee8 100644
--- a/cookbooks/web/recipes/rails.rb
+++ b/cookbooks/web/recipes/rails.rb
@@ -66,3 +66,27 @@ rails_port "www.openstreetmap.org" do
   oauth_key web_passwords["oauth_key"]
   piwik_configuration piwik_configuration
 end
+
+gem_package "apachelogregex"
+gem_package "file-tail"
+
+template "/usr/local/bin/api-statistics" do
+  source "api-statistics.erb"
+  owner "root"
+  group "root"
+  mode 0755
+end
+
+template "/etc/init.d/api-statistics" do
+  source "api-statistics.init.erb"
+  owner "root"
+  group "root"
+  mode 0755
+end
+
+service "api-statistics" do
+  action [:enable, :start]
+  supports :restart => true
+  subscribes :restart, "template[/usr/local/bin/api-statistics]"
+  subscribes :restart, "template[/etc/init.d/api-statistics]"
+end
diff --git a/cookbooks/web/templates/default/api-statistics.erb b/cookbooks/web/templates/default/api-statistics.erb
new file mode 100644
index 000000000..af577f3e9
--- /dev/null
+++ b/cookbooks/web/templates/default/api-statistics.erb
@@ -0,0 +1,42 @@
+#!/usr/bin/ruby
+
+require "apache_log_regex"
+require "file-tail"
+require "json"
+
+def categorise_uri(line)
+  uri = line.split(" ")[1]
+
+  case uri
+  when %r{api/0\.6/map} then :map
+  when %r{api/0\.6/changeset/[0-9]*/upload} then :upload
+  when %r{api/0\.6/amf} then :amf
+  when %r{api/0\.6/(node|way|relation)/[0-9]*/history} then :history
+  when %r{api/0\.6/(node|way|relation)/[0-9]*/full} then :full
+  when %r{api/0\.6/trackpoints} then :trkpts
+  when %r{api/0\.6/} then :other
+  else :web
+  end
+end
+
+parser = ApacheLogRegex.new('%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %Ts')
+last_write = Time.now
+statistics = { :status => Hash.new(0), :uri => Hash.new(0) }
+
+File::Tail::Logfile.tail("/var/log/apache2/access.log") do |line|
+  begin
+    hash = parser.parse(line)
+    status = hash["%>s"]
+    uri = categorise_uri(hash["%r"])
+
+    statistics[:status][status] += 1
+    statistics[:uri][uri] += 1
+  rescue ApacheLogRegex::ParseError
+    # nil
+  end
+
+  if Time.now - last_write > 10
+    File.write("/srv/www.openstreetmap.org/rails/tmp/statistics.json", statistics.to_json)
+    last_write = Time.now
+  end
+end
diff --git a/cookbooks/web/templates/default/api-statistics.init.erb b/cookbooks/web/templates/default/api-statistics.init.erb
new file mode 100644
index 000000000..880f1dad9
--- /dev/null
+++ b/cookbooks/web/templates/default/api-statistics.init.erb
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+start() {
+  start-stop-daemon --start --chuid rails:adm --background --make-pidfile --pidfile /var/run/api-statistics.pid --exec /usr/local/bin/api-statistics
+}
+
+stop() {
+  start-stop-daemon --stop --retry 300 --pidfile /var/run/api-statistics.pid
+}
+
+case "$1" in
+  start)
+    start
+    ;;
+  stop)
+    stop
+    ;;
+  restart)
+    stop || exit $?
+    start
+    ;;
+esac
-- 
2.43.2