Merge pull request #179 from mozilla-services/dev

Sprint Oct 30

Author: Mike Trinkala

heka/CMakeLists.txt

@@ -3,7 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 cmake_minimum_required(VERSION 3.0)
-project(heka VERSION 1.1.12 LANGUAGES C)
+project(heka VERSION 1.1.13 LANGUAGES C)
 set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Utility modules for Heka sandboxes")
 set(CPACK_DEBIAN_PACKAGE_DEPENDS "${PACKAGE_PREFIX}-cjson (>= 2.1)")
 string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS})

heka/modules/heka/alert.lua

@@ -21,6 +21,9 @@ alert = {
 }
 
 ```
+## Variables
+
+* thresholds - the thresholds configuration table
 
 ## Functions
 
@@ -95,6 +98,8 @@ local inject_message = inject_message
 local alert_cfg = read_config("alert")
 assert(type(alert_cfg) == "table", "alert configuration must be a table")
 assert(type(alert_cfg.modules) == "table", "alert.modules configuration must be a table")
+if type(alert_cfg.thresholds) == "nil" then alert_cfg.thresholds = {} end
+assert(type(alert_cfg.thresholds) == "table", "alert.thresholds configuration must be nil or a table")
 
 alert_cfg.throttle = alert_cfg.throttle or 90
 assert(type(alert_cfg.throttle) == "number" and alert_cfg.throttle > 0, "alert.throttle configuration must be a number > 0 ")
@@ -148,6 +153,7 @@ function get_dashboard_uri(id, ext)
 end
 
 
+thresholds = alert_cfg.thresholds -- expose the entire table
 function get_threshold(id)
     local at = alert_cfg.thresholds[id]
     if not at then

moz_logging/CMakeLists.txt

@@ -3,6 +3,6 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 cmake_minimum_required(VERSION 3.0)
-project(moz-logging VERSION 0.0.2 LANGUAGES C)
+project(moz-logging VERSION 0.0.3 LANGUAGES C)
 set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Mozilla Infrastructure Logging Module")
 include(sandbox_module)

moz_logging/io_modules/decoders/moz_logging/json_heka.lua

@@ -75,11 +75,13 @@ function decode(data, dh)
                  msg.Type = dh.Type
              end
 
-             local agent = msg.Fields.agent
-             if agent then
-                 msg.Fields.user_agent_browser,
-                 msg.Fields.user_agent_version,
-                 msg.Fields.user_agent_os = clf.normalize_user_agent(agent)
+             if msg.Fields then
+                 local agent = msg.Fields.agent
+                 if agent then
+                     msg.Fields.user_agent_browser,
+                     msg.Fields.user_agent_version,
+                     msg.Fields.user_agent_os = clf.normalize_user_agent(agent)
+                 end
              end
 
              ok, msg = pcall(inject_message, msg)

moz_logging/io_modules/decoders/moz_logging/json_heka_fields.lua

@@ -57,7 +57,7 @@ function decode(data, dh)
     for line in string.gmatch(data, "([^\n]+)\n*") do
         local ok, msg = pcall(cjson.decode, line)
         if ok then
-            local msg = {Fields = cjson.decode(line)}
+            msg = {Fields = msg}
             if dh then
                 msg.Uuid       = dh.Uuid
                 msg.Logger     = dh.Logger

moz_logging/io_modules/decoders/moz_logging/line_splitter.lua

@@ -16,7 +16,6 @@ decoders_line_splitter = {
 }
 ```
 
-
 ## Functions
 
 ### decode
@@ -56,6 +55,8 @@ local read_config    = read_config
 local M = {}
 setfenv(1, M) -- Remove external access to contain everything in the module
 
+local msg = {}
+
 local sub_decoder
 if cfg.sub_decoder:match("^decoders%.") then
     sub_decoder = sd_module.decode
@@ -64,13 +65,27 @@ else
     local grammar = sd_module.grammar or sd_module.syslog_grammar
     assert(type(grammar) == "userdata", "sub_decoders, no grammar defined")
     sub_decoder = function(data, dh)
-        if not dh then dh = {} end
-        local fields = grammar:match(data)
-        if not fields then return "parse failed" end
-        for k,v in pairs(fields) do
-            dh.Fields[k] = v
+        msg.Fields = grammar:match(data)
+        if not msg.Fields then return "parse failed" end
+        if dh then
+            msg.Uuid        = dh.Uuid
+            msg.Logger      = dh.Logger
+            msg.Hostname    = dh.Hostname
+            msg.Timestamp   = dh.Timestamp
+            msg.Type        = dh.Type
+            msg.Payload     = dh.Payload
+            msg.EnvVersion  = dh.EnvVersion
+            msg.Pid         = dh.Pid
+            msg.Severity    = dh.Severity
+            if type(dh.Fields) == "table" then
+                for k,v in pairs(dh.Fields) do
+                    if msg.Fields[k] == nil then
+                        msg.Fields[k] = v
+                    end
+                end
+            end
         end
-        inject_message(dh)
+        inject_message(msg)
     end
 end
 

moz_logging/sandboxes/heka/analysis/moz_logging_ingestion_error_monitor.lua

@@ -23,12 +23,9 @@ alert = {
   modules = {
     email = {recipients = {"[email protected]"}},
   },
-  ingestion_error = {
-    -- Logger = {inactivity_timeout = 5, percent = 0.5}
-    ["*"] = { -- configures the default
-      inactivity_timeout = 5, -- minutes
-      percent = 0.5
-    }
+  thresholds = {
+    -- ["input.bastion_systemd_sshd"] = {inactivity_timeout = 60, percent = 0.5} -- a timeout of 60 or more disables the check as the alert window is only one hour
+    _default_ = {inactivity_timeout = 5, -- minutes percent = 0.5} -- if not specified the default is no monitoring
   }
 }
 ```
@@ -53,16 +50,13 @@ local CREATED           = 1
 local INGESTED          = 2
 local ERROR             = 3
 
-local ie = read_config("alert").ingestion_error
-assert(type(ie) == "table", "alert.ingestion_error must be a table")
-for k,v in pairs(ie) do
+local cnt = 0
+for k,v in pairs(alert.thresholds) do
     assert(type(v.inactivity_timeout) == "number", "inactivity_timeout must be a number")
     assert(type(v.percent) == "number", "percent must be a number")
-    if k == "*" then
-        local mt = {__index = function(t, k) return v end }
-        setmetatable(ie, mt);
-    end
+    cnt = cnt + 1
 end
+assert(cnt > 0, "at least one alert threshold must be set")
 
 inputs = {}
 local function get_input(logger, rows, spr)
@@ -133,10 +127,10 @@ local function diagnostic_prune(ns, diags)
 end
 
 
-local function inactivity_alert(ns, k, vcnt, cb, e)
+local function inactivity_alert(ns, th, k, vcnt, cb, e)
     if vcnt == 0 then return false end
 
-    local iato = ie[k].inactivity_timeout
+    local iato = th.inactivity_timeout
     if MINS_IN_HOUR - vcnt > iato then
         local _, cnt = stats.sum(cb:get_range(INGESTED, e - ((iato - 1) * 60e9))) -- include the current minute
         if cnt == 0 then
@@ -177,21 +171,21 @@ function timer_event(ns, shutdown)
         local diags = v[2]
         diagnostic_prune(ns, diags)
 
-        if not alert.throttled(k) then
+        local th = alert.get_threshold(k)
+        if th and not alert.throttled(k) then
             local e = cb:current_time() - 60e9 -- exclude the current minute
             local s = e - ((MINS_IN_HOUR - 1) * 60e9)
             local array = cb:get_range(INGESTED, s, e)
             local isum, vcnt = stats.sum(array)
-            if not inactivity_alert(ns, k, vcnt, cb, e) then
+            if not inactivity_alert(ns, th, k, vcnt, cb, e) then
                 array = cb:get_range(ERROR, s, e)
                 local esum = stats.sum(array)
                 if isum > 1000 or esum > 1000 then
-                    local mpe = ie[k].percent
                     local pe  = esum / (isum + esum) * 100
-                    if pe > mpe then
+                    if pe > th.percent then
                         if alert.send(k, "ingestion error",
-                                      string.format(ingestion_error_template, isum, esum, pe, mpe,
-                                                    alert.get_dashboard_uri(k),
+                                      string.format(ingestion_error_template, isum, esum, pe,
+                                                    th.percent, alert.get_dashboard_uri(k),
                                                     diagnostic_dump(diags))) then
                             cb:annotate(ns, ERROR, "alert", string.format("%.4g%%", pe))
                         end

moz_security/CMakeLists.txt

@@ -3,10 +3,8 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 cmake_minimum_required(VERSION 3.0)
-project(moz-security VERSION 0.0.2 LANGUAGES C)
+project(moz-security VERSION 0.0.3 LANGUAGES C)
 set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Mozilla Infrastructure Security Analysis")
-set(CPACK_DEBIAN_PACKAGE_DEPENDS "${PACKAGE_PREFIX}-date (>= 0.0.1)")
-string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS})
 include(sandbox_module)
 
 add_test(NAME ${MODULE_NAME}_hindsight

moz_security/sandboxes/heka/analysis/moz_security_heavy_hitters.lua

@@ -0,0 +1,70 @@
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this
+-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+--[[
+#  Mozilla Security Heavy Hitters
+
+See: https://cacm.acm.org/magazines/2009/10/42481-finding-the-frequent-items-in-streams-of-data/abstract
+
+## Sample Configuration
+```lua
+filename = "moz_security_heavy_hitters.lua"
+message_matcher = "Logger == 'input.nginx'"
+ticker_interval = 60
+preserve_data = true
+
+id_field = "Fields[remote_addr]"
+-- id_field_capture = ",? *([^,]+)$"a -- optional e.g. extract the last entry in a comma delimited list
+
+-- hh_items = 10000 -- optional, defaults to 10000 (maximum number of heavy hitter IDs to track)
+```
+--]]
+require "string"
+
+local id_field  = read_config("id_field") or error("id_field must be configured")
+local id_fieldc = read_config("id_field_capture")
+local hh_items  = read_config("hh_items") or 10000
+
+hh = {}
+hh_size = 0
+
+function process_message()
+    local id = read_message(id_field)
+    if not id then return -1, "no id_field" end
+    if id_fieldc then
+        id = string.match(id, id_fieldc)
+        if not id then return 0 end -- no error as the capture may intentionally reject entries
+    end
+
+    local cnt = hh[id]
+    if cnt then
+        hh[id] = cnt + 1
+    else
+        if hh_size >= hh_items then
+            for k, cnt in pairs(hh) do
+                cnt = cnt - 1
+                if 0 == cnt then
+                    hh[k] = nil
+                    hh_size = hh_size - 1
+                else
+                    hh[k] = cnt
+                end
+            end
+        else
+            hh_size = hh_size + 1
+            hh[id] = 1
+        end
+    end
+    return 0
+end
+
+
+function timer_event(ns)
+    for k, cnt in pairs(hh) do
+        if cnt > 5 then
+            add_to_payload(cnt, "\t", k, "\n")
+        end
+    end
+    inject_payload("tsv", "alltime")
+end