Merge pull request #191 from mozilla-services/1423309

Bugzilla 1423309 - Add the submission and user agent fields

Author: Mike Trinkala

moz_ingest/CMakeLists.txt

@@ -3,7 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 cmake_minimum_required(VERSION 3.0)
-project(moz-ingest VERSION 0.0.2 LANGUAGES C)
+project(moz-ingest VERSION 0.0.3 LANGUAGES C)
 set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Mozilla Nginx Ingestion Data Processing")
 set(CPACK_DEBIAN_PACKAGE_DEPENDS "${PACKAGE_PREFIX}-rjson (>= 1.1.1), ${PACKAGE_PREFIX}-lpeg (>= 1.0.0), ${PACKAGE_PREFIX}-lfs (>= 1.6.4), ${PACKAGE_PREFIX}-zlib (>= 0.3.1)")
 string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS})

moz_ingest/io_modules/decoders/moz_ingest/json.lua

@@ -3,7 +3,7 @@
 -- file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 --[[
-# Mozilla Telemetry Decoder Module
+# Mozilla Telemetry JSON Decoder Module
 
 ## Decoder Configuration Table
 ```lua
@@ -17,6 +17,16 @@ decoders_moz_ingest_json = {
 
     -- array of namespace directories to ignore
     -- namespace_ignore = {"heka", "metadata", "pioneer-study", "telemetry"},
+
+    -- Transform the User-Agent header into user_agent_browser, user_agent_version, user_agent_os.
+    -- user_agent_transform = false, -- default
+
+    -- Always preserve the User-Agent header if transform is enabled.
+    -- user_agent_keep = false, -- default
+
+    -- Only preserve the User-Agent header if transform is enabled and fails.
+    -- user_agent_conditional = false, -- default
+
 }
 ```
 
@@ -52,6 +62,7 @@ local module_cfg    = string.gsub(module_name, "%.", "_")
 local rjson  = require "rjson"
 local miu    = require "moz_ingest.util"
 local lfs    = require "lfs"
+local clf    = require "lpeg.common_log_format"
 
 local read_config          = read_config
 local assert               = assert
@@ -87,8 +98,8 @@ end
 local namespaces = load_namespaces(cfg.namespace_path)
 
 
-local submissionField = {value = nil, representation = "json"}
 local doc = rjson.parse("{}") -- reuse this object to avoid creating a lot of GC
+local submissionField = {value = doc, representation = "json"}
 local function process_json(hsr, msg)
     local ok, err = pcall(doc.parse_message, doc, hsr, "Fields[content]", nil, nil, true)
     if not ok then
@@ -108,6 +119,7 @@ local function process_json(hsr, msg)
         error(string.format("json\tnamespace: %s schema: %s version: %d error: %s",
                             msg.Logger, msg.Fields.docType, msg.Fields.docVersion, err), 0)
     end
+    msg.Fields.submission = submissionField
 end
 
 
@@ -119,8 +131,18 @@ function transform_message(hsr, msg)
     process_json(hsr, msg)
 
     -- Migrate the original message data after the validation (avoids Field duplication in the error message)
-    msg.Hostname    = hsr:read_message("Hostname")
-    msg.Fields.Host = hsr:read_message("Fields[Host]")
+    msg.Hostname                = hsr:read_message("Hostname")
+    msg.Fields.Host             = hsr:read_message("Fields[Host]")
+    msg.Fields["User-Agent"]    = hsr:read_message("Fields[User-Agent]")
+
+    if msg.Fields["User-Agent"] and cfg.user_agent_transform then
+        msg.Fields.user_agent_browser,
+        msg.Fields.user_agent_version,
+        msg.Fields.user_agent_os = clf.normalize_user_agent(msg.Fields["User-Agent"])
+        if not ((cfg.user_agent_conditional and not msg.Fields.user_agent_browser) or cfg.user_agent_keep) then
+            msg.Fields["User-Agent"] = nil
+        end
+    end
 
     local ok, err = pcall(inject_message, msg)
     if not ok then

moz_ingest/tests/hindsight/input.hpb

@@ -1,32 +1,32 @@
 `
-:�E�ӛIߣs�S>u�����۽��
+�GG���E�-���R��撝���
 common.raw"
 moz_ingestJexample.comR
 remote_addr"192.30.255.112p
-6�.��mCu�pN�~������۽��
+�?d��B��{m#�H`�ɩ璝���
 common.raw"
 moz_ingestJexample.comR
 uri"/foobarR
 remote_addr"192.30.255.112�

-�틩��I,���V�-j4����۽��
+��*0(C�3�&��xL�璝���
 common.raw"
 moz_ingestJexample.comRC
 uri"</submit/common/foobar/1/0055FAC4-8A1A-4FCA-B380-EBFDC8571A01R
 remote_addr"192.30.255.112�

-ӘӥKߧi��X�[����۽��
+�`��"C���x	���璝���
 common.raw"
 moz_ingestJexample.comRC
 uri"</submit/common/foobar/1/0055FAC4-8A1A-4FCA-B380-EBFDC8571A01R
 remote_addr"192.30.255.112�

-FJ��K�{]��I|���۽��
+&3�NB�D�������璝���
 common.raw"
 moz_ingestJexample.comR
 geoCity"HalifaxR
 remote_addr"192.30.255.112RD
 uri"=/submit/common/widget/99/0055FAC4-8A1A-4FCA-B380-EBFDC8571A02R
 
 geoCountry"CA�

-_!e3<D����+L~������۽��
+�����C?�·�A�6g��璝���
 common.raw"
 moz_ingestJexample.comR
 remote_addr"192.30.255.112RD

moz_ingest/tests/hindsight/json.hpb

@@ -14,7 +14,11 @@ local messages = {
         docType = "bar",
         geoCity = "New York",
         geoCountry = "US",
-        documentId = "0055FAC4-8A1A-4FCA-B380-EBFDC8571A01"
+        documentId = "0055FAC4-8A1A-4FCA-B380-EBFDC8571A01",
+        submission = [[{"exampleString":"string one"}]],
+        user_agent_browser = "Firefox",
+        user_agent_version = 59,
+        user_agent_os      = "Linux"
         }
     },
     {Logger = "foo", Type = "error", Hostname = "example.com", Fields = {

moz_ingest/tests/hindsight/run/analysis/verify_json_decoder.lua

@@ -48,6 +48,7 @@ function process_message()
     msg.Type = "json.raw"
     msg.Fields.uri = "/submit/foo/bar/1/0055FAC4-8A1A-4FCA-B380-EBFDC8571A01"
     msg.Fields.content = [[{"exampleString":"string one"}]]
+    msg.Fields["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0"
     inject_message(msg)
 
     -- fails parsing

moz_ingest/tests/hindsight/run/input/generate_data.lua

@@ -18,4 +18,5 @@ decoders_moz_ingest_common = {
 
 decoders_moz_ingest_json = {
     namespace_path = "namespaces",
+    user_agent_transform = true,
 }