Skip to content

Commit 14ecde1

Browse files
author
Mike Trinkala
authored
Merge pull request #189 from mozilla-services/dev
Sprint Nov 13
2 parents b887cc9 + 81dc58f commit 14ecde1

File tree

28 files changed

+101
-86
lines changed

28 files changed

+101
-86
lines changed

‎hyperloglog/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
44

55
cmake_minimum_required(VERSION 3.0)
6-
project(hyperloglog VERSION 1.0.0 LANGUAGES C)
6+
project(hyperloglog VERSION 1.0.1 LANGUAGES C)
77
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Lua hyperloglog module (distinct count)")
88
set(MODULE_SRCS hyperloglog.c redis_hyperloglog.c hyperloglog.def)
99
include(sandbox_module)

‎hyperloglog/hyperloglog.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ static int hll_new(lua_State *lua)
4545
hyperloglog *hll = lua_newuserdata(lua, nbytes);
4646
memcpy(hll->magic, hll_magic, sizeof(hll->magic));
4747
hll->encoding = HLL_DENSE;
48+
memset(hll->card, 0, sizeof(hll->card));
4849
HLL_INVALIDATE_CACHE(hll);
4950
memset(hll->notused, 0, sizeof(hll->notused));
5051
memset(hll->registers, 0, HLL_REGISTERS_SIZE);

‎hyperloglog/tests/test.lua

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
require "hyperloglog"
66
require "string"
7-
assert(hyperloglog.version() == "1.0.0", hyperloglog.version())
7+
assert(hyperloglog.version() == "1.0.1", hyperloglog.version())
88

99
local hll = hyperloglog.new()
1010
local hll1 = hyperloglog.new()

‎kafka/index.md

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ local consumer = kafka.consumer(brokerlist, topics, consumer_conf, topic_conf)
6161
*Arguments*
6262
* brokerlist (string) - [librdkafka broker string](https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h#L2205)
6363
* topics (array of 'topic[:partition]' strings) - Balanced consumer group mode a
64-
consumer can only subscribe on topics, not topics:partitions. The partition
64+
consumer can only subscribe on topics, not topics:partitions. The partition
6565
syntax is only used for manual assignments (without balanced consumer groups).
6666
* consumer_conf (table) - must contain 'group.id' see: [librdkafka consumer configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#global-configuration-properties)
6767
* topic_conf (table, optional) - [librdkafka topic configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#topic-configuration-properties)
@@ -91,12 +91,13 @@ Returns a string with the running version of Kafka module.
9191
Creates a topic to be used by a producer, no-op if the topic already exists.
9292

9393
```lua
94-
producer:create_topic(topic) -- creates the topic if it does not exist
94+
producer:create_topic(topic, topic_conf) -- creates the topic if it does not exist
9595

9696
```
9797

9898
*Arguments*
9999
* topic (string) - Name of the topic
100+
* topic_conf (table, optional) - [librdkafka topic configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#topic-configuration-properties)
100101

101102
*Return*
102103
* none
@@ -146,14 +147,14 @@ local ret = producer:send(topic, -1, sequence_id, message)
146147
*Arguments*
147148
* topic (string) - Name of the topic
148149
* partition (number) - Topic partition number (-1 for automatic assignment)
149-
* sequence_id
150+
* sequence_id
150151
* lua_sandbox (lightuserdata/nil/none) - Opaque pointer for checkpointing
151152
* Lua 5.1 (number/nil/none) - range: zero to UINTPTR_MAX
152153
* message
153154
* heka_sandbox (string/table)
154155
* string - message to send
155156
* table - zero copy specifier (table of read_message arguments)
156-
* Lua 5.1 (string) - Message to send
157+
* Lua 5.1 (string) - Message to send
157158

158159

159160
*Return*

‎moz_ingest/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
44

55
cmake_minimum_required(VERSION 3.0)
6-
project(moz-ingest VERSION 0.0.2 LANGUAGES C)
6+
project(moz-ingest VERSION 0.0.3 LANGUAGES C)
77
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Mozilla Nginx Ingestion Data Processing")
88
set(CPACK_DEBIAN_PACKAGE_DEPENDS "${PACKAGE_PREFIX}-rjson (>= 1.1.1), ${PACKAGE_PREFIX}-lpeg (>= 1.0.0), ${PACKAGE_PREFIX}-lfs (>= 1.6.4), ${PACKAGE_PREFIX}-zlib (>= 0.3.1)")
99
string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS})

‎moz_ingest/io_modules/decoders/moz_ingest/json.lua

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
44

55
--[[
6-
# Mozilla Telemetry Decoder Module
6+
# Mozilla Telemetry JSON Decoder Module
77
88
## Decoder Configuration Table
99
```lua
@@ -17,6 +17,16 @@ decoders_moz_ingest_json = {
1717
1818
-- array of namespace directories to ignore
1919
-- namespace_ignore = {"heka", "metadata", "pioneer-study", "telemetry"},
20+
21+
-- Transform the User-Agent header into user_agent_browser, user_agent_version, user_agent_os.
22+
-- user_agent_transform = false, -- default
23+
24+
-- Always preserve the User-Agent header if transform is enabled.
25+
-- user_agent_keep = false, -- default
26+
27+
-- Only preserve the User-Agent header if transform is enabled and fails.
28+
-- user_agent_conditional = false, -- default
29+
2030
}
2131
```
2232
@@ -52,6 +62,7 @@ local module_cfg = string.gsub(module_name, "%.", "_")
5262
local rjson = require "rjson"
5363
local miu = require "moz_ingest.util"
5464
local lfs = require "lfs"
65+
local clf = require "lpeg.common_log_format"
5566

5667
local read_config = read_config
5768
local assert = assert
@@ -87,8 +98,8 @@ end
8798
local namespaces = load_namespaces(cfg.namespace_path)
8899

89100

90-
local submissionField = {value = nil, representation = "json"}
91101
local doc = rjson.parse("{}") -- reuse this object to avoid creating a lot of GC
102+
local submissionField = {value = doc, representation = "json"}
92103
local function process_json(hsr, msg)
93104
local ok, err = pcall(doc.parse_message, doc, hsr, "Fields[content]", nil, nil, true)
94105
if not ok then
@@ -108,6 +119,7 @@ local function process_json(hsr, msg)
108119
error(string.format("json\tnamespace: %s schema: %s version: %d error: %s",
109120
msg.Logger, msg.Fields.docType, msg.Fields.docVersion, err), 0)
110121
end
122+
msg.Fields.submission = submissionField
111123
end
112124

113125

@@ -119,8 +131,18 @@ function transform_message(hsr, msg)
119131
process_json(hsr, msg)
120132

121133
-- Migrate the original message data after the validation (avoids Field duplication in the error message)
122-
msg.Hostname = hsr:read_message("Hostname")
123-
msg.Fields.Host = hsr:read_message("Fields[Host]")
134+
msg.Hostname = hsr:read_message("Hostname")
135+
msg.Fields.Host = hsr:read_message("Fields[Host]")
136+
msg.Fields["User-Agent"] = hsr:read_message("Fields[User-Agent]")
137+
138+
if msg.Fields["User-Agent"] and cfg.user_agent_transform then
139+
msg.Fields.user_agent_browser,
140+
msg.Fields.user_agent_version,
141+
msg.Fields.user_agent_os = clf.normalize_user_agent(msg.Fields["User-Agent"])
142+
if not ((cfg.user_agent_conditional and not msg.Fields.user_agent_browser) or cfg.user_agent_keep) then
143+
msg.Fields["User-Agent"] = nil
144+
end
145+
end
124146

125147
local ok, err = pcall(inject_message, msg)
126148
if not ok then

‎moz_ingest/tests/hindsight/input.hpb

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,32 @@
11
`
2-
:�E�ӛIߣs�S>u�����۽��
2+
�GG���E�-���R��撝���
33
common.raw"
44
moz_ingestJ example.comR
55
remote_addr"192.30.255.112p
6-
6�.��mCu�pN�~������۽��
6+
�?d��B��{m#�H`�ɩ璝���
77
common.raw"
88
moz_ingestJ example.comR
99
uri"/foobarR
1010
remote_addr"192.30.255.112�
11-
�틩��I,���V�-j4����۽��
11+
��*0(C�3�&��xL�璝���
1212
common.raw"
1313
moz_ingestJ example.comRC
1414
uri"</submit/common/foobar/1/0055FAC4-8A1A-4FCA-B380-EBFDC8571A01R
1515
remote_addr"192.30.255.112�
16-
ӘӥKߧi��X�[����۽��
16+
�`��"C���x ���璝���
1717
common.raw"
1818
moz_ingestJ example.comRC
1919
uri"</submit/common/foobar/1/0055FAC4-8A1A-4FCA-B380-EBFDC8571A01R
2020
remote_addr"192.30.255.112�
21-
FJ��K�{]��I|���۽��
21+
&3�NB�D�������璝���
2222
common.raw"
2323
moz_ingestJ example.comR
2424
geoCity"HalifaxR
2525
remote_addr"192.30.255.112RD
2626
uri"=/submit/common/widget/99/0055FAC4-8A1A-4FCA-B380-EBFDC8571A02R
2727

2828
geoCountry"CA�
29-
_!e3<D����+L~������۽��
29+
�����C?�·�A�6g��璝���
3030
common.raw"
3131
moz_ingestJ example.comR
3232
remote_addr"192.30.255.112RD

‎moz_ingest/tests/hindsight/json.hpb

336 Bytes
Binary file not shown.

‎moz_ingest/tests/hindsight/run/analysis/verify_json_decoder.lua

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,11 @@ local messages = {
1414
docType = "bar",
1515
geoCity = "New York",
1616
geoCountry = "US",
17-
documentId = "0055FAC4-8A1A-4FCA-B380-EBFDC8571A01"
17+
documentId = "0055FAC4-8A1A-4FCA-B380-EBFDC8571A01",
18+
submission = [[{"exampleString":"string one"}]],
19+
user_agent_browser = "Firefox",
20+
user_agent_version = 59,
21+
user_agent_os = "Linux"
1822
}
1923
},
2024
{Logger = "foo", Type = "error", Hostname = "example.com", Fields = {

‎moz_ingest/tests/hindsight/run/input/generate_data.lua

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ function process_message()
4848
msg.Type = "json.raw"
4949
msg.Fields.uri = "/submit/foo/bar/1/0055FAC4-8A1A-4FCA-B380-EBFDC8571A01"
5050
msg.Fields.content = [[{"exampleString":"string one"}]]
51+
msg.Fields["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0"
5152
inject_message(msg)
5253

5354
-- fails parsing

‎moz_ingest/tests/hindsight/run/input/test_json_decoder.cfg

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ decoders_moz_ingest_common = {
1818

1919
decoders_moz_ingest_json = {
2020
namespace_path = "namespaces",
21+
user_agent_transform = true,
2122
}

‎moz_security/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
44

55
cmake_minimum_required(VERSION 3.0)
6-
project(moz-security VERSION 0.0.3 LANGUAGES C)
6+
project(moz-security VERSION 0.0.4 LANGUAGES C)
77
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Mozilla Infrastructure Security Analysis")
88
include(sandbox_module)
99

‎moz_security/sandboxes/heka/analysis/moz_security_sshd_login_monitor.lua

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,12 @@ message_matcher = "Type == 'logging.shared.bastion.systemd.sshd' && Fields[sshd_
1515
ticker_interval = 0
1616
process_message_inject_limit = 1
1717
18-
-- default_email = "foxsec-alerts@mozilla.com"
18+
-- default_email = "foxsec-dump+OutOfHours@mozilla.com"
1919
```
2020
--]]
2121
require "string"
2222

23-
local default_email = read_config("default_email") or "foxsec-alerts@mozilla.com"
23+
local default_email = read_config("default_email") or "foxsec-dump+OutOfHours@mozilla.com"
2424
local msg = {
2525
Type = "alert",
2626
Payload = "",
@@ -35,10 +35,8 @@ local msg = {
3535
function process_message()
3636
local user = read_message("Fields[remote_user]")
3737
local ip = read_message("Fields[remote_addr]")
38-
local id = string.format("%X%X%X%X%X%X%X%X%X%X%X%X%X%X%X%X",
39-
string.byte(read_message("Uuid"), 1, 16))
4038

41-
msg.Fields[2].value = string.format("%s logged into bastion from %s id:%s", user, ip, id)
39+
msg.Fields[2].value = string.format("%s logged into bastion from %s", user, ip)
4240
msg.Fields[3].value[2] = string.format("<manatee-%[email protected]>", user)
4341
inject_message(msg)
4442
return 0

‎moz_security/tests/hindsight/run/analysis/hh_monitor.cfg

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
1-
filename = "moz_security_heavy_hitters_monitor.lua"
1+
filename = "moz_security_heavy_hitters.lua"
22
message_matcher = "Logger == 'input.hh'"
33
ticker_interval = 0
44

55
id_field = "Fields[id]"
6-
cf_items = 2000
76
hh_items = 10

‎moz_security/tests/hindsight/run/analysis/sshd_login_monitor.cfg

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -2,32 +2,3 @@ filename = "moz_security_sshd_login_monitor.lua"
22
message_matcher = "Logger == 'input.syslog' && Fields[programname] == 'sshd' && Fields[sshd_authmsg] == 'Accepted'"
33
ticker_interval = 0
44
process_message_inject_limit = 1
5-
6-
closed = {
7-
tz = "America/Los_Angeles",
8-
days = {"Sat", "Sun"},
9-
holidays = {
10-
"2017-09-04", -- Labor Day
11-
"2017-11-23", -- Thanksgiving Holiday
12-
"2017-11-24", -- Thanksgiving Holiday + 1
13-
"2017-12-25", -- Christmas Day
14-
"2017-12-26", -- Christmas + 1
15-
"2017-01-01", -- New Year's Day
16-
"2017-01-15", -- Martin Luther King, Jr. Day
17-
"2017-02-19", -- Presidents' Day
18-
"2017-05-28", -- Memorial Day
19-
},
20-
hours = {open = "09:00", close = "17:00"},
21-
}
22-
23-
alert = {
24-
prefix = false,
25-
throttle = 1,
26-
modules = {
27-
email = {recipients = {"[email protected]"}}
28-
}
29-
}
30-
31-
user_map = {
32-
trink = "mtrinkala"
33-
}

‎moz_security/tests/hindsight/run/input/generate_sshd.lua

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,9 @@
66
# Generates test data for moz_security_sshd_login_monitor
77
--]]
88

9-
require "date"
109

11-
local fmt = "%Y-%m-%d %H:%M:%S"
1210
local tests = {
13-
{"2017-07-24 09:20:01", "trink" , "192.168.1.1"}, -- ok
14-
{"2017-07-22 02:33:44", "sat" , "192.168.1.2"}, -- Saturday
15-
{"2017-07-23 01:11:12", "sun" , "192.168.1.3"}, -- Sunday
16-
{"2017-07-23 17:00:00", "abh" , "192.168.1.4"}, -- after business hours
17-
{"2017-07-23 08:59:59", "bbh" , "192.168.1.5"}, -- before business hours
18-
{"2017-09-04 10:11:12", "trink" , "192.168.1.6"}, -- Labor Day with user_map
19-
{"2017-05-28 10:11:12", "root" , "192.168.1.7"}, -- Memorial Day
11+
{"11111111-1111-1111-1111-111111111111", "trink" , "192.168.1.1"},
2012
}
2113

2214
local msg = {
@@ -32,7 +24,7 @@ local msg = {
3224

3325
function process_message()
3426
for i,v in ipairs(tests) do
35-
msg.Timestamp = date.time(v[1], fmt, "America/Los_Angeles")
27+
msg.Uuid = v[1]
3628
msg.Fields.remote_user = v[2]
3729
msg.Fields.remote_addr = v[3]
3830
inject_message(msg)

‎moz_security/tests/hindsight/run/output/hh_verification.lua

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
44

55
--[[
6-
# Validates the moz_telemetry_heavy_hitters_monitor output
6+
# Validates the moz_security_heavy_hitters output
77
--]]
88

99
require "string"

‎moz_security/tests/hindsight/run/output/sshd_login_verification.lua

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,31 +3,39 @@
33
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
44

55
--[[
6-
# Validates the moz_telemetry_doctype_monitor alerts
6+
# Validates the moz_security_sshd_login alerts
77
--]]
88

99
require "string"
1010

1111
local results = {
12-
"user: sat\nip: 192.168.1.2\n",
13-
"user: sun\nip: 192.168.1.3\n",
14-
"user: abh\nip: 192.168.1.4\n",
15-
"user: bbh\nip: 192.168.1.5\n",
16-
"user: mtrinkala\nip: 192.168.1.6\n",
17-
"user: root\nip: 192.168.1.7\n",
12+
{ summary = "trink logged into bastion from 192.168.1.1",
13+
recipients = {"<[email protected]>", "<[email protected]>"}
14+
}
1815
}
1916

2017
local cnt = 0
2118
function process_message()
22-
local payload = read_message("Payload")
23-
if results[cnt + 1] ~= payload then
24-
error(string.format("test:%d result:%s", cnt + 1, payload))
25-
end
19+
local summary = read_message("Fields[summary]") or "nil"
20+
local dflt_recip = read_message("Fields[email.recipients]") or "nil"
21+
local user_recip = read_message("Fields[email.recipients]", 0, 1) or "nil"
2622
cnt = cnt + 1
23+
local er = results[cnt]
24+
assert(er, "too many messages")
25+
26+
if er.summary ~= summary then
27+
error(string.format("test:%d result:%s", cnt, summary))
28+
end
29+
if er.recipients[1] ~= dflt_recip then
30+
error(string.format("test:%d result:%s", cnt, dflt_recip))
31+
end
32+
if er.recipients[2] ~= user_recip then
33+
error(string.format("test:%d result:%s", cnt, user_recip))
34+
end
2735
return 0
2836
end
2937

3038

3139
function timer_event()
32-
assert(cnt == 6, string.format("%d out of 6 tests ran", cnt))
40+
assert(cnt == 1, string.format("%d out of 1 tests ran", cnt))
3341
end

‎moz_telemetry/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
44

55
cmake_minimum_required(VERSION 3.0)
6-
project(moz-telemetry VERSION 1.2.15 LANGUAGES C)
6+
project(moz-telemetry VERSION 1.2.16 LANGUAGES C)
77
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Mozilla Firefox Telemetry Data Processing")
88
set(CPACK_DEBIAN_PACKAGE_DEPENDS "${PACKAGE_PREFIX}-moz-ingest (>= 0.0.1), ${PACKAGE_PREFIX}-lsb (>= 1.1.0), ${PACKAGE_PREFIX}-circular-buffer (>= 1.0.2), ${PACKAGE_PREFIX}-heka (>= 1.1.9), ${PACKAGE_PREFIX}-elasticsearch (>= 1.0.3), ${PACKAGE_PREFIX}-rjson (>= 1.1.0), ${PACKAGE_PREFIX}-lfs (>= 1.6.4)")
99
string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS})

‎moz_telemetry/io_modules/decoders/moz_ingest/telemetry.lua

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,7 @@ local function process_json(hsr, msg)
224224
msg.Fields.appUpdateChannel = doc:value(doc:find(app, "channel"))
225225
msg.Fields.appVendor = doc:value(doc:find(app, "vendor"))
226226
msg.Fields.normalizedChannel = mtn.channel(msg.Fields.appUpdateChannel)
227+
msg.Fields.normalizedOSVersion = mtn.os_version(doc:value(doc:find("environment", "system", "os", "version")))
227228

228229
remove_objects(msg, doc, "environment", environment_objects)
229230
remove_objects(msg, doc, "payload", extract_payload_objects[msg.Fields.docType])

0 commit comments

Comments
 (0)