Skip to content

Commit 44cf39e

Browse files
trinktrink
committed
Add basic integration test for moz_logging
1 parent 754a5d3 commit 44cf39e

File tree

20 files changed

+232
-7
lines changed

20 files changed

+232
-7
lines changed

‎common/test_verify_message.lua

+1-1
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ function verify_msg_headers(expected, received, id, symmetric)
8383
for k,v in pairs(expected) do
8484
if k == "Fields" then
8585
if not received[k] then
86-
error(string.format("test: %d header: %s not found", id))
86+
error(string.format("test: %d header: %s not found", id, k))
8787
end
8888
else
8989
if v ~= received[k] then

‎moz_logging/CMakeLists.txt

+14-1
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,21 @@
33
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
44

55
cmake_minimum_required(VERSION 3.0)
6-
project(moz-logging VERSION 0.0.5 LANGUAGES C)
6+
project(moz-logging VERSION 0.0.6 LANGUAGES C)
77
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Mozilla Infrastructure Logging Module")
88
set(CPACK_DEBIAN_PACKAGE_DEPENDS "${PACKAGE_PREFIX}-lpeg (>= 1.0.8)")
99
string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS})
1010
include(sandbox_module)
11+
12+
set(integration_tests
13+
decoder_json_heka
14+
decoder_json_heka_fields
15+
decoder_line_splitter
16+
)
17+
18+
foreach(test IN LISTS integration_tests)
19+
add_test(NAME ${PROJECT_NAME}_${test}
20+
COMMAND ../run.sh
21+
CONFIGURATIONS integration
22+
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/integration/${test})
23+
endforeach()

‎moz_logging/io_modules/decoders/moz_logging/line_splitter.lua

+3-5
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ into multiple messages.
1111
## Decoder Configuration Table
1212
1313
```lua
14-
decoders_line_splitter = {
14+
decoders_moz_logging_line_splitter = {
1515
-- printf_messages (table/nil) see: https://mozilla-services.github.io/lua_sandbox_extensions/lpeg/modules/lpeg/printf.html
1616
-- sub_decoder (string/table) see: https://mozilla-services.github.io/lua_sandbox_extensions/lpeg/io_modules/lpeg/sub_decoder_util.html
1717
}
@@ -32,8 +32,6 @@ Decode and inject the resulting message
3232
http://mozilla-services.github.io/lua_sandbox/heka/message.html. In the case
3333
of multiple decoders this may be the message from the previous input/decoding
3434
step.
35-
- mutable (bool/nil/none) - Flag indicating if the decoder can modify the
36-
default_headers/msg structure in place or if it has to be copied first.
3735
3836
*Return*
3937
- err (nil, string) or throws an error on invalid data or an inject message
@@ -73,9 +71,9 @@ local err_msg = {
7371
}
7472

7573

76-
function decode(data, dh, mutable)
74+
function decode(data, dh)
7775
for line in string.gmatch(data, "([^\n]+)\n*") do
78-
local err = sub_decoder(line, dh, mutable)
76+
local err = sub_decoder(line, dh)
7977
if err then
8078
err_msg.Payload = err
8179
err_msg.Fields.data = line

‎moz_logging/tests/integration/decoder_json_heka/run/analysis/verify_decoder.cfg

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
filename = "verify_decoder.lua"
2+
message_matcher = "TRUE"

‎moz_logging/tests/integration/decoder_json_heka/run/analysis/verify_decoder.lua

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
-- This Source Code Form is subject to the terms of the Mozilla Public
2+
-- License, v. 2.0. If a copy of the MPL was not distributed with this
3+
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
5+
require "string"
6+
local test = require "test_verify_message"
7+
8+
local messages = {
9+
{Logger = "input.test_decoder", Type = "error", Hostname = "integration_test",
10+
Payload = "inject_message() failed: field name must be a string",
11+
Fields = {
12+
data = '{"Logger":"input1", "Fields":[{}]}'
13+
}
14+
},
15+
{Logger = "input.test_decoder|input1", Type = "default", Hostname = "integration_test"},
16+
{Logger = "input.test_decoder|input2", Type = "default|type2", Hostname = "integration_test"},
17+
{Logger = "input.test_decoder|input2", Type = "default|type2", Hostname = "integration_test",
18+
Fields = {
19+
user_agent_browser = "Firefox",
20+
agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:27.0) Gecko/20100101 Firefox/27.0",
21+
user_agent_os = "Linux",
22+
user_agent_version = 27
23+
}
24+
},
25+
{Logger = "input.test_decoder|input2", Type = "default|type2", Hostname = "integration_test",
26+
Fields = {
27+
foo = "bar",
28+
}
29+
},
30+
{Logger = "input.test_decoder", Type = "default", Hostname = "integration_test",
31+
Fields = {
32+
["deep.level1.level2.level3"] = '{"level4":"value"}',
33+
Timestamp = 123456789,
34+
foo = "bar",
35+
["nested.level1"] = "l1"
36+
}
37+
},
38+
}
39+
40+
local cnt = 0
41+
function process_message()
42+
cnt = cnt + 1
43+
local received = decode_message(read_message("raw"))
44+
test.fields_array_to_hash(received)
45+
test.verify_msg(messages[cnt], received, cnt)
46+
return 0
47+
end
48+
49+
function timer_event(ns)
50+
assert(cnt == #messages, string.format("%d of %d tests ran", cnt, #messages))
51+
end

‎moz_logging/tests/integration/decoder_json_heka/run/input/test_decoder.cfg

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
filename = "test_decoder.lua"

‎moz_logging/tests/integration/decoder_json_heka/run/input/test_decoder.lua

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
-- This Source Code Form is subject to the terms of the Mozilla Public
2+
-- License, v. 2.0. If a copy of the MPL was not distributed with this
3+
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
5+
local tests = {
6+
{[[
7+
{"Logger":"input1", "Fields":[{}]}
8+
{"Logger":"input1"}
9+
{"Logger":"input2", "Type":"type2"}
10+
{"Logger":"input2", "Type":"type2", "Fields":{"agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:27.0) Gecko/20100101 Firefox/27.0"}}
11+
{"Logger":"input2", "Type":"type2", "Fields":{"foo":"bar"}}
12+
{{invalid}
13+
{"foo":"bar", "Timestamp":123456789, "nested":{"level1":"l1"}, "deep":{"level1":{"level2":{"level3":{"level4":"value"}}}}}]]
14+
, {Type = "default"}}
15+
}
16+
17+
local dm = require("decoders.moz_logging.json_heka").decode
18+
19+
function process_message()
20+
for i,v in ipairs(tests) do
21+
dm(unpack(v))
22+
end
23+
return 0
24+
end

‎moz_logging/tests/integration/decoder_json_heka/run/output/placeholder.txt

Whitespace-only changes.

‎moz_logging/tests/integration/decoder_json_heka_fields/run/analysis/verify_decoder.cfg

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
filename = "verify_decoder.lua"
2+
message_matcher = "TRUE"

‎moz_logging/tests/integration/decoder_json_heka_fields/run/analysis/verify_decoder.lua

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
-- This Source Code Form is subject to the terms of the Mozilla Public
2+
-- License, v. 2.0. If a copy of the MPL was not distributed with this
3+
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
5+
require "string"
6+
local test = require "test_verify_message"
7+
8+
local messages = {
9+
{Logger = "input.test_decoder", Type = "error", Hostname = "integration_test",
10+
Payload = "inject_message() failed: unsupported array type: table",
11+
Fields = {
12+
data = '{"Logger":"input1", "Items":[{}]}'
13+
}
14+
},
15+
{Logger = "input.test_decoder", Type = "default", Hostname = "integration_test",
16+
Fields = {
17+
Logger = "input2",
18+
Type = "type2"
19+
}
20+
},
21+
{Logger = "input.test_decoder", Type = "default", Hostname = "integration_test",
22+
Fields = {
23+
Timestamp = 123456789,
24+
bstring = {value = "binary", value_type = 1},
25+
foo = "bar"
26+
}
27+
},
28+
{Logger = "input.test_decoder", Type = "default", Hostname = "integration_test",
29+
Fields = {
30+
array = {value={1,2,3}, value_type=2, representation="count"}
31+
}
32+
},
33+
}
34+
35+
local cnt = 0
36+
function process_message()
37+
cnt = cnt + 1
38+
local received = decode_message(read_message("raw"))
39+
test.fields_array_to_hash(received)
40+
test.verify_msg(messages[cnt], received, cnt)
41+
return 0
42+
end
43+
44+
function timer_event(ns)
45+
assert(cnt == #messages, string.format("%d of %d tests ran", cnt, #messages))
46+
end

‎moz_logging/tests/integration/decoder_json_heka_fields/run/input/test_decoder.cfg

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
filename = "test_decoder.lua"

‎moz_logging/tests/integration/decoder_json_heka_fields/run/input/test_decoder.lua

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
-- This Source Code Form is subject to the terms of the Mozilla Public
2+
-- License, v. 2.0. If a copy of the MPL was not distributed with this
3+
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
5+
local tests = {
6+
{[[
7+
{"Logger":"input1", "Items":[{}]}
8+
{"Logger":"input2", "Type":"type2"}
9+
{{invalid}
10+
{"foo":"bar", "Timestamp":123456789, "bstring":{"value":"binary", "value_type":1}}
11+
{"array":{"value":[1,2,3], "value_type":2, "representation":"count"}}]]
12+
, {Type = "default"}}
13+
}
14+
15+
local dm = require("decoders.moz_logging.json_heka_fields").decode
16+
17+
function process_message()
18+
for i,v in ipairs(tests) do
19+
dm(unpack(v))
20+
end
21+
return 0
22+
end

‎moz_logging/tests/integration/decoder_json_heka_fields/run/output/placeholder.txt

Whitespace-only changes.

‎moz_logging/tests/integration/decoder_line_splitter/run/analysis/verify_decoder.cfg

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
filename = "verify_decoder.lua"
2+
message_matcher = "TRUE"

‎moz_logging/tests/integration/decoder_line_splitter/run/analysis/verify_decoder.lua

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
-- This Source Code Form is subject to the terms of the Mozilla Public
2+
-- License, v. 2.0. If a copy of the MPL was not distributed with this
3+
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
5+
require "string"
6+
local test = require "test_verify_message"
7+
8+
local messages = {
9+
{Logger = "input.test_decoder", Type = "default", Hostname = "integration_test", Payload = "line one"},
10+
{Logger = "input.test_decoder", Type = "default", Hostname = "integration_test", Payload = "line two"},
11+
{Logger = "input.test_decoder", Type = "default", Hostname = "integration_test", Payload = "line three"},
12+
}
13+
14+
local cnt = 0
15+
function process_message()
16+
cnt = cnt + 1
17+
local received = decode_message(read_message("raw"))
18+
test.fields_array_to_hash(received)
19+
test.verify_msg(messages[cnt], received, cnt)
20+
return 0
21+
end
22+
23+
function timer_event(ns)
24+
assert(cnt == #messages, string.format("%d of %d tests ran", cnt, #messages))
25+
end

‎moz_logging/tests/integration/decoder_line_splitter/run/input/test_decoder.cfg

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
filename = "test_decoder.lua"
2+
3+
decoders_moz_logging_line_splitter = {
4+
sub_decoder = "decoders.payload"
5+
}

‎moz_logging/tests/integration/decoder_line_splitter/run/input/test_decoder.lua

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
-- This Source Code Form is subject to the terms of the Mozilla Public
2+
-- License, v. 2.0. If a copy of the MPL was not distributed with this
3+
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
5+
local tests = {
6+
{[[
7+
line one
8+
line two
9+
line three]], {Type = "default"}}
10+
}
11+
12+
local dm = require("decoders.moz_logging.line_splitter").decode
13+
14+
function process_message()
15+
for i,v in ipairs(tests) do
16+
dm(unpack(v))
17+
end
18+
return 0
19+
end

‎moz_logging/tests/integration/decoder_line_splitter/run/output/placeholder.txt

Whitespace-only changes.

‎moz_logging/tests/integration/hindsight.cfg

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
output_path = "output"
2+
sandbox_run_path = "run"
3+
analysis_lua_path = "../../../../common/?.lua;/usr/lib/luasandbox/modules/?.lua"
4+
analysis_lua_cpath = "/usr/lib/luasandbox/modules/?.so"
5+
io_lua_path = analysis_lua_path .. ";/usr/lib/luasandbox/io_modules/?.lua"
6+
io_lua_cpath = analysis_lua_cpath .. ";/usr/lib/luasandbox/io_modules/?.so"
7+
hostname = "integration_test"
8+
9+
analysis_defaults = {}
10+
input_defaults = {}
11+
output_defaults = {}

‎moz_logging/tests/integration/run.sh

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
rm -rf output
3+
hindsight_cli ../hindsight.cfg 7

0 commit comments

Comments
 (0)