Skip to content

Commit a842f3e

Browse files
Gwynbleidd0241Sergey Lazarenko
Gwynbleidd0241
and
Sergey Lazarenko
authored
resolve decoder:auto only once (#941)
* resolve decoder:auto only once * eliminate repetitive AUTO decoder checks in method In * centralized decoder initialization logic * standardize params type * used decoder.Params * Delete testdata/config/welcome.yaml * returned welcome.yaml --------- Co-authored-by: Sergey Lazarenko <[email protected]>
1 parent a372f19 commit a842f3e

16 files changed

+172
-104
lines changed

‎decoder/csv.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ type CSVDecoder struct {
5555
buffersPool sync.Pool
5656
}
5757

58-
func NewCSVDecoder(params map[string]any) (Decoder, error) {
58+
func NewCSVDecoder(params Params) (Decoder, error) {
5959
p, err := extractCSVParams(params)
6060
if err != nil {
6161
return nil, fmt.Errorf("can't extract params: %w", err)
@@ -245,7 +245,7 @@ func (d *CSVDecoder) GenerateColumnName(i int) string {
245245
return d.params.prefix + strconv.Itoa(i)
246246
}
247247

248-
func extractCSVParams(params map[string]any) (CSVParams, error) {
248+
func extractCSVParams(params Params) (CSVParams, error) {
249249
columnNames := make([]string, 0)
250250
if columnNamesRaw, ok := params[columnNamesParam]; ok {
251251
columnNamesRawSlice, ok := columnNamesRaw.([]any)

‎decoder/csv_test.go‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ func TestDecodeCSV(t *testing.T) {
1212
name string
1313

1414
input string
15-
params map[string]any
15+
params Params
1616

1717
want []string
1818
wantCreateErr bool
@@ -31,31 +31,31 @@ func TestDecodeCSV(t *testing.T) {
3131
{
3232
name: "custom_delimiter",
3333
input: `a b "c"` + "\n",
34-
params: map[string]any{
34+
params: Params{
3535
delimiterParam: "\t",
3636
},
3737
want: CSVRow{"a", "b", "c"},
3838
},
3939
{
4040
name: "invalid_columns",
4141
input: "",
42-
params: map[string]any{
42+
params: Params{
4343
columnNamesParam: "name",
4444
},
4545
wantCreateErr: true,
4646
},
4747
{
4848
name: "invalid_delimiter_1",
4949
input: "",
50-
params: map[string]any{
50+
params: Params{
5151
delimiterParam: ",,",
5252
},
5353
wantCreateErr: true,
5454
},
5555
{
5656
name: "invalid_delimiter_2",
5757
input: "",
58-
params: map[string]any{
58+
params: Params{
5959
delimiterParam: "\n",
6060
},
6161
wantCreateErr: true,
@@ -105,7 +105,7 @@ func TestDecodeToJsonCSV(t *testing.T) {
105105
name string
106106

107107
input string
108-
params map[string]any
108+
params Params
109109

110110
want string
111111
wantDecodeErr bool
@@ -118,15 +118,15 @@ func TestDecodeToJsonCSV(t *testing.T) {
118118
{
119119
name: "custom_columns",
120120
input: `"a","""b""","c"` + "\n",
121-
params: map[string]any{
121+
params: Params{
122122
columnNamesParam: []any{"service", "version", "info"},
123123
},
124124
want: `{"service":"a","version":"\"b\"","info":"c"}`,
125125
},
126126
{
127127
name: "custom_prefix",
128128
input: `"a";"""b""";"c"` + "\n",
129-
params: map[string]any{
129+
params: Params{
130130
prefixParam: "csv_",
131131
delimiterParam: ";",
132132
},
@@ -135,15 +135,15 @@ func TestDecodeToJsonCSV(t *testing.T) {
135135
{
136136
name: "wrong_number_of_fields_1",
137137
input: "a,b,c,d" + "\n",
138-
params: map[string]any{
138+
params: Params{
139139
columnNamesParam: []any{"column_a", "column_b", "column_c"},
140140
},
141141
wantDecodeErr: true,
142142
},
143143
{
144144
name: "wrong_number_of_fields_2",
145145
input: "a,b,c,d" + "\n",
146-
params: map[string]any{
146+
params: Params{
147147
columnNamesParam: []any{"column_a", "column_b", "column_c"},
148148
invalidLineModeParam: "continue",
149149
prefixParam: "csv_",
@@ -153,7 +153,7 @@ func TestDecodeToJsonCSV(t *testing.T) {
153153
{
154154
name: "wrong_number_of_fields_3",
155155
input: "a,b" + "\n",
156-
params: map[string]any{
156+
params: Params{
157157
columnNamesParam: []any{"column_a", "column_b", "column_c"},
158158
invalidLineModeParam: "continue",
159159
},

‎decoder/decoder.go‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
package decoder
22

3-
import insaneJSON "github.com/ozontech/insane-json"
3+
import (
4+
"fmt"
5+
6+
insaneJSON "github.com/ozontech/insane-json"
7+
)
8+
9+
type Params map[string]any
410

511
type Type int
612

@@ -23,3 +29,51 @@ type Decoder interface {
2329
DecodeToJson(root *insaneJSON.Root, data []byte) error
2430
Decode(data []byte, args ...any) (any, error)
2531
}
32+
33+
func TypeFromString(s string) Type {
34+
switch s {
35+
case "json":
36+
return JSON
37+
case "raw":
38+
return RAW
39+
case "cri":
40+
return CRI
41+
case "postgres":
42+
return POSTGRES
43+
case "nginx_error":
44+
return NGINX_ERROR
45+
case "protobuf":
46+
return PROTOBUF
47+
case "syslog_rfc3164":
48+
return SYSLOG_RFC3164
49+
case "syslog_rfc5424":
50+
return SYSLOG_RFC5424
51+
case "csv":
52+
return CSV
53+
case "auto":
54+
return AUTO
55+
default:
56+
return NO
57+
}
58+
}
59+
60+
func New(t Type, params Params) (Decoder, error) {
61+
switch t {
62+
case JSON:
63+
return NewJsonDecoder(params)
64+
case NGINX_ERROR:
65+
return NewNginxErrorDecoder(params)
66+
case PROTOBUF:
67+
return NewProtobufDecoder(params)
68+
case SYSLOG_RFC3164:
69+
return NewSyslogRFC3164Decoder(params)
70+
case SYSLOG_RFC5424:
71+
return NewSyslogRFC5424Decoder(params)
72+
case CSV:
73+
return NewCSVDecoder(params)
74+
case RAW, CRI, POSTGRES, AUTO:
75+
return nil, nil
76+
default:
77+
return nil, fmt.Errorf("unknown decoder type: %v", t)
78+
}
79+
}

‎decoder/json.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ type jsonDecoder struct {
3030
mu *sync.Mutex
3131
}
3232

33-
func NewJsonDecoder(params map[string]any) (Decoder, error) {
33+
func NewJsonDecoder(params Params) (Decoder, error) {
3434
p, err := extractJsonParams(params)
3535
if err != nil {
3636
return nil, fmt.Errorf("can't extract params: %w", err)
@@ -132,7 +132,7 @@ func (d *jsonDecoder) cutFieldsBySize(data []byte) []byte {
132132
return data
133133
}
134134

135-
func extractJsonParams(params map[string]any) (jsonParams, error) {
135+
func extractJsonParams(params Params) (jsonParams, error) {
136136
maxFieldsSize := make(map[string]int)
137137
if maxFieldsSizeRaw, ok := params[jsonMaxFieldsSizeParam]; ok {
138138
maxFieldsSizeMap, ok := maxFieldsSizeRaw.(map[string]any)

‎decoder/json_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ func TestJson(t *testing.T) {
1818
name string
1919

2020
input string
21-
params map[string]any
21+
params Params
2222

2323
want map[string]string
2424
wantCreateErr bool
@@ -39,7 +39,7 @@ func TestJson(t *testing.T) {
3939
{
4040
name: "valid_max_fields_size",
4141
input: inputJson,
42-
params: map[string]any{
42+
params: Params{
4343
jsonMaxFieldsSizeParam: map[string]any{
4444
"": json.Number("1"),
4545
"not_exists": json.Number("100"),
@@ -61,7 +61,7 @@ func TestJson(t *testing.T) {
6161
{
6262
name: "valid_max_fields_size_single",
6363
input: inputJson,
64-
params: map[string]any{
64+
params: Params{
6565
jsonMaxFieldsSizeParam: map[string]any{
6666
"f2.f2_2.f2_2_2": json.Number("4"),
6767
},
@@ -77,14 +77,14 @@ func TestJson(t *testing.T) {
7777
},
7878
{
7979
name: "invalid_create_1",
80-
params: map[string]any{
80+
params: Params{
8181
jsonMaxFieldsSizeParam: "not_map",
8282
},
8383
wantCreateErr: true,
8484
},
8585
{
8686
name: "invalid_create_2",
87-
params: map[string]any{
87+
params: Params{
8888
jsonMaxFieldsSizeParam: map[string]any{
8989
"test": json.Number("not_num"),
9090
},
@@ -93,7 +93,7 @@ func TestJson(t *testing.T) {
9393
},
9494
{
9595
name: "invalid_create_3",
96-
params: map[string]any{
96+
params: Params{
9797
jsonMaxFieldsSizeParam: map[string]any{
9898
"test": json.Number("1.2"),
9999
},

‎decoder/nginx.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ type nginxErrorDecoder struct {
3131
params nginxErrorParams
3232
}
3333

34-
func NewNginxErrorDecoder(params map[string]any) (Decoder, error) {
34+
func NewNginxErrorDecoder(params Params) (Decoder, error) {
3535
p, err := extractNginxErrorParams(params)
3636
if err != nil {
3737
return nil, fmt.Errorf("can't extract params: %w", err)
@@ -198,7 +198,7 @@ func (d *nginxErrorDecoder) extractCustomFields(data []byte) ([]byte, map[string
198198
return data, fields
199199
}
200200

201-
func extractNginxErrorParams(params map[string]any) (nginxErrorParams, error) {
201+
func extractNginxErrorParams(params Params) (nginxErrorParams, error) {
202202
withCustomFields := false
203203
if withCustomFieldsRaw, ok := params[nginxWithCustomFieldsParam]; ok {
204204
withCustomFields, ok = withCustomFieldsRaw.(bool)

‎decoder/nginx_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ func TestNginxError(t *testing.T) {
1111
name string
1212

1313
input string
14-
params map[string]any
14+
params Params
1515

1616
want NginxErrorRow
1717
wantCreateErr bool
@@ -64,7 +64,7 @@ func TestNginxError(t *testing.T) {
6464
{
6565
name: "valid_custom_fields",
6666
input: `2022/08/18 09:29:37 [error] 844935#844935: *44934601 upstream timed out (110: Operation timed out), while connecting to upstream, client: 10.125.172.251, server: , request: "POST /download HTTP/1.1", upstream: "http://10.117.246.15:84/download", host: "mpm-youtube-downloader-38.name.tldn:84", test:`,
67-
params: map[string]any{
67+
params: Params{
6868
nginxWithCustomFieldsParam: true,
6969
},
7070
want: NginxErrorRow{
@@ -86,7 +86,7 @@ func TestNginxError(t *testing.T) {
8686
},
8787
{
8888
name: "invalid_create",
89-
params: map[string]any{
89+
params: Params{
9090
nginxWithCustomFieldsParam: "not bool",
9191
},
9292
wantCreateErr: true,

‎decoder/protobuf.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ type protobufDecoder struct {
3232
msgDesc protoreflect.MessageDescriptor
3333
}
3434

35-
func NewProtobufDecoder(params map[string]any) (Decoder, error) {
35+
func NewProtobufDecoder(params Params) (Decoder, error) {
3636
p, err := extractProtobufParams(params)
3737
if err != nil {
3838
return nil, fmt.Errorf("can't extract params: %w", err)
@@ -104,7 +104,7 @@ func (d *protobufDecoder) Decode(data []byte, _ ...any) (any, error) {
104104
return msgJson, nil
105105
}
106106

107-
func extractProtobufParams(params map[string]any) (protobufParams, error) {
107+
func extractProtobufParams(params Params) (protobufParams, error) {
108108
fileRaw, ok := params[protoFileParam]
109109
if !ok {
110110
return protobufParams{}, fmt.Errorf("%q not set", protoFileParam)

0 commit comments

Comments
 (0)