Skip to content
This repository was archived by the owner on Nov 2, 2023. It is now read-only.

Commit 23bbad3

Browse files
author
Julio Guerra
committed
v0.15.0
New Feature: - (#149) Add activity monitoring of RASP protections. The number of times a protection has been used in the application is now displayed on the `Activity` tab of each RASP protection dashboard page. Fixes: - (#148) Fix the usage of the Go agent in a reverse proxy server: avoid automatically reading a POST request's body because of the former usage of `Request.ParseForm()` in Sqreen's middleware functions, and rather get POST form values from `Request.PostForm`, and URL query values from `Request.URL.Query()`. Note that since `Request.PostForm`'s value is assigned by `Request.ParseForm()`, the In-WAF and RASP protections will now consider POST form values when the request handler will have called `Request.ParseForm()` itself for its own needs. Therefore, the In-App WAF is now also attached to `ParseForm()` to monitor the resulting POST form values, which can return a non-nil error when an attack is detected (cf. <https://docs.sqreen.com/go/integration> for more Go integration details). - (ef81fc2) Enforce a request body reader to ignore it when blocked by the In-App WAF by returning it 0 bytes read along with the current non-nil error. This allows for example `io.LimitReader` not to copy the body buffer despite the non-nil error returned by the In-App WAF protection.
2 parents af87910 + be856ea commit 23bbad3

23 files changed

+157
-75
lines changed

‎CHANGELOG.md

+29
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,32 @@
1+
# v0.15.0 - 9 September 2020
2+
3+
## New Feature
4+
5+
- (#149) Add activity monitoring of RASP protections. The number of times a
6+
protection has been used in the application is now displayed on the `Activity`
7+
tab of each RASP protection dashboard page.
8+
9+
## Fixes
10+
11+
- (#148) Fix the usage of the Go agent in a reverse proxy server: avoid
12+
automatically reading a POST request's body because of the former usage of
13+
`Request.ParseForm()` in Sqreen's middleware functions, and rather get
14+
POST form values from `Request.PostForm`, and URL query values from
15+
`Request.URL.Query()`.
16+
Note that since `Request.PostForm`'s value is assigned by
17+
`Request.ParseForm()`, the In-WAF and RASP protections will now only consider
18+
POST form values when the request handler will have called
19+
`Request.ParseForm()` itself for its own needs.
20+
Therefore, the In-App WAF is now also attached to `ParseForm()` to monitor
21+
the resulting POST form values, and returns a non-nil error when an attack is detected
22+
(cf. <https://docs.sqreen.com/go/integration> for more Go integration details).
23+
24+
- (ef81fc2) Enforce a request body reader to ignore it when blocked by the
25+
In-App WAF by returning it 0 bytes read along with the current non-nil error.
26+
This allows for example `io.LimitReader` not to copy the body buffer despite
27+
the non-nil error returned by the In-App WAF protection.
28+
29+
130
# v0.14.0 - 2 September 2020
231

332
## New Feature

‎internal/adapter.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ func (a *httpRequestAPIAdapter) GetParameters() api.RequestRecord_Request_Parame
136136
rawBody = "<Redacted By Sqreen>"
137137
}
138138
return api.RequestRecord_Request_Parameters{
139-
Query: req.Form(),
139+
Query: req.QueryForm(),
140140
Form: req.PostForm(),
141141
Params: req.Params(),
142142
RawBody: rawBody,

‎internal/agent.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,13 @@ type AgentType struct {
193193
}
194194

195195
type staticMetrics struct {
196-
sdkUserLoginSuccess, sdkUserLoginFailure, sdkUserSignup, allowedIP, allowedPath, errors *metrics.Store
196+
sdkUserLoginSuccess,
197+
sdkUserLoginFailure,
198+
sdkUserSignup,
199+
allowedIP,
200+
allowedPath,
201+
errors,
202+
callCounts *metrics.Store
197203
}
198204

199205
// Error channel buffer length.

‎internal/backend/api/api.go

+12-11
Original file line numberDiff line numberDiff line change
@@ -186,17 +186,18 @@ type BatchRequest_Event struct {
186186
}
187187

188188
type Rule struct {
189-
Name string `json:"name"`
190-
Hookpoint Hookpoint `json:"hookpoint"`
191-
Data RuleData `json:"data"`
192-
Metrics []MetricDefinition `json:"metrics"`
193-
Signature RuleSignature `json:"signature"`
194-
Conditions RuleConditions `json:"conditions"`
195-
Callbacks RuleCallbacks `json:"callbacks"`
196-
Test bool `json:"test"`
197-
Block bool `json:"block"`
198-
AttackType string `json:"attack_type"`
199-
Priority int `json:"priority"`
189+
Name string `json:"name"`
190+
Hookpoint Hookpoint `json:"hookpoint"`
191+
Data RuleData `json:"data"`
192+
Metrics []MetricDefinition `json:"metrics"`
193+
Signature RuleSignature `json:"signature"`
194+
Conditions RuleConditions `json:"conditions"`
195+
Callbacks RuleCallbacks `json:"callbacks"`
196+
Test bool `json:"test"`
197+
Block bool `json:"block"`
198+
AttackType string `json:"attack_type"`
199+
Priority int `json:"priority"`
200+
CallCountInterval int `json:"call_count_interval"`
200201
}
201202

202203
type RuleConditions struct{}

‎internal/protection/http/bindingaccessor.go

+9-6
Original file line numberDiff line numberDiff line change
@@ -59,14 +59,17 @@ func (r *RequestBindingAccessorContext) FromContext(v interface{}) (*RequestBind
5959
}
6060

6161
func (r *RequestBindingAccessorContext) FilteredParams() RequestParamMap {
62-
form := r.Form()
62+
queryForm := r.QueryForm()
63+
postForm := r.PostForm()
6364
params := r.RequestReader.Params()
64-
if len(form) == 0 {
65-
return params
66-
}
6765

68-
res := make(types.RequestParamMap, 1+len(params))
69-
res.Add("Form", form)
66+
res := make(types.RequestParamMap, 2+len(params))
67+
if len(postForm) > 0 {
68+
res.Add("PostForm", postForm)
69+
}
70+
if len(queryForm) > 0 {
71+
res.Add("QueryForm", queryForm)
72+
}
7073
for k, v := range params {
7174
res.Add(k, v)
7275
}

‎internal/protection/http/bindingaccessors_test.go

+17-17
Original file line numberDiff line numberDiff line change
@@ -48,14 +48,14 @@ func TestRequestBindingAccessors(t *testing.T) {
4848
{
4949
Title: "GET with URL parameters",
5050
Method: "GET",
51-
URL: "http://sqreen.com/admin?user=root&password=root",
51+
URL: "http://sqreen.com/admin?user=uid&password=pwd",
5252
BindingAccessors: map[string]interface{}{
5353
`#.Method`: "GET",
5454
`#.Host`: "sqreen.com",
5555
`#.ClientIP`: expectedClientIP,
56-
`#.URL.RequestURI`: "/admin?user=root&password=root",
57-
`#.FilteredParams | flat_values`: FlattenedResult{"root", "root"},
58-
`#.FilteredParams | flat_keys`: FlattenedResult{"Form", "user", "password"},
56+
`#.URL.RequestURI`: "/admin?user=uid&password=pwd",
57+
`#.FilteredParams | flat_values`: FlattenedResult{"uid", "pwd"},
58+
`#.FilteredParams | flat_keys`: FlattenedResult{"QueryForm", "user", "password"},
5959
`#.Body.String`: "",
6060
`#.Body.Bytes`: []byte(nil),
6161
},
@@ -78,7 +78,7 @@ func TestRequestBindingAccessors(t *testing.T) {
7878
{
7979
Title: "POST with multipart form data and URL parameters",
8080
Method: "POST",
81-
URL: "http://sqreen.com/admin/news?user=root&password=root",
81+
URL: "http://sqreen.com/admin/news?user=uid&password=pwd",
8282
Headers: http.Header{
8383
"Content-Type": []string{multipartContentTypeHeader},
8484
},
@@ -88,9 +88,9 @@ func TestRequestBindingAccessors(t *testing.T) {
8888
`#.Host`: "sqreen.com",
8989
`#.ClientIP`: expectedClientIP,
9090
`#.Headers['Content-Type']`: []string{multipartContentTypeHeader},
91-
`#.URL.RequestURI`: "/admin/news?user=root&password=root",
92-
`#.FilteredParams | flat_values`: FlattenedResult{"root", "root", "value 1"}, // The multipart form data is not included for now
93-
`#.FilteredParams | flat_keys`: FlattenedResult{"Form", "user", "password", "field 1"}, // The multipart form data is not included for now
91+
`#.URL.RequestURI`: "/admin/news?user=uid&password=pwd",
92+
`#.FilteredParams | flat_values`: FlattenedResult{"uid", "pwd", "value 1"}, // The multipart form data is not included for now
93+
`#.FilteredParams | flat_keys`: FlattenedResult{"QueryForm", "user", "password", "PostForm", "field 1"}, // The multipart form data is not included for now
9494
`#.Body.String`: string(multipartExpectedBody),
9595
`#.Body.Bytes`: multipartExpectedBody,
9696
},
@@ -110,7 +110,7 @@ func TestRequestBindingAccessors(t *testing.T) {
110110
`#.Headers['Content-Type']`: []string{`application/x-www-form-urlencoded`},
111111
`#.URL.RequestURI`: "/admin/news",
112112
`#.FilteredParams | flat_values`: FlattenedResult{"post", "y", "2", "nokey", "", ""},
113-
`#.FilteredParams | flat_keys`: FlattenedResult{"Form", "z", "both", "prio", "", "orphan", "empty"},
113+
`#.FilteredParams | flat_keys`: FlattenedResult{"PostForm", "z", "both", "prio", "", "orphan", "empty"},
114114
`#.Body.String`: `z=post&both=y&prio=2&=nokey&orphan;empty=&`,
115115
`#.Body.Bytes`: []byte(`z=post&both=y&prio=2&=nokey&orphan;empty=&`),
116116
},
@@ -130,7 +130,7 @@ func TestRequestBindingAccessors(t *testing.T) {
130130
`#.Headers['Content-Type']`: []string{`application/x-www-form-urlencoded`},
131131
`#.URL.RequestURI`: "/admin/news?sqreen=okay",
132132
`#.FilteredParams | flat_values`: FlattenedResult{"post", "y", "2", "nokey", "", "", "okay"},
133-
`#.FilteredParams | flat_keys`: FlattenedResult{"Form", "empty", "z", "both", "prio", "", "orphan", "sqreen"},
133+
`#.FilteredParams | flat_keys`: FlattenedResult{"PostForm", "empty", "z", "both", "prio", "", "orphan", "QueryForm", "sqreen"},
134134
`#.Body.String`: `z=post&both=y&prio=2&=nokey&orphan;empty=&`,
135135
`#.Body.Bytes`: []byte(`z=post&both=y&prio=2&=nokey&orphan;empty=&`),
136136
},
@@ -171,7 +171,7 @@ func TestRequestBindingAccessors(t *testing.T) {
171171
{
172172
Title: "Extra request params",
173173
Method: "GET",
174-
URL: "http://sqreen.com/admin?user=root&password=root",
174+
URL: "http://sqreen.com/admin?user=uid&password=pwd",
175175
RequestParams: types.RequestParamMap{
176176
"json": types.RequestParamValueSlice{
177177
map[string]interface{}{
@@ -185,12 +185,12 @@ func TestRequestBindingAccessors(t *testing.T) {
185185
`#.Method`: "GET",
186186
`#.Host`: "sqreen.com",
187187
`#.ClientIP`: expectedClientIP,
188-
`#.URL.RequestURI`: "/admin?user=root&password=root",
188+
`#.URL.RequestURI`: "/admin?user=uid&password=pwd",
189189
`#.FilteredParams`: http_protection.RequestParamMap{
190-
"Form": []interface{}{
190+
"QueryForm": []interface{}{
191191
url.Values{
192-
"user": []string{"root"},
193-
"password": []string{"root"},
192+
"user": []string{"uid"},
193+
"password": []string{"pwd"},
194194
},
195195
},
196196
"json": types.RequestParamValueSlice{
@@ -319,8 +319,8 @@ func (r requestReaderImpl) IsTLS() bool {
319319
return r.r.TLS != nil
320320
}
321321

322-
func (r requestReaderImpl) Form() url.Values {
323-
return r.r.Form
322+
func (r requestReaderImpl) QueryForm() url.Values {
323+
return r.r.URL.Query()
324324
}
325325

326326
func (r requestReaderImpl) PostForm() url.Values {

‎internal/protection/http/http_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ func (r *RequestReaderMockup) Referer() string {
9999
return r.Called().String(0)
100100
}
101101

102-
func (r *RequestReaderMockup) Form() url.Values {
102+
func (r *RequestReaderMockup) QueryForm() url.Values {
103103
v, _ := r.Called().Get(0).(url.Values)
104104
return v
105105
}

‎internal/protection/http/request.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,9 @@ func (t rawBodyWAF) Read(p []byte) (n int, err error) {
7676

7777
if err == io.EOF {
7878
if wafErr := t.c.bodyWAF(); wafErr != nil {
79+
// Return 0 and the sqreen error so that the caller doesn't take anything
80+
// into account.
81+
n = 0
7982
err = wafErr
8083
}
8184
}
@@ -226,7 +229,7 @@ type handledRequest struct {
226229
isTLS bool
227230
userAgent string
228231
referer string
229-
form url.Values
232+
queryForm url.Values
230233
postForm url.Values
231234
clientIP net.IP
232235
params types.RequestParamMap
@@ -242,7 +245,7 @@ func (h *handledRequest) RemoteAddr() string { return h.remoteAddr }
242245
func (h *handledRequest) IsTLS() bool { return h.isTLS }
243246
func (h *handledRequest) UserAgent() string { return h.userAgent }
244247
func (h *handledRequest) Referer() string { return h.referer }
245-
func (h *handledRequest) Form() url.Values { return h.form }
248+
func (h *handledRequest) QueryForm() url.Values { return h.queryForm }
246249
func (h *handledRequest) PostForm() url.Values { return h.postForm }
247250
func (h *handledRequest) ClientIP() net.IP { return h.clientIP }
248251
func (h *handledRequest) Params() types.RequestParamMap { return h.params }
@@ -270,7 +273,7 @@ func copyRequest(reader types.RequestReader) types.RequestReader {
270273
isTLS: reader.IsTLS(),
271274
userAgent: reader.UserAgent(),
272275
referer: reader.Referer(),
273-
form: reader.Form(),
276+
queryForm: reader.QueryForm(),
274277
postForm: reader.PostForm(),
275278
clientIP: reader.ClientIP(),
276279
params: reader.Params(),

‎internal/protection/http/types/types.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ type RequestReader interface {
2525
IsTLS() bool
2626
UserAgent() string
2727
Referer() string
28-
Form() url.Values
28+
QueryForm() url.Values
2929
PostForm() url.Values
3030
ClientIP() net.IP
3131
// Params returns the request parameters parsed by the handler so far at the

‎internal/rule/callback.go

+37-14
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
package rule
88

99
import (
10+
"fmt"
1011
"reflect"
1112
"time"
1213

@@ -29,16 +30,18 @@ import (
2930
//}
3031

3132
type CallbackContext struct {
32-
metricsStores map[string]*metrics.Store
33-
defaultMetricsStore *metrics.Store
34-
errorMetricsStore *metrics.Store
35-
name string
36-
testMode bool
37-
config callback.NativeCallbackConfig
38-
attackType string
33+
metricsStores map[string]*metrics.Store
34+
defaultMetricsStore *metrics.Store
35+
errorMetricsStore *metrics.Store
36+
callCountsMetricsStore *metrics.Store
37+
preCallCounter string
38+
name string
39+
testMode bool
40+
config callback.NativeCallbackConfig
41+
attackType string
3942
}
4043

41-
func NewCallbackContext(r *api.Rule, metricsEngine *metrics.Engine, errorMetricsStore *metrics.Store) (*CallbackContext, error) {
44+
func NewCallbackContext(r *api.Rule, rulepackID string, metricsEngine *metrics.Engine, errorMetricsStore *metrics.Store) (*CallbackContext, error) {
4245
var (
4346
metricsStores map[string]*metrics.Store
4447
defaultMetricsStore *metrics.Store
@@ -51,13 +54,24 @@ func NewCallbackContext(r *api.Rule, metricsEngine *metrics.Engine, errorMetrics
5154
defaultMetricsStore = metricsStores[r.Metrics[0].Name]
5255
}
5356

57+
var (
58+
callCountsMetricsStore *metrics.Store
59+
preCallCounter string
60+
)
61+
if r.CallCountInterval != 0 {
62+
callCountsMetricsStore = metricsEngine.GetStore("sqreen_call_counts", 60*time.Second)
63+
preCallCounter = fmt.Sprintf("%s/%s/pre", rulepackID, r.Name)
64+
}
65+
5466
return &CallbackContext{
55-
metricsStores: metricsStores,
56-
defaultMetricsStore: defaultMetricsStore,
57-
errorMetricsStore: errorMetricsStore,
58-
name: r.Name,
59-
testMode: r.Test,
60-
attackType: r.AttackType,
67+
metricsStores: metricsStores,
68+
defaultMetricsStore: defaultMetricsStore,
69+
errorMetricsStore: errorMetricsStore,
70+
name: r.Name,
71+
testMode: r.Test,
72+
attackType: r.AttackType,
73+
preCallCounter: preCallCounter,
74+
callCountsMetricsStore: callCountsMetricsStore,
6175
}, nil
6276
}
6377

@@ -89,6 +103,15 @@ func (d *CallbackContext) NewAttackEvent(blocked bool, info interface{}, st erro
89103
}
90104
}
91105

106+
func (d *CallbackContext) MonitorPre() {
107+
// TODO: execution time monitoring and cap
108+
if d.callCountsMetricsStore != nil {
109+
if err := d.callCountsMetricsStore.Add(d.preCallCounter, 1); err != nil {
110+
// TODO: log the error
111+
}
112+
}
113+
}
114+
92115
type (
93116
reflectedCallbackConfig struct {
94117
callback.NativeCallbackConfig

‎internal/rule/callback/callback_test.go

+4
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ type RuleContextMockup struct {
3434
mock.Mock
3535
}
3636

37+
func (r *RuleContextMockup) MonitorPre() {
38+
r.Called()
39+
}
40+
3741
func (r *RuleContextMockup) PushMetricsValue(key interface{}, value int64) error {
3842
return r.Called(key, value).Error(0)
3943
}

‎internal/rule/callback/ip-denylist_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ func (r *RequestReaderMock) Referer() string {
115115
return r.Called().String(0)
116116
}
117117

118-
func (r *RequestReaderMock) Form() (v url.Values) {
118+
func (r *RequestReaderMock) QueryForm() (v url.Values) {
119119
v, _ = r.Called().Get(0).(url.Values)
120120
return
121121
}

‎internal/rule/callback/javascript.go

+1
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ func NewJSExecCallback(rule RuleFace, cfg JSReflectedCallbackConfig) (sqhook.Ref
4848
//}
4949

5050
if vm.hasPre() {
51+
rule.MonitorPre()
5152
result, err := vm.callPre(baCtx)
5253
if err != nil {
5354
// TODO: api adding more information to the error such as the

‎internal/rule/callback/shellshock.go

+2
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ func newShellshockPrologCallback(rule RuleFace, blockingMode bool, regexps []*re
6868
return nil, nil
6969
}
7070

71+
rule.MonitorPre()
72+
7173
env := (*attr).Env
7274
if env == nil {
7375
env = os.Environ()

‎internal/rule/callback/types.go

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ type RuleFace interface {
2424
PushMetricsValue(key interface{}, value int64) error
2525
// TODO: variadic options api
2626
NewAttackEvent(blocked bool, info interface{}, st errors.StackTrace) *event.AttackEvent
27+
MonitorPre()
2728
}
2829

2930
// Config is the interface of the rule configuration.

0 commit comments

Comments
 (0)