v0.15.0

New Feature: - (#149) Add activity monitoring of RASP protections. The number of times a protection has been used in the application is now displayed on the `Activity` tab of each RASP protection dashboard page. Fixes: - (#148) Fix the usage of the Go agent in a reverse proxy server: avoid automatically reading a POST request's body because of the former usage of `Request.ParseForm()` in Sqreen's middleware functions, and rather get POST form values from `Request.PostForm`, and URL query values from `Request.URL.Query()`. Note that since `Request.PostForm`'s value is assigned by `Request.ParseForm()`, the In-WAF and RASP protections will now consider POST form values when the request handler will have called `Request.ParseForm()` itself for its own needs. Therefore, the In-App WAF is now also attached to `ParseForm()` to monitor the resulting POST form values, which can return a non-nil error when an attack is detected (cf. <https://docs.sqreen.com/go/integration> for more Go integration details). - (ef81fc2) Enforce a request body reader to ignore it when blocked by the In-App WAF by returning it 0 bytes read along with the current non-nil error. This allows for example `io.LimitReader` not to copy the body buffer despite the non-nil error returned by the In-App WAF protection.
sqreen · Sep 10, 2020 · 23bbad3 · 23bbad3
2 parents af87910 + be856ea
commit 23bbad3
Show file tree

Hide file tree

Showing 23 changed files with 157 additions and 75 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,32 @@
+# v0.15.0 - 9 September 2020
+
+## New Feature
+
+- (#149) Add activity monitoring of RASP protections. The number of times a
+  protection has been used in the application is now displayed on the `Activity`
+  tab of each RASP protection dashboard page.
+
+## Fixes
+
+- (#148) Fix the usage of the Go agent in a reverse proxy server: avoid
+  automatically reading a POST request's body because of the former usage of
+  `Request.ParseForm()` in Sqreen's middleware functions, and rather get
+  POST form values from `Request.PostForm`, and URL query values from
+  `Request.URL.Query()`.  
+  Note that since `Request.PostForm`'s value is assigned by
+  `Request.ParseForm()`, the In-WAF and RASP protections will now only consider
+  POST form values when the request handler will have called
+  `Request.ParseForm()` itself for its own needs.  
+  Therefore, the In-App WAF is now also attached to `ParseForm()` to monitor
+  the resulting POST form values, and returns a non-nil error when an attack is detected
+  (cf. <https://docs.sqreen.com/go/integration> for more Go integration details).
+
+- (ef81fc2) Enforce a request body reader to ignore it when blocked by the
+  In-App WAF by returning it 0 bytes read along with the current non-nil error.
+  This allows for example `io.LimitReader` not to copy the body buffer despite
+  the non-nil error returned by the In-App WAF protection.
+
+
 # v0.14.0 - 2 September 2020
 
 ## New Feature

diff --git a/internal/adapter.go b/internal/adapter.go
@@ -136,7 +136,7 @@ func (a *httpRequestAPIAdapter) GetParameters() api.RequestRecord_Request_Parame
 		rawBody = "<Redacted By Sqreen>"
 	}
 	return api.RequestRecord_Request_Parameters{
-		Query:   req.Form(),
+		Query:   req.QueryForm(),
 		Form:    req.PostForm(),
 		Params:  req.Params(),
 		RawBody: rawBody,

diff --git a/internal/agent.go b/internal/agent.go
@@ -193,7 +193,13 @@ type AgentType struct {
 }
 
 type staticMetrics struct {
-	sdkUserLoginSuccess, sdkUserLoginFailure, sdkUserSignup, allowedIP, allowedPath, errors *metrics.Store
+	sdkUserLoginSuccess,
+	sdkUserLoginFailure,
+	sdkUserSignup,
+	allowedIP,
+	allowedPath,
+	errors,
+	callCounts *metrics.Store
 }
 
 // Error channel buffer length.

diff --git a/internal/backend/api/api.go b/internal/backend/api/api.go
@@ -186,17 +186,18 @@ type BatchRequest_Event struct {
 }
 
 type Rule struct {
-	Name       string             `json:"name"`
-	Hookpoint  Hookpoint          `json:"hookpoint"`
-	Data       RuleData           `json:"data"`
-	Metrics    []MetricDefinition `json:"metrics"`
-	Signature  RuleSignature      `json:"signature"`
-	Conditions RuleConditions     `json:"conditions"`
-	Callbacks  RuleCallbacks      `json:"callbacks"`
-	Test       bool               `json:"test"`
-	Block      bool               `json:"block"`
-	AttackType string             `json:"attack_type"`
-	Priority   int                `json:"priority"`
+	Name              string             `json:"name"`
+	Hookpoint         Hookpoint          `json:"hookpoint"`
+	Data              RuleData           `json:"data"`
+	Metrics           []MetricDefinition `json:"metrics"`
+	Signature         RuleSignature      `json:"signature"`
+	Conditions        RuleConditions     `json:"conditions"`
+	Callbacks         RuleCallbacks      `json:"callbacks"`
+	Test              bool               `json:"test"`
+	Block             bool               `json:"block"`
+	AttackType        string             `json:"attack_type"`
+	Priority          int                `json:"priority"`
+	CallCountInterval int                `json:"call_count_interval"`
 }
 
 type RuleConditions struct{}

diff --git a/internal/protection/http/bindingaccessor.go b/internal/protection/http/bindingaccessor.go
@@ -59,14 +59,17 @@ func (r *RequestBindingAccessorContext) FromContext(v interface{}) (*RequestBind
 }
 
 func (r *RequestBindingAccessorContext) FilteredParams() RequestParamMap {
-	form := r.Form()
+	queryForm := r.QueryForm()
+	postForm := r.PostForm()
 	params := r.RequestReader.Params()
-	if len(form) == 0 {
-		return params
-	}
 
-	res := make(types.RequestParamMap, 1+len(params))
-	res.Add("Form", form)
+	res := make(types.RequestParamMap, 2+len(params))
+	if len(postForm) > 0 {
+		res.Add("PostForm", postForm)
+	}
+	if len(queryForm) > 0 {
+		res.Add("QueryForm", queryForm)
+	}
 	for k, v := range params {
 		res.Add(k, v)
 	}

diff --git a/internal/protection/http/bindingaccessors_test.go b/internal/protection/http/bindingaccessors_test.go
@@ -48,14 +48,14 @@ func TestRequestBindingAccessors(t *testing.T) {
 		{
 			Title:  "GET with URL parameters",
 			Method: "GET",
-			URL:    "http://sqreen.com/admin?user=root&password=root",
+			URL:    "http://sqreen.com/admin?user=uid&password=pwd",
 			BindingAccessors: map[string]interface{}{
 				`#.Method`:                       "GET",
 				`#.Host`:                         "sqreen.com",
 				`#.ClientIP`:                     expectedClientIP,
-				`#.URL.RequestURI`:               "/admin?user=root&password=root",
-				`#.FilteredParams | flat_values`: FlattenedResult{"root", "root"},
-				`#.FilteredParams | flat_keys`:   FlattenedResult{"Form", "user", "password"},
+				`#.URL.RequestURI`:               "/admin?user=uid&password=pwd",
+				`#.FilteredParams | flat_values`: FlattenedResult{"uid", "pwd"},
+				`#.FilteredParams | flat_keys`:   FlattenedResult{"QueryForm", "user", "password"},
 				`#.Body.String`:                  "",
 				`#.Body.Bytes`:                   []byte(nil),
 			},
@@ -78,7 +78,7 @@ func TestRequestBindingAccessors(t *testing.T) {
 		{
 			Title:  "POST with multipart form data and URL parameters",
 			Method: "POST",
-			URL:    "http://sqreen.com/admin/news?user=root&password=root",
+			URL:    "http://sqreen.com/admin/news?user=uid&password=pwd",
 			Headers: http.Header{
 				"Content-Type": []string{multipartContentTypeHeader},
 			},
@@ -88,9 +88,9 @@ func TestRequestBindingAccessors(t *testing.T) {
 				`#.Host`:                         "sqreen.com",
 				`#.ClientIP`:                     expectedClientIP,
 				`#.Headers['Content-Type']`:      []string{multipartContentTypeHeader},
-				`#.URL.RequestURI`:               "/admin/news?user=root&password=root",
-				`#.FilteredParams | flat_values`: FlattenedResult{"root", "root", "value 1"},             // The multipart form data is not included for now
-				`#.FilteredParams | flat_keys`:   FlattenedResult{"Form", "user", "password", "field 1"}, // The multipart form data is not included for now
+				`#.URL.RequestURI`:               "/admin/news?user=uid&password=pwd",
+				`#.FilteredParams | flat_values`: FlattenedResult{"uid", "pwd", "value 1"},                                // The multipart form data is not included for now
+				`#.FilteredParams | flat_keys`:   FlattenedResult{"QueryForm", "user", "password", "PostForm", "field 1"}, // The multipart form data is not included for now
 				`#.Body.String`:                  string(multipartExpectedBody),
 				`#.Body.Bytes`:                   multipartExpectedBody,
 			},
@@ -110,7 +110,7 @@ func TestRequestBindingAccessors(t *testing.T) {
 				`#.Headers['Content-Type']`:      []string{`application/x-www-form-urlencoded`},
 				`#.URL.RequestURI`:               "/admin/news",
 				`#.FilteredParams | flat_values`: FlattenedResult{"post", "y", "2", "nokey", "", ""},
-				`#.FilteredParams | flat_keys`:   FlattenedResult{"Form", "z", "both", "prio", "", "orphan", "empty"},
+				`#.FilteredParams | flat_keys`:   FlattenedResult{"PostForm", "z", "both", "prio", "", "orphan", "empty"},
 				`#.Body.String`:                  `z=post&both=y&prio=2&=nokey&orphan;empty=&`,
 				`#.Body.Bytes`:                   []byte(`z=post&both=y&prio=2&=nokey&orphan;empty=&`),
 			},
@@ -130,7 +130,7 @@ func TestRequestBindingAccessors(t *testing.T) {
 				`#.Headers['Content-Type']`:      []string{`application/x-www-form-urlencoded`},
 				`#.URL.RequestURI`:               "/admin/news?sqreen=okay",
 				`#.FilteredParams | flat_values`: FlattenedResult{"post", "y", "2", "nokey", "", "", "okay"},
-				`#.FilteredParams | flat_keys`:   FlattenedResult{"Form", "empty", "z", "both", "prio", "", "orphan", "sqreen"},
+				`#.FilteredParams | flat_keys`:   FlattenedResult{"PostForm", "empty", "z", "both", "prio", "", "orphan", "QueryForm", "sqreen"},
 				`#.Body.String`:                  `z=post&both=y&prio=2&=nokey&orphan;empty=&`,
 				`#.Body.Bytes`:                   []byte(`z=post&both=y&prio=2&=nokey&orphan;empty=&`),
 			},
@@ -171,7 +171,7 @@ func TestRequestBindingAccessors(t *testing.T) {
 		{
 			Title:  "Extra request params",
 			Method: "GET",
-			URL:    "http://sqreen.com/admin?user=root&password=root",
+			URL:    "http://sqreen.com/admin?user=uid&password=pwd",
 			RequestParams: types.RequestParamMap{
 				"json": types.RequestParamValueSlice{
 					map[string]interface{}{
@@ -185,12 +185,12 @@ func TestRequestBindingAccessors(t *testing.T) {
 				`#.Method`:         "GET",
 				`#.Host`:           "sqreen.com",
 				`#.ClientIP`:       expectedClientIP,
-				`#.URL.RequestURI`: "/admin?user=root&password=root",
+				`#.URL.RequestURI`: "/admin?user=uid&password=pwd",
 				`#.FilteredParams`: http_protection.RequestParamMap{
-					"Form": []interface{}{
+					"QueryForm": []interface{}{
 						url.Values{
-							"user":     []string{"root"},
-							"password": []string{"root"},
+							"user":     []string{"uid"},
+							"password": []string{"pwd"},
 						},
 					},
 					"json": types.RequestParamValueSlice{
@@ -319,8 +319,8 @@ func (r requestReaderImpl) IsTLS() bool {
 	return r.r.TLS != nil
 }
 
-func (r requestReaderImpl) Form() url.Values {
-	return r.r.Form
+func (r requestReaderImpl) QueryForm() url.Values {
+	return r.r.URL.Query()
 }
 
 func (r requestReaderImpl) PostForm() url.Values {

diff --git a/internal/protection/http/http_test.go b/internal/protection/http/http_test.go
@@ -99,7 +99,7 @@ func (r *RequestReaderMockup) Referer() string {
 	return r.Called().String(0)
 }
 
-func (r *RequestReaderMockup) Form() url.Values {
+func (r *RequestReaderMockup) QueryForm() url.Values {
 	v, _ := r.Called().Get(0).(url.Values)
 	return v
 }

diff --git a/internal/protection/http/request.go b/internal/protection/http/request.go
@@ -76,6 +76,9 @@ func (t rawBodyWAF) Read(p []byte) (n int, err error) {
 
 	if err == io.EOF {
 		if wafErr := t.c.bodyWAF(); wafErr != nil {
+			// Return 0 and the sqreen error so that the caller doesn't take anything
+			// into account.
+			n = 0
 			err = wafErr
 		}
 	}
@@ -226,7 +229,7 @@ type handledRequest struct {
 	isTLS      bool
 	userAgent  string
 	referer    string
-	form       url.Values
+	queryForm  url.Values
 	postForm   url.Values
 	clientIP   net.IP
 	params     types.RequestParamMap
@@ -242,7 +245,7 @@ func (h *handledRequest) RemoteAddr() string            { return h.remoteAddr }
 func (h *handledRequest) IsTLS() bool                   { return h.isTLS }
 func (h *handledRequest) UserAgent() string             { return h.userAgent }
 func (h *handledRequest) Referer() string               { return h.referer }
-func (h *handledRequest) Form() url.Values              { return h.form }
+func (h *handledRequest) QueryForm() url.Values         { return h.queryForm }
 func (h *handledRequest) PostForm() url.Values          { return h.postForm }
 func (h *handledRequest) ClientIP() net.IP              { return h.clientIP }
 func (h *handledRequest) Params() types.RequestParamMap { return h.params }
@@ -270,7 +273,7 @@ func copyRequest(reader types.RequestReader) types.RequestReader {
 		isTLS:      reader.IsTLS(),
 		userAgent:  reader.UserAgent(),
 		referer:    reader.Referer(),
-		form:       reader.Form(),
+		queryForm:  reader.QueryForm(),
 		postForm:   reader.PostForm(),
 		clientIP:   reader.ClientIP(),
 		params:     reader.Params(),

diff --git a/internal/protection/http/types/types.go b/internal/protection/http/types/types.go
@@ -25,7 +25,7 @@ type RequestReader interface {
 	IsTLS() bool
 	UserAgent() string
 	Referer() string
-	Form() url.Values
+	QueryForm() url.Values
 	PostForm() url.Values
 	ClientIP() net.IP
 	// Params returns the request parameters parsed by the handler so far at the

diff --git a/internal/rule/callback.go b/internal/rule/callback.go
@@ -7,6 +7,7 @@
 package rule
 
 import (
+	"fmt"
 	"reflect"
 	"time"
 
@@ -29,16 +30,18 @@ import (
 //}
 
 type CallbackContext struct {
-	metricsStores       map[string]*metrics.Store
-	defaultMetricsStore *metrics.Store
-	errorMetricsStore   *metrics.Store
-	name                string
-	testMode            bool
-	config              callback.NativeCallbackConfig
-	attackType          string
+	metricsStores          map[string]*metrics.Store
+	defaultMetricsStore    *metrics.Store
+	errorMetricsStore      *metrics.Store
+	callCountsMetricsStore *metrics.Store
+	preCallCounter         string
+	name                   string
+	testMode               bool
+	config                 callback.NativeCallbackConfig
+	attackType             string
 }
 
-func NewCallbackContext(r *api.Rule, metricsEngine *metrics.Engine, errorMetricsStore *metrics.Store) (*CallbackContext, error) {
+func NewCallbackContext(r *api.Rule, rulepackID string, metricsEngine *metrics.Engine, errorMetricsStore *metrics.Store) (*CallbackContext, error) {
 	var (
 		metricsStores       map[string]*metrics.Store
 		defaultMetricsStore *metrics.Store
@@ -51,13 +54,24 @@ func NewCallbackContext(r *api.Rule, metricsEngine *metrics.Engine, errorMetrics
 		defaultMetricsStore = metricsStores[r.Metrics[0].Name]
 	}
 
+	var (
+		callCountsMetricsStore *metrics.Store
+		preCallCounter         string
+	)
+	if r.CallCountInterval != 0 {
+		callCountsMetricsStore = metricsEngine.GetStore("sqreen_call_counts", 60*time.Second)
+		preCallCounter = fmt.Sprintf("%s/%s/pre", rulepackID, r.Name)
+	}
+
 	return &CallbackContext{
-		metricsStores:       metricsStores,
-		defaultMetricsStore: defaultMetricsStore,
-		errorMetricsStore:   errorMetricsStore,
-		name:                r.Name,
-		testMode:            r.Test,
-		attackType:          r.AttackType,
+		metricsStores:          metricsStores,
+		defaultMetricsStore:    defaultMetricsStore,
+		errorMetricsStore:      errorMetricsStore,
+		name:                   r.Name,
+		testMode:               r.Test,
+		attackType:             r.AttackType,
+		preCallCounter:         preCallCounter,
+		callCountsMetricsStore: callCountsMetricsStore,
 	}, nil
 }
 
@@ -89,6 +103,15 @@ func (d *CallbackContext) NewAttackEvent(blocked bool, info interface{}, st erro
 	}
 }
 
+func (d *CallbackContext) MonitorPre() {
+	// TODO: execution time monitoring and cap
+	if d.callCountsMetricsStore != nil {
+		if err := d.callCountsMetricsStore.Add(d.preCallCounter, 1); err != nil {
+			// TODO: log the error
+		}
+	}
+}
+
 type (
 	reflectedCallbackConfig struct {
 		callback.NativeCallbackConfig

diff --git a/internal/rule/callback/callback_test.go b/internal/rule/callback/callback_test.go
@@ -34,6 +34,10 @@ type RuleContextMockup struct {
 	mock.Mock
 }
 
+func (r *RuleContextMockup) MonitorPre() {
+	r.Called()
+}
+
 func (r *RuleContextMockup) PushMetricsValue(key interface{}, value int64) error {
 	return r.Called(key, value).Error(0)
 }

diff --git a/internal/rule/callback/ip-denylist_test.go b/internal/rule/callback/ip-denylist_test.go
@@ -115,7 +115,7 @@ func (r *RequestReaderMock) Referer() string {
 	return r.Called().String(0)
 }
 
-func (r *RequestReaderMock) Form() (v url.Values) {
+func (r *RequestReaderMock) QueryForm() (v url.Values) {
 	v, _ = r.Called().Get(0).(url.Values)
 	return
 }

diff --git a/internal/rule/callback/javascript.go b/internal/rule/callback/javascript.go
@@ -48,6 +48,7 @@ func NewJSExecCallback(rule RuleFace, cfg JSReflectedCallbackConfig) (sqhook.Ref
 			//}
 
 			if vm.hasPre() {
+				rule.MonitorPre()
 				result, err := vm.callPre(baCtx)
 				if err != nil {
 					// TODO: api adding more information to the error such as the

diff --git a/internal/rule/callback/shellshock.go b/internal/rule/callback/shellshock.go
@@ -68,6 +68,8 @@ func newShellshockPrologCallback(rule RuleFace, blockingMode bool, regexps []*re
 			return nil, nil
 		}
 
+		rule.MonitorPre()
+
 		env := (*attr).Env
 		if env == nil {
 			env = os.Environ()

diff --git a/internal/rule/callback/types.go b/internal/rule/callback/types.go
@@ -24,6 +24,7 @@ type RuleFace interface {
 	PushMetricsValue(key interface{}, value int64) error
 	// TODO: variadic options api
 	NewAttackEvent(blocked bool, info interface{}, st errors.StackTrace) *event.AttackEvent
+	MonitorPre()
 }
 
 // Config is the interface of the rule configuration.