v1.0.1

- Fix Sqreen's overhead rate calculation when the observed execution time is 0.
  This situation can happen when the execution timer wasn't precise enough in
  order to observe fast execution times.

- Add a new backend url configuration key `ingestion_url` for validation
  testing.

CHANGELOG.md

@@ -1,3 +1,16 @@
+# v1.0.1 - 16 February 2021
+
+## Fix
+
+- Fix Sqreen's overhead rate calculation when the observed execution time is 0.
+  This situation can happen when the execution timer wasn't precise enough in
+  order to observe fast execution times.
+
+## Internal Changes
+
+- Add a new backend url configuration key `ingestion_url` for validation
+  testing.
+
 # v1.0.0 - 19 November 2020
 
 ## New Features

examples/docker/alpine/Dockerfile

@@ -8,6 +8,10 @@ FROM golang:$GO_VERSION AS build
 WORKDIR /app
 COPY . .
 
+# Update the go.mod and go.sum dependencies
+RUN go get -d github.com/sqreen/go-agent/sdk/sqreen-instrumentation-tool
+RUN go get -d ./...
+
 # Install Sqreen's instrumentation tool.
 # Go modules make it easier by correctly choosing the version of your go.mod.
 RUN go build -v github.com/sqreen/go-agent/sdk/sqreen-instrumentation-tool

examples/docker/debian/Dockerfile

@@ -8,8 +8,11 @@ FROM golang:$GO_VERSION AS build
 WORKDIR /app
 COPY . .
 
+# Update the go.mod and go.sum dependencies
+RUN go get -d github.com/sqreen/go-agent/sdk/sqreen-instrumentation-tool
+RUN go get -d ./...
+
 # Install Sqreen's instrumentation tool.
-# Go modules make it easier by correctly choosing the version of your go.mod.
 RUN go build -v github.com/sqreen/go-agent/sdk/sqreen-instrumentation-tool
 
 # Compile the app with the previously built tool.

examples/docker/scratch/Dockerfile

@@ -9,6 +9,10 @@ FROM golang:$GO_VERSION AS build
 WORKDIR /app
 COPY . .
 
+# Update the go.mod and go.sum dependencies
+RUN go get -d github.com/sqreen/go-agent/sdk/sqreen-instrumentation-tool
+RUN go get -d ./...
+
 # Install Sqreen's instrumentation tool.
 # Go modules make it easier by correctly choosing the version of your go.mod.
 RUN go build -v github.com/sqreen/go-agent/sdk/sqreen-instrumentation-tool

go.mod

@@ -22,7 +22,6 @@ require (
 	github.com/mattn/go-colorable v0.1.8 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/mxschmitt/golang-combinations v1.1.0
 	github.com/onsi/ginkgo v1.7.0
 	github.com/onsi/gomega v1.4.3
 	github.com/pkg/errors v0.9.1
@@ -31,6 +30,7 @@ require (
 	github.com/spf13/viper v1.3.2
 	github.com/sqreen/go-libsqreen v0.7.1
 	github.com/sqreen/go-sdk/signal v1.2.0
+	github.com/stretchr/objx v0.3.0 // indirect
 	github.com/stretchr/testify v1.6.1
 	go.elastic.co/apm/module/apmsql v1.9.0
 	golang.org/x/crypto v0.0.0-20201116153603-4be66e5b6582 // indirect
@@ -42,4 +42,5 @@ require (
 	gopkg.in/go-playground/assert.v1 v1.2.1
 	gopkg.in/go-playground/validator.v8 v8.18.2 // indirect
 	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )

go.sum

@@ -161,8 +161,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/mxschmitt/golang-combinations v1.1.0 h1:WlIZCnDm+Xlb2pRPf+R/qPKlGOU1w8lpN69/uy5z+Zg=
-github.com/mxschmitt/golang-combinations v1.1.0/go.mod h1:RbMhWvfCelHR6WROvT2bVfxJvZHoEvBj71SKe+H0MYU=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -215,6 +213,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
+github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -360,6 +360,8 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M=
 howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=

internal/agent.go

@@ -11,6 +11,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"math"
+	"net/url"
 	"os"
 	"runtime"
 	"sync"
@@ -194,7 +196,7 @@ type staticMetrics struct {
 	allowedIP,
 	allowedPath,
 	callCounts *metrics.TimeHistogram
-	requestTime, sqreenTime, sqreenOverheadPercentage *metrics.PerfHistogram
+	requestTime, sqreenTime, sqreenOverheadRate *metrics.PerfHistogram
 }
 
 // Error channel buffer length.
@@ -256,7 +258,12 @@ func New(cfg *config.Config) *AgentType {
 	if err != nil {
 		logger.Error(sqerrors.Wrap(err, "`req` performance histogram constructor error"))
 	}
-	sqOverheadPercentage, err := metrics.PerfHistogram("pct", perfHistogramUnit, perfHistogramBase, perfHistogramPeriod)
+
+	const (
+		sqreenOverheadRateBase = 1.3
+		sqreenOverheadRateUnit = 1.0
+	)
+	sqOverheadRate, err := metrics.PerfHistogram("pct", sqreenOverheadRateUnit, sqreenOverheadRateBase, perfHistogramPeriod)
 	if err != nil {
 		logger.Error(sqerrors.Wrap(err, "`pct` performance histogram constructor error"))
 	}
@@ -269,14 +276,14 @@ func New(cfg *config.Config) *AgentType {
 		isDone:        make(chan struct{}),
 		metrics:       metrics,
 		staticMetrics: staticMetrics{
-			sdkUserLoginSuccess:      metrics.TimeHistogram("sdk-login-success", sdkMetricsPeriod, 60000),
-			sdkUserLoginFailure:      metrics.TimeHistogram("sdk-login-fail", sdkMetricsPeriod, 60000),
-			sdkUserSignup:            metrics.TimeHistogram("sdk-signup", sdkMetricsPeriod, 60000),
-			allowedIP:                metrics.TimeHistogram("whitelisted", sdkMetricsPeriod, 60000),
-			allowedPath:              metrics.TimeHistogram("whitelisted_paths", sdkMetricsPeriod, 60000),
-			requestTime:              req,
-			sqreenTime:               sq,
-			sqreenOverheadPercentage: sqOverheadPercentage,
+			sdkUserLoginSuccess: metrics.TimeHistogram("sdk-login-success", sdkMetricsPeriod, 60000),
+			sdkUserLoginFailure: metrics.TimeHistogram("sdk-login-fail", sdkMetricsPeriod, 60000),
+			sdkUserSignup:       metrics.TimeHistogram("sdk-signup", sdkMetricsPeriod, 60000),
+			allowedIP:           metrics.TimeHistogram("whitelisted", sdkMetricsPeriod, 60000),
+			allowedPath:         metrics.TimeHistogram("whitelisted_paths", sdkMetricsPeriod, 60000),
+			requestTime:         req,
+			sqreenTime:          sq,
+			sqreenOverheadRate:  sqOverheadRate,
 		},
 		ctx:         ctx,
 		cancel:      cancel,
@@ -322,9 +329,10 @@ func (a *AgentType) sendClosedHTTPProtectionContext(ctx http_protection_types.Cl
 		a.logger.Error(sqerrors.Wrap(err, "could not add sqreen's execution time"))
 	}
 
-	sqOverhead := 100 * sq / (req - sq)
-	if err := a.staticMetrics.sqreenOverheadPercentage.Add(sqOverhead); err != nil {
-		a.logger.Error(sqerrors.Wrap(err, "could not add sqreen's execution time"))
+	if overheadRate, err := overheadRate(req, sq); err == nil {
+		if err := a.staticMetrics.sqreenOverheadRate.Add(overheadRate); err != nil {
+			a.logger.Error(sqerrors.Wrap(err, "could not add sqreen overhead rate"))
+		}
 	}
 
 	event := newClosedHTTPRequestContextEvent(a.RulespackID(), start, finish, ctx.Response(), ctx.Request(), events)
@@ -334,6 +342,27 @@ func (a *AgentType) sendClosedHTTPProtectionContext(ctx http_protection_types.Cl
 	a.eventMng.send(event)
 }
 
+func overheadRate(req float64, sq float64) (rate float64, err error) {
+	if req <= 0 || math.IsNaN(req) || math.IsInf(req, 0) {
+		return 0, sqerrors.Errorf("unexpected req value `%v`", req)
+	}
+	if sq <= 0 || math.IsNaN(sq) || math.IsInf(sq, 0) {
+		return 0, sqerrors.Errorf("unexpected sq value `%v`", sq)
+	}
+	if req < sq {
+		return 0, sqerrors.Errorf("unexpected req `%v` and sq `%v` values: req should be greater or equal to sq", req, sq)
+	}
+	if req == sq {
+		return 100, nil
+	}
+
+	userTime := req - sq
+	if userTime == 0 {
+		return 100, nil
+	}
+	return 100 * sq / userTime, nil
+}
+
 type withNotificationError struct {
 	error
 }
@@ -349,7 +378,8 @@ func (a *AgentType) Serve() error {
 
 	token := a.config.BackendHTTPAPIToken()
 	appName := a.config.AppName()
-	appLoginRes, err := appLogin(a.ctx, a.logger, a.client, token, appName, a.appInfo, a.config.DisableSignalBackend())
+	ingestionUrl, _ := url.Parse(a.config.IngestionBackendHTTPAPIBaseURL())
+	appLoginRes, err := appLogin(a.ctx, a.logger, a.client, token, appName, a.appInfo, a.config.DisableSignalBackend(), ingestionUrl)
 	if err != nil {
 		if xerrors.Is(err, context.Canceled) {
 			a.logger.Debug(err)

internal/agent_test.go

@@ -0,0 +1,37 @@
+// Copyright (c) 2016 - 2020 Sqreen. All Rights Reserved.
+// Please refer to our terms for more information:
+// https://www.sqreen.io/terms.html
+
+package internal
+
+import (
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_overheadRate(t *testing.T) {
+	t.Run("no panics nor special float numbers", func(t *testing.T) {
+		limits := []float64{
+			math.MaxFloat64,
+			-math.MaxFloat64,
+			math.NaN(),
+			math.Inf(1),
+			math.Inf(-1),
+			0,
+			1,
+			-1,
+			-1e100,
+			1e100,
+		}
+		for _, req := range limits {
+			for _, sq := range limits {
+				require.NotPanics(t, func() {
+					rate, _ := overheadRate(req, sq)
+					require.False(t, math.IsNaN(rate) || math.IsInf(rate, 0))
+				})
+			}
+		}
+	})
+}

internal/backend/api/signal/signal.go

@@ -239,14 +239,14 @@ func newAgentExceptionPayload(klass, message string, infos interface{}) *api.Sig
 }
 
 func FromLegacyMetrics(metrics []legacy_api.MetricsTimeBucket, agentVersion string, logger plog.ErrorLogger) api.Batch {
-	batch := make(api.Batch, len(metrics))
-	for i, metric := range metrics {
+	batch := make(api.Batch, 0, len(metrics))
+	for _, metric := range metrics {
 		metric, err := convertLegacyMetrics(&metric, agentVersion)
 		if err != nil {
 			logger.Error(err)
-		} else {
-			batch[i] = metric
+			continue
 		}
+		batch = append(batch, metric)
 	}
 	return batch
 }

internal/backend/client.go

@@ -115,7 +115,7 @@ func (c *Client) Health() HealthStatus {
 	return health
 }
 
-func (c *Client) AppLogin(req *api.AppLoginRequest, token string, appName string, disableSignalBackend bool) (*api.AppLoginResponse, error) {
+func (c *Client) AppLogin(req *api.AppLoginRequest, token string, appName string, disableSignalBackend bool, defaultIngestionUrl *url.URL) (*api.AppLoginResponse, error) {
 	httpReq, err := c.newRequest(&config.BackendHTTPAPIEndpoint.AppLogin)
 	if err != nil {
 		return nil, err
@@ -138,6 +138,9 @@ func (c *Client) AppLogin(req *api.AppLoginRequest, token string, appName string
 
 	if !disableSignalBackend && res.Features.UseSignals {
 		c.signalClient = client.NewClient(c.client, c.session)
+		if defaultIngestionUrl != nil {
+			c.signalClient.BaseURL = defaultIngestionUrl
+		}
 
 		// If the default signal URL is not healthy, fallback to the general
 		// backend URL.

internal/backend/client_test.go

@@ -170,7 +170,7 @@ func initFakeServerSession(endpointCfg *config.HTTPAPIEndpoint, request, respons
 
 	token := testlib.RandHTTPHeaderValue(2, 50)
 	appName := testlib.RandHTTPHeaderValue(2, 50)
-	_, err = client.AppLogin(loginReq, token, appName, false)
+	_, err = client.AppLogin(loginReq, token, appName, false, nil)
 	if err != nil {
 		panic(err)
 	}

internal/client.go

@@ -12,6 +12,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/pkg/errors"
@@ -53,7 +54,7 @@ func (e LoginError) Unwrap() error {
 
 // Login to the backend. When the API request fails, retry for ever and after
 // sleeping some time.
-func appLogin(ctx context.Context, logger plog.DebugLevelLogger, client *backend.Client, token string, appName string, appInfo *app.Info, disableSignalBackend bool) (*api.AppLoginResponse, error) {
+func appLogin(ctx context.Context, logger plog.DebugLevelLogger, client *backend.Client, token string, appName string, appInfo *app.Info, disableSignalBackend bool, defaultIngestionUrl *url.URL) (*api.AppLoginResponse, error) {
 	_, bundleSignature, err := appInfo.Dependencies()
 	if err != nil {
 		logger.Error(withNotificationError{sqerrors.Wrap(err, "could not retrieve the program dependencies")})
@@ -83,7 +84,7 @@ func appLogin(ctx context.Context, logger plog.DebugLevelLogger, client *backend
 		case <-ctx.Done():
 			return nil, ctx.Err()
 		default:
-			appLoginRes, err = client.AppLogin(&appLoginReq, token, appName, disableSignalBackend)
+			appLoginRes, err = client.AppLogin(&appLoginReq, token, appName, disableSignalBackend, defaultIngestionUrl)
 			if err == nil && appLoginRes.Status {
 				return appLoginRes, nil
 			}

internal/config/config.go

@@ -215,26 +215,29 @@ const (
 const (
 	configEnvKeyConfigFile = `config_file`
 
-	configKeyBackendHTTPAPIBaseURL     = `url`
-	configKeyBackendHTTPAPIToken       = `token`
-	configKeyLogLevel                  = `log_level`
-	configKeyAppName                   = `app_name`
-	configKeyHTTPClientIPHeader        = `ip_header`
-	configKeyHTTPClientIPHeaderFormat  = `ip_header_format`
-	configKeyBackendHTTPAPIProxy       = `proxy`
-	configKeyDisable                   = `disable`
-	configKeyStripHTTPReferer          = `strip_http_referer`
-	configKeyRules                     = `rules`
-	configKeySDKMetricsPeriod          = `sdk_metrics_period`
-	configKeyMaxMetricsStoreLength     = `max_metrics_store_length`
-	configKeyDisableSignalBackend      = `disable_signal_backend`
-	configKeyStripSensitiveKeyRegexp   = `strip_sensitive_key_regexp`
-	configKeyStripSensitiveValueRegexp = `strip_sensitive_value_regexp`
+	configKeyBackendHTTPAPIBaseURL          = `url`
+	configKeyIngestionBackendHTTPAPIBaseURL = `ingestion_url`
+	configKeyBackendHTTPAPIToken            = `token`
+	configKeyLogLevel                       = `log_level`
+	configKeyAppName                        = `app_name`
+	configKeyHTTPClientIPHeader             = `ip_header`
+	configKeyHTTPClientIPHeaderFormat       = `ip_header_format`
+	configKeyBackendHTTPAPIProxy            = `proxy`
+	configKeyDisable                        = `disable`
+	configKeyStripHTTPReferer               = `strip_http_referer`
+	configKeyRules                          = `rules`
+	configKeySDKMetricsPeriod               = `sdk_metrics_period`
+	configKeyMaxMetricsStoreLength          = `max_metrics_store_length`
+	configKeyDisableSignalBackend           = `disable_signal_backend`
+	configKeyStripSensitiveKeyRegexp        = `strip_sensitive_key_regexp`
+	configKeyStripSensitiveValueRegexp      = `strip_sensitive_value_regexp`
 )
 
 // User configuration's default values.
 const (
-	configDefaultBackendHTTPAPIBaseURL = `https://back.sqreen.com`
+	configDefaultBackendHTTPAPIBaseURL          = `https://back.sqreen.com`
+	configDefaultIngestionBackendHTTPAPIBaseURL = "https://ingestion.sqreen.com/"
+
 	configDefaultLogLevel              = `info`
 	configDefaultSDKMetricsPeriod      = 60
 	configDefaultMaxMetricsStoreLength = 100 * 1024 * 1024
@@ -265,6 +268,7 @@ func New(logger *plog.Logger) (*Config, error) {
 		hidden         bool
 	}{
 		{key: configKeyBackendHTTPAPIBaseURL, defaultValue: configDefaultBackendHTTPAPIBaseURL},
+		{key: configKeyIngestionBackendHTTPAPIBaseURL, defaultValue: configDefaultIngestionBackendHTTPAPIBaseURL},
 		{key: configKeyLogLevel, defaultValue: configDefaultLogLevel},
 		{key: configKeyBackendHTTPAPIToken, defaultValue: "", secretFromChar: len(BackendHTTPAPIOrganizationTokenSubstr) + 3},
 		{key: configKeyAppName, defaultValue: ""},
@@ -345,6 +349,11 @@ func (c *Config) BackendHTTPAPIBaseURL() string {
 	return sanitizeString(c.GetString(configKeyBackendHTTPAPIBaseURL))
 }
 
+// IngestionBackendHTTPAPIBaseURL returns the base URL of the backend HTTP API.
+func (c *Config) IngestionBackendHTTPAPIBaseURL() string {
+	return sanitizeString(c.GetString(configKeyIngestionBackendHTTPAPIBaseURL))
+}
+
 // BackendHTTPAPIToken returns the access token to the backend API.
 func (c *Config) BackendHTTPAPIToken() string {
 	return sanitizeString(c.GetString(configKeyBackendHTTPAPIToken))