diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 63e93eeaaa..2f293a93b3 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -278,9 +278,6 @@ type config struct { // Value from DD_DYNAMIC_INSTRUMENTATION_ENABLED, default false. dynamicInstrumentationEnabled bool - // globalSampleRate holds sample rate read from environment variables. - globalSampleRate float64 - // ciVisibilityEnabled controls if the tracer is loaded with CI Visibility mode. default false ciVisibilityEnabled bool @@ -292,6 +289,40 @@ type config struct { } var ( + // globalSampleRate holds sample rate read from environment variables. + globalSampleRate = knobs.Register(&knobs.Definition[float64]{ + Default: math.NaN(), + // Out of scope: detecting if both DD_TRACE_SAMPLE_RATE and OTEL_TRACES_SAMPLER are set + // and reporting warning log and telemetry "otel.env.hiding" if so. + EnvVars: []knobs.EnvVar{ + { + Key: "DD_TRACE_SAMPLE_RATE", + }, + { + Key: "OTEL_TRACES_SAMPLER", + Transform: func(v string) string { + v, err := mapSampleRate(v) + if err != nil { + log.Warn("ignoring OTEL_TRACES_SAMPLER, error: %v", err) + return "" + } + return v + }, + }, + }, + Parse: func(s string) (float64, error) { + sampleRate, err := strconv.ParseFloat(s, 64) + if err != nil { + log.Warn("ignoring DD_TRACE_SAMPLE_RATE, error: %v", err) + return 0.0, knobs.ErrInvalidValue + } else if sampleRate < 0.0 || sampleRate > 1.0 { + log.Warn("ignoring DD_TRACE_SAMPLE_RATE: out of range %f", sampleRate) + return 0.0, knobs.ErrInvalidValue + } + return sampleRate, nil + }, + }) + // partialFlushEnabled specifices whether the tracer should enable partial flushing. Value // from DD_TRACE_PARTIAL_FLUSH_ENABLED, default false. partialFlushEnabled = knobs.Register(&knobs.Definition[bool]{ @@ -351,19 +382,6 @@ func newConfig(opts ...StartOption) *config { c := new(config) c.Scope = knobs.NewScope() c.sampler = NewAllSampler() - sampleRate := math.NaN() - if r := getDDorOtelConfig("sampleRate"); r != "" { - var err error - sampleRate, err = strconv.ParseFloat(r, 64) - if err != nil { - log.Warn("ignoring DD_TRACE_SAMPLE_RATE, error: %v", err) - sampleRate = math.NaN() - } else if sampleRate < 0.0 || sampleRate > 1.0 { - log.Warn("ignoring DD_TRACE_SAMPLE_RATE: out of range %f", sampleRate) - sampleRate = math.NaN() - } - } - c.globalSampleRate = sampleRate c.httpClientTimeout = time.Second * 10 // 10 seconds if v := os.Getenv("OTEL_LOGS_EXPORTER"); v != "" { diff --git a/ddtrace/tracer/sampler_test.go b/ddtrace/tracer/sampler_test.go index 117518b533..3e3473ce9b 100644 --- a/ddtrace/tracer/sampler_test.go +++ b/ddtrace/tracer/sampler_test.go @@ -20,6 +20,7 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/internal" "gopkg.in/DataDog/dd-trace-go.v1/internal/samplernames" + "github.com/darccio/knobs" "github.com/stretchr/testify/assert" "golang.org/x/time/rate" ) @@ -201,7 +202,8 @@ func TestRuleEnvVars(t *testing.T) { {in: "1point0", out: math.NaN()}, // default if invalid value } { t.Setenv("DD_TRACE_SAMPLE_RATE", tt.in) - res := newConfig().globalSampleRate + c := newConfig() + res := knobs.GetScope(c.Scope, globalSampleRate) if math.IsNaN(tt.out) { assert.True(math.IsNaN(res)) } else { @@ -226,7 +228,9 @@ func TestRuleEnvVars(t *testing.T) { assert := assert.New(t) t.Setenv("OTEL_TRACES_SAMPLER", tt.config) t.Setenv("OTEL_TRACES_SAMPLER_ARG", fmt.Sprintf("%f", tt.rate)) - assert.Equal(tt.rate, newConfig().globalSampleRate) + c := newConfig() + sampleRate := knobs.GetScope(c.Scope, globalSampleRate) + assert.Equal(tt.rate, sampleRate) }) } }) @@ -477,7 +481,9 @@ func TestRulesSampler(t *testing.T) { } t.Run("no-rules", func(t *testing.T) { assert := assert.New(t) - rs := newRulesSampler(nil, nil, newConfig().globalSampleRate) + c := newConfig() + sampleRate := knobs.GetScope(c.Scope, globalSampleRate) + rs := newRulesSampler(nil, nil, sampleRate) span := makeSpan("http.request", "test-service") result := rs.SampleTrace(span) @@ -542,7 +548,9 @@ func TestRulesSampler(t *testing.T) { assert.Nil(t, err) assert := assert.New(t) - rs := newRulesSampler(rules, nil, newConfig().globalSampleRate) + c := newConfig() + sampleRate := knobs.GetScope(c.Scope, globalSampleRate) + rs := newRulesSampler(rules, nil, sampleRate) span := makeFinishedSpan(tt.spanName, tt.spanSrv, tt.spanRsc, tt.spanTags) @@ -566,7 +574,9 @@ func TestRulesSampler(t *testing.T) { for _, v := range traceRules { t.Run("", func(t *testing.T) { assert := assert.New(t) - rs := newRulesSampler(v, nil, newConfig().globalSampleRate) + c := newConfig() + sampleRate := knobs.GetScope(c.Scope, globalSampleRate) + rs := newRulesSampler(v, nil, sampleRate) span := makeSpan("http.request", "test-service") result := rs.SampleTrace(span) @@ -592,7 +602,9 @@ func TestRulesSampler(t *testing.T) { for _, v := range traceRules { t.Run("", func(t *testing.T) { assert := assert.New(t) - rs := newRulesSampler(v, nil, newConfig().globalSampleRate) + c := newConfig() + sampleRate := knobs.GetScope(c.Scope, globalSampleRate) + rs := newRulesSampler(v, nil, sampleRate) span := makeSpan("http.request", "test-service") result := rs.SampleTrace(span) @@ -638,7 +650,9 @@ func TestRulesSampler(t *testing.T) { _, rules, err := samplingRulesFromEnv() assert.Nil(t, err) assert := assert.New(t) - rs := newRulesSampler(nil, rules, newConfig().globalSampleRate) + c := newConfig() + sampleRate := knobs.GetScope(c.Scope, globalSampleRate) + rs := newRulesSampler(nil, rules, sampleRate) span := makeFinishedSpan(tt.spanName, tt.spanSrv, "res-10", map[string]interface{}{"hostname": "hn-30"}) @@ -761,7 +775,8 @@ func TestRulesSampler(t *testing.T) { t.Run(fmt.Sprintf("%v", i), func(t *testing.T) { assert := assert.New(t) c := newConfig(WithSamplingRules(tt.rules)) - rs := newRulesSampler(nil, c.spanRules, newConfig().globalSampleRate) + sampleRate := knobs.GetScope(c.Scope, globalSampleRate) + rs := newRulesSampler(nil, c.spanRules, sampleRate) span := makeFinishedSpan(tt.spanName, tt.spanSrv, "res-10", map[string]interface{}{"hostname": "hn-30", "tag": 20.1, @@ -829,7 +844,8 @@ func TestRulesSampler(t *testing.T) { _, rules, _ := samplingRulesFromEnv() assert := assert.New(t) - sampleRate := newConfig().globalSampleRate + c := newConfig() + sampleRate := knobs.GetScope(c.Scope, globalSampleRate) rs := newRulesSampler(nil, rules, sampleRate) span := makeFinishedSpan(tt.spanName, tt.spanSrv, tt.resName, map[string]interface{}{"hostname": "hn-30"}) @@ -940,7 +956,8 @@ func TestRulesSampler(t *testing.T) { t.Run("", func(t *testing.T) { assert := assert.New(t) c := newConfig(WithSamplingRules(tt.rules)) - rs := newRulesSampler(nil, c.spanRules, newConfig().globalSampleRate) + sampleRate := knobs.GetScope(c.Scope, globalSampleRate) + rs := newRulesSampler(nil, c.spanRules, sampleRate) span := makeFinishedSpan(tt.spanName, tt.spanSrv, "res-10", map[string]interface{}{"hostname": "hn-30", "tag": 20.1, @@ -969,7 +986,9 @@ func TestRulesSampler(t *testing.T) { t.Run("", func(t *testing.T) { assert := assert.New(t) t.Setenv("DD_TRACE_SAMPLE_RATE", fmt.Sprint(rate)) - rs := newRulesSampler(nil, rules, newConfig().globalSampleRate) + c := newConfig() + sampleRate := knobs.GetScope(c.Scope, globalSampleRate) + rs := newRulesSampler(nil, rules, sampleRate) span := makeSpan("http.request", "test-service") result := rs.SampleTrace(span) @@ -1250,7 +1269,9 @@ func TestRulesSamplerInternals(t *testing.T) { t.Run("full-rate", func(t *testing.T) { assert := assert.New(t) now := time.Now() - rs := newRulesSampler(nil, nil, newConfig().globalSampleRate) + c := newConfig() + sampleRate := knobs.GetScope(c.Scope, globalSampleRate) + rs := newRulesSampler(nil, nil, sampleRate) // set samplingLimiter to specific state rs.traces.limiter.prevTime = now.Add(-1 * time.Second) rs.traces.limiter.allowed = 1 @@ -1265,7 +1286,9 @@ func TestRulesSamplerInternals(t *testing.T) { t.Run("limited-rate", func(t *testing.T) { assert := assert.New(t) now := time.Now() - rs := newRulesSampler(nil, nil, newConfig().globalSampleRate) + c := newConfig() + sampleRate := knobs.GetScope(c.Scope, globalSampleRate) + rs := newRulesSampler(nil, nil, sampleRate) // force sampling limiter to 1.0 spans/sec rs.traces.limiter.limiter = rate.NewLimiter(rate.Limit(1.0), 1) rs.traces.limiter.prevTime = now.Add(-1 * time.Second) diff --git a/ddtrace/tracer/tracer.go b/ddtrace/tracer/tracer.go index 89277045de..357ccd4c9c 100644 --- a/ddtrace/tracer/tracer.go +++ b/ddtrace/tracer/tracer.go @@ -34,6 +34,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/obfuscate" "github.com/DataDog/go-runtime-metrics-internal/pkg/runtimemetrics" + "github.com/darccio/knobs" ) var _ ddtrace.Tracer = (*tracer)(nil) @@ -267,13 +268,14 @@ func newUnstartedTracer(opts ...StartOption) *tracer { if spans != nil { c.spanRules = spans } - rulesSampler := newRulesSampler(c.traceRules, c.spanRules, c.globalSampleRate) - c.traceSampleRate = newDynamicConfig("trace_sample_rate", c.globalSampleRate, rulesSampler.traces.setGlobalSampleRate, equal[float64]) + gsr := knobs.GetScope(c.Scope, globalSampleRate) + rulesSampler := newRulesSampler(c.traceRules, c.spanRules, gsr) + c.traceSampleRate = newDynamicConfig("trace_sample_rate", gsr, rulesSampler.traces.setGlobalSampleRate, equal[float64]) // If globalSampleRate returns NaN, it means the environment variable was not set or valid. // We could always set the origin to "env_var" inconditionally, but then it wouldn't be possible // to distinguish between the case where the environment variable was not set and the case where // it default to NaN. - if !math.IsNaN(c.globalSampleRate) { + if !math.IsNaN(gsr) { c.traceSampleRate.cfgOrigin = telemetry.OriginEnvVar } c.traceSampleRules = newDynamicConfig("trace_sample_rules", c.traceRules,