Skip to content

Commit

Permalink
poc(ddtrace/tracer): migrate globalSampleRate
Browse files Browse the repository at this point in the history
  • Loading branch information
@darccio
darccio committed Dec 12, 2024
1 parent 6b7054d commit ecf23b8
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 32 deletions.
50 changes: 34 additions & 16 deletions ddtrace/tracer/option.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,9 +278,6 @@ type config struct {
// Value from DD_DYNAMIC_INSTRUMENTATION_ENABLED, default false.
dynamicInstrumentationEnabled bool

// globalSampleRate holds sample rate read from environment variables.
globalSampleRate float64

// ciVisibilityEnabled controls if the tracer is loaded with CI Visibility mode. default false
ciVisibilityEnabled bool

Expand All @@ -292,6 +289,40 @@ type config struct {
}

var (
// globalSampleRate holds sample rate read from environment variables.
globalSampleRate = knobs.Register(&knobs.Definition[float64]{
Default: math.NaN(),
// Out of scope: detecting if both DD_TRACE_SAMPLE_RATE and OTEL_TRACES_SAMPLER are set
// and reporting warning log and telemetry "otel.env.hiding" if so.
EnvVars: []knobs.EnvVar{
{
Key: "DD_TRACE_SAMPLE_RATE",
},
{
Key: "OTEL_TRACES_SAMPLER",
Transform: func(v string) string {
v, err := mapSampleRate(v)
if err != nil {
log.Warn("ignoring OTEL_TRACES_SAMPLER, error: %v", err)
return ""
}
return v
},
},
},
Parse: func(s string) (float64, error) {
sampleRate, err := strconv.ParseFloat(s, 64)
if err != nil {
log.Warn("ignoring DD_TRACE_SAMPLE_RATE, error: %v", err)
return 0.0, knobs.ErrInvalidValue
} else if sampleRate < 0.0 || sampleRate > 1.0 {
log.Warn("ignoring DD_TRACE_SAMPLE_RATE: out of range %f", sampleRate)
return 0.0, knobs.ErrInvalidValue
}
return sampleRate, nil
},
})

// partialFlushEnabled specifices whether the tracer should enable partial flushing. Value
// from DD_TRACE_PARTIAL_FLUSH_ENABLED, default false.
partialFlushEnabled = knobs.Register(&knobs.Definition[bool]{
Expand Down Expand Up @@ -351,19 +382,6 @@ func newConfig(opts ...StartOption) *config {
c := new(config)
c.Scope = knobs.NewScope()
c.sampler = NewAllSampler()
sampleRate := math.NaN()
if r := getDDorOtelConfig("sampleRate"); r != "" {
var err error
sampleRate, err = strconv.ParseFloat(r, 64)
if err != nil {
log.Warn("ignoring DD_TRACE_SAMPLE_RATE, error: %v", err)
sampleRate = math.NaN()
} else if sampleRate < 0.0 || sampleRate > 1.0 {
log.Warn("ignoring DD_TRACE_SAMPLE_RATE: out of range %f", sampleRate)
sampleRate = math.NaN()
}
}
c.globalSampleRate = sampleRate
c.httpClientTimeout = time.Second * 10 // 10 seconds

if v := os.Getenv("OTEL_LOGS_EXPORTER"); v != "" {
Expand Down
49 changes: 36 additions & 13 deletions ddtrace/tracer/sampler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/internal"
"gopkg.in/DataDog/dd-trace-go.v1/internal/samplernames"

"github.com/darccio/knobs"
"github.com/stretchr/testify/assert"
"golang.org/x/time/rate"
)
Expand Down Expand Up @@ -201,7 +202,8 @@ func TestRuleEnvVars(t *testing.T) {
{in: "1point0", out: math.NaN()}, // default if invalid value
} {
t.Setenv("DD_TRACE_SAMPLE_RATE", tt.in)
res := newConfig().globalSampleRate
c := newConfig()
res := knobs.GetScope(c.Scope, globalSampleRate)
if math.IsNaN(tt.out) {
assert.True(math.IsNaN(res))
} else {
Expand All @@ -226,7 +228,9 @@ func TestRuleEnvVars(t *testing.T) {
assert := assert.New(t)
t.Setenv("OTEL_TRACES_SAMPLER", tt.config)
t.Setenv("OTEL_TRACES_SAMPLER_ARG", fmt.Sprintf("%f", tt.rate))
assert.Equal(tt.rate, newConfig().globalSampleRate)
c := newConfig()
sampleRate := knobs.GetScope(c.Scope, globalSampleRate)
assert.Equal(tt.rate, sampleRate)
})
}
})
Expand Down Expand Up @@ -477,7 +481,9 @@ func TestRulesSampler(t *testing.T) {
}
t.Run("no-rules", func(t *testing.T) {
assert := assert.New(t)
rs := newRulesSampler(nil, nil, newConfig().globalSampleRate)
c := newConfig()
sampleRate := knobs.GetScope(c.Scope, globalSampleRate)
rs := newRulesSampler(nil, nil, sampleRate)

span := makeSpan("http.request", "test-service")
result := rs.SampleTrace(span)
Expand Down Expand Up @@ -542,7 +548,9 @@ func TestRulesSampler(t *testing.T) {
assert.Nil(t, err)

assert := assert.New(t)
rs := newRulesSampler(rules, nil, newConfig().globalSampleRate)
c := newConfig()
sampleRate := knobs.GetScope(c.Scope, globalSampleRate)
rs := newRulesSampler(rules, nil, sampleRate)

span := makeFinishedSpan(tt.spanName, tt.spanSrv, tt.spanRsc, tt.spanTags)

Expand All @@ -566,7 +574,9 @@ func TestRulesSampler(t *testing.T) {
for _, v := range traceRules {
t.Run("", func(t *testing.T) {
assert := assert.New(t)
rs := newRulesSampler(v, nil, newConfig().globalSampleRate)
c := newConfig()
sampleRate := knobs.GetScope(c.Scope, globalSampleRate)
rs := newRulesSampler(v, nil, sampleRate)

span := makeSpan("http.request", "test-service")
result := rs.SampleTrace(span)
Expand All @@ -592,7 +602,9 @@ func TestRulesSampler(t *testing.T) {
for _, v := range traceRules {
t.Run("", func(t *testing.T) {
assert := assert.New(t)
rs := newRulesSampler(v, nil, newConfig().globalSampleRate)
c := newConfig()
sampleRate := knobs.GetScope(c.Scope, globalSampleRate)
rs := newRulesSampler(v, nil, sampleRate)

span := makeSpan("http.request", "test-service")
result := rs.SampleTrace(span)
Expand Down Expand Up @@ -638,7 +650,9 @@ func TestRulesSampler(t *testing.T) {
_, rules, err := samplingRulesFromEnv()
assert.Nil(t, err)
assert := assert.New(t)
rs := newRulesSampler(nil, rules, newConfig().globalSampleRate)
c := newConfig()
sampleRate := knobs.GetScope(c.Scope, globalSampleRate)
rs := newRulesSampler(nil, rules, sampleRate)

span := makeFinishedSpan(tt.spanName, tt.spanSrv, "res-10", map[string]interface{}{"hostname": "hn-30"})

Expand Down Expand Up @@ -761,7 +775,8 @@ func TestRulesSampler(t *testing.T) {
t.Run(fmt.Sprintf("%v", i), func(t *testing.T) {
assert := assert.New(t)
c := newConfig(WithSamplingRules(tt.rules))
rs := newRulesSampler(nil, c.spanRules, newConfig().globalSampleRate)
sampleRate := knobs.GetScope(c.Scope, globalSampleRate)
rs := newRulesSampler(nil, c.spanRules, sampleRate)

span := makeFinishedSpan(tt.spanName, tt.spanSrv, "res-10", map[string]interface{}{"hostname": "hn-30",
"tag": 20.1,
Expand Down Expand Up @@ -829,7 +844,8 @@ func TestRulesSampler(t *testing.T) {
_, rules, _ := samplingRulesFromEnv()

assert := assert.New(t)
sampleRate := newConfig().globalSampleRate
c := newConfig()
sampleRate := knobs.GetScope(c.Scope, globalSampleRate)
rs := newRulesSampler(nil, rules, sampleRate)

span := makeFinishedSpan(tt.spanName, tt.spanSrv, tt.resName, map[string]interface{}{"hostname": "hn-30"})
Expand Down Expand Up @@ -940,7 +956,8 @@ func TestRulesSampler(t *testing.T) {
t.Run("", func(t *testing.T) {
assert := assert.New(t)
c := newConfig(WithSamplingRules(tt.rules))
rs := newRulesSampler(nil, c.spanRules, newConfig().globalSampleRate)
sampleRate := knobs.GetScope(c.Scope, globalSampleRate)
rs := newRulesSampler(nil, c.spanRules, sampleRate)

span := makeFinishedSpan(tt.spanName, tt.spanSrv, "res-10", map[string]interface{}{"hostname": "hn-30",
"tag": 20.1,
Expand Down Expand Up @@ -969,7 +986,9 @@ func TestRulesSampler(t *testing.T) {
t.Run("", func(t *testing.T) {
assert := assert.New(t)
t.Setenv("DD_TRACE_SAMPLE_RATE", fmt.Sprint(rate))
rs := newRulesSampler(nil, rules, newConfig().globalSampleRate)
c := newConfig()
sampleRate := knobs.GetScope(c.Scope, globalSampleRate)
rs := newRulesSampler(nil, rules, sampleRate)

span := makeSpan("http.request", "test-service")
result := rs.SampleTrace(span)
Expand Down Expand Up @@ -1250,7 +1269,9 @@ func TestRulesSamplerInternals(t *testing.T) {
t.Run("full-rate", func(t *testing.T) {
assert := assert.New(t)
now := time.Now()
rs := newRulesSampler(nil, nil, newConfig().globalSampleRate)
c := newConfig()
sampleRate := knobs.GetScope(c.Scope, globalSampleRate)
rs := newRulesSampler(nil, nil, sampleRate)
// set samplingLimiter to specific state
rs.traces.limiter.prevTime = now.Add(-1 * time.Second)
rs.traces.limiter.allowed = 1
Expand All @@ -1265,7 +1286,9 @@ func TestRulesSamplerInternals(t *testing.T) {
t.Run("limited-rate", func(t *testing.T) {
assert := assert.New(t)
now := time.Now()
rs := newRulesSampler(nil, nil, newConfig().globalSampleRate)
c := newConfig()
sampleRate := knobs.GetScope(c.Scope, globalSampleRate)
rs := newRulesSampler(nil, nil, sampleRate)
// force sampling limiter to 1.0 spans/sec
rs.traces.limiter.limiter = rate.NewLimiter(rate.Limit(1.0), 1)
rs.traces.limiter.prevTime = now.Add(-1 * time.Second)
Expand Down
8 changes: 5 additions & 3 deletions ddtrace/tracer/tracer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (

"github.com/DataDog/datadog-agent/pkg/obfuscate"
"github.com/DataDog/go-runtime-metrics-internal/pkg/runtimemetrics"
"github.com/darccio/knobs"
)

var _ ddtrace.Tracer = (*tracer)(nil)
Expand Down Expand Up @@ -267,13 +268,14 @@ func newUnstartedTracer(opts ...StartOption) *tracer {
if spans != nil {
c.spanRules = spans
}
rulesSampler := newRulesSampler(c.traceRules, c.spanRules, c.globalSampleRate)
c.traceSampleRate = newDynamicConfig("trace_sample_rate", c.globalSampleRate, rulesSampler.traces.setGlobalSampleRate, equal[float64])
gsr := knobs.GetScope(c.Scope, globalSampleRate)
rulesSampler := newRulesSampler(c.traceRules, c.spanRules, gsr)
c.traceSampleRate = newDynamicConfig("trace_sample_rate", gsr, rulesSampler.traces.setGlobalSampleRate, equal[float64])
// If globalSampleRate returns NaN, it means the environment variable was not set or valid.
// We could always set the origin to "env_var" inconditionally, but then it wouldn't be possible
// to distinguish between the case where the environment variable was not set and the case where
// it default to NaN.
if !math.IsNaN(c.globalSampleRate) {
if !math.IsNaN(gsr) {
c.traceSampleRate.cfgOrigin = telemetry.OriginEnvVar
}
c.traceSampleRules = newDynamicConfig("trace_sample_rules", c.traceRules,
Expand Down

0 comments on commit ecf23b8

Please sign in to comment.