Merge pull request #290 from form3tech-oss/metrics_enrichments

Injection of static label/values to the F1 metrics

Author: mlornac

go.mod

@@ -5,7 +5,7 @@ go 1.22.0
 require (
 	github.com/guptarohit/asciigraph v0.7.2
 	github.com/mattn/go-isatty v0.0.20
-	github.com/prometheus/client_golang v1.20.3
+	github.com/prometheus/client_golang v1.20.4
 	github.com/prometheus/client_model v0.6.1
 	github.com/prometheus/common v0.59.1
 	github.com/sirupsen/logrus v1.9.3
@@ -24,6 +24,7 @@ require (
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect

go.sum

@@ -21,14 +21,16 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4=
-github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI=
+github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
 github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
 github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0=

internal/metrics/metrics.go

@@ -12,8 +12,6 @@ const (
 	metricSubsystem = "loadtest"
 )
 
-const IterationMetricName = "form3_loadtest_iteration"
-
 const (
 	TestNameLabel = "test"
 	StageLabel    = "stage"
@@ -27,6 +25,7 @@ type Metrics struct {
 	Iteration               *prometheus.SummaryVec
 	Registry                *prometheus.Registry
 	IterationMetricsEnabled bool
+	staticMetricLabelValues []string
 }
 
 //nolint:gochecknoglobals // removing the global Instance is a breaking change
@@ -35,7 +34,7 @@ var (
 	once sync.Once
 )
 
-func buildMetrics() *Metrics {
+func buildMetrics(staticMetrics map[string]string) *Metrics {
 	percentileObjectives := map[float64]float64{
 		0.5: 0.05, 0.75: 0.05, 0.9: 0.01, 0.95: 0.001, 0.99: 0.001, 0.9999: 0.00001, 1.0: 0.00001,
 	}
@@ -47,37 +46,44 @@ func buildMetrics() *Metrics {
 			Name:       "setup",
 			Help:       "Duration of setup functions.",
 			Objectives: percentileObjectives,
-		}, []string{TestNameLabel, ResultLabel}),
+		}, append([]string{TestNameLabel, ResultLabel}, getStaticMetricLabelKeys(staticMetrics)...)),
 		Iteration: prometheus.NewSummaryVec(prometheus.SummaryOpts{
 			Namespace:  metricNamespace,
 			Subsystem:  metricSubsystem,
 			Name:       "iteration",
 			Help:       "Duration of iteration functions.",
 			Objectives: percentileObjectives,
-		}, []string{TestNameLabel, StageLabel, ResultLabel}),
+		}, append([]string{TestNameLabel, StageLabel, ResultLabel}, getStaticMetricLabelKeys(staticMetrics)...)),
 	}
 }
 
-func NewInstance(registry *prometheus.Registry, iterationMetricsEnabled bool) *Metrics {
-	i := buildMetrics()
+func NewInstance(registry *prometheus.Registry,
+	iterationMetricsEnabled bool,
+	staticMetrics map[string]string,
+) *Metrics {
+	i := buildMetrics(staticMetrics)
 	i.Registry = registry
 
 	i.Registry.MustRegister(
 		i.Setup,
 		i.Iteration,
 	)
 	i.IterationMetricsEnabled = iterationMetricsEnabled
-
+	i.staticMetricLabelValues = getStaticMetricLabelValues(staticMetrics)
 	return i
 }
 
 func Init(iterationMetricsEnabled bool) {
+	InitWithStaticMetrics(iterationMetricsEnabled, nil)
+}
+
+func InitWithStaticMetrics(iterationMetricsEnabled bool, staticMetrics map[string]string) {
 	once.Do(func() {
 		defaultRegistry, ok := prometheus.DefaultRegisterer.(*prometheus.Registry)
 		if !ok {
 			panic(errors.New("casting prometheus.DefaultRegisterer to Registry"))
 		}
-		m = NewInstance(defaultRegistry, iterationMetricsEnabled)
+		m = NewInstance(defaultRegistry, iterationMetricsEnabled, staticMetrics)
 	})
 }
 
@@ -91,21 +97,38 @@ func (metrics *Metrics) Reset() {
 }
 
 func (metrics *Metrics) RecordSetupResult(name string, result ResultType, nanoseconds int64) {
-	metrics.Setup.WithLabelValues(name, result.String()).Observe(float64(nanoseconds))
+	labels := append([]string{name, result.String()}, metrics.staticMetricLabelValues...)
+	metrics.Setup.WithLabelValues(labels...).Observe(float64(nanoseconds))
 }
 
 func (metrics *Metrics) RecordIterationResult(name string, result ResultType, nanoseconds int64) {
 	if !metrics.IterationMetricsEnabled {
 		return
 	}
-
-	metrics.Iteration.WithLabelValues(name, IterationStage, result.String()).Observe(float64(nanoseconds))
+	labels := append([]string{name, IterationStage, result.String()}, metrics.staticMetricLabelValues...)
+	metrics.Iteration.WithLabelValues(labels...).Observe(float64(nanoseconds))
 }
 
 func (metrics *Metrics) RecordIterationStage(name string, stage string, result ResultType, nanoseconds int64) {
 	if !metrics.IterationMetricsEnabled {
 		return
 	}
+	labels := append([]string{name, stage, result.String()}, metrics.staticMetricLabelValues...)
+	metrics.Iteration.WithLabelValues(labels...).Observe(float64(nanoseconds))
+}
 
-	metrics.Iteration.WithLabelValues(name, stage, result.String()).Observe(float64(nanoseconds))
+func getStaticMetricLabelKeys(staticMetrics map[string]string) []string {
+	data := make([]string, 0, len(staticMetrics))
+	for k := range staticMetrics {
+		data = append(data, k)
+	}
+	return data
+}
+
+func getStaticMetricLabelValues(staticMetrics map[string]string) []string {
+	data := make([]string, 0, len(staticMetrics))
+	for _, v := range staticMetrics {
+		data = append(data, v)
+	}
+	return data
 }

internal/metrics/metrics_test.go

@@ -3,22 +3,39 @@ package metrics_test
 import (
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/form3tech-oss/f1/v2/internal/metrics"
 )
 
 func TestMetrics_Init_IsSafe(t *testing.T) {
 	t.Parallel()
-
-	metrics.Init(true)
-
-	// race detector assertion
+	metrics.InitWithStaticMetrics(true, map[string]string{
+		"product": "fps",
+		"f1_id":   "myid",
+	}) // race detector assertion
 	for range 10 {
 		go func() {
-			metrics.Init(false)
+			metrics.InitWithStaticMetrics(true, map[string]string{
+				"product": "fps",
+				"f1_id":   "myid",
+			})
 		}()
 	}
-
 	assert.True(t, metrics.Instance().IterationMetricsEnabled)
+	metrics.Instance().RecordIterationResult("test1", metrics.SuccessResult, 1)
+	assert.Equal(t, 1, testutil.CollectAndCount(metrics.Instance().Iteration, "form3_loadtest_iteration"))
+	o, err := metrics.Instance().Iteration.MetricVec.GetMetricWith(prometheus.Labels{
+		metrics.TestNameLabel: "test1",
+		metrics.StageLabel:    metrics.IterationStage,
+		metrics.ResultLabel:   metrics.SuccessResult.String(),
+		"product":             "fps",
+		"f1_id":               "myid",
+	})
+	require.NoError(t, err)
+	assert.Contains(t, o.Desc().String(), "product")
+	assert.Contains(t, o.Desc().String(), "f1_id")
 }

internal/metrics/result.go

@@ -3,7 +3,7 @@ package metrics
 type ResultType string
 
 const (
-	SucessResult  ResultType = "success"
+	SuccessResult ResultType = "success"
 	FailedResult  ResultType = "fail"
 	DroppedResult ResultType = "dropped"
 	UnknownResult ResultType = "unknown"
@@ -17,5 +17,5 @@ func Result(failed bool) ResultType {
 	if failed {
 		return FailedResult
 	}
-	return SucessResult
+	return SuccessResult
 }

internal/progress/stats.go

@@ -16,7 +16,7 @@ type Stats struct {
 
 func (s *Stats) Record(result metrics.ResultType, nanoseconds int64) {
 	switch result {
-	case metrics.SucessResult:
+	case metrics.SuccessResult:
 		s.successfulIterationDurations.Record(nanoseconds)
 	case metrics.FailedResult:
 		s.failedIterationDurations.Record(nanoseconds)

internal/run/run_stage_test.go

@@ -118,7 +118,7 @@ func NewRunTestStage(t *testing.T) (*RunTestStage, *RunTestStage, *RunTestStage)
 		settings:                 envsettings.Get(),
 		metricData:               NewMetricData(),
 		output:                   ui.NewDiscardOutput(),
-		metrics:                  metrics.NewInstance(prometheus.NewRegistry(), true),
+		metrics:                  metrics.NewInstance(prometheus.NewRegistry(), true, nil),
 		stdout:                   syncWriter{writer: &bytes.Buffer{}},
 		stderr:                   syncWriter{writer: &bytes.Buffer{}},
 		waitForCompletionTimeout: 5 * time.Second,

pkg/f1/f1.go

@@ -27,10 +27,15 @@ const (
 // Represents an F1 CLI instance. Instantiate this struct to create an instance
 // of the F1 CLI and to register new test scenarios.
 type F1 struct {
-	output    *ui.Output
 	scenarios *scenarios.Scenarios
 	profiling *profiling
 	settings  envsettings.Settings
+	options   *f1Options
+}
+
+type f1Options struct {
+	output        *ui.Output
+	staticMetrics map[string]string
 }
 
 // New instantiates a new instance of an F1 CLI.
@@ -41,7 +46,9 @@ func New() *F1 {
 		scenarios: scenarios.New(),
 		profiling: &profiling{},
 		settings:  settings,
-		output:    ui.NewDefaultOutput(settings.Log.SlogLevel(), settings.Log.IsFormatJSON()),
+		options: &f1Options{
+			output: ui.NewDefaultOutput(settings.Log.SlogLevel(), settings.Log.IsFormatJSON()),
+		},
 	}
 }
 
@@ -52,7 +59,13 @@ func New() *F1 {
 //
 // The logger will be used for non-interactive output, file logs or when `--verbose` is specified.
 func (f *F1) WithLogger(logger *slog.Logger) *F1 {
-	f.output = ui.NewDefaultOutputWithLogger(logger)
+	f.options.output = ui.NewDefaultOutputWithLogger(logger)
+	return f
+}
+
+// WithStaticMetrics registers additional labels with fixed values to the f1 metrics
+func (f *F1) WithStaticMetrics(labels map[string]string) *F1 {
+	f.options.staticMetrics = labels
 	return f
 }
 
@@ -107,7 +120,7 @@ func newSignalContext(stopCh <-chan struct{}) context.Context {
 }
 
 func (f *F1) execute(args []string) error {
-	rootCmd, err := buildRootCmd(f.scenarios, f.settings, f.profiling, f.output)
+	rootCmd, err := buildRootCmd(f.scenarios, f.settings, f.profiling, f.options.output, f.options.staticMetrics)
 	if err != nil {
 		return fmt.Errorf("building root command: %w", err)
 	}
@@ -138,7 +151,7 @@ func (f *F1) execute(args []string) error {
 // function.
 func (f *F1) Execute() {
 	if err := f.execute(nil); err != nil {
-		f.output.Display(ui.ErrorMessage{Message: "f1 failed", Error: err})
+		f.options.output.Display(ui.ErrorMessage{Message: "f1 failed", Error: err})
 		os.Exit(1)
 	}
 }

pkg/f1/root_cmd.go

@@ -26,6 +26,7 @@ func buildRootCmd(
 	settings envsettings.Settings,
 	p *profiling,
 	output *ui.Output,
+	staticMetrics map[string]string,
 ) (*cobra.Command, error) {
 	rootCmd := &cobra.Command{
 		Use:               getCmdName(),
@@ -43,7 +44,7 @@ func buildRootCmd(
 		return nil, fmt.Errorf("marking flag as filename: %w", err)
 	}
 
-	metrics.Init(settings.PrometheusEnabled())
+	metrics.InitWithStaticMetrics(settings.PrometheusEnabled(), staticMetrics)
 	metricsInstance := metrics.Instance()
 
 	builders := trigger.GetBuilders(output)