diff --git a/go.mod b/go.mod index a0732976..759f2863 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22.0 require ( github.com/guptarohit/asciigraph v0.7.2 github.com/mattn/go-isatty v0.0.20 - github.com/prometheus/client_golang v1.20.3 + github.com/prometheus/client_golang v1.20.4 github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.59.1 github.com/sirupsen/logrus v1.9.3 @@ -24,6 +24,7 @@ require ( github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect diff --git a/go.sum b/go.sum index e2a92f11..c35b06d7 100644 --- a/go.sum +++ b/go.sum @@ -21,14 +21,16 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4= -github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= +github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0= diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index f581d569..17faadb0 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -12,8 +12,6 @@ const ( metricSubsystem = "loadtest" ) -const IterationMetricName = "form3_loadtest_iteration" - const ( TestNameLabel = "test" StageLabel = "stage" @@ -27,6 +25,7 @@ type Metrics struct { Iteration *prometheus.SummaryVec Registry *prometheus.Registry IterationMetricsEnabled bool + staticMetricLabelValues []string } //nolint:gochecknoglobals // removing the global Instance is a breaking change @@ -35,7 +34,7 @@ var ( once sync.Once ) -func buildMetrics() *Metrics { +func buildMetrics(staticMetrics map[string]string) *Metrics { percentileObjectives := map[float64]float64{ 0.5: 0.05, 0.75: 0.05, 0.9: 0.01, 0.95: 0.001, 0.99: 0.001, 0.9999: 0.00001, 1.0: 0.00001, } @@ -47,19 +46,22 @@ func buildMetrics() *Metrics { Name: "setup", Help: "Duration of setup functions.", Objectives: percentileObjectives, - }, []string{TestNameLabel, ResultLabel}), + }, append([]string{TestNameLabel, ResultLabel}, getStaticMetricLabelKeys(staticMetrics)...)), Iteration: prometheus.NewSummaryVec(prometheus.SummaryOpts{ Namespace: metricNamespace, Subsystem: metricSubsystem, Name: "iteration", Help: "Duration of iteration functions.", Objectives: percentileObjectives, - }, []string{TestNameLabel, StageLabel, ResultLabel}), + }, append([]string{TestNameLabel, StageLabel, ResultLabel}, getStaticMetricLabelKeys(staticMetrics)...)), } } -func NewInstance(registry *prometheus.Registry, iterationMetricsEnabled bool) *Metrics { - i := buildMetrics() +func NewInstance(registry *prometheus.Registry, + iterationMetricsEnabled bool, + staticMetrics map[string]string, +) *Metrics { + i := buildMetrics(staticMetrics) i.Registry = registry i.Registry.MustRegister( @@ -67,17 +69,21 @@ func NewInstance(registry *prometheus.Registry, iterationMetricsEnabled bool) *M i.Iteration, ) i.IterationMetricsEnabled = iterationMetricsEnabled - + i.staticMetricLabelValues = getStaticMetricLabelValues(staticMetrics) return i } func Init(iterationMetricsEnabled bool) { + InitWithStaticMetrics(iterationMetricsEnabled, nil) +} + +func InitWithStaticMetrics(iterationMetricsEnabled bool, staticMetrics map[string]string) { once.Do(func() { defaultRegistry, ok := prometheus.DefaultRegisterer.(*prometheus.Registry) if !ok { panic(errors.New("casting prometheus.DefaultRegisterer to Registry")) } - m = NewInstance(defaultRegistry, iterationMetricsEnabled) + m = NewInstance(defaultRegistry, iterationMetricsEnabled, staticMetrics) }) } @@ -91,21 +97,38 @@ func (metrics *Metrics) Reset() { } func (metrics *Metrics) RecordSetupResult(name string, result ResultType, nanoseconds int64) { - metrics.Setup.WithLabelValues(name, result.String()).Observe(float64(nanoseconds)) + labels := append([]string{name, result.String()}, metrics.staticMetricLabelValues...) + metrics.Setup.WithLabelValues(labels...).Observe(float64(nanoseconds)) } func (metrics *Metrics) RecordIterationResult(name string, result ResultType, nanoseconds int64) { if !metrics.IterationMetricsEnabled { return } - - metrics.Iteration.WithLabelValues(name, IterationStage, result.String()).Observe(float64(nanoseconds)) + labels := append([]string{name, IterationStage, result.String()}, metrics.staticMetricLabelValues...) + metrics.Iteration.WithLabelValues(labels...).Observe(float64(nanoseconds)) } func (metrics *Metrics) RecordIterationStage(name string, stage string, result ResultType, nanoseconds int64) { if !metrics.IterationMetricsEnabled { return } + labels := append([]string{name, stage, result.String()}, metrics.staticMetricLabelValues...) + metrics.Iteration.WithLabelValues(labels...).Observe(float64(nanoseconds)) +} - metrics.Iteration.WithLabelValues(name, stage, result.String()).Observe(float64(nanoseconds)) +func getStaticMetricLabelKeys(staticMetrics map[string]string) []string { + data := make([]string, 0, len(staticMetrics)) + for k := range staticMetrics { + data = append(data, k) + } + return data +} + +func getStaticMetricLabelValues(staticMetrics map[string]string) []string { + data := make([]string, 0, len(staticMetrics)) + for _, v := range staticMetrics { + data = append(data, v) + } + return data } diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index abd6aab1..c173d80c 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -3,22 +3,39 @@ package metrics_test import ( "testing" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/form3tech-oss/f1/v2/internal/metrics" ) func TestMetrics_Init_IsSafe(t *testing.T) { t.Parallel() - - metrics.Init(true) - - // race detector assertion + metrics.InitWithStaticMetrics(true, map[string]string{ + "product": "fps", + "f1_id": "myid", + }) // race detector assertion for range 10 { go func() { - metrics.Init(false) + metrics.InitWithStaticMetrics(true, map[string]string{ + "product": "fps", + "f1_id": "myid", + }) }() } - assert.True(t, metrics.Instance().IterationMetricsEnabled) + metrics.Instance().RecordIterationResult("test1", metrics.SuccessResult, 1) + assert.Equal(t, 1, testutil.CollectAndCount(metrics.Instance().Iteration, "form3_loadtest_iteration")) + o, err := metrics.Instance().Iteration.MetricVec.GetMetricWith(prometheus.Labels{ + metrics.TestNameLabel: "test1", + metrics.StageLabel: metrics.IterationStage, + metrics.ResultLabel: metrics.SuccessResult.String(), + "product": "fps", + "f1_id": "myid", + }) + require.NoError(t, err) + assert.Contains(t, o.Desc().String(), "product") + assert.Contains(t, o.Desc().String(), "f1_id") } diff --git a/internal/metrics/result.go b/internal/metrics/result.go index c4b1c987..4152f759 100644 --- a/internal/metrics/result.go +++ b/internal/metrics/result.go @@ -3,7 +3,7 @@ package metrics type ResultType string const ( - SucessResult ResultType = "success" + SuccessResult ResultType = "success" FailedResult ResultType = "fail" DroppedResult ResultType = "dropped" UnknownResult ResultType = "unknown" @@ -17,5 +17,5 @@ func Result(failed bool) ResultType { if failed { return FailedResult } - return SucessResult + return SuccessResult } diff --git a/internal/progress/stats.go b/internal/progress/stats.go index 8b825c6d..85aa9d2b 100644 --- a/internal/progress/stats.go +++ b/internal/progress/stats.go @@ -16,7 +16,7 @@ type Stats struct { func (s *Stats) Record(result metrics.ResultType, nanoseconds int64) { switch result { - case metrics.SucessResult: + case metrics.SuccessResult: s.successfulIterationDurations.Record(nanoseconds) case metrics.FailedResult: s.failedIterationDurations.Record(nanoseconds) diff --git a/internal/run/run_stage_test.go b/internal/run/run_stage_test.go index 7dcde624..5ccbff59 100644 --- a/internal/run/run_stage_test.go +++ b/internal/run/run_stage_test.go @@ -118,7 +118,7 @@ func NewRunTestStage(t *testing.T) (*RunTestStage, *RunTestStage, *RunTestStage) settings: envsettings.Get(), metricData: NewMetricData(), output: ui.NewDiscardOutput(), - metrics: metrics.NewInstance(prometheus.NewRegistry(), true), + metrics: metrics.NewInstance(prometheus.NewRegistry(), true, nil), stdout: syncWriter{writer: &bytes.Buffer{}}, stderr: syncWriter{writer: &bytes.Buffer{}}, waitForCompletionTimeout: 5 * time.Second, diff --git a/pkg/f1/f1.go b/pkg/f1/f1.go index 97c66fd3..5a832653 100644 --- a/pkg/f1/f1.go +++ b/pkg/f1/f1.go @@ -27,10 +27,15 @@ const ( // Represents an F1 CLI instance. Instantiate this struct to create an instance // of the F1 CLI and to register new test scenarios. type F1 struct { - output *ui.Output scenarios *scenarios.Scenarios profiling *profiling settings envsettings.Settings + options *f1Options +} + +type f1Options struct { + output *ui.Output + staticMetrics map[string]string } // New instantiates a new instance of an F1 CLI. @@ -41,7 +46,9 @@ func New() *F1 { scenarios: scenarios.New(), profiling: &profiling{}, settings: settings, - output: ui.NewDefaultOutput(settings.Log.SlogLevel(), settings.Log.IsFormatJSON()), + options: &f1Options{ + output: ui.NewDefaultOutput(settings.Log.SlogLevel(), settings.Log.IsFormatJSON()), + }, } } @@ -52,7 +59,13 @@ func New() *F1 { // // The logger will be used for non-interactive output, file logs or when `--verbose` is specified. func (f *F1) WithLogger(logger *slog.Logger) *F1 { - f.output = ui.NewDefaultOutputWithLogger(logger) + f.options.output = ui.NewDefaultOutputWithLogger(logger) + return f +} + +// WithStaticMetrics registers additional labels with fixed values to the f1 metrics +func (f *F1) WithStaticMetrics(labels map[string]string) *F1 { + f.options.staticMetrics = labels return f } @@ -107,7 +120,7 @@ func newSignalContext(stopCh <-chan struct{}) context.Context { } func (f *F1) execute(args []string) error { - rootCmd, err := buildRootCmd(f.scenarios, f.settings, f.profiling, f.output) + rootCmd, err := buildRootCmd(f.scenarios, f.settings, f.profiling, f.options.output, f.options.staticMetrics) if err != nil { return fmt.Errorf("building root command: %w", err) } @@ -138,7 +151,7 @@ func (f *F1) execute(args []string) error { // function. func (f *F1) Execute() { if err := f.execute(nil); err != nil { - f.output.Display(ui.ErrorMessage{Message: "f1 failed", Error: err}) + f.options.output.Display(ui.ErrorMessage{Message: "f1 failed", Error: err}) os.Exit(1) } } diff --git a/pkg/f1/root_cmd.go b/pkg/f1/root_cmd.go index b5946c9f..5393271c 100644 --- a/pkg/f1/root_cmd.go +++ b/pkg/f1/root_cmd.go @@ -26,6 +26,7 @@ func buildRootCmd( settings envsettings.Settings, p *profiling, output *ui.Output, + staticMetrics map[string]string, ) (*cobra.Command, error) { rootCmd := &cobra.Command{ Use: getCmdName(), @@ -43,7 +44,7 @@ func buildRootCmd( return nil, fmt.Errorf("marking flag as filename: %w", err) } - metrics.Init(settings.PrometheusEnabled()) + metrics.InitWithStaticMetrics(settings.PrometheusEnabled(), staticMetrics) metricsInstance := metrics.Instance() builders := trigger.GetBuilders(output)