Skip to content

Commit c6f3147

Browse files
NexucisNexucis
committed
use set to optimize usage comparison
Signed-off-by: Augustin Husson <[email protected]>
1 parent c0a0c95 commit c6f3147

File tree

12 files changed

+214
-175
lines changed

12 files changed

+214
-175
lines changed

‎database/database.go

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import (
2121

2222
"github.com/perses/metrics-usage/config"
2323
v1 "github.com/perses/metrics-usage/pkg/api/v1"
24-
"github.com/perses/metrics-usage/utils"
2524
"github.com/sirupsen/logrus"
2625
)
2726

@@ -145,7 +144,9 @@ func (d *db) watchMetricsQueue() {
145144
for _, metricName := range metricsName {
146145
if _, ok := d.metrics[metricName]; !ok {
147146
// As this queue only serves the purpose of storing missing metrics, we are only looking for the one not already present in the database.
148-
d.metrics[metricName] = &v1.Metric{}
147+
d.metrics[metricName] = &v1.Metric{
148+
Labels: make(v1.Set[string]),
149+
}
149150
// Since it's a new metric, potentially we already have a usage stored in the buffer.
150151
if usage, usageExists := d.usage[metricName]; usageExists {
151152
// TODO at some point we need to erase the usage map because it will cause a memory leak
@@ -204,10 +205,14 @@ func (d *db) watchLabelsQueue() {
204205
if _, ok := d.metrics[metricName]; !ok {
205206
// In this case, we should add the metric, because it means the metrics has been found from another source.
206207
d.metrics[metricName] = &v1.Metric{
207-
Labels: labels,
208+
Labels: v1.NewSet(labels...),
208209
}
209210
} else {
210-
d.metrics[metricName].Labels = utils.Merge(d.metrics[metricName].Labels, labels)
211+
if d.metrics[metricName].Labels == nil {
212+
d.metrics[metricName].Labels = v1.NewSet(labels...)
213+
} else {
214+
d.metrics[metricName].Labels.Add(labels...)
215+
}
211216
}
212217
}
213218
d.metricsMutex.Unlock()
@@ -249,9 +254,8 @@ func mergeUsage(old, new *v1.MetricUsage) *v1.MetricUsage {
249254
if new == nil {
250255
return old
251256
}
252-
return &v1.MetricUsage{
253-
Dashboards: utils.Merge(old.Dashboards, new.Dashboards),
254-
RecordingRules: utils.Merge(old.RecordingRules, new.RecordingRules),
255-
AlertRules: utils.Merge(old.AlertRules, new.AlertRules),
256-
}
257+
old.Dashboards.Merge(new.Dashboards)
258+
old.AlertRules.Merge(new.AlertRules)
259+
old.RecordingRules.Merge(new.RecordingRules)
260+
return old
257261
}

‎pkg/analyze/grafana/grafana.go

Lines changed: 28 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import (
2121
"github.com/perses/metrics-usage/pkg/analyze/parser"
2222
"github.com/perses/metrics-usage/pkg/analyze/prometheus"
2323
modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1"
24-
"github.com/perses/metrics-usage/utils"
2524
)
2625

2726
type variableTuple struct {
@@ -146,24 +145,26 @@ var (
146145
variableReplacer = strings.NewReplacer(generateGrafanaTupleVariableSyntaxReplacer(globalVariableList)...)
147146
)
148147

149-
func Analyze(dashboard *SimplifiedDashboard) ([]string, []string, []*modelAPIV1.LogError) {
148+
func Analyze(dashboard *SimplifiedDashboard) (modelAPIV1.Set[string], modelAPIV1.Set[string], []*modelAPIV1.LogError) {
150149
staticVariables := strings.NewReplacer(generateGrafanaVariableSyntaxReplacer(extractStaticVariables(dashboard.Templating.List))...)
151150
allVariableNames := collectAllVariableName(dashboard.Templating.List)
152151
m1, inv1, err1 := extractMetricsFromPanels(dashboard.Panels, staticVariables, allVariableNames, dashboard)
153152
for _, r := range dashboard.Rows {
154153
m2, inv2, err2 := extractMetricsFromPanels(r.Panels, staticVariables, allVariableNames, dashboard)
155-
m1 = utils.Merge(m1, m2)
156-
inv1 = utils.Merge(inv1, inv2)
154+
m1.Merge(m2)
155+
inv1.Merge(inv2)
157156
err1 = append(err1, err2...)
158157
}
159158
m3, inv3, err3 := extractMetricsFromVariables(dashboard.Templating.List, staticVariables, allVariableNames, dashboard)
160-
return utils.Merge(m1, m3), utils.Merge(inv1, inv3), append(err1, err3...)
159+
m1.Merge(m3)
160+
inv1.Merge(inv3)
161+
return m1, inv1, append(err1, err3...)
161162
}
162163

163-
func extractMetricsFromPanels(panels []Panel, staticVariables *strings.Replacer, allVariableNames []string, dashboard *SimplifiedDashboard) ([]string, []string, []*modelAPIV1.LogError) {
164+
func extractMetricsFromPanels(panels []Panel, staticVariables *strings.Replacer, allVariableNames modelAPIV1.Set[string], dashboard *SimplifiedDashboard) (modelAPIV1.Set[string], modelAPIV1.Set[string], []*modelAPIV1.LogError) {
164165
var errs []*modelAPIV1.LogError
165-
var result []string
166-
var invalidMetricsResult []string
166+
result := modelAPIV1.Set[string]{}
167+
invalidMetricsResult := modelAPIV1.Set[string]{}
167168
for _, p := range panels {
168169
for _, t := range extractTarget(p) {
169170
if len(t.Expr) == 0 {
@@ -174,11 +175,11 @@ func extractMetricsFromPanels(panels []Panel, staticVariables *strings.Replacer,
174175
if err != nil {
175176
otherMetrics := parser.ExtractMetricNameWithVariable(exprWithVariableReplaced)
176177
if len(otherMetrics) > 0 {
177-
for _, m := range otherMetrics {
178+
for m := range otherMetrics {
178179
if prometheus.IsValidMetricName(m) {
179-
result = utils.InsertIfNotPresent(result, m)
180+
result.Add(m)
180181
} else {
181-
invalidMetricsResult = utils.InsertIfNotPresent(invalidMetricsResult, formatVariableInMetricName(m, allVariableNames))
182+
invalidMetricsResult.Add(formatVariableInMetricName(m, allVariableNames))
182183
}
183184
}
184185
} else {
@@ -188,18 +189,18 @@ func extractMetricsFromPanels(panels []Panel, staticVariables *strings.Replacer,
188189
})
189190
}
190191
} else {
191-
result = utils.Merge(result, metrics)
192-
invalidMetricsResult = utils.Merge(invalidMetricsResult, invalidMetrics)
192+
result.Merge(metrics)
193+
invalidMetricsResult.Merge(invalidMetrics)
193194
}
194195
}
195196
}
196197
return result, invalidMetricsResult, errs
197198
}
198199

199-
func extractMetricsFromVariables(variables []templateVar, staticVariables *strings.Replacer, allVariableNames []string, dashboard *SimplifiedDashboard) ([]string, []string, []*modelAPIV1.LogError) {
200+
func extractMetricsFromVariables(variables []templateVar, staticVariables *strings.Replacer, allVariableNames modelAPIV1.Set[string], dashboard *SimplifiedDashboard) (modelAPIV1.Set[string], modelAPIV1.Set[string], []*modelAPIV1.LogError) {
200201
var errs []*modelAPIV1.LogError
201-
var result []string
202-
var invalidMetricsResult []string
202+
result := modelAPIV1.Set[string]{}
203+
invalidMetricsResult := modelAPIV1.Set[string]{}
203204
for _, v := range variables {
204205
if v.Type != "query" {
205206
continue
@@ -234,11 +235,11 @@ func extractMetricsFromVariables(variables []templateVar, staticVariables *strin
234235
if err != nil {
235236
otherMetrics := parser.ExtractMetricNameWithVariable(exprWithVariableReplaced)
236237
if len(otherMetrics) > 0 {
237-
for _, m := range otherMetrics {
238+
for m := range otherMetrics {
238239
if prometheus.IsValidMetricName(m) {
239-
result = utils.InsertIfNotPresent(result, m)
240+
result.Add(m)
240241
} else {
241-
invalidMetricsResult = utils.InsertIfNotPresent(invalidMetricsResult, formatVariableInMetricName(m, allVariableNames))
242+
invalidMetricsResult.Add(formatVariableInMetricName(m, allVariableNames))
242243
}
243244
}
244245
} else {
@@ -248,8 +249,8 @@ func extractMetricsFromVariables(variables []templateVar, staticVariables *strin
248249
})
249250
}
250251
} else {
251-
result = utils.Merge(result, metrics)
252-
invalidMetricsResult = utils.Merge(invalidMetricsResult, invalidMetrics)
252+
result.Merge(metrics)
253+
invalidMetricsResult.Merge(invalidMetrics)
253254
}
254255
}
255256
return result, invalidMetricsResult, errs
@@ -273,10 +274,10 @@ func extractStaticVariables(variables []templateVar) map[string]string {
273274
return result
274275
}
275276

276-
func collectAllVariableName(variables []templateVar) []string {
277-
result := make([]string, len(variables))
278-
for i, v := range variables {
279-
result[i] = v.Name
277+
func collectAllVariableName(variables []templateVar) modelAPIV1.Set[string] {
278+
result := modelAPIV1.Set[string]{}
279+
for _, v := range variables {
280+
result.Add(v.Name)
280281
}
281282
return result
282283
}
@@ -291,8 +292,8 @@ func replaceVariables(expr string, staticVariables *strings.Replacer) string {
291292

292293
// formatVariableInMetricName will replace the syntax of the variable by another one that can actually be parsed.
293294
// It will be useful for later when we want to know which metrics, this metric with variable is covered.
294-
func formatVariableInMetricName(metric string, variables []string) string {
295-
for _, v := range variables {
295+
func formatVariableInMetricName(metric string, variables modelAPIV1.Set[string]) string {
296+
for v := range variables {
296297
metric = strings.Replace(metric, fmt.Sprintf("$%s", v), fmt.Sprintf("${%s}", v), -1)
297298
}
298299
return metric

‎pkg/analyze/grafana/grafana_test.go

Lines changed: 46 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ package grafana
1616
import (
1717
"encoding/json"
1818
"os"
19+
"slices"
1920
"testing"
2021

2122
modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1"
@@ -58,59 +59,59 @@ func TestAnalyze(t *testing.T) {
5859
name: "variable in metrics",
5960
dashboardFile: "tests/d4.json",
6061
resultMetrics: []string{
62+
"otelcol_exporter_queue_capacity",
63+
"otelcol_exporter_queue_size",
6164
"otelcol_process_memory_rss",
62-
"otelcol_rpc_client_request_size_bucket",
63-
"otelcol_rpc_server_request_size_bucket",
64-
"otelcol_rpc_client_duration_bucket",
65-
"otelcol_rpc_server_duration_bucket",
66-
"otelcol_rpc_client_responses_per_rpc_count",
67-
"otelcol_rpc_server_responses_per_rpc_count",
6865
"otelcol_process_runtime_heap_alloc_bytes",
6966
"otelcol_process_runtime_total_sys_memory_bytes",
70-
"otelcol_exporter_queue_size",
71-
"otelcol_exporter_queue_capacity",
72-
"otelcol_processor_batch_batch_send_size_sum",
73-
"otelcol_processor_batch_batch_send_size_count",
7467
"otelcol_processor_batch_batch_send_size_bucket",
68+
"otelcol_processor_batch_batch_send_size_count",
69+
"otelcol_processor_batch_batch_send_size_sum",
70+
"otelcol_rpc_client_duration_bucket",
71+
"otelcol_rpc_client_request_size_bucket",
72+
"otelcol_rpc_client_responses_per_rpc_count",
73+
"otelcol_rpc_server_duration_bucket",
74+
"otelcol_rpc_server_request_size_bucket",
75+
"otelcol_rpc_server_responses_per_rpc_count",
7576
},
7677
invalidMetrics: []string{
77-
"otelcol_process_uptime.+",
7878
"otelcol_exporter_.+",
79-
"otelcol_processor_.+",
80-
"otelcol_receiver_.+",
81-
"otelcol_receiver_accepted_spans${suffix}",
82-
"otelcol_receiver_refused_spans${suffix}",
83-
"otelcol_receiver_accepted_metric_points${suffix}",
84-
"otelcol_receiver_refused_metric_points${suffix}",
85-
"otelcol_receiver_accepted_log_records${suffix}",
86-
"otelcol_receiver_refused_log_records${suffix}",
87-
"otelcol_processor_accepted_spans${suffix}",
88-
"otelcol_processor_refused_spans${suffix}",
89-
"otelcol_processor_dropped_spans${suffix}",
90-
"otelcol_processor_accepted_metric_points${suffix}",
91-
"otelcol_processor_refused_metric_points${suffix}",
92-
"otelcol_processor_dropped_metric_points${suffix}",
93-
"otelcol_processor_accepted_log_records${suffix}",
94-
"otelcol_processor_refused_log_records${suffix}",
95-
"otelcol_processor_dropped_log_records${suffix}",
96-
"otelcol_processor_batch_batch_size_trigger_send${suffix}",
97-
"otelcol_processor_batch_timeout_trigger_send${suffix}",
98-
"otelcol_exporter_sent_spans${suffix}",
99-
"otelcol_exporter_enqueue_failed_spans${suffix}",
100-
"otelcol_exporter_send_failed_spans${suffix}",
101-
"otelcol_exporter_sent_metric_points${suffix}",
79+
"otelcol_exporter_enqueue_failed_log_records${suffix}",
10280
"otelcol_exporter_enqueue_failed_metric_points${suffix}",
81+
"otelcol_exporter_enqueue_failed_spans${suffix}",
82+
"otelcol_exporter_send_failed_log_records${suffix}",
10383
"otelcol_exporter_send_failed_metric_points${suffix}",
84+
"otelcol_exporter_send_failed_spans${suffix}",
10485
"otelcol_exporter_sent_log_records${suffix}",
105-
"otelcol_exporter_enqueue_failed_log_records${suffix}",
106-
"otelcol_exporter_send_failed_log_records${suffix}",
107-
"otelcol_process_cpu_seconds${suffix}",
108-
"otelcol_process_uptime${suffix}",
86+
"otelcol_exporter_sent_metric_points${suffix}",
87+
"otelcol_exporter_sent_spans${suffix}",
10988
"otelcol_otelsvc_k8s_namespace_added${suffix}",
11089
"otelcol_otelsvc_k8s_namespace_updated${suffix}",
11190
"otelcol_otelsvc_k8s_pod_added${suffix}",
112-
"otelcol_otelsvc_k8s_pod_updated${suffix}",
11391
"otelcol_otelsvc_k8s_pod_deleted${suffix}",
92+
"otelcol_otelsvc_k8s_pod_updated${suffix}",
93+
"otelcol_process_cpu_seconds${suffix}",
94+
"otelcol_process_uptime${suffix}",
95+
"otelcol_process_uptime.+",
96+
"otelcol_processor_.+",
97+
"otelcol_processor_accepted_log_records${suffix}",
98+
"otelcol_processor_accepted_metric_points${suffix}",
99+
"otelcol_processor_accepted_spans${suffix}",
100+
"otelcol_processor_batch_batch_size_trigger_send${suffix}",
101+
"otelcol_processor_batch_timeout_trigger_send${suffix}",
102+
"otelcol_processor_dropped_log_records${suffix}",
103+
"otelcol_processor_dropped_metric_points${suffix}",
104+
"otelcol_processor_dropped_spans${suffix}",
105+
"otelcol_processor_refused_log_records${suffix}",
106+
"otelcol_processor_refused_metric_points${suffix}",
107+
"otelcol_processor_refused_spans${suffix}",
108+
"otelcol_receiver_.+",
109+
"otelcol_receiver_accepted_log_records${suffix}",
110+
"otelcol_receiver_accepted_metric_points${suffix}",
111+
"otelcol_receiver_accepted_spans${suffix}",
112+
"otelcol_receiver_refused_log_records${suffix}",
113+
"otelcol_receiver_refused_metric_points${suffix}",
114+
"otelcol_receiver_refused_spans${suffix}",
114115
},
115116
},
116117
}
@@ -121,8 +122,12 @@ func TestAnalyze(t *testing.T) {
121122
t.Fatal(err)
122123
}
123124
metrics, invalidMetrics, errs := Analyze(dashboard)
124-
assert.Equal(t, tt.resultMetrics, metrics)
125-
assert.Equal(t, tt.invalidMetrics, invalidMetrics)
125+
metricsAsSlice := metrics.TransformAsSlice()
126+
invalidMetricsAsSlice := invalidMetrics.TransformAsSlice()
127+
slices.Sort(metricsAsSlice)
128+
slices.Sort(invalidMetricsAsSlice)
129+
assert.Equal(t, tt.resultMetrics, metricsAsSlice)
130+
assert.Equal(t, tt.invalidMetrics, invalidMetricsAsSlice)
126131
assert.Equal(t, tt.resultErrs, errs)
127132
})
128133
}

‎pkg/analyze/parser/parser.go

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,21 @@
1313

1414
package parser
1515

16-
func ExtractMetricNameWithVariable(expr string) []string {
17-
p := &parser{}
16+
import modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1"
17+
18+
func ExtractMetricNameWithVariable(expr string) modelAPIV1.Set[string] {
19+
p := &parser{
20+
metrics: modelAPIV1.Set[string]{},
21+
}
1822
return p.parse(expr)
1923
}
2024

2125
type parser struct {
22-
metrics []string
26+
metrics modelAPIV1.Set[string]
2327
currentMetric string
2428
}
2529

26-
func (p *parser) parse(expr string) []string {
30+
func (p *parser) parse(expr string) modelAPIV1.Set[string] {
2731
query := []rune(expr)
2832
for i := 0; i < len(query); i++ {
2933
char := query[i]
@@ -64,7 +68,7 @@ func (p *parser) parse(expr string) []string {
6468
if char == '{' {
6569
if len(p.currentMetric) > 0 {
6670
// That means we reached the end of a metric, so we can save it
67-
p.metrics = append(p.metrics, p.currentMetric)
71+
p.metrics.Add(p.currentMetric)
6872
p.currentMetric = ""
6973
}
7074
}

‎pkg/analyze/parser/parser_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ func TestExtractMetricNameWithVariable(t *testing.T) {
4444
for _, test := range tests {
4545
t.Run(test.title, func(t *testing.T) {
4646
result := ExtractMetricNameWithVariable(test.expr)
47-
assert.Equal(t, test.result, result)
47+
assert.Equal(t, test.result, result.TransformAsSlice())
4848
})
4949
}
5050
}

0 commit comments

Comments
 (0)