Skip to content

Commit 49c2fbd

Browse files
authored
Merge pull request #23 from everesio/label_dimensions_inconsistent
Fill missing metrics labels with empty string to avoid label dimensions inconsistent failure
2 parents de048a0 + 0d41153 commit 49c2fbd

File tree

4 files changed

+280
-28
lines changed

4 files changed

+280
-28
lines changed

collectors/fnv.go

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package collectors
2+
3+
const separatorByte = 255
4+
5+
// https://github.com/prometheus/client_golang/blob/master/prometheus/fnv.go
6+
// Inline and byte-free variant of hash/fnv's fnv64a.
7+
8+
const (
9+
offset64 = 14695981039346656037
10+
prime64 = 1099511628211
11+
)
12+
13+
// hashNew initializies a new fnv64a hash value.
14+
func hashNew() uint64 {
15+
return offset64
16+
}
17+
18+
// hashAdd adds a string to a fnv64a hash value, returning the updated hash.
19+
func hashAdd(h uint64, s string) uint64 {
20+
for i := 0; i < len(s); i++ {
21+
h ^= uint64(s[i])
22+
h *= prime64
23+
}
24+
return h
25+
}
26+
27+
// hashAddByte adds a byte to a fnv64a hash value, returning the updated hash.
28+
func hashAddByte(h uint64, b byte) uint64 {
29+
h ^= uint64(b)
30+
h *= prime64
31+
return h
32+
}

collectors/monitoring_collector.go

Lines changed: 13 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ type MonitoringCollector struct {
2727
lastScrapeErrorMetric prometheus.Gauge
2828
lastScrapeTimestampMetric prometheus.Gauge
2929
lastScrapeDurationSecondsMetric prometheus.Gauge
30+
collectorFillMissingLabels bool
3031
}
3132

3233
func NewMonitoringCollector(
@@ -35,6 +36,7 @@ func NewMonitoringCollector(
3536
metricsInterval time.Duration,
3637
metricsOffset time.Duration,
3738
monitoringService *monitoring.Service,
39+
collectorFillMissingLabels bool,
3840
) (*MonitoringCollector, error) {
3941
apiCallsTotalMetric := prometheus.NewCounter(
4042
prometheus.CounterOpts{
@@ -108,6 +110,7 @@ func NewMonitoringCollector(
108110
lastScrapeErrorMetric: lastScrapeErrorMetric,
109111
lastScrapeTimestampMetric: lastScrapeTimestampMetric,
110112
lastScrapeDurationSecondsMetric: lastScrapeDurationSecondsMetric,
113+
collectorFillMissingLabels: collectorFillMissingLabels,
111114
}
112115

113116
return monitoringCollector, nil
@@ -231,8 +234,14 @@ func (c *MonitoringCollector) reportTimeSeriesMetrics(
231234
var metricValue float64
232235
var metricValueType prometheus.ValueType
233236
var newestTSPoint *monitoring.Point
234-
var metricDesc *prometheus.Desc
235237

238+
timeSeriesMetrics := &TimeSeriesMetrics{
239+
metricDescriptor: metricDescriptor,
240+
ch: ch,
241+
fillMissingLabels: c.collectorFillMissingLabels,
242+
constMetrics: make(map[string][]ConstMetric),
243+
histogramMetrics: make(map[string][]HistogramMetric),
244+
}
236245
for _, timeSeries := range page.TimeSeries {
237246
newestEndTime := time.Unix(0, 0)
238247
for _, point := range timeSeries.Points {
@@ -245,7 +254,6 @@ func (c *MonitoringCollector) reportTimeSeriesMetrics(
245254
newestTSPoint = point
246255
}
247256
}
248-
249257
labelKeys := []string{"unit"}
250258
labelValues := []string{metricDescriptor.Unit}
251259

@@ -263,17 +271,6 @@ func (c *MonitoringCollector) reportTimeSeriesMetrics(
263271
labelValues = append(labelValues, value)
264272
}
265273

266-
// The metric name to report is composed by the 3 parts:
267-
// 1. namespace is a constant prefix (stackdriver)
268-
// 2. subsystem is the monitored resource type (ie gce_instance)
269-
// 3. name is the metric type (ie compute.googleapis.com/instance/cpu/usage_time)
270-
metricDesc = prometheus.NewDesc(
271-
prometheus.BuildFQName("stackdriver", utils.NormalizeMetricName(timeSeries.Resource.Type), utils.NormalizeMetricName(timeSeries.Metric.Type)),
272-
metricDescriptor.Description,
273-
labelKeys,
274-
prometheus.Labels{},
275-
)
276-
277274
switch timeSeries.MetricKind {
278275
case "GAUGE":
279276
metricValueType = prometheus.GaugeValue
@@ -299,13 +296,7 @@ func (c *MonitoringCollector) reportTimeSeriesMetrics(
299296
dist := newestTSPoint.Value.DistributionValue
300297
buckets, err := c.generateHistogramBuckets(dist)
301298
if err == nil {
302-
ch <- prometheus.MustNewConstHistogram(
303-
metricDesc,
304-
uint64(dist.Count),
305-
dist.Mean*float64(dist.Count), // Stackdriver does not provide the sum, but we can fake it
306-
buckets,
307-
labelValues...,
308-
)
299+
timeSeriesMetrics.CollectNewConstHistogram(timeSeries, labelKeys, dist, buckets, labelValues)
309300
} else {
310301
log.Debugf("Discarding resource %s metric %s: %s", timeSeries.Resource.Type, timeSeries.Metric.Type, err)
311302
}
@@ -315,14 +306,9 @@ func (c *MonitoringCollector) reportTimeSeriesMetrics(
315306
continue
316307
}
317308

318-
ch <- prometheus.MustNewConstMetric(
319-
metricDesc,
320-
metricValueType,
321-
metricValue,
322-
labelValues...,
323-
)
309+
timeSeriesMetrics.CollectNewConstMetric(timeSeries, labelKeys, metricValueType, metricValue, labelValues)
324310
}
325-
311+
timeSeriesMetrics.Complete()
326312
return nil
327313
}
328314

collectors/monitoring_metrics.go

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
package collectors
2+
3+
import (
4+
"github.com/prometheus/client_golang/prometheus"
5+
"google.golang.org/api/monitoring/v3"
6+
7+
"github.com/frodenas/stackdriver_exporter/utils"
8+
"sort"
9+
)
10+
11+
func buildFQName(timeSeries *monitoring.TimeSeries) string {
12+
// The metric name to report is composed by the 3 parts:
13+
// 1. namespace is a constant prefix (stackdriver)
14+
// 2. subsystem is the monitored resource type (ie gce_instance)
15+
// 3. name is the metric type (ie compute.googleapis.com/instance/cpu/usage_time)
16+
return prometheus.BuildFQName("stackdriver", utils.NormalizeMetricName(timeSeries.Resource.Type), utils.NormalizeMetricName(timeSeries.Metric.Type))
17+
}
18+
19+
type TimeSeriesMetrics struct {
20+
metricDescriptor *monitoring.MetricDescriptor
21+
ch chan<- prometheus.Metric
22+
23+
fillMissingLabels bool
24+
constMetrics map[string][]ConstMetric
25+
histogramMetrics map[string][]HistogramMetric
26+
}
27+
28+
func (t *TimeSeriesMetrics) newMetricDesc(fqName string, labelKeys []string) *prometheus.Desc {
29+
return prometheus.NewDesc(
30+
fqName,
31+
t.metricDescriptor.Description,
32+
labelKeys,
33+
prometheus.Labels{},
34+
)
35+
}
36+
37+
type ConstMetric struct {
38+
fqName string
39+
labelKeys []string
40+
valueType prometheus.ValueType
41+
value float64
42+
labelValues []string
43+
44+
keysHash uint64
45+
}
46+
47+
type HistogramMetric struct {
48+
fqName string
49+
labelKeys []string
50+
dist *monitoring.Distribution
51+
buckets map[float64]uint64
52+
labelValues []string
53+
54+
keysHash uint64
55+
}
56+
57+
func (t *TimeSeriesMetrics) CollectNewConstHistogram(timeSeries *monitoring.TimeSeries, labelKeys []string, dist *monitoring.Distribution, buckets map[float64]uint64, labelValues []string) {
58+
fqName := buildFQName(timeSeries)
59+
60+
if t.fillMissingLabels {
61+
vs, ok := t.histogramMetrics[fqName]
62+
if !ok {
63+
vs = make([]HistogramMetric, 0)
64+
}
65+
v := HistogramMetric{
66+
fqName: fqName,
67+
labelKeys: labelKeys,
68+
dist: dist,
69+
buckets: buckets,
70+
labelValues: labelValues,
71+
72+
keysHash: hashLabelKeys(labelKeys),
73+
}
74+
t.histogramMetrics[fqName] = append(vs, v)
75+
return
76+
}
77+
t.ch <- t.newConstHistogram(fqName, labelKeys, dist, buckets, labelValues)
78+
}
79+
80+
func (t *TimeSeriesMetrics) newConstHistogram(fqName string, labelKeys []string, dist *monitoring.Distribution, buckets map[float64]uint64, labelValues []string) prometheus.Metric {
81+
return prometheus.MustNewConstHistogram(
82+
t.newMetricDesc(fqName, labelKeys),
83+
uint64(dist.Count),
84+
dist.Mean*float64(dist.Count), // Stackdriver does not provide the sum, but we can fake it
85+
buckets,
86+
labelValues...,
87+
)
88+
}
89+
90+
func (t *TimeSeriesMetrics) CollectNewConstMetric(timeSeries *monitoring.TimeSeries, labelKeys []string, metricValueType prometheus.ValueType, metricValue float64, labelValues []string) {
91+
fqName := buildFQName(timeSeries)
92+
93+
if t.fillMissingLabels {
94+
vs, ok := t.constMetrics[fqName]
95+
if !ok {
96+
vs = make([]ConstMetric, 0)
97+
}
98+
v := ConstMetric{
99+
fqName: fqName,
100+
labelKeys: labelKeys,
101+
valueType: metricValueType,
102+
value: metricValue,
103+
labelValues: labelValues,
104+
105+
keysHash: hashLabelKeys(labelKeys),
106+
}
107+
t.constMetrics[fqName] = append(vs, v)
108+
return
109+
}
110+
t.ch <- t.newConstMetric(fqName, labelKeys, metricValueType, metricValue, labelValues)
111+
}
112+
113+
func (t *TimeSeriesMetrics) newConstMetric(fqName string, labelKeys []string, metricValueType prometheus.ValueType, metricValue float64, labelValues []string) prometheus.Metric {
114+
return prometheus.MustNewConstMetric(
115+
t.newMetricDesc(fqName, labelKeys),
116+
metricValueType,
117+
metricValue,
118+
labelValues...,
119+
)
120+
}
121+
122+
func hashLabelKeys(labelKeys []string) uint64 {
123+
dh := hashNew()
124+
sortedKeys := make([]string, len(labelKeys))
125+
copy(sortedKeys, labelKeys)
126+
sort.Strings(sortedKeys)
127+
for _, key := range sortedKeys {
128+
dh = hashAdd(dh, key)
129+
dh = hashAddByte(dh, separatorByte)
130+
}
131+
return dh
132+
}
133+
134+
func (t *TimeSeriesMetrics) Complete() {
135+
t.completeConstMetrics()
136+
t.completeHistogramMetrics()
137+
}
138+
139+
func (t *TimeSeriesMetrics) completeConstMetrics() {
140+
for _, vs := range t.constMetrics {
141+
if len(vs) > 1 {
142+
var needFill bool
143+
for i := 1; i < len(vs); i++ {
144+
if vs[0].keysHash != vs[i].keysHash {
145+
needFill = true
146+
}
147+
}
148+
if needFill {
149+
vs = fillConstMetricsLabels(vs)
150+
}
151+
}
152+
153+
for _, v := range vs {
154+
t.ch <- t.newConstMetric(v.fqName, v.labelKeys, v.valueType, v.value, v.labelValues)
155+
}
156+
}
157+
}
158+
159+
func (t *TimeSeriesMetrics) completeHistogramMetrics() {
160+
for _, vs := range t.histogramMetrics {
161+
if len(vs) > 1 {
162+
var needFill bool
163+
for i := 1; i < len(vs); i++ {
164+
if vs[0].keysHash != vs[i].keysHash {
165+
needFill = true
166+
}
167+
}
168+
if needFill {
169+
vs = fillHistogramMetricsLabels(vs)
170+
}
171+
}
172+
for _, v := range vs {
173+
t.ch <- t.newConstHistogram(v.fqName, v.labelKeys, v.dist, v.buckets, v.labelValues)
174+
}
175+
}
176+
}
177+
178+
func fillConstMetricsLabels(metrics []ConstMetric) []ConstMetric {
179+
allKeys := make(map[string]struct{})
180+
for _, metric := range metrics {
181+
for _, key := range metric.labelKeys {
182+
allKeys[key] = struct{}{}
183+
}
184+
}
185+
result := make([]ConstMetric, len(metrics))
186+
for i, metric := range metrics {
187+
if len(metric.labelKeys) != len(allKeys) {
188+
metricKeys := make(map[string]struct{})
189+
for _, key := range metric.labelKeys {
190+
metricKeys[key] = struct{}{}
191+
}
192+
for key := range allKeys {
193+
if _, ok := metricKeys[key]; !ok {
194+
metric.labelKeys = append(metric.labelKeys, key)
195+
metric.labelValues = append(metric.labelValues, "")
196+
}
197+
}
198+
}
199+
result[i] = metric
200+
}
201+
202+
return result
203+
}
204+
205+
func fillHistogramMetricsLabels(metrics []HistogramMetric) []HistogramMetric {
206+
allKeys := make(map[string]struct{})
207+
for _, metric := range metrics {
208+
for _, key := range metric.labelKeys {
209+
allKeys[key] = struct{}{}
210+
}
211+
}
212+
result := make([]HistogramMetric, len(metrics))
213+
for i, metric := range metrics {
214+
if len(metric.labelKeys) != len(allKeys) {
215+
metricKeys := make(map[string]struct{})
216+
for _, key := range metric.labelKeys {
217+
metricKeys[key] = struct{}{}
218+
}
219+
for key := range allKeys {
220+
if _, ok := metricKeys[key]; !ok {
221+
metric.labelKeys = append(metric.labelKeys, key)
222+
metric.labelValues = append(metric.labelValues, "")
223+
}
224+
}
225+
}
226+
result[i] = metric
227+
}
228+
229+
return result
230+
}

stackdriver_exporter.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,10 @@ var (
4141
metricsPath = kingpin.Flag(
4242
"web.telemetry-path", "Path under which to expose Prometheus metrics ($STACKDRIVER_EXPORTER_WEB_TELEMETRY_PATH).",
4343
).Envar("STACKDRIVER_EXPORTER_WEB_TELEMETRY_PATH").Default("/metrics").String()
44+
45+
collectorFillMissingLabels = kingpin.Flag(
46+
"collector.fill-missing-labels", "Fill missing metrics labels with empty string to avoid label dimensions inconsistent failure ($STACKDRIVER_EXPORTER_COLLECTOR_FILL_MISSING_LABELS).",
47+
).Envar("STACKDRIVER_EXPORTER_COLLECTOR_FILL_MISSING_LABELS").Default("true").Bool()
4448
)
4549

4650
func init() {
@@ -89,7 +93,7 @@ func main() {
8993
os.Exit(1)
9094
}
9195

92-
monitoringCollector, err := collectors.NewMonitoringCollector(*projectID, metricsTypePrefixes, *monitoringMetricsInterval, *monitoringMetricsOffset, monitoringService)
96+
monitoringCollector, err := collectors.NewMonitoringCollector(*projectID, metricsTypePrefixes, *monitoringMetricsInterval, *monitoringMetricsOffset, monitoringService, *collectorFillMissingLabels)
9397
if err != nil {
9498
log.Error(err)
9599
os.Exit(1)

0 commit comments

Comments
 (0)