From f49ceb070f99a7893b1dae64d093d4b5b880e4bf Mon Sep 17 00:00:00 2001 From: Michal Budzyn Date: Thu, 15 Mar 2018 00:05:50 +0100 Subject: [PATCH 1/3] Collect metrics to fill missing labels --- collectors/fnv.go | 32 ++++ collectors/monitoring_collector.go | 269 ++++++++++++++++++++++++++--- 2 files changed, 274 insertions(+), 27 deletions(-) create mode 100644 collectors/fnv.go diff --git a/collectors/fnv.go b/collectors/fnv.go new file mode 100644 index 00000000..2af3133b --- /dev/null +++ b/collectors/fnv.go @@ -0,0 +1,32 @@ +package collectors + +const separatorByte = 255 + +// https://github.com/prometheus/client_golang/blob/master/prometheus/fnv.go +// Inline and byte-free variant of hash/fnv's fnv64a. + +const ( + offset64 = 14695981039346656037 + prime64 = 1099511628211 +) + +// hashNew initializies a new fnv64a hash value. +func hashNew() uint64 { + return offset64 +} + +// hashAdd adds a string to a fnv64a hash value, returning the updated hash. +func hashAdd(h uint64, s string) uint64 { + for i := 0; i < len(s); i++ { + h ^= uint64(s[i]) + h *= prime64 + } + return h +} + +// hashAddByte adds a byte to a fnv64a hash value, returning the updated hash. +func hashAddByte(h uint64, b byte) uint64 { + h ^= uint64(b) + h *= prime64 + return h +} diff --git a/collectors/monitoring_collector.go b/collectors/monitoring_collector.go index b0486421..3a092cdb 100644 --- a/collectors/monitoring_collector.go +++ b/collectors/monitoring_collector.go @@ -13,6 +13,7 @@ import ( "google.golang.org/api/monitoring/v3" "github.com/frodenas/stackdriver_exporter/utils" + "sort" ) type MonitoringCollector struct { @@ -231,8 +232,14 @@ func (c *MonitoringCollector) reportTimeSeriesMetrics( var metricValue float64 var metricValueType prometheus.ValueType var newestTSPoint *monitoring.Point - var metricDesc *prometheus.Desc + timeSeriesMetrics := &TimeSeriesMetrics{ + metricDescriptor: metricDescriptor, + ch: ch, + fillMissingLabels: true, //TODO: this can be configurable + constMetrics: make(map[string][]ConstMetric), + histogramMetrics: make(map[string][]HistogramMetric), + } for _, timeSeries := range page.TimeSeries { newestEndTime := time.Unix(0, 0) for _, point := range timeSeries.Points { @@ -245,7 +252,6 @@ func (c *MonitoringCollector) reportTimeSeriesMetrics( newestTSPoint = point } } - labelKeys := []string{"unit"} labelValues := []string{metricDescriptor.Unit} @@ -263,17 +269,6 @@ func (c *MonitoringCollector) reportTimeSeriesMetrics( labelValues = append(labelValues, value) } - // The metric name to report is composed by the 3 parts: - // 1. namespace is a constant prefix (stackdriver) - // 2. subsystem is the monitored resource type (ie gce_instance) - // 3. name is the metric type (ie compute.googleapis.com/instance/cpu/usage_time) - metricDesc = prometheus.NewDesc( - prometheus.BuildFQName("stackdriver", utils.NormalizeMetricName(timeSeries.Resource.Type), utils.NormalizeMetricName(timeSeries.Metric.Type)), - metricDescriptor.Description, - labelKeys, - prometheus.Labels{}, - ) - switch timeSeries.MetricKind { case "GAUGE": metricValueType = prometheus.GaugeValue @@ -299,13 +294,7 @@ func (c *MonitoringCollector) reportTimeSeriesMetrics( dist := newestTSPoint.Value.DistributionValue buckets, err := c.generateHistogramBuckets(dist) if err == nil { - ch <- prometheus.MustNewConstHistogram( - metricDesc, - uint64(dist.Count), - dist.Mean*float64(dist.Count), // Stackdriver does not provide the sum, but we can fake it - buckets, - labelValues..., - ) + timeSeriesMetrics.CollectNewConstHistogram(timeSeries, labelKeys, dist, buckets, labelValues) } else { log.Debugf("Discarding resource %s metric %s: %s", timeSeries.Resource.Type, timeSeries.Metric.Type, err) } @@ -315,14 +304,9 @@ func (c *MonitoringCollector) reportTimeSeriesMetrics( continue } - ch <- prometheus.MustNewConstMetric( - metricDesc, - metricValueType, - metricValue, - labelValues..., - ) + timeSeriesMetrics.CollectNewConstMetric(timeSeries, labelKeys, metricValueType, metricValue, labelValues) } - + timeSeriesMetrics.Complete() return nil } @@ -378,3 +362,234 @@ func (c *MonitoringCollector) generateHistogramBuckets( } return buckets, nil } + +func buildFQName(timeSeries *monitoring.TimeSeries) string { + // The metric name to report is composed by the 3 parts: + // 1. namespace is a constant prefix (stackdriver) + // 2. subsystem is the monitored resource type (ie gce_instance) + // 3. name is the metric type (ie compute.googleapis.com/instance/cpu/usage_time) + return prometheus.BuildFQName("stackdriver", utils.NormalizeMetricName(timeSeries.Resource.Type), utils.NormalizeMetricName(timeSeries.Metric.Type)) +} + +type TimeSeriesMetrics struct { + metricDescriptor *monitoring.MetricDescriptor + ch chan<- prometheus.Metric + + fillMissingLabels bool + constMetrics map[string][]ConstMetric + histogramMetrics map[string][]HistogramMetric +} + +func (t *TimeSeriesMetrics) NewMetricDesc(fqName string, labelKeys []string) *prometheus.Desc { + return prometheus.NewDesc( + fqName, + t.metricDescriptor.Description, + labelKeys, + prometheus.Labels{}, + ) +} + +type ConstMetric struct { + fqName string + labelKeys []string + valueType prometheus.ValueType + value float64 + labelValues []string + + keysHash uint64 +} + +type HistogramMetric struct { + fqName string + labelKeys []string + dist *monitoring.Distribution + buckets map[float64]uint64 + labelValues []string + + keysHash uint64 +} + +func (t *TimeSeriesMetrics) CollectNewConstHistogram(timeSeries *monitoring.TimeSeries, labelKeys []string, dist *monitoring.Distribution, buckets map[float64]uint64, labelValues []string) { + fqName := buildFQName(timeSeries) + + if t.fillMissingLabels { + vs, ok := t.histogramMetrics[fqName] + if !ok { + vs = make([]HistogramMetric, 0) + } + v := HistogramMetric{ + fqName: fqName, + labelKeys: labelKeys, + dist: dist, + buckets: buckets, + labelValues: labelValues, + + keysHash: hashLabelKeys(labelKeys), + } + t.histogramMetrics[fqName] = append(vs, v) + return + } + + metric := prometheus.MustNewConstHistogram( + t.NewMetricDesc(fqName, labelKeys), + uint64(dist.Count), + dist.Mean*float64(dist.Count), // Stackdriver does not provide the sum, but we can fake it + buckets, + labelValues..., + ) + t.ch <- metric +} + +func (t *TimeSeriesMetrics) CollectNewConstMetric(timeSeries *monitoring.TimeSeries, labelKeys []string, metricValueType prometheus.ValueType, metricValue float64, labelValues []string) { + fqName := buildFQName(timeSeries) + + if t.fillMissingLabels { + vs, ok := t.constMetrics[fqName] + if !ok { + vs = make([]ConstMetric, 0) + } + v := ConstMetric{ + fqName: fqName, + labelKeys: labelKeys, + valueType: metricValueType, + value: metricValue, + labelValues: labelValues, + + keysHash: hashLabelKeys(labelKeys), + } + t.constMetrics[fqName] = append(vs, v) + return + } + + metric := prometheus.MustNewConstMetric( + t.NewMetricDesc(fqName, labelKeys), + metricValueType, + metricValue, + labelValues..., + ) + t.ch <- metric +} + +func hashLabelKeys(labelKeys []string) uint64 { + dh := hashNew() + sortedKeys := make([]string, len(labelKeys)) + copy(sortedKeys, labelKeys) + sort.Strings(sortedKeys) + for _, key := range sortedKeys { + dh = hashAdd(dh, key) + dh = hashAddByte(dh, separatorByte) + } + return dh +} + +func (t *TimeSeriesMetrics) Complete() { + t.CompleteConstMetrics() + t.CompleteHistogramMetrics() +} + +func (t *TimeSeriesMetrics) CompleteConstMetrics() { + for _, vs := range t.constMetrics { + if len(vs) > 1 { + var needFill bool + for i := 1; i < len(vs); i++ { + if vs[0].keysHash != vs[i].keysHash { + needFill = true + } + } + if needFill { + vs = fillConstMetricsLabels(vs) + } + } + + for _, v := range vs { + metric := prometheus.MustNewConstMetric( + t.NewMetricDesc(v.fqName, v.labelKeys), + v.valueType, + v.value, + v.labelValues..., + ) + t.ch <- metric + } + } +} + +func (t *TimeSeriesMetrics) CompleteHistogramMetrics() { + for _, vs := range t.histogramMetrics { + if len(vs) > 1 { + var needFill bool + for i := 1; i < len(vs); i++ { + if vs[0].keysHash != vs[i].keysHash { + needFill = true + } + } + if needFill { + vs = fillHistogramMetricsLabels(vs) + } + } + + for _, v := range vs { + metric := prometheus.MustNewConstHistogram( + t.NewMetricDesc(v.fqName, v.labelKeys), + uint64(v.dist.Count), + v.dist.Mean*float64(v.dist.Count), // Stackdriver does not provide the sum, but we can fake it + v.buckets, + v.labelValues..., + ) + t.ch <- metric + } + } +} + +func fillConstMetricsLabels(metrics []ConstMetric) []ConstMetric { + allKeys := make(map[string]struct{}) + for _, metric := range metrics { + for _, key := range metric.labelKeys { + allKeys[key] = struct{}{} + } + } + result := make([]ConstMetric, len(metrics)) + for i, metric := range metrics { + if len(metric.labelKeys) != len(allKeys) { + metricKeys := make(map[string]struct{}) + for _, key := range metric.labelKeys { + metricKeys[key] = struct{}{} + } + for key := range allKeys { + if _, ok := metricKeys[key]; !ok { + metric.labelKeys = append(metric.labelKeys, key) + metric.labelValues = append(metric.labelValues, "") + } + } + } + result[i] = metric + } + + return result +} + +func fillHistogramMetricsLabels(metrics []HistogramMetric) []HistogramMetric { + allKeys := make(map[string]struct{}) + for _, metric := range metrics { + for _, key := range metric.labelKeys { + allKeys[key] = struct{}{} + } + } + result := make([]HistogramMetric, len(metrics)) + for i, metric := range metrics { + if len(metric.labelKeys) != len(allKeys) { + metricKeys := make(map[string]struct{}) + for _, key := range metric.labelKeys { + metricKeys[key] = struct{}{} + } + for key := range allKeys { + if _, ok := metricKeys[key]; !ok { + metric.labelKeys = append(metric.labelKeys, key) + metric.labelValues = append(metric.labelValues, "") + } + } + } + result[i] = metric + } + + return result +} From 3da4cbdb0a0b6a03314878b838033a4b496d9415 Mon Sep 17 00:00:00 2001 From: Michal Budzyn Date: Thu, 15 Mar 2018 12:44:32 +0100 Subject: [PATCH 2/3] Add missing labals --- collectors/monitoring_collector.go | 237 +---------------------------- collectors/monitoring_metrics.go | 230 ++++++++++++++++++++++++++++ stackdriver_exporter.go | 6 +- 3 files changed, 239 insertions(+), 234 deletions(-) create mode 100644 collectors/monitoring_metrics.go diff --git a/collectors/monitoring_collector.go b/collectors/monitoring_collector.go index 3a092cdb..2255fe27 100644 --- a/collectors/monitoring_collector.go +++ b/collectors/monitoring_collector.go @@ -13,7 +13,6 @@ import ( "google.golang.org/api/monitoring/v3" "github.com/frodenas/stackdriver_exporter/utils" - "sort" ) type MonitoringCollector struct { @@ -28,6 +27,7 @@ type MonitoringCollector struct { lastScrapeErrorMetric prometheus.Gauge lastScrapeTimestampMetric prometheus.Gauge lastScrapeDurationSecondsMetric prometheus.Gauge + collectorFillMissingLabels bool } func NewMonitoringCollector( @@ -36,6 +36,7 @@ func NewMonitoringCollector( metricsInterval time.Duration, metricsOffset time.Duration, monitoringService *monitoring.Service, + collectorFillMissingLabels bool, ) (*MonitoringCollector, error) { apiCallsTotalMetric := prometheus.NewCounter( prometheus.CounterOpts{ @@ -109,6 +110,7 @@ func NewMonitoringCollector( lastScrapeErrorMetric: lastScrapeErrorMetric, lastScrapeTimestampMetric: lastScrapeTimestampMetric, lastScrapeDurationSecondsMetric: lastScrapeDurationSecondsMetric, + collectorFillMissingLabels: collectorFillMissingLabels, } return monitoringCollector, nil @@ -236,7 +238,7 @@ func (c *MonitoringCollector) reportTimeSeriesMetrics( timeSeriesMetrics := &TimeSeriesMetrics{ metricDescriptor: metricDescriptor, ch: ch, - fillMissingLabels: true, //TODO: this can be configurable + fillMissingLabels: c.collectorFillMissingLabels, constMetrics: make(map[string][]ConstMetric), histogramMetrics: make(map[string][]HistogramMetric), } @@ -362,234 +364,3 @@ func (c *MonitoringCollector) generateHistogramBuckets( } return buckets, nil } - -func buildFQName(timeSeries *monitoring.TimeSeries) string { - // The metric name to report is composed by the 3 parts: - // 1. namespace is a constant prefix (stackdriver) - // 2. subsystem is the monitored resource type (ie gce_instance) - // 3. name is the metric type (ie compute.googleapis.com/instance/cpu/usage_time) - return prometheus.BuildFQName("stackdriver", utils.NormalizeMetricName(timeSeries.Resource.Type), utils.NormalizeMetricName(timeSeries.Metric.Type)) -} - -type TimeSeriesMetrics struct { - metricDescriptor *monitoring.MetricDescriptor - ch chan<- prometheus.Metric - - fillMissingLabels bool - constMetrics map[string][]ConstMetric - histogramMetrics map[string][]HistogramMetric -} - -func (t *TimeSeriesMetrics) NewMetricDesc(fqName string, labelKeys []string) *prometheus.Desc { - return prometheus.NewDesc( - fqName, - t.metricDescriptor.Description, - labelKeys, - prometheus.Labels{}, - ) -} - -type ConstMetric struct { - fqName string - labelKeys []string - valueType prometheus.ValueType - value float64 - labelValues []string - - keysHash uint64 -} - -type HistogramMetric struct { - fqName string - labelKeys []string - dist *monitoring.Distribution - buckets map[float64]uint64 - labelValues []string - - keysHash uint64 -} - -func (t *TimeSeriesMetrics) CollectNewConstHistogram(timeSeries *monitoring.TimeSeries, labelKeys []string, dist *monitoring.Distribution, buckets map[float64]uint64, labelValues []string) { - fqName := buildFQName(timeSeries) - - if t.fillMissingLabels { - vs, ok := t.histogramMetrics[fqName] - if !ok { - vs = make([]HistogramMetric, 0) - } - v := HistogramMetric{ - fqName: fqName, - labelKeys: labelKeys, - dist: dist, - buckets: buckets, - labelValues: labelValues, - - keysHash: hashLabelKeys(labelKeys), - } - t.histogramMetrics[fqName] = append(vs, v) - return - } - - metric := prometheus.MustNewConstHistogram( - t.NewMetricDesc(fqName, labelKeys), - uint64(dist.Count), - dist.Mean*float64(dist.Count), // Stackdriver does not provide the sum, but we can fake it - buckets, - labelValues..., - ) - t.ch <- metric -} - -func (t *TimeSeriesMetrics) CollectNewConstMetric(timeSeries *monitoring.TimeSeries, labelKeys []string, metricValueType prometheus.ValueType, metricValue float64, labelValues []string) { - fqName := buildFQName(timeSeries) - - if t.fillMissingLabels { - vs, ok := t.constMetrics[fqName] - if !ok { - vs = make([]ConstMetric, 0) - } - v := ConstMetric{ - fqName: fqName, - labelKeys: labelKeys, - valueType: metricValueType, - value: metricValue, - labelValues: labelValues, - - keysHash: hashLabelKeys(labelKeys), - } - t.constMetrics[fqName] = append(vs, v) - return - } - - metric := prometheus.MustNewConstMetric( - t.NewMetricDesc(fqName, labelKeys), - metricValueType, - metricValue, - labelValues..., - ) - t.ch <- metric -} - -func hashLabelKeys(labelKeys []string) uint64 { - dh := hashNew() - sortedKeys := make([]string, len(labelKeys)) - copy(sortedKeys, labelKeys) - sort.Strings(sortedKeys) - for _, key := range sortedKeys { - dh = hashAdd(dh, key) - dh = hashAddByte(dh, separatorByte) - } - return dh -} - -func (t *TimeSeriesMetrics) Complete() { - t.CompleteConstMetrics() - t.CompleteHistogramMetrics() -} - -func (t *TimeSeriesMetrics) CompleteConstMetrics() { - for _, vs := range t.constMetrics { - if len(vs) > 1 { - var needFill bool - for i := 1; i < len(vs); i++ { - if vs[0].keysHash != vs[i].keysHash { - needFill = true - } - } - if needFill { - vs = fillConstMetricsLabels(vs) - } - } - - for _, v := range vs { - metric := prometheus.MustNewConstMetric( - t.NewMetricDesc(v.fqName, v.labelKeys), - v.valueType, - v.value, - v.labelValues..., - ) - t.ch <- metric - } - } -} - -func (t *TimeSeriesMetrics) CompleteHistogramMetrics() { - for _, vs := range t.histogramMetrics { - if len(vs) > 1 { - var needFill bool - for i := 1; i < len(vs); i++ { - if vs[0].keysHash != vs[i].keysHash { - needFill = true - } - } - if needFill { - vs = fillHistogramMetricsLabels(vs) - } - } - - for _, v := range vs { - metric := prometheus.MustNewConstHistogram( - t.NewMetricDesc(v.fqName, v.labelKeys), - uint64(v.dist.Count), - v.dist.Mean*float64(v.dist.Count), // Stackdriver does not provide the sum, but we can fake it - v.buckets, - v.labelValues..., - ) - t.ch <- metric - } - } -} - -func fillConstMetricsLabels(metrics []ConstMetric) []ConstMetric { - allKeys := make(map[string]struct{}) - for _, metric := range metrics { - for _, key := range metric.labelKeys { - allKeys[key] = struct{}{} - } - } - result := make([]ConstMetric, len(metrics)) - for i, metric := range metrics { - if len(metric.labelKeys) != len(allKeys) { - metricKeys := make(map[string]struct{}) - for _, key := range metric.labelKeys { - metricKeys[key] = struct{}{} - } - for key := range allKeys { - if _, ok := metricKeys[key]; !ok { - metric.labelKeys = append(metric.labelKeys, key) - metric.labelValues = append(metric.labelValues, "") - } - } - } - result[i] = metric - } - - return result -} - -func fillHistogramMetricsLabels(metrics []HistogramMetric) []HistogramMetric { - allKeys := make(map[string]struct{}) - for _, metric := range metrics { - for _, key := range metric.labelKeys { - allKeys[key] = struct{}{} - } - } - result := make([]HistogramMetric, len(metrics)) - for i, metric := range metrics { - if len(metric.labelKeys) != len(allKeys) { - metricKeys := make(map[string]struct{}) - for _, key := range metric.labelKeys { - metricKeys[key] = struct{}{} - } - for key := range allKeys { - if _, ok := metricKeys[key]; !ok { - metric.labelKeys = append(metric.labelKeys, key) - metric.labelValues = append(metric.labelValues, "") - } - } - } - result[i] = metric - } - - return result -} diff --git a/collectors/monitoring_metrics.go b/collectors/monitoring_metrics.go new file mode 100644 index 00000000..193ea965 --- /dev/null +++ b/collectors/monitoring_metrics.go @@ -0,0 +1,230 @@ +package collectors + +import ( + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/api/monitoring/v3" + + "github.com/frodenas/stackdriver_exporter/utils" + "sort" +) + +func buildFQName(timeSeries *monitoring.TimeSeries) string { + // The metric name to report is composed by the 3 parts: + // 1. namespace is a constant prefix (stackdriver) + // 2. subsystem is the monitored resource type (ie gce_instance) + // 3. name is the metric type (ie compute.googleapis.com/instance/cpu/usage_time) + return prometheus.BuildFQName("stackdriver", utils.NormalizeMetricName(timeSeries.Resource.Type), utils.NormalizeMetricName(timeSeries.Metric.Type)) +} + +type TimeSeriesMetrics struct { + metricDescriptor *monitoring.MetricDescriptor + ch chan<- prometheus.Metric + + fillMissingLabels bool + constMetrics map[string][]ConstMetric + histogramMetrics map[string][]HistogramMetric +} + +func (t *TimeSeriesMetrics) newMetricDesc(fqName string, labelKeys []string) *prometheus.Desc { + return prometheus.NewDesc( + fqName, + t.metricDescriptor.Description, + labelKeys, + prometheus.Labels{}, + ) +} + +type ConstMetric struct { + fqName string + labelKeys []string + valueType prometheus.ValueType + value float64 + labelValues []string + + keysHash uint64 +} + +type HistogramMetric struct { + fqName string + labelKeys []string + dist *monitoring.Distribution + buckets map[float64]uint64 + labelValues []string + + keysHash uint64 +} + +func (t *TimeSeriesMetrics) CollectNewConstHistogram(timeSeries *monitoring.TimeSeries, labelKeys []string, dist *monitoring.Distribution, buckets map[float64]uint64, labelValues []string) { + fqName := buildFQName(timeSeries) + + if t.fillMissingLabels { + vs, ok := t.histogramMetrics[fqName] + if !ok { + vs = make([]HistogramMetric, 0) + } + v := HistogramMetric{ + fqName: fqName, + labelKeys: labelKeys, + dist: dist, + buckets: buckets, + labelValues: labelValues, + + keysHash: hashLabelKeys(labelKeys), + } + t.histogramMetrics[fqName] = append(vs, v) + return + } + t.ch <- t.newConstHistogram(fqName, labelKeys, dist, buckets, labelValues) +} + +func (t *TimeSeriesMetrics) newConstHistogram(fqName string, labelKeys []string, dist *monitoring.Distribution, buckets map[float64]uint64, labelValues []string) prometheus.Metric { + return prometheus.MustNewConstHistogram( + t.newMetricDesc(fqName, labelKeys), + uint64(dist.Count), + dist.Mean*float64(dist.Count), // Stackdriver does not provide the sum, but we can fake it + buckets, + labelValues..., + ) +} + +func (t *TimeSeriesMetrics) CollectNewConstMetric(timeSeries *monitoring.TimeSeries, labelKeys []string, metricValueType prometheus.ValueType, metricValue float64, labelValues []string) { + fqName := buildFQName(timeSeries) + + if t.fillMissingLabels { + vs, ok := t.constMetrics[fqName] + if !ok { + vs = make([]ConstMetric, 0) + } + v := ConstMetric{ + fqName: fqName, + labelKeys: labelKeys, + valueType: metricValueType, + value: metricValue, + labelValues: labelValues, + + keysHash: hashLabelKeys(labelKeys), + } + t.constMetrics[fqName] = append(vs, v) + return + } + t.ch <- t.newConstMetric(fqName, labelKeys, metricValueType, metricValue, labelValues) +} + +func (t *TimeSeriesMetrics) newConstMetric(fqName string, labelKeys []string, metricValueType prometheus.ValueType, metricValue float64, labelValues []string) prometheus.Metric { + return prometheus.MustNewConstMetric( + t.newMetricDesc(fqName, labelKeys), + metricValueType, + metricValue, + labelValues..., + ) +} + +func hashLabelKeys(labelKeys []string) uint64 { + dh := hashNew() + sortedKeys := make([]string, len(labelKeys)) + copy(sortedKeys, labelKeys) + sort.Strings(sortedKeys) + for _, key := range sortedKeys { + dh = hashAdd(dh, key) + dh = hashAddByte(dh, separatorByte) + } + return dh +} + +func (t *TimeSeriesMetrics) Complete() { + t.completeConstMetrics() + t.completeHistogramMetrics() +} + +func (t *TimeSeriesMetrics) completeConstMetrics() { + for _, vs := range t.constMetrics { + if len(vs) > 1 { + var needFill bool + for i := 1; i < len(vs); i++ { + if vs[0].keysHash != vs[i].keysHash { + needFill = true + } + } + if needFill { + vs = fillConstMetricsLabels(vs) + } + } + + for _, v := range vs { + t.ch <- t.newConstMetric(v.fqName, v.labelKeys, v.valueType, v.value, v.labelValues) + } + } +} + +func (t *TimeSeriesMetrics) completeHistogramMetrics() { + for _, vs := range t.histogramMetrics { + if len(vs) > 1 { + var needFill bool + for i := 1; i < len(vs); i++ { + if vs[0].keysHash != vs[i].keysHash { + needFill = true + } + } + if needFill { + vs = fillHistogramMetricsLabels(vs) + } + } + for _, v := range vs { + t.ch <- t.newConstHistogram(v.fqName, v.labelKeys, v.dist, v.buckets, v.labelValues) + } + } +} + +func fillConstMetricsLabels(metrics []ConstMetric) []ConstMetric { + allKeys := make(map[string]struct{}) + for _, metric := range metrics { + for _, key := range metric.labelKeys { + allKeys[key] = struct{}{} + } + } + result := make([]ConstMetric, len(metrics)) + for i, metric := range metrics { + if len(metric.labelKeys) != len(allKeys) { + metricKeys := make(map[string]struct{}) + for _, key := range metric.labelKeys { + metricKeys[key] = struct{}{} + } + for key := range allKeys { + if _, ok := metricKeys[key]; !ok { + metric.labelKeys = append(metric.labelKeys, key) + metric.labelValues = append(metric.labelValues, "") + } + } + } + result[i] = metric + } + + return result +} + +func fillHistogramMetricsLabels(metrics []HistogramMetric) []HistogramMetric { + allKeys := make(map[string]struct{}) + for _, metric := range metrics { + for _, key := range metric.labelKeys { + allKeys[key] = struct{}{} + } + } + result := make([]HistogramMetric, len(metrics)) + for i, metric := range metrics { + if len(metric.labelKeys) != len(allKeys) { + metricKeys := make(map[string]struct{}) + for _, key := range metric.labelKeys { + metricKeys[key] = struct{}{} + } + for key := range allKeys { + if _, ok := metricKeys[key]; !ok { + metric.labelKeys = append(metric.labelKeys, key) + metric.labelValues = append(metric.labelValues, "") + } + } + } + result[i] = metric + } + + return result +} diff --git a/stackdriver_exporter.go b/stackdriver_exporter.go index cbf81dd1..620c02bf 100644 --- a/stackdriver_exporter.go +++ b/stackdriver_exporter.go @@ -41,6 +41,10 @@ var ( metricsPath = kingpin.Flag( "web.telemetry-path", "Path under which to expose Prometheus metrics ($STACKDRIVER_EXPORTER_WEB_TELEMETRY_PATH).", ).Envar("STACKDRIVER_EXPORTER_WEB_TELEMETRY_PATH").Default("/metrics").String() + + collectorFillMissingLabels = kingpin.Flag( + "collector.fill-missing-labels", "Fill missing metrics labels with empty string to avoid label dimensions inconsistent failure ($STACKDRIVER_EXPORTER_COLLECTOR_FILL_MISSING_LABELS).", + ).Envar("STACKDRIVER_EXPORTER_COLLECTOR_FILL_MISSING_LABELS").Default("false").Bool() ) func init() { @@ -89,7 +93,7 @@ func main() { os.Exit(1) } - monitoringCollector, err := collectors.NewMonitoringCollector(*projectID, metricsTypePrefixes, *monitoringMetricsInterval, *monitoringMetricsOffset, monitoringService) + monitoringCollector, err := collectors.NewMonitoringCollector(*projectID, metricsTypePrefixes, *monitoringMetricsInterval, *monitoringMetricsOffset, monitoringService, *collectorFillMissingLabels) if err != nil { log.Error(err) os.Exit(1) From 0d4115326e25b5a40fae12708962d6d3265b6730 Mon Sep 17 00:00:00 2001 From: Michal Budzyn Date: Wed, 16 May 2018 23:38:15 +0200 Subject: [PATCH 3/3] Change default value of STACKDRIVER_EXPORTER_COLLECTOR_FILL_MISSING_LABELS to true --- stackdriver_exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdriver_exporter.go b/stackdriver_exporter.go index 620c02bf..3d3e420a 100644 --- a/stackdriver_exporter.go +++ b/stackdriver_exporter.go @@ -44,7 +44,7 @@ var ( collectorFillMissingLabels = kingpin.Flag( "collector.fill-missing-labels", "Fill missing metrics labels with empty string to avoid label dimensions inconsistent failure ($STACKDRIVER_EXPORTER_COLLECTOR_FILL_MISSING_LABELS).", - ).Envar("STACKDRIVER_EXPORTER_COLLECTOR_FILL_MISSING_LABELS").Default("false").Bool() + ).Envar("STACKDRIVER_EXPORTER_COLLECTOR_FILL_MISSING_LABELS").Default("true").Bool() ) func init() {