Feat: Add data transformations.

Signed-off-by: Matt Kruse <[email protected]>

Author: MattK360

README.md

@@ -50,6 +50,20 @@ animal_population{name="deer",predator="false"} 456
 animal_population{name="lion",predator="true"} 123
 animal_population{name="pigeon",predator="false"} 789
 
+## Test transform module
+$ curl "http://localhost:7979/probe?module=transform&target=http://localhost:8000/examples/transform-data.json"
+
+# HELP origin_health Health of each origin in the pool
+# TYPE origin_health untyped
+origin_health{address="10.0.0.1",endpoint_name="origin3",pool_id="2",pool_name="pool2"} 1
+origin_health{address="10.0.0.1",endpoint_name="origin4",pool_id="3",pool_name="pool3"} 0
+origin_health{address="127.0.0.1",endpoint_name="origin1",pool_id="1",pool_name="pool1"} 1
+origin_health{address="192.168.1.1",endpoint_name="origin2",pool_id="1",pool_name="pool1"} 0
+# HELP pool_health Health of the pools
+# TYPE pool_health untyped
+pool_health{pool_id="1",pool_name="pool1"} 1
+pool_health{pool_id="2",pool_name="pool2"} 1
+pool_health{pool_id="3",pool_name="pool3"} 0
 
 ## TEST through prometheus:
 

config/config.go

@@ -14,6 +14,7 @@
 package config
 
 import (
+	"github.com/prometheus-community/json_exporter/transformers"
 	"os"
 
 	pconfig "github.com/prometheus/common/config"
@@ -22,14 +23,15 @@ import (
 
 // Metric contains values that define a metric
 type Metric struct {
-	Name           string
-	Path           string
-	Labels         map[string]string
-	Type           ScrapeType
-	ValueType      ValueType
-	EpochTimestamp string
-	Help           string
-	Values         map[string]string
+	Name            string
+	Path            string
+	Labels          map[string]string
+	Type            ScrapeType
+	ValueType       ValueType
+	EpochTimestamp  string
+	Help            string
+	Values          map[string]string
+	Transformations []transformers.TransformationConfig
 }
 
 type ScrapeType string

examples/config.yml

@@ -43,6 +43,37 @@ modules:
       values:
         population: '{ .population }'
 
+  transform:
+    metrics:
+    - name: origin
+      transformations:
+        - type: jq
+          query: |-
+            .result[] | .name as $poolName | .id as $poolId | .origins[] | (  .name as $name | .healthy as $endpointHealth | {endpointName: $name, endpointHealthy: .healthy, poolName: $poolName, address:.address, poolId:$poolId, address:.address} )
+      help: Health of each origin in the pool
+      path: '{ [*] }'
+      type: object
+      labels:
+        pool_id: '{.poolId}'
+        pool_name: '{.poolName}'
+        address: '{.address}'
+        endpoint_name: '{.endpointName}'
+      values:
+        health: '{.endpointHealthy}'  # Extract only the `healthy` field
+
+
+    - name: pool
+      type: object
+      help: Health of the pools
+      path: '{.result[*]}'
+      labels:
+        pool_name: '{.name}'
+        pool_id: '{.id}'
+      values:
+        health: '{.healthy}'
+
+
+
     ## HTTP connection configurations can be set in 'modules.<module_name>.http_client_config' field. For full http client config parameters, ref: https://pkg.go.dev/github.com/prometheus/common/config?tab=doc#HTTPClientConfig
     #
     # http_client_config:

examples/prometheus.yml

@@ -32,3 +32,18 @@ scrape_configs:
       ## Location of the json exporter's real <hostname>:<port>
       replacement: host.docker.internal:7979 # equivalent to "localhost:7979"
 
+## Gather metrics from transform-data JSON source using the 'transform' module
+- job_name: json_transform
+  metrics_path: /probe
+  params:
+    module: [transform]  # Use the 'transform' module
+  static_configs:
+    - targets:
+        - http://localhost:8000/examples/transform-data.json
+  relabel_configs:
+    - source_labels: [__address__]
+      target_label: __param_target
+    - source_labels: [__param_target]
+      target_label: instance
+    - target_label: __address__
+      replacement: host.docker.internal:7979  # json_exporter instance

examples/transform-data.json

@@ -0,0 +1,45 @@
+{
+  "result": [
+    {
+      "name": "pool1",
+      "id": "1",
+      "healthy": true,
+      "origins": [
+        {
+          "name": "origin1",
+          "address": "127.0.0.1",
+          "healthy": true
+        },
+        {
+          "name": "origin2",
+          "address": "192.168.1.1",
+          "healthy": false
+        }
+      ]
+    },
+    {
+      "name": "pool2",
+      "id": "2",
+      "healthy": true,
+      "origins": [
+        {
+          "name": "origin3",
+          "address": "10.0.0.1",
+          "healthy": true
+        }
+      ]
+    },
+    {
+      "name": "pool3",
+      "id": "3",
+      "healthy": false,
+      "origins": [
+        {
+          "name": "origin4",
+          "address": "10.0.0.1",
+          "healthy": false
+        }
+      ]
+    }
+  ]
+}

exporter/collector.go

@@ -16,6 +16,7 @@ package exporter
 import (
 	"bytes"
 	"encoding/json"
+	"github.com/prometheus-community/json_exporter/transformers"
 	"time"
 
 	"github.com/go-kit/log"
@@ -39,6 +40,7 @@ type JSONMetric struct {
 	LabelsJSONPaths        []string
 	ValueType              prometheus.ValueType
 	EpochTimestampJSONPath string
+	Transformers           []transformers.Transformer
 }
 
 func (mc JSONMetricCollector) Describe(ch chan<- *prometheus.Desc) {
@@ -49,11 +51,22 @@ func (mc JSONMetricCollector) Describe(ch chan<- *prometheus.Desc) {
 
 func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 	for _, m := range mc.JSONMetrics {
+
+		jsonData := mc.Data
+		for _, transformer := range m.Transformers {
+			transformedData, err := transformer.Transform(jsonData)
+			if err != nil {
+				level.Error(mc.Logger).Log("msg", "Transformation failed", "err", err)
+				continue
+			}
+			jsonData = transformedData
+		}
+
 		switch m.Type {
 		case config.ValueScrape:
-			value, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, false)
+			value, err := extractValue(mc.Logger, jsonData, m.KeyJSONPath, false)
 			if err != nil {
-				level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
+				level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc, "data", jsonData)
 				continue
 			}
 
@@ -62,16 +75,17 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 					m.Desc,
 					m.ValueType,
 					floatValue,
-					extractLabels(mc.Logger, mc.Data, m.LabelsJSONPaths)...,
+					extractLabels(mc.Logger, jsonData, m.LabelsJSONPaths)...,
 				)
-				ch <- timestampMetric(mc.Logger, m, mc.Data, metric)
+				ch <- timestampMetric(mc.Logger, m, jsonData, metric)
 			} else {
-				level.Error(mc.Logger).Log("msg", "Failed to convert extracted value to float64", "path", m.KeyJSONPath, "value", value, "err", err, "metric", m.Desc)
+				level.Error(mc.Logger).Log("msg", "Failed to convert extracted value to float64", "path", m.KeyJSONPath, "value", value, "err", err, "metric", m.Desc) //  was getting error here!
 				continue
 			}
 
 		case config.ObjectScrape:
-			values, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true)
+			values, err := extractValue(mc.Logger, jsonData, m.KeyJSONPath, true)
+
 			if err != nil {
 				level.Error(mc.Logger).Log("msg", "Failed to extract json objects for metric", "err", err, "metric", m.Desc)
 				continue

exporter/util.go

@@ -17,6 +17,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/prometheus-community/json_exporter/transformers"
 	"io"
 	"math"
 	"net/http"
@@ -80,6 +81,22 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 		valueType prometheus.ValueType
 	)
 	for _, metric := range c.Metrics {
+		metricTransfomers := []transformers.Transformer{}
+		for _, tConfig := range metric.Transformations {
+			transformer, err := transformers.NewTransformer(tConfig) // Use the package reference here
+			if err != nil {
+				return nil, err
+			}
+			metricTransfomers = append(metricTransfomers, transformer)
+		}
+		switch metric.ValueType {
+		case config.ValueTypeGauge:
+			valueType = prometheus.GaugeValue
+		case config.ValueTypeCounter:
+			valueType = prometheus.CounterValue
+		default:
+			valueType = prometheus.UntypedValue
+		}
 		switch metric.ValueType {
 		case config.ValueTypeGauge:
 			valueType = prometheus.GaugeValue
@@ -107,6 +124,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 				LabelsJSONPaths:        variableLabelsValues,
 				ValueType:              valueType,
 				EpochTimestampJSONPath: metric.EpochTimestamp,
+				Transformers:           metricTransfomers, // Add transformers here
 			}
 			metrics = append(metrics, jsonMetric)
 		case config.ObjectScrape:
@@ -130,6 +148,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 					LabelsJSONPaths:        variableLabelsValues,
 					ValueType:              valueType,
 					EpochTimestampJSONPath: metric.EpochTimestamp,
+					Transformers:           metricTransfomers, // Add transformers here
 				}
 				metrics = append(metrics, jsonMetric)
 			}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/alecthomas/kingpin/v2 v2.4.0
 	github.com/go-kit/log v0.2.1
+	github.com/itchyny/gojq v0.12.16
 	github.com/prometheus/client_golang v1.19.1
 	github.com/prometheus/common v0.55.0
 	github.com/prometheus/exporter-toolkit v0.11.0
@@ -24,6 +25,7 @@ require (
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/huandu/xstrings v1.3.3 // indirect
 	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/itchyny/timefmt-go v0.1.6 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/mitchellh/copystructure v1.0.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.0 // indirect

go.sum

@@ -31,6 +31,10 @@ github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4
 github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g=
+github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM=
+github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q=
+github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=

transformers/jq_transformer.go

@@ -0,0 +1,68 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transformers
+
+import (
+	"encoding/json"
+	"github.com/itchyny/gojq"
+)
+
+// JQTransformer struct for jq transformation
+type JQTransformer struct {
+	Query string
+}
+
+// NewJQTransformer creates a new JQTransformer with a given query
+func NewJQTransformer(query string) JQTransformer {
+	return JQTransformer{Query: query}
+}
+
+// Transform applies the jq filter transformation to the input data
+func (jq JQTransformer) Transform(data []byte) ([]byte, error) {
+	return applyJQFilter(data, jq.Query)
+}
+
+// applyJQFilter uses gojq to apply a jq transformation to the input data
+func applyJQFilter(jsonData []byte, jqQuery string) ([]byte, error) {
+	var input interface{}
+	if err := json.Unmarshal(jsonData, &input); err != nil {
+		return nil, err
+	}
+
+	query, err := gojq.Parse(jqQuery)
+	if err != nil {
+		return nil, err
+	}
+
+	iter := query.Run(input)
+	var results []interface{}
+	for {
+		v, ok := iter.Next()
+		if !ok {
+			break
+		}
+		if err, ok := v.(error); ok {
+			return nil, err
+		}
+		results = append(results, v)
+	}
+
+	// Convert the transformed result back to JSON []byte
+	transformedJSON, err := json.Marshal(results)
+	if err != nil {
+		return nil, err
+	}
+
+	return transformedJSON, nil
+}

transformers/transformers.go

@@ -0,0 +1,34 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transformers
+
+import "fmt"
+
+type Transformer interface {
+	Transform(data []byte) ([]byte, error)
+}
+
+type TransformationConfig struct {
+	Type  string
+	Query string
+}
+
+func NewTransformer(config TransformationConfig) (Transformer, error) {
+	switch config.Type {
+	case "jq":
+		return NewJQTransformer(config.Query), nil
+	default:
+		return nil, fmt.Errorf("unsupported transformer type: %s", config.Type)
+	}
+}