WIP: Initial support for CEL

This adds initial support for queries using the common expression
language (CEL).

ValueToJSON function is adapted from
https://github.com/google/cel-go/blob/cfbf821f1b458533051306305a39b743db7c4bdb/codelab/codelab.go#L274
(Apache-2.0 Licensed)

Signed-off-by: Manuel Rüger <[email protected]>

Author: mrueg

README.md

@@ -2,7 +2,7 @@ json_exporter
 ========================
 [![CircleCI](https://circleci.com/gh/prometheus-community/json_exporter.svg?style=svg)](https://circleci.com/gh/prometheus-community/json_exporter)
 
-A [prometheus](https://prometheus.io/) exporter which scrapes remote JSON by JSONPath.
+A [prometheus](https://prometheus.io/) exporter which scrapes remote JSON by JSONPath or [CEL (Common Expression Language)](https://github.com/google/cel-spec).
 
 - [Supported JSONPath Syntax](https://kubernetes.io/docs/reference/kubectl/jsonpath/)
 - [Examples configurations](/examples)
@@ -21,6 +21,24 @@ Serving HTTP on :: port 8000 (http://[::]:8000/) ...
 ## TEST with 'default' module
 
 $ curl "http://localhost:7979/probe?module=default&target=http://localhost:8000/examples/data.json"
+# HELP example_cel_global_value Example of a top-level global value scrape in the json using cel
+# TYPE example_cel_global_value gauge
+example_cel_global_value{environment="beta",location="planet-mars"} 1234
+# HELP example_cel_timestamped_value_count Example of a timestamped value scrape in the json
+# TYPE example_cel_timestamped_value_count untyped
+example_cel_timestamped_value_count{environment="beta"} 2
+# HELP example_cel_value_active Example of sub-level value scrapes from a json
+# TYPE example_cel_value_active untyped
+example_cel_value_active{environment="beta",id="id-A"} 1
+example_cel_value_active{environment="beta",id="id-C"} 1
+# HELP example_cel_value_boolean Example of sub-level value scrapes from a json
+# TYPE example_cel_value_boolean untyped
+example_cel_value_boolean{environment="beta",id="id-A"} 1
+example_cel_value_boolean{environment="beta",id="id-C"} 0
+# HELP example_cel_value_count Example of sub-level value scrapes from a json
+# TYPE example_cel_value_count untyped
+example_cel_value_count{environment="beta",id="id-A"} 1
+example_cel_value_count{environment="beta",id="id-C"} 3
 # HELP example_global_value Example of a top-level global value scrape in the json
 # TYPE example_global_value untyped
 example_global_value{environment="beta",location="planet-mars"} 1234

config/config.go

@@ -23,6 +23,7 @@ import (
 // Metric contains values that define a metric
 type Metric struct {
 	Name           string
+	Engine         EngineType
 	Path           string
 	Labels         map[string]string
 	Type           ScrapeType
@@ -44,7 +45,14 @@ type ValueType string
 const (
 	ValueTypeGauge   ValueType = "gauge"
 	ValueTypeCounter ValueType = "counter"
-	ValueTypeUntyped ValueType = "untyped"
+	ValueTypeUntyped ValueType = "untyped" // default
+)
+
+type EngineType string
+
+const (
+	EngineTypeJSONPath EngineType = "jsonpath" // default
+	EngineTypeCEL      EngineType = "cel"
 )
 
 // Config contains multiple modules.
@@ -89,6 +97,9 @@ func LoadConfig(configPath string) (Config, error) {
 			if module.Metrics[i].ValueType == "" {
 				module.Metrics[i].ValueType = ValueTypeUntyped
 			}
+			if module.Metrics[i].Engine == "" {
+				module.Metrics[i].Engine = EngineTypeJSONPath
+			}
 		}
 	}
 

examples/config.yml

@@ -9,7 +9,17 @@ modules:
       help: Example of a top-level global value scrape in the json
       labels:
         environment: beta # static label
-        location: 'planet-{.location}'          # dynamic label
+        location: 'planet-{ .location }' # dynamic label
+
+    - name: example_cel_global_value
+      engine: cel
+      path: '.counter'
+      help: Example of a top-level global value scrape in the json using cel
+      valuetype: 'gauge'
+      labels:
+        environment: "\"beta\"" # static label. Quotes need to be escaped for CEL
+        location: "\"planet-\"+.location" # dynamic label. Quotes need to be escaped for CEL
+
     - name: example_timestamped_value
       type: object
       path: '{ .values[?(@.state == "INACTIVE")] }'
@@ -18,18 +28,44 @@ modules:
       labels:
         environment: beta # static label
       values:
-        count: '{.count}' # dynamic value
+        count: '{ .count }' # dynamic value
+
+    - name: example_cel_timestamped_value
+      type: object
+      engine: cel
+      path: ".values.filter(i, i.state == \"INACTIVE\")"
+      epochTimestamp: '.timestamp'
+      help: Example of a timestamped value scrape in the json
+      labels:
+        environment: "\"beta\"" # static label
+      values:
+        count: '.count' # dynamic value
+
     - name: example_value
       type: object
       help: Example of sub-level value scrapes from a json
-      path: '{.values[?(@.state == "ACTIVE")]}'
+      path: '{ .values[?(@.state == "ACTIVE")] }'
       labels:
         environment: beta # static label
-        id: '{.id}'       # dynamic label
+        id: '{ .id }'       # dynamic label
       values:
         active: 1         # static value
-        count: '{.count}' # dynamic value
-        boolean: '{.some_boolean}'
+        count: '{ .count }' # dynamic value
+        boolean: '{ .some_boolean }'
+
+    - name: example_cel_value
+      type: object
+      engine: cel
+      help: Example of sub-level value scrapes from a json
+      path: ".values.filter(i, i.state == \"ACTIVE\")"
+      labels:
+        environment: "\"beta\"" # static label
+        id: '.id' # dynamic label
+      values:
+        active: 1 # static value
+        count: '.count' # dynamic value
+        boolean: '.some_boolean'
+
 
   animals:
     metrics:
@@ -43,6 +79,17 @@ modules:
       values:
         population: '{ .population }'
 
+    - name: animal_cel
+      type: object
+      engine: cel
+      help: Example of top-level lists in a separate module
+      path: '[*]'
+      labels:
+        name: '.noun'
+        predator: '.predator'
+      values:
+        population: '.population'
+
     ## HTTP connection configurations can be set in 'modules.<module_name>.http_client_config' field. For full http client config parameters, ref: https://pkg.go.dev/github.com/prometheus/common/config?tab=doc#HTTPClientConfig
     #
     # http_client_config:
@@ -59,11 +106,11 @@ modules:
     ## If 'modueles.<module_name>.body' field is set, it will be sent by the exporter as the body content in the scrape request. The HTTP method will also be set as 'POST' in this case.
     # body:
     #   content: |
-    #     {"time_diff": "1m25s", "anotherVar": "some value"}
+    #     { "time_diff": "1m25s", "anotherVar": "some value" }
 
     ## The body content can also be a Go Template (https://golang.org/pkg/text/template), with all the functions from the Sprig library (https://masterminds.github.io/sprig/) available. All the query parameters sent by prometheus in the scrape query to the exporter, are available in the template.
     # body:
     #   content: |
-    #     {"time_diff": "{{ duration `95` }}","anotherVar": "{{ .myVal | first }}"}
+    #     { "time_diff": "{{ duration `95` }}","anotherVar": "{{ .myVal | first }}" }
     #   templatize: true
 

exporter/collector.go

@@ -16,12 +16,19 @@ package exporter
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
+	"reflect"
 	"time"
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/common/types/ref"
 	"github.com/prometheus-community/json_exporter/config"
 	"github.com/prometheus/client_golang/prometheus"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
+	structpb "google.golang.org/protobuf/types/known/structpb"
 	"k8s.io/client-go/util/jsonpath"
 )
 
@@ -34,6 +41,7 @@ type JSONMetricCollector struct {
 type JSONMetric struct {
 	Desc                   *prometheus.Desc
 	Type                   config.ScrapeType
+	EngineType             config.EngineType
 	KeyJSONPath            string
 	ValueJSONPath          string
 	LabelsJSONPaths        []string
@@ -51,7 +59,8 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 	for _, m := range mc.JSONMetrics {
 		switch m.Type {
 		case config.ValueScrape:
-			value, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, false)
+			level.Debug(mc.Logger).Log("msg", "Extracting value for metric", "path", m.KeyJSONPath, "metric", m.Desc)
+			value, err := extractValue(mc.Logger, m.EngineType, mc.Data, m.KeyJSONPath, false)
 			if err != nil {
 				level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
 				continue
@@ -62,7 +71,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 					m.Desc,
 					m.ValueType,
 					floatValue,
-					extractLabels(mc.Logger, mc.Data, m.LabelsJSONPaths)...,
+					extractLabels(mc.Logger, m.EngineType, mc.Data, m.LabelsJSONPaths)...,
 				)
 				ch <- timestampMetric(mc.Logger, m, mc.Data, metric)
 			} else {
@@ -71,7 +80,8 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 			}
 
 		case config.ObjectScrape:
-			values, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true)
+			level.Debug(mc.Logger).Log("msg", "Extracting object for metric", "path", m.KeyJSONPath, "metric", m.Desc)
+			values, err := extractValue(mc.Logger, m.EngineType, mc.Data, m.KeyJSONPath, true)
 			if err != nil {
 				level.Error(mc.Logger).Log("msg", "Failed to extract json objects for metric", "err", err, "metric", m.Desc)
 				continue
@@ -85,7 +95,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 						level.Error(mc.Logger).Log("msg", "Failed to marshal data to json", "path", m.ValueJSONPath, "err", err, "metric", m.Desc, "data", data)
 						continue
 					}
-					value, err := extractValue(mc.Logger, jdata, m.ValueJSONPath, false)
+					value, err := extractValue(mc.Logger, m.EngineType, jdata, m.ValueJSONPath, false)
 					if err != nil {
 						level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.ValueJSONPath, "err", err, "metric", m.Desc)
 						continue
@@ -96,7 +106,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 							m.Desc,
 							m.ValueType,
 							floatValue,
-							extractLabels(mc.Logger, jdata, m.LabelsJSONPaths)...,
+							extractLabels(mc.Logger, m.EngineType, jdata, m.LabelsJSONPaths)...,
 						)
 						ch <- timestampMetric(mc.Logger, m, jdata, metric)
 					} else {
@@ -105,7 +115,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 					}
 				}
 			} else {
-				level.Error(mc.Logger).Log("msg", "Failed to convert extracted objects to json", "err", err, "metric", m.Desc)
+				level.Error(mc.Logger).Log("msg", "Failed to convert extracted objects to json", "value", values, "err", err, "metric", m.Desc)
 				continue
 			}
 		default:
@@ -115,8 +125,19 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 	}
 }
 
+func extractValue(logger log.Logger, engine config.EngineType, data []byte, path string, enableJSONOutput bool) (string, error) {
+	switch engine {
+	case config.EngineTypeJSONPath:
+		return extractValueJSONPath(logger, data, path, enableJSONOutput)
+	case config.EngineTypeCEL:
+		return extractValueCEL(logger, data, path, enableJSONOutput)
+	default:
+		return "", fmt.Errorf("Unknown engine type: %s", engine)
+	}
+}
+
 // Returns the last matching value at the given json path
-func extractValue(logger log.Logger, data []byte, path string, enableJSONOutput bool) (string, error) {
+func extractValueJSONPath(logger log.Logger, data []byte, path string, enableJSONOutput bool) (string, error) {
 	var jsonData interface{}
 	buf := new(bytes.Buffer)
 
@@ -148,11 +169,70 @@ func extractValue(logger log.Logger, data []byte, path string, enableJSONOutput
 	return buf.String(), nil
 }
 
+// Returns the last matching value at the given json path
+func extractValueCEL(logger log.Logger, data []byte, expression string, enableJSONOutput bool) (string, error) {
+
+	var jsonData map[string]any
+
+	err := json.Unmarshal(data, &jsonData)
+	if err != nil {
+		level.Error(logger).Log("msg", "Failed to unmarshal data to json", "err", err, "data", data)
+		return "", err
+	}
+
+	inputVars := make([]cel.EnvOption, 0, len(jsonData))
+	for k := range jsonData {
+		inputVars = append(inputVars, cel.Variable(k, cel.DynType))
+	}
+
+	env, err := cel.NewEnv(inputVars...)
+
+	if err != nil {
+		level.Error(logger).Log("msg", "Failed to set up CEL environment", "err", err, "data", data)
+		return "", err
+	}
+
+	ast, issues := env.Compile(expression)
+	if issues != nil && issues.Err() != nil {
+		level.Error(logger).Log("CEL type-check error", issues.Err(), "expression", expression)
+		return "", err
+	}
+	prg, err := env.Program(ast)
+	if err != nil {
+		level.Error(logger).Log("CEL program construction error", err)
+		return "", err
+	}
+
+	out, _, err := prg.Eval(jsonData)
+	if err != nil {
+		level.Error(logger).Log("msg", "Failed to evaluate cel query", "err", err, "expression", expression, "data", jsonData)
+		return "", err
+	}
+
+	// Since we are finally going to extract only float64, unquote if necessary
+
+	//res, err := jsonpath.UnquoteExtend(fmt.Sprintf("%g", out))
+	//if err == nil {
+	//	level.Error(logger).Log("msg","Triggered")
+	//	return res, nil
+	//}
+	level.Error(logger).Log("msg", "Triggered later", "val", out)
+	if enableJSONOutput {
+		res, err := valueToJSON(out)
+		if err != nil {
+			return "", err
+		}
+		return res, nil
+	}
+
+	return fmt.Sprintf("%v", out), nil
+}
+
 // Returns the list of labels created from the list of provided json paths
-func extractLabels(logger log.Logger, data []byte, paths []string) []string {
+func extractLabels(logger log.Logger, engine config.EngineType, data []byte, paths []string) []string {
 	labels := make([]string, len(paths))
 	for i, path := range paths {
-		if result, err := extractValue(logger, data, path, false); err == nil {
+		if result, err := extractValue(logger, engine, data, path, false); err == nil {
 			labels[i] = result
 		} else {
 			level.Error(logger).Log("msg", "Failed to extract label value", "err", err, "path", path, "data", data)
@@ -165,7 +245,7 @@ func timestampMetric(logger log.Logger, m JSONMetric, data []byte, pm prometheus
 	if m.EpochTimestampJSONPath == "" {
 		return pm
 	}
-	ts, err := extractValue(logger, data, m.EpochTimestampJSONPath, false)
+	ts, err := extractValue(logger, m.EngineType, data, m.EpochTimestampJSONPath, false)
 	if err != nil {
 		level.Error(logger).Log("msg", "Failed to extract timestamp for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
 		return pm
@@ -178,3 +258,18 @@ func timestampMetric(logger log.Logger, m JSONMetric, data []byte, pm prometheus
 	timestamp := time.UnixMilli(epochTime)
 	return prometheus.NewMetricWithTimestamp(timestamp, pm)
 }
+
+// valueToJSON converts the CEL type to a protobuf JSON representation and
+// marshals the result to a string.
+func valueToJSON(val ref.Val) (string, error) {
+	v, err := val.ConvertToNative(reflect.TypeOf(&structpb.Value{}))
+	if err != nil {
+		return "", err
+	}
+	marshaller := protojson.MarshalOptions{Indent: "    "}
+	bytes, err := marshaller.Marshal(v.(proto.Message))
+	if err != nil {
+		return "", err
+	}
+	return string(bytes), err
+}

exporter/util.go

@@ -103,6 +103,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 					variableLabels,
 					nil,
 				),
+				EngineType:             metric.Engine,
 				KeyJSONPath:            metric.Path,
 				LabelsJSONPaths:        variableLabelsValues,
 				ValueType:              valueType,
@@ -125,6 +126,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 						variableLabels,
 						nil,
 					),
+					EngineType:             metric.Engine,
 					KeyJSONPath:            metric.Path,
 					ValueJSONPath:          valuePath,
 					LabelsJSONPaths:        variableLabelsValues,
@@ -134,7 +136,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 				metrics = append(metrics, jsonMetric)
 			}
 		default:
-			return nil, fmt.Errorf("Unknown metric type: '%s', for metric: '%s'", metric.Type, metric.Name)
+			return nil, fmt.Errorf("unknown metric type: '%s', for metric: '%s'", metric.Type, metric.Name)
 		}
 	}
 	return metrics, nil