Skip to content

Commit 5a26a4a

Browse files
mruegmrueg
committed
WIP: Initial support for CEL
This adds initial support for queries using the common expression language (CEL). Signed-off-by: Manuel Rüger <[email protected]>
1 parent 15d4535 commit 5a26a4a

File tree

6 files changed

+178
-15
lines changed

6 files changed

+178
-15
lines changed

‎config/config.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
// Metric contains values that define a metric
2424
type Metric struct {
2525
Name string
26+
Engine EngineType
2627
Path string
2728
Labels map[string]string
2829
Type ScrapeType
@@ -44,7 +45,14 @@ type ValueType string
4445
const (
4546
ValueTypeGauge ValueType = "gauge"
4647
ValueTypeCounter ValueType = "counter"
47-
ValueTypeUntyped ValueType = "untyped"
48+
ValueTypeUntyped ValueType = "untyped" // default
49+
)
50+
51+
type EngineType string
52+
53+
const (
54+
EngineTypeJSONPath EngineType = "jsonpath" // default
55+
EngineTypeCEL EngineType = "cel"
4856
)
4957

5058
// Config contains multiple modules.
@@ -89,6 +97,9 @@ func LoadConfig(configPath string) (Config, error) {
8997
if module.Metrics[i].ValueType == "" {
9098
module.Metrics[i].ValueType = ValueTypeUntyped
9199
}
100+
if module.Metrics[i].Engine == "" {
101+
module.Metrics[i].Engine = EngineTypeJSONPath
102+
}
92103
}
93104
}
94105

‎examples/config.yml

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,17 @@ modules:
99
help: Example of a top-level global value scrape in the json
1010
labels:
1111
environment: beta # static label
12-
location: 'planet-{.location}' # dynamic label
12+
location: 'planet-{ .location }' # dynamic label
13+
14+
- name: example_cel_global_value
15+
engine: cel
16+
path: '.counter'
17+
help: Example of a top-level global value scrape in the json using cel
18+
valuetype: 'gauge'
19+
labels:
20+
environment: "\"beta\"" # static label. Quotes need to be escaped for CEL
21+
location: "\"planet-\"+.location" # dynamic label. Quotes need to be escaped for CEL
22+
1323
- name: example_timestamped_value
1424
type: object
1525
path: '{ .values[?(@.state == "INACTIVE")] }'
@@ -18,7 +28,19 @@ modules:
1828
labels:
1929
environment: beta # static label
2030
values:
21-
count: '{.count}' # dynamic value
31+
count: '{ .count }' # dynamic value
32+
33+
- name: example_cel_timestamped_value
34+
type: object
35+
engine: cel
36+
path: ".values.filter(i, i.state == \"INACTIVE\")"
37+
epochTimestamp: '.timestamp'
38+
help: Example of a timestamped value scrape in the json
39+
labels:
40+
environment: "\"beta\"" # static label
41+
values:
42+
count: '.count' # dynamic value
43+
2244
- name: example_value
2345
type: object
2446
help: Example of sub-level value scrapes from a json
@@ -31,6 +53,20 @@ modules:
3153
count: '{.count}' # dynamic value
3254
boolean: '{.some_boolean}'
3355

56+
- name: example_cel_value
57+
type: object
58+
engine: cel
59+
help: Example of sub-level value scrapes from a json
60+
path: ".values.filter(i, i.state == \"ACTIVE\")"
61+
labels:
62+
environment: "\"beta\"" # static label
63+
id: '.id' # dynamic label
64+
values:
65+
active: 1 # static value
66+
count: '.count' # dynamic value
67+
boolean: '.some_boolean'
68+
69+
3470
animals:
3571
metrics:
3672
- name: animal

‎exporter/collector.go

Lines changed: 106 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,19 @@ package exporter
1616
import (
1717
"bytes"
1818
"encoding/json"
19+
"fmt"
20+
"reflect"
1921
"time"
2022

2123
"github.com/go-kit/log"
2224
"github.com/go-kit/log/level"
25+
"github.com/google/cel-go/cel"
26+
"github.com/google/cel-go/common/types/ref"
2327
"github.com/prometheus-community/json_exporter/config"
2428
"github.com/prometheus/client_golang/prometheus"
29+
"google.golang.org/protobuf/encoding/protojson"
30+
"google.golang.org/protobuf/proto"
31+
structpb "google.golang.org/protobuf/types/known/structpb"
2532
"k8s.io/client-go/util/jsonpath"
2633
)
2734

@@ -34,6 +41,7 @@ type JSONMetricCollector struct {
3441
type JSONMetric struct {
3542
Desc *prometheus.Desc
3643
Type config.ScrapeType
44+
EngineType config.EngineType
3745
KeyJSONPath string
3846
ValueJSONPath string
3947
LabelsJSONPaths []string
@@ -51,7 +59,8 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
5159
for _, m := range mc.JSONMetrics {
5260
switch m.Type {
5361
case config.ValueScrape:
54-
value, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, false)
62+
level.Debug(mc.Logger).Log("msg", "Extracting value for metric", "path", m.KeyJSONPath, "metric", m.Desc)
63+
value, err := extractValue(mc.Logger, m.EngineType, mc.Data, m.KeyJSONPath, false)
5564
if err != nil {
5665
level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
5766
continue
@@ -62,7 +71,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
6271
m.Desc,
6372
m.ValueType,
6473
floatValue,
65-
extractLabels(mc.Logger, mc.Data, m.LabelsJSONPaths)...,
74+
extractLabels(mc.Logger, m.EngineType, mc.Data, m.LabelsJSONPaths)...,
6675
)
6776
ch <- timestampMetric(mc.Logger, m, mc.Data, metric)
6877
} else {
@@ -71,7 +80,8 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
7180
}
7281

7382
case config.ObjectScrape:
74-
values, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true)
83+
level.Debug(mc.Logger).Log("msg", "Extracting object for metric", "path", m.KeyJSONPath, "metric", m.Desc)
84+
values, err := extractValue(mc.Logger, m.EngineType, mc.Data, m.KeyJSONPath, true)
7585
if err != nil {
7686
level.Error(mc.Logger).Log("msg", "Failed to extract json objects for metric", "err", err, "metric", m.Desc)
7787
continue
@@ -85,7 +95,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
8595
level.Error(mc.Logger).Log("msg", "Failed to marshal data to json", "path", m.ValueJSONPath, "err", err, "metric", m.Desc, "data", data)
8696
continue
8797
}
88-
value, err := extractValue(mc.Logger, jdata, m.ValueJSONPath, false)
98+
value, err := extractValue(mc.Logger, m.EngineType, jdata, m.ValueJSONPath, false)
8999
if err != nil {
90100
level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.ValueJSONPath, "err", err, "metric", m.Desc)
91101
continue
@@ -96,7 +106,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
96106
m.Desc,
97107
m.ValueType,
98108
floatValue,
99-
extractLabels(mc.Logger, jdata, m.LabelsJSONPaths)...,
109+
extractLabels(mc.Logger, m.EngineType, jdata, m.LabelsJSONPaths)...,
100110
)
101111
ch <- timestampMetric(mc.Logger, m, jdata, metric)
102112
} else {
@@ -105,7 +115,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
105115
}
106116
}
107117
} else {
108-
level.Error(mc.Logger).Log("msg", "Failed to convert extracted objects to json", "err", err, "metric", m.Desc)
118+
level.Error(mc.Logger).Log("msg", "Failed to convert extracted objects to json", "value", values, "err", err, "metric", m.Desc)
109119
continue
110120
}
111121
default:
@@ -115,8 +125,19 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
115125
}
116126
}
117127

128+
func extractValue(logger log.Logger, engine config.EngineType, data []byte, path string, enableJSONOutput bool) (string, error) {
129+
switch engine {
130+
case config.EngineTypeJSONPath:
131+
return extractValueJSONPath(logger, data, path, enableJSONOutput)
132+
case config.EngineTypeCEL:
133+
return extractValueCEL(logger, data, path, enableJSONOutput)
134+
default:
135+
return "", fmt.Errorf("Unknown engine type: %s", engine)
136+
}
137+
}
138+
118139
// Returns the last matching value at the given json path
119-
func extractValue(logger log.Logger, data []byte, path string, enableJSONOutput bool) (string, error) {
140+
func extractValueJSONPath(logger log.Logger, data []byte, path string, enableJSONOutput bool) (string, error) {
120141
var jsonData interface{}
121142
buf := new(bytes.Buffer)
122143

@@ -148,11 +169,71 @@ func extractValue(logger log.Logger, data []byte, path string, enableJSONOutput
148169
return buf.String(), nil
149170
}
150171

172+
// Returns the last matching value at the given json path
173+
func extractValueCEL(logger log.Logger, data []byte, expression string, enableJSONOutput bool) (string, error) {
174+
175+
var jsonData map[string]any
176+
177+
err := json.Unmarshal(data, &jsonData)
178+
if err != nil {
179+
level.Error(logger).Log("msg", "Failed to unmarshal data to json", "err", err, "data", data)
180+
return "", err
181+
}
182+
183+
inputVars := make([]cel.EnvOption, 0, len(jsonData))
184+
for k := range jsonData {
185+
inputVars = append(inputVars, cel.Variable(k, cel.DynType))
186+
}
187+
188+
env, err := cel.NewEnv(inputVars...)
189+
190+
if err != nil {
191+
level.Error(logger).Log("msg", "Failed to set up CEL environment", "err", err, "data", data)
192+
return "", err
193+
}
194+
195+
ast, issues := env.Compile(expression)
196+
if issues != nil && issues.Err() != nil {
197+
level.Error(logger).Log("CEL type-check error", issues.Err(), "expression", expression)
198+
return "", err
199+
}
200+
prg, err := env.Program(ast)
201+
if err != nil {
202+
level.Error(logger).Log("CEL program construction error", err)
203+
return "", err
204+
}
205+
206+
out, _, err := prg.Eval(jsonData)
207+
if err != nil {
208+
level.Error(logger).Log("msg", "Failed to evaluate cel query", "err", err, "expression", expression, "data", jsonData)
209+
return "", err
210+
}
211+
212+
// Since we are finally going to extract only float64, unquote if necessary
213+
214+
//res, err := jsonpath.UnquoteExtend(fmt.Sprintf("%g", out))
215+
//if err == nil {
216+
// level.Error(logger).Log("msg","Triggered")
217+
// return res, nil
218+
//}
219+
level.Error(logger).Log("msg", "Triggered later", "val", out)
220+
if enableJSONOutput {
221+
res, err := valueToJSON(out)
222+
if err != nil {
223+
return "", err
224+
} else {
225+
return res, nil
226+
}
227+
}
228+
229+
return fmt.Sprintf("%v", out), nil
230+
}
231+
151232
// Returns the list of labels created from the list of provided json paths
152-
func extractLabels(logger log.Logger, data []byte, paths []string) []string {
233+
func extractLabels(logger log.Logger, engine config.EngineType, data []byte, paths []string) []string {
153234
labels := make([]string, len(paths))
154235
for i, path := range paths {
155-
if result, err := extractValue(logger, data, path, false); err == nil {
236+
if result, err := extractValue(logger, engine, data, path, false); err == nil {
156237
labels[i] = result
157238
} else {
158239
level.Error(logger).Log("msg", "Failed to extract label value", "err", err, "path", path, "data", data)
@@ -165,7 +246,7 @@ func timestampMetric(logger log.Logger, m JSONMetric, data []byte, pm prometheus
165246
if m.EpochTimestampJSONPath == "" {
166247
return pm
167248
}
168-
ts, err := extractValue(logger, data, m.EpochTimestampJSONPath, false)
249+
ts, err := extractValue(logger, m.EngineType, data, m.EpochTimestampJSONPath, false)
169250
if err != nil {
170251
level.Error(logger).Log("msg", "Failed to extract timestamp for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
171252
return pm
@@ -178,3 +259,18 @@ func timestampMetric(logger log.Logger, m JSONMetric, data []byte, pm prometheus
178259
timestamp := time.UnixMilli(epochTime)
179260
return prometheus.NewMetricWithTimestamp(timestamp, pm)
180261
}
262+
263+
// valueToJSON converts the CEL type to a protobuf JSON representation and
264+
// marshals the result to a string.
265+
func valueToJSON(val ref.Val) (string, error) {
266+
v, err := val.ConvertToNative(reflect.TypeOf(&structpb.Value{}))
267+
if err != nil {
268+
return "", err
269+
}
270+
marshaller := protojson.MarshalOptions{Indent: " "}
271+
bytes, err := marshaller.Marshal(v.(proto.Message))
272+
if err != nil {
273+
return "", err
274+
}
275+
return string(bytes), err
276+
}

‎exporter/util.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
103103
variableLabels,
104104
nil,
105105
),
106+
EngineType: metric.Engine,
106107
KeyJSONPath: metric.Path,
107108
LabelsJSONPaths: variableLabelsValues,
108109
ValueType: valueType,
@@ -125,6 +126,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
125126
variableLabels,
126127
nil,
127128
),
129+
EngineType: metric.Engine,
128130
KeyJSONPath: metric.Path,
129131
ValueJSONPath: valuePath,
130132
LabelsJSONPaths: variableLabelsValues,
@@ -134,7 +136,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
134136
metrics = append(metrics, jsonMetric)
135137
}
136138
default:
137-
return nil, fmt.Errorf("Unknown metric type: '%s', for metric: '%s'", metric.Type, metric.Name)
139+
return nil, fmt.Errorf("unknown metric type: '%s', for metric: '%s'", metric.Type, metric.Name)
138140
}
139141
}
140142
return metrics, nil

‎go.mod

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@ require (
66
github.com/Masterminds/sprig/v3 v3.2.3
77
github.com/alecthomas/kingpin/v2 v2.3.2
88
github.com/go-kit/log v0.2.1
9+
github.com/google/cel-go v0.18.2
910
github.com/prometheus/client_golang v1.17.0
1011
github.com/prometheus/common v0.45.0
1112
github.com/prometheus/exporter-toolkit v0.10.0
13+
google.golang.org/protobuf v1.31.0
1214
gopkg.in/yaml.v2 v2.4.0
1315
k8s.io/client-go v0.28.3
1416
)
@@ -17,6 +19,7 @@ require (
1719
github.com/Masterminds/goutils v1.1.1 // indirect
1820
github.com/Masterminds/semver/v3 v3.2.0 // indirect
1921
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
22+
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
2023
github.com/beorn7/perks v1.0.1 // indirect
2124
github.com/cespare/xxhash/v2 v2.2.0 // indirect
2225
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
@@ -35,13 +38,16 @@ require (
3538
github.com/prometheus/procfs v0.11.1 // indirect
3639
github.com/shopspring/decimal v1.2.0 // indirect
3740
github.com/spf13/cast v1.3.1 // indirect
41+
github.com/stoewer/go-strcase v1.2.0 // indirect
3842
github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
3943
golang.org/x/crypto v0.14.0 // indirect
44+
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
4045
golang.org/x/net v0.17.0 // indirect
4146
golang.org/x/oauth2 v0.12.0 // indirect
4247
golang.org/x/sync v0.3.0 // indirect
4348
golang.org/x/sys v0.13.0 // indirect
4449
golang.org/x/text v0.13.0 // indirect
4550
google.golang.org/appengine v1.6.7 // indirect
46-
google.golang.org/protobuf v1.31.0 // indirect
51+
google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect
52+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect
4753
)

0 commit comments

Comments
 (0)