feature: count json objects with same value

if we want to count json objects with same values we can use
countbylabel type in the metric configuration

Signed-off-by: Parsa <[email protected]>
Signed-off-by: Parsa <[email protected]>

Author: Parsa

README.md

@@ -21,6 +21,10 @@ Serving HTTP on :: port 8000 (http://[::]:8000/) ...
 ## TEST with 'default' module
 
 $ curl "http://localhost:7979/probe?module=default&target=http://localhost:8000/examples/data.json"
+# HELP example_count Example of count object from a json
+# TYPE example_count untyped
+example_count{environment="beta",state="ACTIVE"} 2
+example_count{environment="beta",state="INACTIVE"} 1
 # HELP example_global_value Example of a top-level global value scrape in the json
 # TYPE example_global_value untyped
 example_global_value{environment="beta",location="planet-mars"} 1234

config/config.go

@@ -30,13 +30,15 @@ type Metric struct {
 	EpochTimestamp string
 	Help           string
 	Values         map[string]string
+	Mincount       int
 }
 
 type ScrapeType string
 
 const (
 	ValueScrape  ScrapeType = "value" // default
 	ObjectScrape ScrapeType = "object"
+	CountScrape  ScrapeType = "countbylabel"
 )
 
 type ValueType string
@@ -89,8 +91,10 @@ func LoadConfig(configPath string) (Config, error) {
 			if module.Metrics[i].ValueType == "" {
 				module.Metrics[i].ValueType = ValueTypeUntyped
 			}
+			if !(module.Metrics[i].Mincount > 0) {
+				module.Metrics[i].Mincount = 1
+			}
 		}
 	}
-
 	return config, nil
 }

examples/config.yml

@@ -30,6 +30,14 @@ modules:
         active: 1         # static value
         count: '{.count}' # dynamic value
         boolean: '{.some_boolean}'
+    - name: example_count
+      type: countbylabel
+      help: Example of count json labels
+      path: '{.values[*].state}'
+      labels:
+        environment: beta # static label
+        state: '{}'       # dynamic label
+      mincount: 1
 
   animals:
     metrics:
@@ -66,4 +74,3 @@ modules:
     #   content: |
     #     {"time_diff": "{{ duration `95` }}","anotherVar": "{{ .myVal | first }}"}
     #   templatize: true
-

exporter/collector.go

@@ -39,6 +39,7 @@ type JSONMetric struct {
 	LabelsJSONPaths        []string
 	ValueType              prometheus.ValueType
 	EpochTimestampJSONPath string
+	Mincount               int
 }
 
 func (mc JSONMetricCollector) Describe(ch chan<- *prometheus.Desc) {
@@ -70,6 +71,42 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 				continue
 			}
 
+		case config.CountScrape:
+			values, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true)
+			if err != nil {
+				level.Error(mc.Logger).Log("msg", "Failed to extract json objects for metric", "err", err, "metric", m.Desc)
+				continue
+			}
+
+			var jsonData []interface{}
+			counts := make(map[interface{}]int)
+
+			if err := json.Unmarshal([]byte(values), &jsonData); err == nil {
+				for _, data := range jsonData {
+					counts[data]++
+				}
+				for data, count := range counts {
+					if count >= m.Mincount {
+						jdata, err := json.Marshal(data)
+						if err != nil {
+							level.Error(mc.Logger).Log("msg", "Failed to marshal data to json", "path", m.ValueJSONPath, "err", err, "metric", m.Desc, "data", data)
+							continue
+						}
+						if err != nil {
+							level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.ValueJSONPath, "err", err, "metric", m.Desc)
+							continue
+						}
+
+						ch <- prometheus.MustNewConstMetric(
+							m.Desc,
+							prometheus.UntypedValue,
+							float64(count),
+							extractLabels(mc.Logger, jdata, m.LabelsJSONPaths)...,
+						)
+					}
+				}
+			}
+
 		case config.ObjectScrape:
 			values, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true)
 			if err != nil {

exporter/util.go

@@ -133,6 +133,28 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 				}
 				metrics = append(metrics, jsonMetric)
 			}
+		case config.CountScrape:
+			var variableLabels, variableLabelsValues []string
+			for k, v := range metric.Labels {
+				variableLabels = append(variableLabels, k)
+				variableLabelsValues = append(variableLabelsValues, v)
+			}
+			jsonMetric := JSONMetric{
+				Type: config.CountScrape,
+				Desc: prometheus.NewDesc(
+					metric.Name,
+					metric.Help,
+					variableLabels,
+					nil,
+				),
+				KeyJSONPath:            metric.Path,
+				Mincount:               metric.Mincount,
+				LabelsJSONPaths:        variableLabelsValues,
+				ValueType:              valueType,
+				EpochTimestampJSONPath: metric.EpochTimestamp,
+			}
+			fmt.Println(jsonMetric)
+			metrics = append(metrics, jsonMetric)
 		default:
 			return nil, fmt.Errorf("Unknown metric type: '%s', for metric: '%s'", metric.Type, metric.Name)
 		}

test/config/good.yml

@@ -22,3 +22,11 @@ modules:
         active: 1      # static value
         count: '{.count}' # dynamic value
         boolean: '{.some_boolean}'
+
+    - name: example_count
+      type: countbylabel
+      help: Example of count object from a json
+      path: '{.values[*].state}'
+      labels:
+        environment: beta # static label
+        state: '{}'          # dynamic label

test/response/good.txt

@@ -1,3 +1,7 @@
+# HELP example_count Example of count object from a json
+# TYPE example_count untyped
+example_count{environment="beta",state="ACTIVE"} 2
+example_count{environment="beta",state="INACTIVE"} 1
 # HELP example_global_value Example of a top-level global value scrape in the json
 # TYPE example_global_value gauge
 example_global_value{environment="beta",location="planet-mars"} 1234