Skip to content

Commit

Permalink
Grouping by with added labels working, but not sorted
Browse files Browse the repository at this point in the history
  • Loading branch information
retrodaredevil committed Feb 24, 2024
1 parent 647ee7c commit 46ce01d
Show file tree
Hide file tree
Showing 6 changed files with 261 additions and 36 deletions.
10 changes: 8 additions & 2 deletions pkg/plugin/datasource.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,14 +185,20 @@ func (d *Datasource) query(ctx context.Context, pCtx backend.PluginContext, quer

// add the frames to the response.
for _, parsingOption := range qm.ParsingOptions {
frame, err := parsing.ParseData(
frames, err, errorType := parsing.ParseData(
graphQLResponse.Data,
parsingOption,
)
if err != nil {
if errorType == parsing.FRIENDLY_ERROR {
return &backend.DataResponse{
Error: err,
Status: status,
}, nil
}
return nil, err
}
response.Frames = append(response.Frames, frame)
response.Frames = append(response.Frames, frames...)
}

return &response, nil
Expand Down
106 changes: 106 additions & 0 deletions pkg/plugin/parsing/framemap/framemap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
package framemap

import (
"fmt"
"github.com/grafana/grafana-plugin-sdk-go/data"
"hash/fnv"
)

type FrameAndLabels struct {
labels data.Labels
fieldMap map[string]interface{}
}

func labelsEqual(labelsA data.Labels, labelsB data.Labels) bool {
if len(labelsA) != len(labelsB) {
return false
}
for key, value := range labelsA {
otherValue, exists := labelsB[key]
if !exists || value != otherValue {
return false
}
}
return true
}
func labelsHash(labels data.Labels) uint64 {
h := fnv.New64a()
for key, value := range labels {
_, err := h.Write([]byte(key))
if err != nil {
panic(fmt.Sprintf("Error writing to hash: %v", err))
}
_, err = h.Write([]byte(value))
if err != nil {
panic(fmt.Sprintf("Error writing to hash: %v", err))
}
}
return h.Sum64()
}

type FrameMap struct {
data map[uint64][]FrameAndLabels
}

func CreateFrameMap() FrameMap {
return FrameMap{
data: map[uint64][]FrameAndLabels{},
}
}

func (f *FrameMap) Get(labels data.Labels) (map[string]interface{}, bool) {
hash := labelsHash(labels)
values, exists := f.data[hash]
if !exists {
return nil, false
}

for _, value := range values {
if labelsEqual(value.labels, labels) {
return value.fieldMap, true
}
}
return nil, false
}
func (f *FrameMap) Put(labels data.Labels, fieldMap map[string]interface{}) {
hash := labelsHash(labels)
values, exists := f.data[hash]
if !exists {
f.data[hash] = []FrameAndLabels{{labels: labels, fieldMap: fieldMap}}
return
}
for index, value := range values {
if labelsEqual(value.labels, labels) {
values[index] = FrameAndLabels{labels: labels, fieldMap: fieldMap}
return
}
}

newValues := append(values, FrameAndLabels{labels: labels, fieldMap: fieldMap})
f.data[hash] = newValues
}

func (f *FrameMap) ToFrames() []*data.Frame {
// create data frame response.
// For an overview on data frames and how grafana handles them:
// https://grafana.com/developers/plugin-tools/introduction/data-frames
// The goal here is to output a long format. If needed, prepare time series can transform it
// https://grafana.com/docs/grafana/latest/panels-visualizations/query-transform-data/transform-data/#prepare-time-series

// NOTE: The order of the frames here determines the order they appear in the legend in Grafana
// A workaround on the frontend is to make the legend in "Table" mode and then sort the "Name" column: https://github.com/grafana/grafana/pull/69490
var r []*data.Frame
for _, values := range f.data {
for _, value := range values {
frameName := fmt.Sprintf("response %v", value.labels)
frame := data.NewFrame(frameName)
for key, values := range value.fieldMap {
frame.Fields = append(frame.Fields,
data.NewField(key, value.labels, values),
)
}
r = append(r, frame)
}
}
return r
}
68 changes: 68 additions & 0 deletions pkg/plugin/parsing/framemap/framemap_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package framemap

import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"reflect"
"testing"
"time"
)

func TestEqualityAndHash(t *testing.T) {
labelsA := data.Labels{
"asdf": "a",
}
labelsB := data.Labels{
"asdf": "b",
}
if labelsEqual(labelsA, labelsB) {
t.Error("labelsA should not equal labelsB")
}
if labelsHash(labelsA) == labelsHash(labelsB) {
t.Error("Should not have the same hashes!")
}
}

func TestFrameMap(t *testing.T) {
fm := CreateFrameMap()
labelsA := data.Labels{
"asdf": "a",
}
labelsB := data.Labels{
"asdf": "b",
}
dataA := map[string]interface{}{
"batteryVoltage": []float64{22.4, 22.5},
"dateMillis": []time.Time{time.UnixMilli(1705974650887), time.UnixMilli(1705974659884)},
}
dataB := map[string]interface{}{
"batteryVoltage": []float64{22.43, 22.51},
"dateMillis": []time.Time{time.UnixMilli(1705974650888), time.UnixMilli(1705974659885)},
}

_, exists := fm.Get(labelsA)
if exists {
t.Error("We haven't put anything in the map! Nothing should exist!")
}

fm.Put(labelsA, dataA)
expectingDataA, exists := fm.Get(labelsA)
if !exists {
t.Error("We expect that fm.get(labelsA) exists now!")
}
if !reflect.DeepEqual(dataA, expectingDataA) {
t.Error("We expect this to be dataA!")
}

fm.Put(labelsB, dataB)
expectingDataB, exists := fm.Get(labelsB)
if !exists {
t.Error("We expect that fm.get(labelsB) exists now!")
}
if !reflect.DeepEqual(dataB, expectingDataB) {
t.Error("We expect this to be dataB!")
}

if len(fm.ToFrames()) != 2 {
t.Error("ToFrames() should result in an array of size 2!")
}
}
93 changes: 62 additions & 31 deletions pkg/plugin/parsing/parsing.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/wildmountainfarms/wild-graphql-datasource/pkg/plugin/parsing/framemap"
"github.com/wildmountainfarms/wild-graphql-datasource/pkg/plugin/querymodel"
"reflect"
"strings"
Expand All @@ -12,29 +13,37 @@ import (

// the purpose of this file is to parse JSON data with configuration from a ParsingOption

func ParseData(graphQlResponseData map[string]interface{}, parsingOption querymodel.ParsingOption) (*data.Frame, error) {
type ParseDataErrorType int

const (
NO_ERROR ParseDataErrorType = 0
FRIENDLY_ERROR ParseDataErrorType = 1
UNKNOWN_ERROR ParseDataErrorType = 2
)

func ParseData(graphQlResponseData map[string]interface{}, parsingOption querymodel.ParsingOption) (data.Frames, error, ParseDataErrorType) {
if len(parsingOption.DataPath) == 0 {
return nil, errors.New("data path cannot be empty")
return nil, errors.New("data path cannot be empty"), FRIENDLY_ERROR
}
split := strings.Split(parsingOption.DataPath, ".")

var currentData map[string]interface{} = graphQlResponseData
for _, part := range split[:len(split)-1] {
newData, exists := currentData[part]
if !exists {
return nil, errors.New(fmt.Sprintf("Part of data path: %s does not exist! dataPath: %s", part, parsingOption.DataPath))
return nil, errors.New(fmt.Sprintf("Part of data path: %s does not exist! dataPath: %s", part, parsingOption.DataPath)), FRIENDLY_ERROR
}
switch value := newData.(type) {
case map[string]interface{}:
currentData = value
default:
return nil, errors.New(fmt.Sprintf("Part of data path: %s is not a nested object! dataPath: %s", part, parsingOption.DataPath))
return nil, errors.New(fmt.Sprintf("Part of data path: %s is not a nested object! dataPath: %s", part, parsingOption.DataPath)), FRIENDLY_ERROR
}
}
// after this for loop, currentData should be an array if everything is going well
finalData, finalDataExists := currentData[split[len(split)-1]]
if !finalDataExists {
return nil, errors.New(fmt.Sprintf("Final part of data path: %s does not exist! dataPath: %s", split[len(split)-1], parsingOption.DataPath))
return nil, errors.New(fmt.Sprintf("Final part of data path: %s does not exist! dataPath: %s", split[len(split)-1], parsingOption.DataPath)), FRIENDLY_ERROR
}

var dataArray []map[string]interface{}
Expand All @@ -46,22 +55,35 @@ func ParseData(graphQlResponseData map[string]interface{}, parsingOption querymo
case map[string]interface{}:
dataArray[i] = typedElement
default:
return nil, errors.New(fmt.Sprintf("One of the elements inside the data array is not an object! element: %d is of type: %v", i, reflect.TypeOf(element)))
return nil, errors.New(fmt.Sprintf("One of the elements inside the data array is not an object! element: %d is of type: %v", i, reflect.TypeOf(element))), FRIENDLY_ERROR
}
}
default:
return nil, errors.New(fmt.Sprintf("Final part of data path: is not an array! dataPath: %s type of result: %v", parsingOption.DataPath, reflect.TypeOf(value)))
return nil, errors.New(fmt.Sprintf("Final part of data path: is not an array! dataPath: %s type of result: %v", parsingOption.DataPath, reflect.TypeOf(value))), FRIENDLY_ERROR
}

// fieldMap is a map of keys to array of data points. Upon first initialization of a particular key's value,
// We store a fieldMap inside of this frameMap.
// fieldMap is a map of keys to array of data points. Upon first initialization of a particular key's value,
// an array should be chosen corresponding to the first value of that key.
// Upon subsequent element insertion, if the type of the array does not match that elements type, an error is thrown.
// This error is never expected to occur because a correct GraphQL response should never have a particular field be of different types
fieldMap := map[string]interface{}{}
fm := framemap.CreateFrameMap()

//labelsToFieldMapMap := map[Labels]map[string]interface{}{}

for _, dataElement := range dataArray {
flatData := map[string]interface{}{}
flattenData(dataElement, "", flatData)
labels, err := getLabelsFromFlatData(flatData, parsingOption)
if err != nil {
return nil, err, FRIENDLY_ERROR // getLabelsFromFlatData must always return a friendly error
}
var fieldMap, fieldMapExists = fm.Get(labels)
if !fieldMapExists {
fieldMap = map[string]interface{}{}
fm.Put(labels, fieldMap)
}

for key, value := range flatData {
existingFieldValues, fieldValuesExist := fieldMap[key]

Expand All @@ -74,24 +96,24 @@ func ParseData(graphQlResponseData map[string]interface{}, parsingOption querymo
// and also consider using time.RFC339Nano instead
parsedTime, err := time.Parse(time.RFC3339, valueValue)
if err != nil {
return nil, errors.New(fmt.Sprintf("Time could not be parsed! Time: %s", valueValue))
return nil, errors.New(fmt.Sprintf("Time could not be parsed! Time: %s", valueValue)), FRIENDLY_ERROR
}
timeValue = parsedTime
case float64:
timeValue = time.UnixMilli(int64(valueValue))
case bool:
return nil, errors.New("time field is a bool")
return nil, errors.New("time field is a bool"), FRIENDLY_ERROR
default:
// This case should never happen because we never expect other types to pop up here
return nil, errors.New(fmt.Sprintf("Unsupported time type! Time: %s type: %v", valueValue, reflect.TypeOf(valueValue)))
return nil, errors.New(fmt.Sprintf("Unsupported time type! Time: %s type: %v", valueValue, reflect.TypeOf(valueValue))), FRIENDLY_ERROR
}
var fieldValues []time.Time
if fieldValuesExist {
switch typedExistingFieldValues := existingFieldValues.(type) {
case []time.Time:
fieldValues = typedExistingFieldValues
default:
return nil, errors.New(fmt.Sprintf("This error should never occur. The existing array for time field values is of the type: %v", reflect.TypeOf(existingFieldValues)))
return nil, errors.New(fmt.Sprintf("This error should never occur. The existing array for time field values is of the type: %v", reflect.TypeOf(existingFieldValues))), UNKNOWN_ERROR
}
} else {
fieldValues = []time.Time{}
Expand All @@ -106,24 +128,24 @@ func ParseData(graphQlResponseData map[string]interface{}, parsingOption querymo
case float64:
fieldMap[key] = append(typedExistingFieldValues, typedValue)
default:
return nil, errors.New(fmt.Sprintf("Existing field values for key: %s is float64, but got value with type: %v", key, reflect.TypeOf(value)))
return nil, errors.New(fmt.Sprintf("Existing field values for key: %s is float64, but got value with type: %v", key, reflect.TypeOf(value))), FRIENDLY_ERROR
}
case []string:
switch typedValue := value.(type) {
case string:
fieldMap[key] = append(typedExistingFieldValues, typedValue)
default:
return nil, errors.New(fmt.Sprintf("Existing field values for key: %s is string, but got value with type: %v", key, reflect.TypeOf(value)))
return nil, errors.New(fmt.Sprintf("Existing field values for key: %s is string, but got value with type: %v", key, reflect.TypeOf(value))), FRIENDLY_ERROR
}
case []bool:
switch typedValue := value.(type) {
case bool:
fieldMap[key] = append(typedExistingFieldValues, typedValue)
default:
return nil, errors.New(fmt.Sprintf("Existing field values for key: %s is bool, but got value with type: %v", key, reflect.TypeOf(value)))
return nil, errors.New(fmt.Sprintf("Existing field values for key: %s is bool, but got value with type: %v", key, reflect.TypeOf(value))), FRIENDLY_ERROR
}
default:
return nil, errors.New(fmt.Sprintf("This error should never occur. The existing array for time field values is of the type: %v", reflect.TypeOf(existingFieldValues)))
return nil, errors.New(fmt.Sprintf("This error should never occur. The existing array for time field values is of the type: %v", reflect.TypeOf(existingFieldValues))), UNKNOWN_ERROR
}
} else {
switch typedValue := value.(type) {
Expand All @@ -134,28 +156,37 @@ func ParseData(graphQlResponseData map[string]interface{}, parsingOption querymo
case bool:
fieldMap[key] = []bool{typedValue}
default:
return nil, errors.New(fmt.Sprintf("Unsupported and unexpected type for key: %s. Type is: %v", key, reflect.TypeOf(value)))
return nil, errors.New(fmt.Sprintf("Unsupported and unexpected type for key: %s. Type is: %v", key, reflect.TypeOf(value))), UNKNOWN_ERROR
}
}
}
}
}

// create data frame response.
// For an overview on data frames and how grafana handles them:
// https://grafana.com/developers/plugin-tools/introduction/data-frames
// The goal here is to output a long format. If needed, prepare time series can transform it
// https://grafana.com/docs/grafana/latest/panels-visualizations/query-transform-data/transform-data/#prepare-time-series

frame := data.NewFrame("response")
return fm.ToFrames(), nil, NO_ERROR
}

for key, values := range fieldMap {
frame.Fields = append(frame.Fields,
data.NewField(key, nil, values),
)
// Given flatData and label options, computes the labels or returns a friendly error
func getLabelsFromFlatData(flatData map[string]interface{}, parsingOption querymodel.ParsingOption) (data.Labels, error) {
labels := map[string]string{}
for _, labelOption := range parsingOption.LabelOptions {
switch labelOption.Type {
case querymodel.CONSTANT:
labels[labelOption.Name] = labelOption.Value
case querymodel.FIELD:
fieldValue, fieldExists := flatData[labelOption.Value]
if !fieldExists {
return nil, errors.New(fmt.Sprintf("Label option: %s could not be satisfied as key %s does not exist", labelOption.Name, labelOption.Value))
}
switch typedFieldValue := fieldValue.(type) {
case string:
labels[labelOption.Name] = typedFieldValue
default:
return nil, errors.New(fmt.Sprintf("Label option: %s could not be satisfied as key %s is not a string. It's type is %v", labelOption.Name, labelOption.Value, reflect.TypeOf(typedFieldValue)))
}
}
}

return frame, nil
return labels, nil
}

func flattenArray[T interface{}](array []T, prefix string, flattenedData map[string]interface{}) {
Expand Down
Loading

0 comments on commit 46ce01d

Please sign in to comment.