swapi dashboard, flattenData ignores nulls

DEVELOPMENT.md

@@ -123,14 +123,18 @@ This section contains notes about dependencies.
 
 ## To-Do
 
-* Make data returned in the same order as the JSON specified it was
-  * GoDS seems like a good solution, but there is a problem with its linked hash map deserializer: https://github.com/emirpasic/gods/issues/171
+* Add a provisioned dashboard and datasource that test a publicly available API
+* Updated `src/README.md`
+* Decide on a simpler default query (https://github.com/wildmountainfarms/wild-graphql-datasource/issues/1)
+* Make annotation queries more intuitive
 * Add ability to move parsing options up and down
 * Add support for secure variable data defined in the data source configuration
   * The variables defined here cannot be overridden for any request - this is for security
   * Also add support for secure HTTP headers
-* See what minimum Grafana version we can support
-* Add support for variables: https://grafana.com/developers/plugin-tools/create-a-plugin/extend-a-plugin/add-support-for-variables
+
+Lower priority To-Dos
+
+* Add support for variables: https://grafana.com/developers/plugin-tools/create-a-plugin/extend-a-plugin/add-support-for-variables#add-support-for-query-variables-to-your-data-source
 * Add metrics to backend component: https://grafana.com/developers/plugin-tools/create-a-plugin/extend-a-plugin/add-logs-metrics-traces-for-backend-plugins#implement-metrics-in-your-plugin
 * Support returning logs data: https://grafana.com/developers/plugin-tools/tutorials/build-a-logs-data-source-plugin
   * We could just add `"logs": true` to `plugin.json`, however we need to support the renaming of fields because sometimes the `body` or `timestamp` fields will be nested
@@ -139,9 +143,11 @@ This section contains notes about dependencies.
   * https://grafana.com/developers/plugin-tools/publish-a-plugin/sign-a-plugin#generate-an-access-policy-token
   * https://grafana.com/legal/plugins/
   * https://grafana.com/developers/plugin-tools/publish-a-plugin/provide-test-environment
-* Publish on a private plugin repository
-  * https://volkovlabs.io/blog/installing-grafana-plugins-from-a-private-repository-805b54a1add3/
 * Create a GraphQL button panel (or a SolarThing app) that has a button panel that can be used to
   * If we create an app, we can follow https://github.com/RedisGrafana/grafana-redis-app
     * https://github.com/RedisGrafana/grafana-redis-app/blob/e093d18a021bb28ba7df3a54d7ad17c2d8e38f88/src/redis-gears-panel/components/RedisGearsPanel/RedisGearsPanel.tsx#L314
 * Auto-populate the data path field by using `documentAST` to recognize the first path to an array
+* Look into Apollo GraphQL
+  * https://studio.apollographql.com/sandbox/explorer
+  * https://www.apollographql.com/docs/graphos/explorer/
+  * https://www.npmjs.com/package/@apollo/explorer

docker-compose.yaml

@@ -15,3 +15,4 @@ services:
     volumes:
       - ./dist:/var/lib/grafana/plugins/retrodaredevil-wildgraphql-datasource
       - ./provisioning:/etc/grafana/provisioning
+      - ./provisioned-dashboards:/provisioned-dashboards

pkg/plugin/datasource.go

@@ -123,14 +123,14 @@ func (d *Datasource) query(ctx context.Context, pCtx backend.PluginContext, quer
 	// use later: pCtx.AppInstanceSettings.DecryptedSecureJSONData
 	variables, _ := queryvariables.ParseVariables(query, qm.Variables)
 
-	graphQLRequest := graphql.GraphQLRequest{
+	graphQLRequest := graphql.Request{
 		Query:         qm.QueryText,
 		OperationName: qm.OperationName,
 		Variables:     variables,
 	}
 	request, err := graphQLRequest.ToRequest(ctx, d.settings.URL)
 	if err != nil {
-		// We don't expect the conversion of the GraphQLRequest into a http.Request to fail
+		// We don't expect the conversion of the graphql.Request into a http.Request to fail
 		return nil, err
 	}
 
@@ -202,7 +202,7 @@ func (d *Datasource) query(ctx context.Context, pCtx backend.PluginContext, quer
 func (d *Datasource) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
 	// test command to do the same thing:
 	//   curl -X POST -H "Content-Type: application/json" -d '{"query":"{\n\t\t  __schema{\n\t\t\tqueryType{name}\n\t\t  }\n\t\t}"}' https://swapi-graphql.netlify.app/.netlify/functions/index
-	graphQLRequest := graphql.GraphQLRequest{
+	graphQLRequest := graphql.Request{
 		Query: `{
 		  __schema{
 		    queryType{name}

pkg/plugin/parsing/framemap/framemap.go

@@ -6,7 +6,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 )
 
-type FrameAndLabels struct {
+type frameAndLabels struct {
 	labels data.Labels
 	// A map of field names to an array of the values of that given column
 	fieldMap *linkedhashmap.Map[string, any]
@@ -17,12 +17,12 @@ func keyOfLabels(labels data.Labels) string {
 }
 
 type FrameMap struct {
-	data *linkedhashmap.Map[string, FrameAndLabels]
+	data *linkedhashmap.Map[string, frameAndLabels]
 }
 
 func New() *FrameMap {
 	return &FrameMap{
-		data: linkedhashmap.New[string, FrameAndLabels](),
+		data: linkedhashmap.New[string, frameAndLabels](),
 	}
 }
 
@@ -38,7 +38,7 @@ func (f *FrameMap) Put(labels data.Labels, fieldMap *linkedhashmap.Map[string, a
 	mapKey := keyOfLabels(labels)
 	f.data.Put(
 		mapKey,
-		FrameAndLabels{
+		frameAndLabels{
 			labels:   labels,
 			fieldMap: fieldMap,
 		},

pkg/plugin/parsing/framemap/framemap_test.go

@@ -1,27 +1,13 @@
 package framemap
 
 import (
+	"github.com/emirpasic/gods/v2/maps/linkedhashmap"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"reflect"
 	"testing"
 	"time"
 )
 
-func TestEqualityAndHash(t *testing.T) {
-	labelsA := data.Labels{
-		"asdf": "a",
-	}
-	labelsB := data.Labels{
-		"asdf": "b",
-	}
-	if labelsEqual(labelsA, labelsB) {
-		t.Error("labelsA should not equal labelsB")
-	}
-	if labelsHash(labelsA) == labelsHash(labelsB) {
-		t.Error("Should not have the same hashes!")
-	}
-}
-
 func TestFrameMap(t *testing.T) {
 	fm := New()
 	labelsA := data.Labels{
@@ -30,14 +16,13 @@ func TestFrameMap(t *testing.T) {
 	labelsB := data.Labels{
 		"asdf": "b",
 	}
-	dataA := map[string]interface{}{
-		"batteryVoltage": []float64{22.4, 22.5},
-		"dateMillis":     []time.Time{time.UnixMilli(1705974650887), time.UnixMilli(1705974659884)},
-	}
-	dataB := map[string]interface{}{
-		"batteryVoltage": []float64{22.43, 22.51},
-		"dateMillis":     []time.Time{time.UnixMilli(1705974650888), time.UnixMilli(1705974659885)},
-	}
+	dataA := linkedhashmap.New[string, any]()
+	dataA.Put("batteryVoltage", []float64{22.4, 22.5})
+	dataA.Put("dateMillis", []time.Time{time.UnixMilli(1705974650887), time.UnixMilli(1705974659884)})
+
+	dataB := linkedhashmap.New[string, any]()
+	dataB.Put("batteryVoltage", []float64{22.43, 22.51})
+	dataB.Put("dateMillis", []time.Time{time.UnixMilli(1705974650888), time.UnixMilli(1705974659885)})
 
 	_, exists := fm.Get(labelsA)
 	if exists {

pkg/plugin/parsing/parsing.go

@@ -81,7 +81,12 @@ func ParseData(graphQlResponseData *jsonnode.Object, parsingOption querymodel.Pa
 
 	for _, dataElement := range dataArray {
 		flatData := jsonnode.NewObject()
-		flattenData(dataElement, "", flatData)
+		if !flattenData(dataElement, "", flatData) {
+			// If we cannot flatten the data, we must skip this entry
+			// TODO consider logging this
+			// Data may not be able to be flattened if a null value is present in the data
+			continue
+		}
 		labels, err := getLabelsFromFlatData(flatData, parsingOption)
 		if err != nil {
 			return nil, err, FRIENDLY_ERROR // getLabelsFromFlatData must always return a friendly error
@@ -99,7 +104,7 @@ func ParseData(graphQlResponseData *jsonnode.Object, parsingOption querymodel.Pa
 			if key == parsingOption.TimePath {
 				var timeValue time.Time
 				switch typedValue := value.(type) {
-				case *jsonnode.String:
+				case jsonnode.String:
 					// TODO allow user to customize time format
 					// Look at https://stackoverflow.com/questions/522251/whats-the-difference-between-iso-8601-and-rfc-3339-date-formats
 					//   and also consider using time.RFC339Nano instead
@@ -108,7 +113,7 @@ func ParseData(graphQlResponseData *jsonnode.Object, parsingOption querymodel.Pa
 						return nil, errors.New(fmt.Sprintf("Time could not be parsed! Time: %s", typedValue)), FRIENDLY_ERROR
 					}
 					timeValue = parsedTime
-				case *jsonnode.Number:
+				case jsonnode.Number:
 					epochMillis, err := typedValue.Int64()
 					if err != nil {
 						return nil, err, UNKNOWN_ERROR
@@ -136,7 +141,7 @@ func ParseData(graphQlResponseData *jsonnode.Object, parsingOption querymodel.Pa
 					switch typedExistingFieldValues := existingFieldValues.(type) {
 					case []float64:
 						switch typedValue := value.(type) {
-						case *jsonnode.Number:
+						case jsonnode.Number:
 							number, err := typedValue.Float64()
 							if err != nil {
 								return nil, err, UNKNOWN_ERROR
@@ -147,14 +152,14 @@ func ParseData(graphQlResponseData *jsonnode.Object, parsingOption querymodel.Pa
 						}
 					case []string:
 						switch typedValue := value.(type) {
-						case *jsonnode.String:
+						case jsonnode.String:
 							fieldMap.Put(key, append(typedExistingFieldValues, typedValue.String()))
 						default:
 							return nil, errors.New(fmt.Sprintf("Existing field values for key: %s is string, but got value with type: %v", key, reflect.TypeOf(value))), FRIENDLY_ERROR
 						}
 					case []bool:
 						switch typedValue := value.(type) {
-						case *jsonnode.Boolean:
+						case jsonnode.Boolean:
 							fieldMap.Put(key, append(typedExistingFieldValues, typedValue.Bool()))
 						default:
 							return nil, errors.New(fmt.Sprintf("Existing field values for key: %s is bool, but got value with type: %v", key, reflect.TypeOf(value))), FRIENDLY_ERROR
@@ -164,18 +169,16 @@ func ParseData(graphQlResponseData *jsonnode.Object, parsingOption querymodel.Pa
 					}
 				} else {
 					switch typedValue := value.(type) {
-					case *jsonnode.Number:
+					case jsonnode.Number:
 						number, err := typedValue.Float64()
 						if err != nil {
 							return nil, err, UNKNOWN_ERROR
 						}
 						fieldMap.Put(key, []float64{number})
-					case *jsonnode.String:
+					case jsonnode.String:
 						fieldMap.Put(key, []string{typedValue.String()})
-					case *jsonnode.Boolean:
+					case jsonnode.Boolean:
 						fieldMap.Put(key, []bool{typedValue.Bool()})
-					case jsonnode.Null:
-						// do nothing for null
 					default:
 						return nil, errors.New(fmt.Sprintf("Unsupported and unexpected type for key: %s. Type is: %v", key, reflect.TypeOf(value))), UNKNOWN_ERROR
 					}
@@ -200,7 +203,7 @@ func getLabelsFromFlatData(flatData *jsonnode.Object, parsingOption querymodel.P
 				return nil, errors.New(fmt.Sprintf("Label option: %s could not be satisfied as key %s does not exist", labelOption.Name, labelOption.Value))
 			}
 			switch typedFieldValue := fieldValue.(type) {
-			case *jsonnode.String:
+			case jsonnode.String:
 				labels[labelOption.Name] = typedFieldValue.String()
 			default:
 				return nil, errors.New(fmt.Sprintf("Label option: %s could not be satisfied as key %s is not a string. It's type is %v", labelOption.Name, labelOption.Value, reflect.TypeOf(typedFieldValue)))
@@ -210,25 +213,47 @@ func getLabelsFromFlatData(flatData *jsonnode.Object, parsingOption querymodel.P
 	return labels, nil
 }
 
-func flattenArray(array *jsonnode.Array, prefix string, flattenedData *jsonnode.Object) {
+func flattenArray(array *jsonnode.Array, prefix string, flattenedData *jsonnode.Object) bool {
 	for key, value := range *array {
-		flattenedData.Put(
-			fmt.Sprintf("%s%d", prefix, key),
-			value,
-		)
+		baseKey := fmt.Sprintf("%s%d", prefix, key)
+		switch typedValue := value.(type) {
+		case *jsonnode.Object:
+			if !flattenData(typedValue, baseKey+".", flattenedData) {
+				return false
+			}
+		case *jsonnode.Array:
+			if !flattenArray(typedValue, baseKey+".", flattenedData) {
+				return false
+			}
+		case jsonnode.Null:
+			return false
+		default:
+			flattenedData.Put(
+				baseKey,
+				value,
+			)
+		}
 	}
+	return true
 }
 
-func flattenData(originalData *jsonnode.Object, prefix string, flattenedData *jsonnode.Object) {
+func flattenData(originalData *jsonnode.Object, prefix string, flattenedData *jsonnode.Object) bool {
 	for _, key := range originalData.Keys() {
 		value := originalData.Get(key)
 		switch typedValue := value.(type) {
 		case *jsonnode.Object:
-			flattenData(typedValue, prefix+key+".", flattenedData)
+			if !flattenData(typedValue, prefix+key+".", flattenedData) {
+				return false
+			}
 		case *jsonnode.Array:
-			flattenArray(typedValue, prefix+key+".", flattenedData)
+			if !flattenArray(typedValue, prefix+key+".", flattenedData) {
+				return false
+			}
+		case jsonnode.Null:
+			return false
 		default:
 			flattenedData.Put(prefix+key, typedValue)
 		}
 	}
+	return true
 }

pkg/plugin/parsing/parsing_test.go

@@ -3,7 +3,6 @@ package parsing
 import (
 	"bytes"
 	"encoding/json"
-	"fmt"
 	"reflect"
 	"testing"
 )
@@ -62,8 +61,8 @@ func TestParsingJsonNumbersAsNumbersGood(t *testing.T) {
 	if !aExists || !bExists {
 		t.Fatal("something does not exist")
 	}
-	t.Log(fmt.Sprintf("Values of a: %d , b: %d", a, b))
-	t.Log(fmt.Sprintf("Types of a: %v , b: %v", reflect.TypeOf(a), reflect.TypeOf(b)))
+	//t.Log(fmt.Sprintf("Values of a: %d , b: %d", a, b))
+	//t.Log(fmt.Sprintf("Types of a: %v , b: %v", reflect.TypeOf(a), reflect.TypeOf(b)))
 
 	if a != json.Number("9223372036854775807") {
 		t.Error("We expect a to be represented precisely")

pkg/util/graphql/graphql.go

@@ -10,19 +10,17 @@ import (
 	"net/http"
 )
 
-// NOTE: We may consider using this library in the future instead: https://github.com/machinebox/graphql
-
-type GraphQLRequest struct {
+type Request struct {
 	Query string `json:"query"`
 	// A map of variable names to the value of that variable. Allowed value types are strings, numeric types, and booleans
 	Variables     map[string]interface{} `json:"variables,omitempty"`
 	OperationName string                 `json:"operationName,omitempty"`
 }
 
-func (request *GraphQLRequest) ToBody() ([]byte, error) {
+func (request *Request) ToBody() ([]byte, error) {
 	return json.Marshal(request)
 }
-func (request *GraphQLRequest) ToRequest(ctx context.Context, url string) (*http.Request, error) {
+func (request *Request) ToRequest(ctx context.Context, url string) (*http.Request, error) {
 	body, err := request.ToBody()
 	if err != nil {
 		return nil, err
@@ -37,34 +35,34 @@ func (request *GraphQLRequest) ToRequest(ctx context.Context, url string) (*http
 	return httpReq, nil
 }
 
-type GraphQLResponse struct {
+type Response struct {
 	// We use a jsonnode.Object here because it maintains the order of the keys in a JSON object
 	Data   jsonnode.Object `json:"data"`
-	Errors []GraphQLError  `json:"errors"`
+	Errors []Error         `json:"errors"`
 }
 
-type GraphQLError struct {
+type Error struct {
 	Message    string                 `json:"message"`
-	Locations  []GraphQLErrorLocation `json:"locations,omitempty"`
+	Locations  []ErrorLocation        `json:"locations,omitempty"`
 	Path       []interface{}          `json:"path,omitempty"`
 	Extensions map[string]interface{} `json:"extensions,omitempty"`
 }
 
-type GraphQLErrorLocation struct {
+type ErrorLocation struct {
 	Line   int `json:"line"`
 	Column int `json:"column"`
 }
 
-func ParseGraphQLResponse(body io.ReadCloser) (*GraphQLResponse, error) {
+func ParseGraphQLResponse(body io.ReadCloser) (*Response, error) {
 	bodyAsBytes, err := io.ReadAll(body)
 	if err != nil {
 		log.DefaultLogger.Error("We don't expect this!")
 		return nil, err
 	}
-	var graphQLResponse GraphQLResponse
+	var graphQLResponse Response
 	err = json.Unmarshal(bodyAsBytes, &graphQLResponse)
 	if err != nil {
-		log.DefaultLogger.Error("Error while parsing GraphQL response to GraphQLResponse")
+		log.DefaultLogger.Error("Error while parsing GraphQL response to graphql.Response")
 		return nil, err
 	}
 	return &graphQLResponse, nil