diff --git a/CHANGELOG.md b/CHANGELOG.md index 3089636..2096d9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,13 @@ # Changelog -## 1.0.0 -### Breaking Changes: +### 1.1.0 + +- Adds proper type reflection to support all databricks data types. (except type `BINARY` which is not supported) +--- + +### 1.0.0 + +#### Breaking Changes: - Datasource configuration (`hostname` & `sql path`) needs to be entered again in the Datasource Settings after upgrading. - Queries made via Visual Editor no longer work. @@ -13,7 +19,6 @@ - Made some config vars non secret - Added alerting capability ---- ### 0.0.9 diff --git a/go.mod b/go.mod index cc38339..10197c8 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,6 @@ require ( github.com/hashicorp/go-hclog v0.14.1 // indirect github.com/hashicorp/go-plugin v1.4.3 // indirect github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d // indirect - github.com/jmoiron/sqlx v1.3.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.13.1 // indirect @@ -41,7 +40,6 @@ require ( github.com/mattn/go-colorable v0.1.4 // indirect github.com/mattn/go-isatty v0.0.10 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/mattn/go-sqlite3 v1.14.16 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/go.sum b/go.sum index 2e31acc..1e90621 100644 --- a/go.sum +++ b/go.sum @@ -112,7 +112,6 @@ github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -200,8 +199,6 @@ github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKe github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= -github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= -github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -228,7 +225,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magefile/mage v1.13.0 h1:XtLJl8bcCM7EFoO8FyH8XK3t7G5hQAeK+i4tq+veT9M= github.com/magefile/mage v1.13.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -245,9 +241,6 @@ github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW1 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= -github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= diff --git a/package.json b/package.json index 1f45772..c52afcf 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mullerpeter-databricks-datasource", "private": true, - "version": "1.0.0", + "version": "1.1.0", "description": "Databricks SQL Connector", "scripts": { "build": "grafana-toolkit plugin:build", diff --git a/pkg/plugin/helper.go b/pkg/plugin/helper.go deleted file mode 100644 index fa51e3f..0000000 --- a/pkg/plugin/helper.go +++ /dev/null @@ -1,65 +0,0 @@ -package plugin - -import ( - "database/sql" - "errors" - "fmt" - "github.com/grafana/grafana-plugin-sdk-go/backend/log" - "github.com/grafana/grafana-plugin-sdk-go/data" - "time" -) - -func initDataframeTypes(columnTypes []*sql.ColumnType, columnNames []string, frame *data.Frame) (*data.Frame, error) { - for i := 0; i < len(columnTypes); i++ { - - cName := columnNames[i] - switch cType := columnTypes[i].DatabaseTypeName(); cType { - case "BIGINT": - frame.Fields = append(frame.Fields, - data.NewField(cName, nil, []int64{}), - ) - case "BOOLEAN": - frame.Fields = append(frame.Fields, - data.NewField(cName, nil, []bool{}), - ) - case "DATE": - frame.Fields = append(frame.Fields, - data.NewField(cName, nil, []string{}), - ) - case "DOUBLE": - frame.Fields = append(frame.Fields, - data.NewField(cName, nil, []float64{}), - ) - case "FLOAT": - frame.Fields = append(frame.Fields, - data.NewField(cName, nil, []float64{}), - ) - case "INT": - frame.Fields = append(frame.Fields, - data.NewField(cName, nil, []int32{}), - ) - case "SMALLINT": - frame.Fields = append(frame.Fields, - data.NewField(cName, nil, []int16{}), - ) - case "STRING": - frame.Fields = append(frame.Fields, - data.NewField(cName, nil, []string{}), - ) - case "TIMESTAMP": - frame.Fields = append(frame.Fields, - data.NewField(cName, nil, []time.Time{}), - ) - case "TINYINT": - frame.Fields = append(frame.Fields, - data.NewField(cName, nil, []int8{}), - ) - default: - err := errors.New(fmt.Sprintf("Unsuported Type %s for column %s", cType, cName)) - log.DefaultLogger.Info("Unsuported Type", "err", err) - return frame, err - } - - } - return frame, nil -} diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index 0f6727d..12525d8 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -11,7 +11,8 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" "github.com/grafana/grafana-plugin-sdk-go/backend/log" "github.com/grafana/grafana-plugin-sdk-go/data" - "github.com/jmoiron/sqlx" + "github.com/grafana/grafana-plugin-sdk-go/data/sqlutil" + "reflect" "strings" "time" ) @@ -31,7 +32,7 @@ var ( _ backend.StreamHandler = (*SampleDatasource)(nil) _ instancemgmt.InstanceDisposer = (*SampleDatasource)(nil) databricksConnectionsString string - databricksDB *sqlx.DB + databricksDB *sql.DB ) type DatasourceSettings struct { @@ -53,7 +54,7 @@ func NewSampleDatasource(settings backend.DataSourceInstanceSettings) (instancem if err != nil { log.DefaultLogger.Info("DB Init Error", "err", err) } else { - databricksDB = sqlx.NewDb(db, "databricks") + databricksDB = db log.DefaultLogger.Info("Store Databricks SQL DB Connection") } } @@ -69,7 +70,7 @@ func RefreshDBConnection() error { log.DefaultLogger.Info("DB Init Error", "err", err) return err } else { - databricksDB = sqlx.NewDb(db, "databricks") + databricksDB = db log.DefaultLogger.Info("Store Databricks SQL DB Connection") return nil } @@ -78,15 +79,15 @@ func RefreshDBConnection() error { return errors.New("no connection string set") } -func ExecuteQueryx(queryString string) (*sqlx.Rows, error) { - rows, err := databricksDB.Queryx(queryString) +func ExecuteQuery(queryString string) (*sql.Rows, error) { + rows, err := databricksDB.Query(queryString) if err != nil { if strings.HasPrefix(err.Error(), "Invalid SessionHandle") { err = RefreshDBConnection() if err != nil { return nil, err } - rows, err = databricksDB.Queryx(queryString) + rows, err = databricksDB.Query(queryString) if err != nil { return nil, err } @@ -162,43 +163,43 @@ func (d *SampleDatasource) query(_ context.Context, pCtx backend.PluginContext, frame := data.NewFrame("response") - rows, err := ExecuteQueryx(queryString) + rows, err := ExecuteQuery(queryString) if err != nil { response.Error = err log.DefaultLogger.Info("Error", "err", err) return response } - colTypes, err := rows.ColumnTypes() - - if err != nil { - response.Error = err - log.DefaultLogger.Info("Error", "err", err) - return response - } - columnNames, err := rows.Columns() - if err != nil { - response.Error = err - log.DefaultLogger.Info("Error", "err", err) - return response + dateConverter := sqlutil.Converter{ + Name: "Databricks date to timestamp converter", + InputScanType: reflect.TypeOf(sql.NullString{}), + InputTypeName: "DATE", + FrameConverter: sqlutil.FrameConverter{ + FieldType: data.FieldTypeNullableTime, + ConverterFunc: func(n interface{}) (interface{}, error) { + v := n.(*sql.NullString) + + if !v.Valid { + return (*time.Time)(nil), nil + } + + f := v.String + date, error := time.Parse("2006-01-02", f) + if error != nil { + return (*time.Time)(nil), error + } + return &date, nil + }, + }, } - frame, err = initDataframeTypes(colTypes, columnNames, frame) + frame, err = sqlutil.FrameFromRows(rows, -1, dateConverter) if err != nil { + log.DefaultLogger.Info("FrameFromRows", "err", err) response.Error = err return response } - for rows.Next() { - res, err := rows.SliceScan() - if err != nil { - response.Error = err - log.DefaultLogger.Info("Error", "err", err) - return response - } - frame.AppendRow(res...) - } - if qm.QuerySettings.ConvertLongToWide { wideFrame, err := data.LongToWide(frame, &data.FillMissing{Value: qm.QuerySettings.FillValue, Mode: qm.QuerySettings.FillMode}) if err != nil { @@ -231,7 +232,7 @@ func (d *SampleDatasource) CheckHealth(_ context.Context, req *backend.CheckHeal }, nil } - rows, err := ExecuteQueryx("SELECT 1") + rows, err := ExecuteQuery("SELECT 1") if err != nil { return &backend.CheckHealthResult{