Add data quality indicators to table Viz (#11307)

Author: marthasharkey

app/gui/src/project-view/components/shared/AgGridTableView.vue

@@ -84,6 +84,8 @@ const _props = defineProps<{
   suppressMoveWhenColumnDragging?: boolean
   textFormatOption?: TextFormatOptions
   processDataFromClipboard?: (params: ProcessDataFromClipboardParams<TData>) => string[][] | null
+  pinnedTopRowData?: TData[]
+  pinnedRowHeightMultiplier?: number
 }>()
 const emit = defineEmits<{
   cellEditingStarted: [event: CellEditingStartedEvent]
@@ -106,6 +108,10 @@ function onGridReady(event: GridReadyEvent<TData>) {
 }
 
 function getRowHeight(params: RowHeightParams): number {
+  if (params.node.rowPinned === 'top') {
+    return DEFAULT_ROW_HEIGHT * (_props.pinnedRowHeightMultiplier ?? 2)
+  }
+
   if (_props.textFormatOption === 'off') {
     return DEFAULT_ROW_HEIGHT
   }
@@ -268,6 +274,7 @@ const { AgGridVue } = await import('ag-grid-vue3')
       :suppressDragLeaveHidesColumns="suppressDragLeaveHidesColumns"
       :suppressMoveWhenColumnDragging="suppressMoveWhenColumnDragging"
       :processDataFromClipboard="processDataFromClipboard"
+      :pinnedTopRowData="pinnedTopRowData"
       @gridReady="onGridReady"
       @firstDataRendered="updateColumnWidths"
       @rowDataUpdated="updateColumnWidths($event), emit('rowDataUpdated', $event)"

app/gui/src/project-view/components/visualizations/TableVisualization.vue

@@ -10,6 +10,7 @@ import type {
   CellClickedEvent,
   ColDef,
   ICellRendererParams,
+  ITooltipParams,
   SortChangedEvent,
 } from 'ag-grid-enterprise'
 import { computed, onMounted, ref, shallowRef, watchEffect, type Ref } from 'vue'
@@ -79,6 +80,12 @@ interface UnknownTable {
   get_child_node_action: string
   get_child_node_link_name: string
   link_value_type: string
+  data_quality_pairs?: DataQualityPairs
+}
+
+interface DataQualityPairs {
+  number_of_nothing: number[]
+  number_of_whitespace: number[]
 }
 
 export type TextFormatOptions = 'full' | 'partial' | 'off'
@@ -118,7 +125,9 @@ const defaultColDef: Ref<ColDef> = ref({
   filter: true,
   resizable: true,
   minWidth: 25,
-  cellRenderer: cellRenderer,
+  cellRenderer: (params: ICellRendererParams) => {
+    return params.node.rowPinned === 'top' ? customCellRenderer(params) : cellRenderer(params)
+  },
   cellClass: cellClass,
   contextMenuItems: [commonContextMenuActions.copy, 'copyWithHeaders', 'separator', 'export'],
 } satisfies ColDef)
@@ -142,6 +151,47 @@ const selectableRowLimits = computed(() => {
   return defaults
 })
 
+const pinnedTopRowData = computed(() => {
+  if (typeof props.data === 'object' && 'data_quality_pairs' in props.data) {
+    const data_ = props.data
+    const headers = data_.header
+    const numberOfNothing = data_.data_quality_pairs!.number_of_nothing
+    const numberOfWhitespace = data_.data_quality_pairs!.number_of_whitespace
+    const total = data_.all_rows_count as number
+    if (headers?.length) {
+      const pairs: Record<string, string> = headers.reduce(
+        (obj: any, key: string, index: number) => {
+          obj[key] = {
+            numberOfNothing: numberOfNothing[index],
+            numberOfWhitespace: numberOfWhitespace[index],
+            total,
+          }
+          return obj
+        },
+        {},
+      )
+      return [{ [INDEX_FIELD_NAME]: 'Data Quality', ...pairs }]
+    }
+    return [
+      {
+        [INDEX_FIELD_NAME]: 'Data Quality',
+        Value: {
+          numberOfNothing: numberOfNothing[0] ?? null,
+          numberOfWhitespace: numberOfWhitespace[0] ?? null,
+          total,
+        },
+      },
+    ]
+  }
+  return []
+})
+
+const pinnedRowHeight = computed(() => {
+  const valueTypes =
+    typeof props.data === 'object' && 'value_type' in props.data ? props.data.value_type : []
+  return valueTypes.some((t) => t.constructor === 'Char' || t.constructor === 'Mixed') ? 2 : 1
+})
+
 const newNodeSelectorValues = computed(() => {
   let tooltipValue
   let headerName
@@ -284,6 +334,39 @@ function cellClass(params: CellClassParams) {
   return null
 }
 
+const createVisual = (value: number) => {
+  let color
+  if (value < 33) {
+    color = 'green'
+  } else if (value < 66) {
+    color = 'orange'
+  } else {
+    color = 'red'
+  }
+  return `
+    <div style="display: inline-block; width: 10px; height: 10px; border-radius: 50%; background-color: ${color}; margin-left: 5px;"></div>
+  `
+}
+
+const customCellRenderer = (params: ICellRendererParams) => {
+  if (params.node.rowPinned === 'top') {
+    const nothingPerecent = (params.value.numberOfNothing / params.value.total) * 100
+    const wsPerecent = (params.value.numberOfWhitespace / params.value.total) * 100
+    const nothingVisibility = params.value.numberOfNothing === null ? 'hidden' : 'visible'
+    const whitespaceVisibility = params.value.numberOfWhitespace === null ? 'hidden' : 'visible'
+
+    return `<div>
+    <div style="visibility:${nothingVisibility};">
+      Nulls/Nothing: ${nothingPerecent.toFixed(2)}% ${createVisual(nothingPerecent)}
+    </div>
+    <div style="visibility:${whitespaceVisibility};">
+      Trailing/Leading Whitespace: ${wsPerecent.toFixed(2)}% ${createVisual(wsPerecent)}
+    </div>
+    </div>`
+  }
+  return null
+}
+
 function cellRenderer(params: ICellRendererParams) {
   // Convert's the value into a display string.
   if (params.value === null) return '<span style="color:grey; font-style: italic;">Nothing</span>'
@@ -404,10 +487,14 @@ function toLinkField(fieldName: string, getChildAction?: string, castValueTypes?
       newNodeSelectorValues.value.headerName ? newNodeSelectorValues.value.headerName : fieldName,
     field: fieldName,
     onCellDoubleClicked: (params) => createNode(params, fieldName, getChildAction, castValueTypes),
-    tooltipValueGetter: () => {
-      return `Double click to view this ${newNodeSelectorValues.value.tooltipValue} in a separate component`
-    },
-    cellRenderer: (params: any) => `<div class='link'> ${params.value} </div>`,
+    tooltipValueGetter: (params: ITooltipParams) =>
+      params.node?.rowPinned === 'top' ?
+        null
+      : `Double click to view this ${newNodeSelectorValues.value.tooltipValue} in a separate component`,
+    cellRenderer: (params: ICellRendererParams) =>
+      params.node.rowPinned === 'top' ?
+        `<div> ${params.value}</div>`
+      : `<div class='link'> ${params.value} </div>`,
   }
 }
 
@@ -639,6 +726,8 @@ config.setToolbar(
         :rowData="rowData"
         :defaultColDef="defaultColDef"
         :textFormatOption="textFormatterSelected"
+        :pinnedTopRowData="pinnedTopRowData"
+        :pinnedRowHeightMultiplier="pinnedRowHeight"
         @sortOrFilterUpdated="(e) => checkSortAndFilter(e)"
       />
     </Suspense>

distribution/lib/Standard/Visualization/0.0.0-dev/src/Table/Visualization.enso

@@ -2,7 +2,7 @@ from Standard.Base import all
 import Standard.Base.Data.Vector.Builder
 
 import Standard.Table.Row.Row
-from Standard.Table import Column, Excel_Workbook, Table
+from Standard.Table import Column, Excel_Workbook, Table, Value_Type
 
 import Standard.Database.DB_Column.DB_Column
 import Standard.Database.DB_Table.DB_Table
@@ -34,12 +34,12 @@ prepare_visualization y max_rows=1000 =
         _ : Table ->
             dataframe = x.take max_rows
             all_rows_count = x.row_count
-            make_json_for_table dataframe all_rows_count True
+            make_json_for_table dataframe all_rows_count True False
         _ : DB_Column -> prepare_visualization x.to_table max_rows
         _ : DB_Table ->
             dataframe = x.read (..First max_rows)
             all_rows_count = x.row_count
-            make_json_for_table dataframe all_rows_count True
+            make_json_for_table dataframe all_rows_count True True
         _ : Function ->
             pairs = [['_display_text_', '[Function '+x.to_text+']']]
             value = JS_Object.from_pairs pairs
@@ -59,6 +59,14 @@ prepare_visualization y max_rows=1000 =
    Column Limit
 max_columns = 250
 
+##PRIVATE
+whitespace_count : Column -> Integer | Nothing
+whitespace_count col =
+    find_whitespace col = 
+        filtered = col.to_vector.filter (c-> c.is_a Text && (c.first.is_whitespace || c.last.is_whitespace))
+        filtered.length
+    if (col.value_type == Value_Type.Mixed || col.value_type.is_text) then find_whitespace col else Nothing 
+
 ## PRIVATE
    Render Vector to JSON
 make_json_for_vector : Vector -> Integer -> JS_Object
@@ -172,8 +180,8 @@ make_json_for_xml_element xml_element max_items type:Text="XML_Element" =
      to display.
    - all_rows_count: the number of all rows in the underlying data, useful if
      only a fragment is displayed.
-make_json_for_table : Table -> Integer -> Boolean -> JS_Object
-make_json_for_table dataframe all_rows_count include_index_col =
+make_json_for_table : Table -> Integer -> Boolean -> Boolean -> JS_Object
+make_json_for_table dataframe all_rows_count include_index_col is_db_table =
     get_vector c = Warning.set (c.to_vector.map v-> make_json_for_value v) []
     columns     = dataframe.columns
     header      = ["header", columns.map .name]
@@ -182,7 +190,10 @@ make_json_for_table dataframe all_rows_count include_index_col =
     all_rows    = ["all_rows_count", all_rows_count]
     has_index_col = ["has_index_col", include_index_col]
     links       = ["get_child_node_action", "get_row"]
-    pairs       = [header, value_type, data, all_rows, has_index_col, links, ["type", "Table"]]
+    number_of_nothing = if is_db_table then Nothing else columns.map c-> c.count_nothing
+    number_of_whitespace= if is_db_table then Nothing else columns.map c-> whitespace_count c
+    data_quality_pairs = JS_Object.from_pairs [["number_of_nothing", number_of_nothing], ["number_of_whitespace", number_of_whitespace]]
+    pairs       = [header, value_type, data, all_rows, has_index_col, links, ["data_quality_pairs", data_quality_pairs] ,["type", "Table"]]
     JS_Object.from_pairs pairs
 
 ## PRIVATE

test/Visualization_Tests/src/Table_Spec.enso

@@ -21,13 +21,17 @@ type Data
 
     t self = self.data.at 0
     t2 self = self.data.at 1
+    t3_with_nulls self = self.data.at 2
+    t4_with_space self = self.data.at 3
 
     setup = Data.Value <|
         connection = Database.connect (SQLite.In_Memory)
         in_mem = Table.new [["A", ['a', 'a', 'a']], ["B", [2, 2, 3]], ["C", [3, 5, 6]]]
         t = in_mem.select_into_database_table connection "T" primary_key=Nothing temporary=True
         t2 = Table.new [["A", [1, 2, 3]], ["B", [4, 5, 6]], ["C", [7, 8, 9]]]
-        [t, t2]
+        t3_with_nulls = Table.new [["A", [1, Nothing, 3]], ["B", [4, Nothing, Nothing]], ["C", [7, Nothing, Nothing]]]
+        t4_with_space = Table.new [["A", ['hello', '   leading space', 'trailing space    ']], ["B", ['a', 'b', 'c']], ["C", [7, 8, 9]]]
+        [t, t2, t3_with_nulls, t4_with_space]
 
 
 type Foo
@@ -43,48 +47,51 @@ type Foo_Link
     to_js_object self = JS_Object.from_pairs [["x", self.x], ["links", ["a", "b", "c"]]]
 
 add_specs suite_builder =
-    make_json header data all_rows value_type has_index_col get_child_node =
+    make_json header data all_rows value_type has_index_col get_child_node number_of_nothing number_of_whitespace =
         p_header      = ["header", header]
         p_data        = ["data",   data]
         p_all_rows    = ["all_rows_count", all_rows]
         p_value_type  = ["value_type", value_type]
         p_has_index_col = ["has_index_col", has_index_col]
         p_get_child_node = ["get_child_node_action", get_child_node]
-        pairs    = [p_header, p_value_type, p_data, p_all_rows, p_has_index_col, p_get_child_node, ["type", "Table"]]
+        p_number_of_nothing = ["number_of_nothing", number_of_nothing]
+        p_number_of_whitespace = ["number_of_whitespace", number_of_whitespace]
+        data_quality_pairs = JS_Object.from_pairs [p_number_of_nothing, p_number_of_whitespace]
+        pairs    = [p_header, p_value_type, p_data, p_all_rows, p_has_index_col, p_get_child_node, ["data_quality_pairs", data_quality_pairs], ["type", "Table"]]
         JS_Object.from_pairs pairs . to_text
-
+    
     suite_builder.group "Table Visualization" group_builder->
         data = Data.setup
 
         group_builder.specify "should visualize database tables" <|
             vis = Visualization.prepare_visualization data.t 1
             value_type_int = JS_Object.from_pairs [["type", "Value_Type"], ["constructor", "Integer"], ["display_text", "Integer (64 bits)"], ["bits", 64]]
             value_type_char = JS_Object.from_pairs [["type", "Value_Type"], ["constructor", "Char"], ["display_text", "Char (variable length, max_size=unlimited)"], ["size", Nothing], ["variable_length", True]]
-            json = make_json header=["A", "B", "C"] data=[['a'], [2], [3]] all_rows=3 value_type=[value_type_char, value_type_int, value_type_int] has_index_col=True get_child_node="get_row"
+            json = make_json header=["A", "B", "C"] data=[['a'], [2], [3]] all_rows=3 value_type=[value_type_char, value_type_int, value_type_int] has_index_col=True get_child_node="get_row" number_of_nothing=Nothing number_of_whitespace=Nothing
             vis . should_equal json
 
         group_builder.specify "should visualize database columns" <|
             vis = Visualization.prepare_visualization (data.t.at "A") 2
             value_type_char = JS_Object.from_pairs [["type", "Value_Type"], ["constructor", "Char"], ["display_text", "Char (variable length, max_size=unlimited)"], ["size", Nothing], ["variable_length", True]]
             value_type_float = JS_Object.from_pairs [["type", "Value_Type"], ["constructor", "Float"], ["display_text", "Float (64 bits)"], ["bits", 64]]
-            json = make_json header=["A"] data=[['a', 'a']] all_rows=3 value_type=[value_type_char] has_index_col=True get_child_node="get_row"
+            json = make_json header=["A"] data=[['a', 'a']] all_rows=3 value_type=[value_type_char] has_index_col=True get_child_node="get_row" number_of_nothing=Nothing number_of_whitespace=Nothing
             vis . should_equal json
 
             g = data.t.aggregate ["A", "B"] [Aggregate_Column.Average "C"] . at "Average C"
             vis2 = Visualization.prepare_visualization g 1
-            json2 = make_json header=["Average C"] data=[[4.0]] all_rows=2 value_type=[value_type_float] has_index_col=True get_child_node="get_row"
+            json2 = make_json header=["Average C"] data=[[4.0]] all_rows=2 value_type=[value_type_float] has_index_col=True get_child_node="get_row" number_of_nothing=Nothing number_of_whitespace=Nothing
             vis2 . should_equal json2
 
         group_builder.specify "should visualize dataframe tables" <|
             vis = Visualization.prepare_visualization data.t2 1
             value_type_int = JS_Object.from_pairs [["type", "Value_Type"], ["constructor", "Integer"], ["display_text", "Integer (64 bits)"], ["bits", 64]]
-            json = make_json header=["A", "B", "C"] data=[[1], [4], [7]] all_rows=3 value_type=[value_type_int, value_type_int, value_type_int] has_index_col=True get_child_node="get_row"
+            json = make_json header=["A", "B", "C"] data=[[1], [4], [7]] all_rows=3 value_type=[value_type_int, value_type_int, value_type_int] has_index_col=True get_child_node="get_row" number_of_nothing=[0,0,0] number_of_whitespace=[Nothing, Nothing, Nothing]
             vis . should_equal json
 
         group_builder.specify "should visualize dataframe columns" <|
             vis = Visualization.prepare_visualization (data.t2.at "A") 2
             value_type_int = JS_Object.from_pairs [["type", "Value_Type"], ["constructor", "Integer"], ["display_text", "Integer (64 bits)"], ["bits", 64]]
-            json = make_json header=["A"] data=[[1, 2]] all_rows=3 value_type=[value_type_int] has_index_col=True get_child_node="get_row"
+            json = make_json header=["A"] data=[[1, 2]] all_rows=3 value_type=[value_type_int] has_index_col=True get_child_node="get_row" number_of_nothing=[0] number_of_whitespace=[Nothing]
             vis . should_equal json
 
         group_builder.specify "should handle Vectors" <|
@@ -124,8 +131,20 @@ add_specs suite_builder =
             Visualization.prepare_visualization Value_Type.Char . should_equal (make_json Value_Type.Char)
             Visualization.prepare_visualization Value_Type.Unsupported_Data_Type . should_equal (make_json Value_Type.Unsupported_Data_Type)
 
+        group_builder.specify "should indicate number of Nothing/Nulls" <|
+            vis = Visualization.prepare_visualization data.t3_with_nulls 3
+            value_type_int = JS_Object.from_pairs [["type", "Value_Type"], ["constructor", "Integer"], ["display_text", "Integer (64 bits)"], ["bits", 64]]
+            json = make_json header=["A", "B", "C"] data=[[1,Nothing,3],[4,Nothing,Nothing],[7,Nothing,Nothing]] all_rows=3 value_type=[value_type_int, value_type_int, value_type_int] has_index_col=True get_child_node="get_row" number_of_nothing=[1, 2, 2] number_of_whitespace=[Nothing, Nothing, Nothing]
+            vis . should_equal json
+
+        group_builder.specify "should indicate number of leading/trailing whitespace" <|
+            vis = Visualization.prepare_visualization data.t4_with_space 3
+            value_type_char = JS_Object.from_pairs [["type", "Value_Type"], ["constructor", "Char"], ["display_text", "Char (variable length, max_size=unlimited)"], ["size", Nothing], ["variable_length", True]]
+            value_type_int = JS_Object.from_pairs [["type", "Value_Type"], ["constructor", "Integer"], ["display_text", "Integer (64 bits)"], ["bits", 64]]
+            json = make_json header=["A", "B", "C"] data=[['hello', '   leading space', 'trailing space    '],['a', 'b', 'c'],[7, 8, 9]] all_rows=3 value_type=[value_type_char, value_type_char, value_type_int] has_index_col=True get_child_node="get_row" number_of_nothing=[0, 0, 0] number_of_whitespace=[2, 0, Nothing]
+            vis . should_equal json
+
 main filter=Nothing =
     suite = Test.build suite_builder->
         add_specs suite_builder
     suite.run_with_filter filter
-