From 582a9aaa3d4415e5d4a3c82a3217712a193faf2f Mon Sep 17 00:00:00 2001 From: marthasharkey Date: Mon, 25 Nov 2024 23:01:09 +0000 Subject: [PATCH] Drive data quality metrics from enso code (#11638) If a data quality metric is added to the array sent in the table viz json for a table/column the metric will be added to the columns tooltip without the need for any frontend/ts changes. This doesn't change anything for the users but here is a screenshot to show the same functionality: ![dqm-enso-driven](https://github.com/user-attachments/assets/7bf83d35-0d63-49ac-8d70-1f86dbedc169) --- .../TableVisualisationTooltip.ts | 33 ++++++++++------- .../visualizations/TableVisualization.vue | 35 +++++++------------ .../0.0.0-dev/src/Table/Visualization.enso | 6 ++-- test/Visualization_Tests/src/Table_Spec.enso | 8 ++--- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/app/gui/src/project-view/components/visualizations/TableVisualisationTooltip.ts b/app/gui/src/project-view/components/visualizations/TableVisualisationTooltip.ts index b9e5e8ca9cbb..9b128bfb9603 100644 --- a/app/gui/src/project-view/components/visualizations/TableVisualisationTooltip.ts +++ b/app/gui/src/project-view/components/visualizations/TableVisualisationTooltip.ts @@ -13,8 +13,7 @@ export class TableVisualisationTooltip implements ITooltipComp { */ init( params: ITooltipParams & { - numberOfNothing: number - numberOfWhitespace: number + dataQualityMetrics: Record[] total: number showDataQuality: boolean }, @@ -32,7 +31,6 @@ export class TableVisualisationTooltip implements ITooltipComp { }) const getPercentage = (value: number) => ((value / params.total) * 100).toFixed(2) - const getDisplay = (value: number) => (value > 0 ? 'block' : 'none') const createIndicator = (value: number) => { const color = value < 33 ? 'green' @@ -41,20 +39,29 @@ export class TableVisualisationTooltip implements ITooltipComp { return `
` } - const dataQualityTemplate = ` -
- Nulls/Nothing: ${getPercentage(params.numberOfNothing)}% ${createIndicator(+getPercentage(params.numberOfWhitespace))} -
-
- Trailing/Leading Whitespace: ${getPercentage(params.numberOfWhitespace)}% ${createIndicator(+getPercentage(params.numberOfWhitespace))} -
- ` + const getDataQualityTemplate = () => { + let template = '' + params.dataQualityMetrics.forEach((obj) => { + const key = Object.keys(obj)[0] + const value = key ? obj[key] : null + if (key && value) { + const metricTemplate = `
+ ${key}: ${getPercentage(value)}% ${createIndicator(+getPercentage(value))} +
` + template = template + metricTemplate + } else { + console.warn( + 'Data quality metric is missing a valid key-value pair. Ensure each object in data_quality_pairs contains a single valid key with a numeric value.', + ) + } + }) + return template + } this.eGui.innerHTML = `
Column value type: ${params.value}
- Data Quality Indicators - ${dataQualityTemplate} + ${getDataQualityTemplate()}
` } diff --git a/app/gui/src/project-view/components/visualizations/TableVisualization.vue b/app/gui/src/project-view/components/visualizations/TableVisualization.vue index 59ba8082f711..3af3b8c393e1 100644 --- a/app/gui/src/project-view/components/visualizations/TableVisualization.vue +++ b/app/gui/src/project-view/components/visualizations/TableVisualization.vue @@ -81,12 +81,12 @@ interface UnknownTable { get_child_node_action: string get_child_node_link_name: string link_value_type: string - data_quality_pairs?: DataQualityPairs + data_quality_metrics?: DataQualityMetric[] } -interface DataQualityPairs { - number_of_nothing: number[] - number_of_whitespace: number[] +type DataQualityMetric = { + name: string + percentage_value: number[] } export type TextFormatOptions = 'full' | 'partial' | 'off' @@ -356,23 +356,15 @@ function toField( const displayValue = valueType ? valueType.display_text : null const icon = valueType ? getValueTypeIcon(valueType.constructor) : null - const dataQuality = - typeof props.data === 'object' && 'data_quality_pairs' in props.data ? - props.data.data_quality_pairs - // eslint-disable-next-line camelcase - : { number_of_nothing: [], number_of_whitespace: [] } - - const nothingIsNonZero = - index != null && dataQuality?.number_of_nothing ? - (dataQuality.number_of_nothing[index] ?? 0) > 0 - : false - - const whitespaceIsNonZero = - index != null && dataQuality?.number_of_nothing ? - (dataQuality.number_of_whitespace[index] ?? 0) > 0 - : false + const dataQualityMetrics = + typeof props.data === 'object' && 'data_quality_metrics' in props.data ? + props.data.data_quality_metrics.map((metric: DataQualityMetric) => { + return { [metric.name]: metric.percentage_value[index!] ?? 0 } + }) + : [] - const showDataQuality = nothingIsNonZero || whitespaceIsNonZero + const showDataQuality = + dataQualityMetrics.filter((obj) => (Object.values(obj)[0] as number) > 0).length > 0 const getSvgTemplate = (icon: string) => ` ` @@ -401,8 +393,7 @@ function toField( tooltipComponent: TableVisualisationTooltip, headerTooltip: displayValue ? displayValue : '', tooltipComponentParams: { - numberOfNothing: index != null ? dataQuality.number_of_nothing[index] : null, - numberOfWhitespace: index != null ? dataQuality.number_of_whitespace[index] : null, + dataQualityMetrics, total: typeof props.data === 'object' ? props.data.all_rows_count : 0, showDataQuality, }, diff --git a/distribution/lib/Standard/Visualization/0.0.0-dev/src/Table/Visualization.enso b/distribution/lib/Standard/Visualization/0.0.0-dev/src/Table/Visualization.enso index 5fc26840f575..121819081711 100644 --- a/distribution/lib/Standard/Visualization/0.0.0-dev/src/Table/Visualization.enso +++ b/distribution/lib/Standard/Visualization/0.0.0-dev/src/Table/Visualization.enso @@ -199,8 +199,10 @@ make_json_for_table dataframe all_rows_count include_index_col is_db_table = links = ["get_child_node_action", "get_row"] number_of_nothing = if is_db_table then Nothing else columns.map c-> c.count_nothing number_of_whitespace= if is_db_table then Nothing else columns.map c-> whitespace_count c - data_quality_pairs = JS_Object.from_pairs [["number_of_nothing", number_of_nothing], ["number_of_whitespace", number_of_whitespace]] - pairs = [header, value_type, data, all_rows, has_index_col, links, ["data_quality_pairs", data_quality_pairs] ,["type", "Table"]] + nothing_p = JS_Object.from_pairs [["name", "Number of nothings"], ["percentage_value", number_of_nothing]] + whitespace_p = JS_Object.from_pairs [["name", "Number of untrimmed whitespace"], ["percentage_value",number_of_whitespace]] + data_quality_metrics = [nothing_p, whitespace_p] + pairs = [header, value_type, data, all_rows, has_index_col, links, ["data_quality_metrics", data_quality_metrics] ,["type", "Table"]] JS_Object.from_pairs pairs ## PRIVATE diff --git a/test/Visualization_Tests/src/Table_Spec.enso b/test/Visualization_Tests/src/Table_Spec.enso index c8c81acca2b0..5743cc8c7fb1 100644 --- a/test/Visualization_Tests/src/Table_Spec.enso +++ b/test/Visualization_Tests/src/Table_Spec.enso @@ -54,10 +54,10 @@ add_specs suite_builder = p_value_type = ["value_type", value_type] p_has_index_col = ["has_index_col", has_index_col] p_get_child_node = ["get_child_node_action", get_child_node] - p_number_of_nothing = ["number_of_nothing", number_of_nothing] - p_number_of_whitespace = ["number_of_whitespace", number_of_whitespace] - data_quality_pairs = JS_Object.from_pairs [p_number_of_nothing, p_number_of_whitespace] - pairs = [p_header, p_value_type, p_data, p_all_rows, p_has_index_col, p_get_child_node, ["data_quality_pairs", data_quality_pairs], ["type", "Table"]] + p_number_of_nothing = JS_Object.from_pairs [["name", "Number of nothings"], ["percentage_value", number_of_nothing]] + p_number_of_whitespace = JS_Object.from_pairs [["name", "Number of untrimmed whitespace"], ["percentage_value", number_of_whitespace]] + data_quality_metrics = [p_number_of_nothing, p_number_of_whitespace] + pairs = [p_header, p_value_type, p_data, p_all_rows, p_has_index_col, p_get_child_node, ["data_quality_metrics", data_quality_metrics], ["type", "Table"]] JS_Object.from_pairs pairs . to_text suite_builder.group "Table Visualization" group_builder->