diff --git a/app/gui/src/project-view/components/GraphEditor/GraphVisualization.vue b/app/gui/src/project-view/components/GraphEditor/GraphVisualization.vue index d868d96b67f0..0362e051029c 100644 --- a/app/gui/src/project-view/components/GraphEditor/GraphVisualization.vue +++ b/app/gui/src/project-view/components/GraphEditor/GraphVisualization.vue @@ -61,6 +61,7 @@ const { setToolbarDefinition, visualizationDefinedToolbar, toolbarOverlay, + executeExpression, } = useVisualizationData({ selectedVis: toRef(props, 'currentType'), dataSource: toRef(props, 'dataSource'), @@ -190,6 +191,7 @@ const visParams = computed(() => { data: effectiveVisualizationData.value, size: contentElementSize.value, nodeType: props.typename, + executeExpression, } }) diff --git a/app/gui/src/project-view/components/GraphEditor/GraphVisualization/visualizationData.ts b/app/gui/src/project-view/components/GraphEditor/GraphVisualization/visualizationData.ts index 5df10df5dc10..7151e8ea9592 100644 --- a/app/gui/src/project-view/components/GraphEditor/GraphVisualization/visualizationData.ts +++ b/app/gui/src/project-view/components/GraphEditor/GraphVisualization/visualizationData.ts @@ -1,6 +1,8 @@ import LoadingErrorVisualization from '@/components/visualizations/LoadingErrorVisualization.vue' import LoadingVisualization from '@/components/visualizations/LoadingVisualization.vue' import type { ToolbarItem } from '@/components/visualizations/toolbar' +import { useGraphStore } from '@/stores/graph' +import { NodeId } from '@/stores/graph/graphDatabase' import { useProjectStore } from '@/stores/project' import type { NodeVisualizationConfiguration } from '@/stores/project/executionContext' import { @@ -56,6 +58,7 @@ export function useVisualizationData({ const projectStore = useProjectStore() const visualizationStore = useVisualizationStore() + const graph = useGraphStore() // Flag used to prevent rendering the visualization with a stale preprocessor while the new preprocessor is being // prepared asynchronously. @@ -92,6 +95,50 @@ export function useVisualizationData({ }, ) + const executeExpression = async ( + visulizationModule: string, + expressionString: string, + formatFunction: + | ((arg: any, tempModule: Ast.MutableModule) => Ast.Owned) + | null, + ...positionalArgumentsExpressions: any[] + ) => { + const dataSourceValue = toValue(dataSource) + if (dataSourceValue?.type !== 'node') return + const graphDb = graph.db + const nodeFirstOurputPort = graphDb.getNodeFirstOutputPort(dataSourceValue.nodeId as NodeId) + const identifier = graphDb.getOutputPortIdentifier(nodeFirstOurputPort) + if (identifier === undefined) return + const contextId = + dataSourceValue.nodeId && + graphDb.nodeIdToNode.get(dataSourceValue.nodeId as NodeId)?.outerAst.externalId + if (contextId === undefined) return + try { + const tempModule = Ast.MutableModule.Transient() + const preprocessorModule = Ast.parseExpression(visulizationModule, tempModule)! + const preprocessorQn = Ast.PropertyAccess.new( + tempModule, + preprocessorModule, + Ast.identifier(expressionString)!, + ) + + const preprocessorInvocation = Ast.App.PositionalSequence(preprocessorQn, [ + Ast.Wildcard.new(tempModule), + ...positionalArgumentsExpressions.map((arg) => { + const parsedArg = + formatFunction ? formatFunction(arg, tempModule) : Ast.parseExpression(arg, tempModule)! + return Ast.Group.new(tempModule, parsedArg) + }), + ]) + const rhs = Ast.parseExpression(identifier, tempModule)! + const expression = Ast.OprApp.new(tempModule, preprocessorInvocation, '<|', rhs) + return projectStore.executeExpression(contextId, expression.code()) + } catch (e) { + console.error(e) + throw e + } + } + const currentType = computed(() => { const selectedTypeValue = toValue(selectedVis) if (selectedTypeValue) return selectedTypeValue @@ -260,5 +307,19 @@ export function useVisualizationData({ (toolbarDefinition.value = definition), visualizationDefinedToolbar: computed(() => toValue(toolbarDefinition.value)), toolbarOverlay, + executeExpression: ( + visulizationModule: string, + expressionString: string, + formatFunction: + | ((arg: any, tempModule: Ast.MutableModule) => Ast.Owned) + | null, + ...positionalArgumentsExpressions: string[] + ) => + executeExpression( + visulizationModule, + expressionString, + formatFunction, + ...positionalArgumentsExpressions, + ), } } diff --git a/app/gui/src/project-view/components/shared/AgGridTableView.vue b/app/gui/src/project-view/components/shared/AgGridTableView.vue index c5ebeedc6e32..8f21b2ae5829 100644 --- a/app/gui/src/project-view/components/shared/AgGridTableView.vue +++ b/app/gui/src/project-view/components/shared/AgGridTableView.vue @@ -83,16 +83,14 @@ import type { ICellEditorComp, IHeaderComp, IHeaderParams, + IServerSideDatasource, MenuItemDef, ProcessDataFromClipboardParams, RowDataUpdatedEvent, RowEditingStartedEvent, RowEditingStoppedEvent, - RowHeightParams, SortChangedEvent, } from 'ag-grid-enterprise' -import * as iter from 'enso-common/src/utilities/data/iter' -import { LINE_BOUNDARIES } from 'enso-common/src/utilities/data/string' import { Component, type ComponentInstance, @@ -110,8 +108,6 @@ import { tableToEnsoExpression, } from '../GraphEditor/widgets/WidgetTableEditor/tableParsing' -const DEFAULT_ROW_HEIGHT = 22 - const props = defineProps<{ rowData: TData[] columnDefs: (ColDef | ColGroupDef)[] | null @@ -124,6 +120,9 @@ const props = defineProps<{ suppressMoveWhenColumnDragging?: boolean textFormatOption?: TextFormatOptions processDataFromClipboard?: (params: ProcessDataFromClipboardParams) => string[][] | null + datasource?: IServerSideDatasource + rowCount?: number + isServerSideModel?: boolean }>() const emit = defineEmits<{ cellEditingStarted: [event: CellEditingStartedEvent] @@ -145,24 +144,7 @@ function onGridReady(event: GridReadyEvent) { gridApi.value = event.api } -function getRowHeight(params: RowHeightParams): number { - if (props.textFormatOption === 'off') { - return DEFAULT_ROW_HEIGHT - } - const rowData = Object.values(params.data) - const textValues = rowData.filter((r): r is string => typeof r === 'string') - - if (!textValues.length) { - return DEFAULT_ROW_HEIGHT - } - - const returnCharsCount = iter.map(textValues, (text) => - iter.count(text.matchAll(LINE_BOUNDARIES)), - ) - - const maxReturnCharsCount = iter.reduce(returnCharsCount, Math.max, 0) - return (maxReturnCharsCount + 1) * DEFAULT_ROW_HEIGHT -} +const rowModelType = computed(() => (props.isServerSideModel ? 'serverSide' : 'clientSide')) watch( () => props.textFormatOption, @@ -360,8 +342,10 @@ const { AgGridVue } = await import('./AgGridTableView/AgGridVue') ref="grid" class="ag-theme-alpine inner" :headerHeight="26" - :getRowHeight="getRowHeight" - :rowData="rowData" + :rowModelType="rowModelType" + :serverSideDatasource="datasource" + :rowCount="rowCount" + :rowData="rowModelType === 'clientSide' ? rowData : null" :columnDefs="columnDefs" :defaultColDef="defaultColDef" :copyHeadersToClipboard="true" diff --git a/app/gui/src/project-view/components/visualizations/TableVisualization.vue b/app/gui/src/project-view/components/visualizations/TableVisualization.vue index ecbd3ec15636..a1ab6efda58a 100644 --- a/app/gui/src/project-view/components/visualizations/TableVisualization.vue +++ b/app/gui/src/project-view/components/visualizations/TableVisualization.vue @@ -2,7 +2,6 @@ import icons from '@/assets/icons.svg' import AgGridTableView, { commonContextMenuActions } from '@/components/shared/AgGridTableView.vue' import { - GridFilterModel, useTableVizToolbar, type SortModel, } from '@/components/visualizations/TableVisualization/tableVizToolbar' @@ -15,11 +14,21 @@ import type { CellDoubleClickedEvent, ColDef, ICellRendererParams, + IServerSideDatasource, + IServerSideGetRowsRequest, ITooltipParams, + SetFilterValuesFuncParams, SortChangedEvent, } from 'ag-grid-enterprise' import { computed, onMounted, ref, shallowRef, watchEffect, type Ref } from 'vue' import { TableVisualisationTooltip } from './TableVisualization/TableVisualisationTooltip' +import { + convertFilterModel, + convertSortModel, + parseArgument, +} from './TableVisualization/TableVizDataSourceUtils' +import { GridFilterModel, makeFilterModelList } from './TableVisualization/tableVizFilterUtils' +import { TableVizStatusBar } from './TableVisualization/TableVizStatusBar' import { getCellValueType, isNumericType } from './TableVisualization/tableVizUtils' export const name = 'Table' @@ -96,6 +105,7 @@ interface UnknownTable { child_label: string visualization_header: string data_quality_metrics?: DataQualityMetric[] + is_using_server_sort_and_filter: boolean } type DataQualityMetric = { @@ -130,7 +140,6 @@ const dataGroupingMap = shallowRef>() const defaultColDef: Ref = ref({ editable: false, sortable: true, - filter: true, resizable: true, minWidth: 25, cellRenderer: cellRenderer, @@ -142,9 +151,33 @@ const defaultColDef: Ref = ref({ 'separator', 'export', ], + autoHeight: true, } satisfies ColDef) const rowData = ref[]>([]) const columnDefs: Ref = ref([]) +const allRowCount = computed(() => + typeof props.data === 'object' && 'all_rows_count' in props.data ? props.data.all_rows_count : 0, +) +const isSSRM = computed( + () => typeof props.data === 'object' && 'is_using_server_sort_and_filter' in props.data && props.data.is_using_server_sort_and_filter, +) +const statusBar = computed(() => + allRowCount.value ? + { + statusPanels: + isSSRM.value ? + [ + { + statusPanel: TableVizStatusBar, + statusPanelParams: { + total: allRowCount.value, + }, + }, + ] + : [], + } + : null, +) const textFormatterSelected = ref('partial') @@ -163,6 +196,17 @@ const selectableRowLimits = computed(() => { return defaults }) +function setRowLimit(newRowLimit: number) { + if (newRowLimit !== rowLimit.value) { + rowLimit.value = newRowLimit + config.setPreprocessor( + 'Standard.Visualization.Table.Visualization', + 'prepare_visualization', + newRowLimit.toString(), + ) + } +} + const isFilterSortNodeEnabled = computed( () => config.nodeType === TABLE_NODE_TYPE || config.nodeType === DB_TABLE_NODE_TYPE, ) @@ -241,14 +285,115 @@ function formatText(params: ICellRendererParams) { return ` ${newString} ` } -function setRowLimit(newRowLimit: number) { - if (newRowLimit !== rowLimit.value) { - rowLimit.value = newRowLimit - config.setPreprocessor( - 'Standard.Visualization.Table.Visualization', - 'prepare_visualization', - newRowLimit.toString(), +const createRowsForTable = (data: unknown[][], shift: number, isSSrm: boolean) => { + const rows = data && data.length > 0 ? (data[0]?.length ?? 0) : 0 + const getIndexInfo = (i: number) => { + return isSSrm ? data?.[0]?.[i] : i + } + return Array.from({ length: rows }, (_, i) => { + return Object.fromEntries( + columnDefs.value.map((h, j) => { + return [ + h.field, + h.field === INDEX_FIELD_NAME ? getIndexInfo(i) : toRender(data?.[j - shift]?.[i]), + ] + }), ) + }) +} + +async function getFilterValues(params: SetFilterValuesFuncParams) { + const colName = params.colDef.field + if (typeof props.data === 'object' && 'header' in props.data) { + const index = props.data.header?.findIndex((h: string) => colName === h) + const server = createServer() + const response = await server.getSetFilterValues(index) + setTimeout(() => { + if (response.success) { + params.success(response.data) + } + }, 500) + } +} + +function createServer() { + return { + getSetFilterValues: async (columnIndex?: number) => { + const response = await config.executeExpression( + 'Standard.Visualization.Table.Visualization', + 'get_distinct_values_for_column', + //null as values dont need parsing + null, + `${columnIndex}`, + ) + return { + success: true, + data: response.value.distinct_vals, + } + }, + getData: async (request: IServerSideGetRowsRequest) => { + const columnHeaders = + typeof props.data === 'object' && 'header' in props.data ? + props.data.header ? + props.data.header + : [] + : [] + + const { sortColIndexes, sortDirections } = convertSortModel(request, columnHeaders) + + const { filterColumnIndexList, filterActions, valueList, toValueList } = convertFilterModel( + request, + columnHeaders, + colTypeMap.value, + ) + + const response = await config.executeExpression( + 'Standard.Visualization.Table.Visualization', + 'get_rows_for_table', + // function that will parse filter values to enso compaible + parseArgument, + //the index of the next bucket of rows to get + `${request.startRow}`, + //column indexes that require a sort + sortColIndexes, + //direction (Ascending/Descending) for the sorts + sortDirections, + //column indexes that require a filter + filterColumnIndexList, + //column actions i.e Greater Than, Between... + filterActions, + //column values, or From Values if using a Between filter + valueList, + // To Values (only used in Between filters will be 'Nothing' for any other filter) + toValueList, + ) + return { + success: true, + data: response.value.rows, + } + }, + } +} + +interface Response { + data: unknown[][] + success: boolean +} +function createServerSideDatasource(): IServerSideDatasource { + return { + getRows: async (params) => { + const server = createServer() + const response: Response = await server.getData(params.request) + const startIndex = params.request.startRow ? params.request.startRow : 0 + const rows = createRowsForTable(response.data, 0, true) + setTimeout(() => { + if (response.success) { + params.success({ rowData: rows }) + } else { + params.fail() + } + }, 500) + }, } } @@ -414,6 +559,7 @@ function toField( filter: filterType, filterParams: { maxNumConditions: 1, + values: getFilterValues, filterOptions: filterOptions, }, headerComponentParams: { @@ -522,6 +668,8 @@ watchEffect(() => { visualization_header: undefined, // eslint-disable-next-line camelcase link_value_type: undefined, + // eslint-disable-next-line camelcase + is_using_server_sort_and_filter: undefined, } if ('error' in data_) { columnDefs.value = [ @@ -631,21 +779,10 @@ watchEffect(() => { ...dataHeader, ] : dataHeader - const rows = data_.data && data_.data.length > 0 ? (data_.data[0]?.length ?? 0) : 0 - rowData.value = Array.from({ length: rows }, (_, i) => { - const shift = data_.has_index_col ? 1 : 0 - return Object.fromEntries( - columnDefs.value.map((h, j) => { - return [ - h.field, - toRender(h.field === INDEX_FIELD_NAME ? i : data_.data?.[j - shift]?.[i]), - ] - }), - ) - }) - isTruncated.value = data_.all_rows_count !== rowData.value.length + if (!data_.is_using_server_sort_and_filter) { + rowData.value = data_.data ? createRowsForTable(data_.data, 0, data_.is_using_server_sort_and_filter) : [] + } } - // Update paging const newRowCount = data_.all_rows_count == null ? 1 : data_.all_rows_count showRowCount.value = !(data_.all_rows_count == null) @@ -656,21 +793,6 @@ watchEffect(() => { page.value = newPageLimit } - if (rowData.value[0]) { - const headers = Object.keys(rowData.value[0]) - const headerGroupingMap = new Map() - headers.forEach((header) => { - const needsGrouping = rowData.value.some((row) => { - if (header in row && row[header] != null) { - const value = typeof row[header] === 'object' ? row[header].value : row[header] - return value > 999999 || value < -999999 - } - }) - headerGroupingMap.set(header, needsGrouping) - }) - dataGroupingMap.value = headerGroupingMap - } - // If data is truncated, we cannot rely on sorting/filtering so will disable. defaultColDef.value.filter = !isTruncated.value defaultColDef.value.sortable = !isTruncated.value @@ -765,18 +887,7 @@ function checkSortAndFilter(e: SortChangedEvent) { } }) .filter((sort) => sort) - const filter = Object.entries(gridFilterModel).map(([key, value]) => { - return { - columnName: key, - filterType: value.filterType, - filterAction: value.type, - filter: value.filter, - filterTo: value.filterTo, - dateFrom: value.dateFrom, - dateTo: value.dateTo, - values: value.values, - } - }) + const filter = makeFilterModelList(gridFilterModel) if (sort.length || filter.length) { isCreateNodeEnabled.value = true sortModel.value = sort as SortModel[] @@ -791,7 +902,6 @@ function checkSortAndFilter(e: SortChangedEvent) { // =============== // === Updates === // =============== - onMounted(() => { setRowLimit(1000) }) @@ -815,28 +925,30 @@ config.setToolbar(