From f88bd90ebb38c0784d4e06c467db733f9ffc5eac Mon Sep 17 00:00:00 2001 From: Kaz Wesley Date: Wed, 24 Jan 2024 11:22:05 -0800 Subject: [PATCH] Refactor for YJs AST (#8840) Some refactoring separated from #8825 for easier review. # Important Notes **ID types** The new *synchronization IDs* will replace `ExprId` for `Ast` references in frontend logic. `ExprId` (now called `ExternalId`) is now used only for module serialization and engine communication. The graph database will maintain an index that is used to translate at the boundaries. For now, this translation is implemented as a type cast, as the IDs have the same values until the next PR. - `AstId`: Identifies an `Ast` node. - `NodeId`: A subtype of `AstId`. - `ExternalId`: UUID used for serialization and engine communication. **Other changes**: - Immediate validation of `Owned` usage. - Eliminate `Ast.RawCode`. - Prepare to remove `IdMap` from yjsModel. --- app/gui2/mock/index.ts | 6 - app/gui2/shared/languageServerTypes.ts | 4 +- app/gui2/shared/yjsModel.ts | 49 +-- app/gui2/src/components/CodeEditor.vue | 9 +- app/gui2/src/components/ComponentBrowser.vue | 2 +- .../ComponentBrowser/__tests__/input.test.ts | 10 +- .../src/components/ComponentBrowser/input.ts | 11 +- app/gui2/src/components/GraphEditor.vue | 11 +- .../src/components/GraphEditor/GraphEdges.vue | 11 +- .../src/components/GraphEditor/GraphNode.vue | 34 +- .../src/components/GraphEditor/GraphNodes.vue | 12 +- .../components/GraphEditor/NodeWidgetTree.vue | 4 +- .../src/components/GraphEditor/collapsing.ts | 49 ++- .../src/components/GraphEditor/dragging.ts | 11 +- .../widgets/WidgetArgumentName.vue | 2 +- .../GraphEditor/widgets/WidgetCheckbox.vue | 6 +- .../GraphEditor/widgets/WidgetFunction.vue | 51 +-- .../GraphEditor/widgets/WidgetHierarchy.vue | 2 +- .../GraphEditor/widgets/WidgetNumber.vue | 4 +- .../GraphEditor/widgets/WidgetPort.vue | 9 +- .../GraphEditor/widgets/WidgetSelection.vue | 4 +- .../GraphEditor/widgets/WidgetVector.vue | 9 +- app/gui2/src/composables/selection.ts | 4 +- app/gui2/src/composables/stackNavigator.ts | 7 +- app/gui2/src/providers/functionInfo.ts | 4 +- app/gui2/src/providers/graphSelection.ts | 8 +- app/gui2/src/providers/portInfo.ts | 4 +- app/gui2/src/providers/widgetRegistry.ts | 8 +- app/gui2/src/providers/widgetTree.ts | 3 +- .../graph/__tests__/graphDatabase.test.ts | 22 +- app/gui2/src/stores/graph/graphDatabase.ts | 143 +++++--- app/gui2/src/stores/graph/imports.ts | 2 +- app/gui2/src/stores/graph/index.ts | 107 +++--- app/gui2/src/stores/project/index.ts | 11 +- app/gui2/src/stores/visualization/index.ts | 6 +- .../src/util/ast/__tests__/abstract.test.ts | 32 +- .../util/ast/__tests__/aliasAnalysis.test.ts | 8 +- app/gui2/src/util/ast/__tests__/match.test.ts | 6 +- app/gui2/src/util/ast/abstract.ts | 304 +++++++++--------- app/gui2/src/util/ast/aliasAnalysis.ts | 8 +- app/gui2/src/util/ast/extended.ts | 8 +- app/gui2/src/util/ast/match.ts | 4 +- app/gui2/src/util/ast/node.ts | 4 +- app/gui2/src/util/ast/prefixes.ts | 2 +- app/gui2/src/util/callTree.ts | 8 +- app/gui2/src/util/data/types.ts | 5 + app/gui2/stories/GraphNode.story.vue | 2 +- app/gui2/ydoc-server/serialization.ts | 6 +- 48 files changed, 558 insertions(+), 478 deletions(-) create mode 100644 app/gui2/src/util/data/types.ts diff --git a/app/gui2/mock/index.ts b/app/gui2/mock/index.ts index 06b7956debe8..41e5aafb8bfa 100644 --- a/app/gui2/mock/index.ts +++ b/app/gui2/mock/index.ts @@ -5,9 +5,7 @@ import { GraphDb, mockNode } from '@/stores/graph/graphDatabase' import { useProjectStore } from '@/stores/project' import { ComputedValueRegistry } from '@/stores/project/computedValueRegistry' import { MockTransport, MockWebSocket } from '@/util/net' -import * as random from 'lib0/random' import { getActivePinia } from 'pinia' -import type { ExprId } from 'shared/yjsModel' import { ref, type App } from 'vue' import { mockDataHandler, mockLSHandler } from './engine' export * as providers from './providers' @@ -76,10 +74,6 @@ export function projectStoreAndGraphStore() { return [projectStore(), graphStore()] satisfies [] | unknown[] } -export function newExprId() { - return random.uuidv4() as ExprId -} - /** This should only be used for supplying as initial props when testing. * Please do {@link GraphDb.mockNode} with a `useGraphStore().db` after mount. */ export function node() { diff --git a/app/gui2/shared/languageServerTypes.ts b/app/gui2/shared/languageServerTypes.ts index 8d21a9affaff..15db6435458d 100644 --- a/app/gui2/shared/languageServerTypes.ts +++ b/app/gui2/shared/languageServerTypes.ts @@ -2,7 +2,7 @@ import type { SuggestionsDatabaseEntry, SuggestionsDatabaseUpdate, } from './languageServerTypes/suggestions' -import type { ExprId, Uuid } from './yjsModel' +import type { ExternalId, Uuid } from './yjsModel' export type { Uuid } @@ -11,7 +11,7 @@ declare const brandChecksum: unique symbol export type Checksum = string & { [brandChecksum]: never } declare const brandContextId: unique symbol export type ContextId = Uuid & { [brandContextId]: never } -export type ExpressionId = ExprId +export type ExpressionId = ExternalId declare const brandUtcDateTime: unique symbol export type UTCDateTime = string & { [brandUtcDateTime]: never } diff --git a/app/gui2/shared/yjsModel.ts b/app/gui2/shared/yjsModel.ts index 7daf78026e68..571f64506643 100644 --- a/app/gui2/shared/yjsModel.ts +++ b/app/gui2/shared/yjsModel.ts @@ -3,8 +3,10 @@ import * as random from 'lib0/random' import * as Y from 'yjs' export type Uuid = `${string}-${string}-${string}-${string}-${string}` -declare const brandExprId: unique symbol -export type ExprId = Uuid & { [brandExprId]: never } + +declare const brandExternalId: unique symbol +/** Identifies an AST node or token. Used in module serialization and communication with the language server. */ +export type ExternalId = Uuid & { [brandExternalId]: never } export type VisualizationModule = | { kind: 'Builtin' } @@ -149,12 +151,12 @@ export class DistributedModule { return this.doc.ydoc.transact(fn, 'local') } - updateNodeMetadata(id: ExprId, meta: Partial): void { + updateNodeMetadata(id: ExternalId, meta: Partial): void { const existing = this.doc.metadata.get(id) ?? { x: 0, y: 0, vis: null } this.transact(() => this.doc.metadata.set(id, { ...existing, ...meta })) } - getNodeMetadata(id: ExprId): NodeMetadata | null { + getNodeMetadata(id: ExternalId): NodeMetadata | null { return this.doc.metadata.get(id) ?? null } @@ -168,11 +170,20 @@ export class DistributedModule { } export type SourceRange = readonly [start: number, end: number] +declare const brandSourceRangeKey: unique symbol +export type SourceRangeKey = string & { [brandSourceRangeKey]: never } + +export function sourceRangeKey(range: SourceRange): SourceRangeKey { + return `${range[0].toString(16)}:${range[1].toString(16)}` as SourceRangeKey +} +export function sourceRangeFromKey(key: SourceRangeKey): SourceRange { + return key.split(':').map((x) => parseInt(x, 16)) as [number, number] +} export class IdMap { - private readonly rangeToExpr: Map + private readonly rangeToExpr: Map - constructor(entries?: [string, ExprId][]) { + constructor(entries?: [string, ExternalId][]) { this.rangeToExpr = new Map(entries ?? []) } @@ -180,38 +191,30 @@ export class IdMap { return new IdMap([]) } - public static keyForRange(range: SourceRange): string { - return `${range[0].toString(16)}:${range[1].toString(16)}` - } - - public static rangeForKey(key: string): SourceRange { - return key.split(':').map((x) => parseInt(x, 16)) as [number, number] - } - - insertKnownId(range: SourceRange, id: ExprId) { - const key = IdMap.keyForRange(range) + insertKnownId(range: SourceRange, id: ExternalId) { + const key = sourceRangeKey(range) this.rangeToExpr.set(key, id) } - getIfExist(range: SourceRange): ExprId | undefined { - const key = IdMap.keyForRange(range) + getIfExist(range: SourceRange): ExternalId | undefined { + const key = sourceRangeKey(range) return this.rangeToExpr.get(key) } - getOrInsertUniqueId(range: SourceRange): ExprId { - const key = IdMap.keyForRange(range) + getOrInsertUniqueId(range: SourceRange): ExternalId { + const key = sourceRangeKey(range) const val = this.rangeToExpr.get(key) if (val !== undefined) { return val } else { - const newId = random.uuidv4() as ExprId + const newId = random.uuidv4() as ExternalId this.rangeToExpr.set(key, newId) return newId } } - entries(): [string, ExprId][] { - return [...this.rangeToExpr] + entries(): [SourceRangeKey, ExternalId][] { + return [...this.rangeToExpr] as [SourceRangeKey, ExternalId][] } get size(): number { diff --git a/app/gui2/src/components/CodeEditor.vue b/app/gui2/src/components/CodeEditor.vue index 6864848ddeac..d9773bb3450f 100644 --- a/app/gui2/src/components/CodeEditor.vue +++ b/app/gui2/src/components/CodeEditor.vue @@ -1,7 +1,8 @@