Refactor for YJs AST (#8840)

Some refactoring separated from #8825 for easier review.

# Important Notes
**ID types**

The new *synchronization IDs* will replace `ExprId` for `Ast` references in frontend logic. `ExprId` (now called `ExternalId`) is now used only for module serialization and engine communication. The graph database will maintain an index that is used to translate at the boundaries. For now, this translation is implemented as a type cast, as the IDs have the same values until the next PR.

- `AstId`: Identifies an `Ast` node.
- `NodeId`: A subtype of `AstId`.
- `ExternalId`: UUID used for serialization and engine communication.

**Other changes**:

- Immediate validation of `Owned` usage.
- Eliminate `Ast.RawCode`.
- Prepare to remove `IdMap` from yjsModel.

app/gui2/mock/index.ts

@@ -5,9 +5,7 @@ import { GraphDb, mockNode } from '@/stores/graph/graphDatabase'
 import { useProjectStore } from '@/stores/project'
 import { ComputedValueRegistry } from '@/stores/project/computedValueRegistry'
 import { MockTransport, MockWebSocket } from '@/util/net'
-import * as random from 'lib0/random'
 import { getActivePinia } from 'pinia'
-import type { ExprId } from 'shared/yjsModel'
 import { ref, type App } from 'vue'
 import { mockDataHandler, mockLSHandler } from './engine'
 export * as providers from './providers'
@@ -76,10 +74,6 @@ export function projectStoreAndGraphStore() {
   return [projectStore(), graphStore()] satisfies [] | unknown[]
 }
 
-export function newExprId() {
-  return random.uuidv4() as ExprId
-}
-
 /** This should only be used for supplying as initial props when testing.
  * Please do {@link GraphDb.mockNode} with a `useGraphStore().db` after mount. */
 export function node() {

app/gui2/shared/languageServerTypes.ts

@@ -2,7 +2,7 @@ import type {
   SuggestionsDatabaseEntry,
   SuggestionsDatabaseUpdate,
 } from './languageServerTypes/suggestions'
-import type { ExprId, Uuid } from './yjsModel'
+import type { ExternalId, Uuid } from './yjsModel'
 
 export type { Uuid }
 
@@ -11,7 +11,7 @@ declare const brandChecksum: unique symbol
 export type Checksum = string & { [brandChecksum]: never }
 declare const brandContextId: unique symbol
 export type ContextId = Uuid & { [brandContextId]: never }
-export type ExpressionId = ExprId
+export type ExpressionId = ExternalId
 declare const brandUtcDateTime: unique symbol
 export type UTCDateTime = string & { [brandUtcDateTime]: never }
 

app/gui2/shared/yjsModel.ts

@@ -3,8 +3,10 @@ import * as random from 'lib0/random'
 import * as Y from 'yjs'
 
 export type Uuid = `${string}-${string}-${string}-${string}-${string}`
-declare const brandExprId: unique symbol
-export type ExprId = Uuid & { [brandExprId]: never }
+
+declare const brandExternalId: unique symbol
+/** Identifies an AST node or token. Used in module serialization and communication with the language server. */
+export type ExternalId = Uuid & { [brandExternalId]: never }
 
 export type VisualizationModule =
   | { kind: 'Builtin' }
@@ -149,12 +151,12 @@ export class DistributedModule {
     return this.doc.ydoc.transact(fn, 'local')
   }
 
-  updateNodeMetadata(id: ExprId, meta: Partial<NodeMetadata>): void {
+  updateNodeMetadata(id: ExternalId, meta: Partial<NodeMetadata>): void {
     const existing = this.doc.metadata.get(id) ?? { x: 0, y: 0, vis: null }
     this.transact(() => this.doc.metadata.set(id, { ...existing, ...meta }))
   }
 
-  getNodeMetadata(id: ExprId): NodeMetadata | null {
+  getNodeMetadata(id: ExternalId): NodeMetadata | null {
     return this.doc.metadata.get(id) ?? null
   }
 
@@ -168,50 +170,51 @@ export class DistributedModule {
 }
 
 export type SourceRange = readonly [start: number, end: number]
+declare const brandSourceRangeKey: unique symbol
+export type SourceRangeKey = string & { [brandSourceRangeKey]: never }
+
+export function sourceRangeKey(range: SourceRange): SourceRangeKey {
+  return `${range[0].toString(16)}:${range[1].toString(16)}` as SourceRangeKey
+}
+export function sourceRangeFromKey(key: SourceRangeKey): SourceRange {
+  return key.split(':').map((x) => parseInt(x, 16)) as [number, number]
+}
 
 export class IdMap {
-  private readonly rangeToExpr: Map<string, ExprId>
+  private readonly rangeToExpr: Map<string, ExternalId>
 
-  constructor(entries?: [string, ExprId][]) {
+  constructor(entries?: [string, ExternalId][]) {
     this.rangeToExpr = new Map(entries ?? [])
   }
 
   static Mock(): IdMap {
     return new IdMap([])
   }
 
-  public static keyForRange(range: SourceRange): string {
-    return `${range[0].toString(16)}:${range[1].toString(16)}`
-  }
-
-  public static rangeForKey(key: string): SourceRange {
-    return key.split(':').map((x) => parseInt(x, 16)) as [number, number]
-  }
-
-  insertKnownId(range: SourceRange, id: ExprId) {
-    const key = IdMap.keyForRange(range)
+  insertKnownId(range: SourceRange, id: ExternalId) {
+    const key = sourceRangeKey(range)
     this.rangeToExpr.set(key, id)
   }
 
-  getIfExist(range: SourceRange): ExprId | undefined {
-    const key = IdMap.keyForRange(range)
+  getIfExist(range: SourceRange): ExternalId | undefined {
+    const key = sourceRangeKey(range)
     return this.rangeToExpr.get(key)
   }
 
-  getOrInsertUniqueId(range: SourceRange): ExprId {
-    const key = IdMap.keyForRange(range)
+  getOrInsertUniqueId(range: SourceRange): ExternalId {
+    const key = sourceRangeKey(range)
     const val = this.rangeToExpr.get(key)
     if (val !== undefined) {
       return val
     } else {
-      const newId = random.uuidv4() as ExprId
+      const newId = random.uuidv4() as ExternalId
       this.rangeToExpr.set(key, newId)
       return newId
     }
   }
 
-  entries(): [string, ExprId][] {
-    return [...this.rangeToExpr]
+  entries(): [SourceRangeKey, ExternalId][] {
+    return [...this.rangeToExpr] as [SourceRangeKey, ExternalId][]
   }
 
   get size(): number {

app/gui2/src/components/CodeEditor.vue

@@ -1,15 +1,16 @@
 <script setup lang="ts">
 import type { Diagnostic, Highlighter } from '@/components/CodeEditor/codemirror'
 import { usePointer } from '@/composables/events'
-import { useGraphStore } from '@/stores/graph'
+import { useGraphStore, type NodeId } from '@/stores/graph'
+import { asNodeId } from '@/stores/graph/graphDatabase'
 import { useProjectStore } from '@/stores/project'
 import { useSuggestionDbStore } from '@/stores/suggestionDatabase'
 import { useAutoBlur } from '@/util/autoBlur'
 import { chain } from '@/util/data/iterable'
 import { unwrap } from '@/util/data/result'
 import { qnJoin, tryQualifiedName } from '@/util/qualifiedName'
 import { useLocalStorage } from '@vueuse/core'
-import { rangeEncloses, type ExprId } from 'shared/yjsModel'
+import { rangeEncloses } from 'shared/yjsModel'
 import { computed, onMounted, ref, watch, watchEffect } from 'vue'
 
 // Use dynamic imports to aid code splitting. The codemirror dependency is quite large.
@@ -52,7 +53,7 @@ const expressionUpdatesDiagnostics = computed(() => {
   for (const id of chain(panics, errors)) {
     const update = updates.get(id)
     if (!update) continue
-    const node = nodeMap.get(id)
+    const node = nodeMap.get(asNodeId(graphStore.db.idFromExternal(id)))
     if (!node) continue
     if (!node.rootSpan.span) continue
     const [from, to] = node.rootSpan.span
@@ -100,7 +101,7 @@ watchEffect(() => {
         hoverTooltip((ast, syn) => {
           const dom = document.createElement('div')
           const astSpan = ast.span()
-          let foundNode: ExprId | undefined
+          let foundNode: NodeId | undefined
           for (const [id, node] of graphStore.db.nodeIdToNode.entries()) {
             if (node.rootSpan.span && rangeEncloses(node.rootSpan.span, astSpan)) {
               foundNode = id

app/gui2/src/components/ComponentBrowser.vue

@@ -177,7 +177,7 @@ const previewDataSource: ComputedRef<VisualizationDataSource | undefined> = comp
   return {
     type: 'expression',
     expression: previewedExpression.value,
-    contextId: body.exprId,
+    contextId: body.externalId,
   }
 })
 

app/gui2/src/components/ComponentBrowser/__tests__/input.test.ts

@@ -14,9 +14,10 @@ import {
   makeType,
   type SuggestionEntry,
 } from '@/stores/suggestionDatabase/entry'
+import type { AstId } from '@/util/ast/abstract'
 import { unwrap } from '@/util/data/result'
 import { tryIdentifier, tryQualifiedName } from '@/util/qualifiedName'
-import type { ExprId } from 'shared/yjsModel'
+import type { ExternalId, Uuid } from 'shared/yjsModel'
 import { expect, test } from 'vitest'
 
 test.each([
@@ -97,10 +98,11 @@ test.each([
       selfArg?: { type: string; typename?: string }
     },
   ) => {
-    const operator1Id: ExprId = '3d0e9b96-3ca0-4c35-a820-7d3a1649de55' as ExprId
-    const operator2Id: ExprId = '5eb16101-dd2b-4034-a6e2-476e8bfa1f2b' as ExprId
+    const operator1Id = '3d0e9b96-3ca0-4c35-a820-7d3a1649de55' as AstId
+    const operator1ExternalId = operator1Id as Uuid as ExternalId
+    const operator2Id = '5eb16101-dd2b-4034-a6e2-476e8bfa1f2b' as AstId
     const computedValueRegistryMock = ComputedValueRegistry.Mock()
-    computedValueRegistryMock.db.set(operator1Id, {
+    computedValueRegistryMock.db.set(operator1ExternalId, {
       typename: 'Standard.Base.Number',
       methodCall: undefined,
       payload: { type: 'Value' },

app/gui2/src/components/ComponentBrowser/input.ts

@@ -1,5 +1,5 @@
 import type { Filter } from '@/components/ComponentBrowser/filtering'
-import { useGraphStore } from '@/stores/graph'
+import { useGraphStore, type NodeId } from '@/stores/graph'
 import type { GraphDb } from '@/stores/graph/graphDatabase'
 import { requiredImportEquals, requiredImports, type RequiredImport } from '@/stores/graph/imports'
 import { useSuggestionDbStore, type SuggestionDb } from '@/stores/suggestionDatabase'
@@ -10,6 +10,7 @@ import {
   type Typename,
 } from '@/stores/suggestionDatabase/entry'
 import { RawAst, RawAstExtended, astContainingChar } from '@/util/ast'
+import type { AstId } from '@/util/ast/abstract.ts'
 import { AliasAnalyzer } from '@/util/ast/aliasAnalysis'
 import { GeneralOprApp, type OperatorChain } from '@/util/ast/opr'
 import { MappedSet } from '@/util/containers'
@@ -21,13 +22,13 @@ import {
   type QualifiedName,
 } from '@/util/qualifiedName'
 import { equalFlat } from 'lib0/array'
-import { IdMap, type ExprId, type SourceRange } from 'shared/yjsModel'
+import { sourceRangeKey, type SourceRange } from 'shared/yjsModel'
 import { computed, ref, type ComputedRef } from 'vue'
 
 /** Information how the component browser is used, needed for proper input initializing. */
 export type Usage =
-  | { type: 'newNode'; sourcePort?: ExprId | undefined }
-  | { type: 'editNode'; node: ExprId; cursorPos: number }
+  | { type: 'newNode'; sourcePort?: AstId | undefined }
+  | { type: 'editNode'; node: NodeId; cursorPos: number }
 
 /** Input's editing context.
  *
@@ -116,7 +117,7 @@ export function useComponentBrowserInput(
         yield* usages
       }
     }
-    return new MappedSet(IdMap.keyForRange, internalUsages())
+    return new MappedSet(sourceRangeKey, internalUsages())
   })
 
   // Filter deduced from the access (`.` operator) chain written by user.

app/gui2/src/components/GraphEditor.vue

@@ -23,17 +23,18 @@ import { provideGraphNavigator } from '@/providers/graphNavigator'
 import { provideGraphSelection } from '@/providers/graphSelection'
 import { provideInteractionHandler, type Interaction } from '@/providers/interactionHandler'
 import { provideWidgetRegistry } from '@/providers/widgetRegistry'
-import { useGraphStore } from '@/stores/graph'
+import { useGraphStore, type NodeId } from '@/stores/graph'
 import type { RequiredImport } from '@/stores/graph/imports'
 import { useProjectStore } from '@/stores/project'
 import { groupColorVar, useSuggestionDbStore } from '@/stores/suggestionDatabase'
 import { bail } from '@/util/assert'
+import type { AstId } from '@/util/ast/abstract.ts'
 import { colorFromString } from '@/util/colors'
 import { Rect } from '@/util/data/rect'
 import { Vec2 } from '@/util/data/vec2'
 import * as set from 'lib0/set'
 import { toast } from 'react-toastify'
-import type { ExprId, NodeMetadata } from 'shared/yjsModel'
+import type { NodeMetadata } from 'shared/yjsModel'
 import { computed, onMounted, onScopeDispose, onUnmounted, ref, watch } from 'vue'
 import { ProjectManagerEvents } from '../../../ide-desktop/lib/dashboard/src/utilities/projectManager'
 import { type Usage } from './ComponentBrowser/input'
@@ -117,7 +118,7 @@ const interactionBindingsHandler = interactionBindings.handler({
 // Return the environment for the placement of a new node. The passed nodes should be the nodes that are
 // used as the source of the placement. This means, for example, the selected nodes when creating from a selection
 // or the node that is being edited when creating from a port double click.
-function environmentForNodes(nodeIds: IterableIterator<ExprId>): Environment {
+function environmentForNodes(nodeIds: IterableIterator<NodeId>): Environment {
   const nodeRects = [...graphStore.nodeRects.values()]
   const selectedNodeRects = [...nodeIds]
     .map((id) => graphStore.nodeRects.get(id))
@@ -577,7 +578,7 @@ async function readNodeFromExcelClipboard(
   return undefined
 }
 
-function handleNodeOutputPortDoubleClick(id: ExprId) {
+function handleNodeOutputPortDoubleClick(id: AstId) {
   componentBrowserUsage.value = { type: 'newNode', sourcePort: id }
   const srcNode = graphStore.db.getPatternExpressionNodeId(id)
   if (srcNode == null) {
@@ -598,7 +599,7 @@ function handleNodeOutputPortDoubleClick(id: ExprId) {
 
 const stackNavigator = useStackNavigator()
 
-function handleEdgeDrop(source: ExprId, position: Vec2) {
+function handleEdgeDrop(source: AstId, position: Vec2) {
   componentBrowserUsage.value = { type: 'newNode', sourcePort: source }
   componentBrowserNodePosition.value = position
   interaction.setCurrent(creatingNodeFromEdgeDrop)

app/gui2/src/components/GraphEditor/GraphEdges.vue

@@ -6,16 +6,17 @@ import { injectInteractionHandler, type Interaction } from '@/providers/interact
 import type { PortId } from '@/providers/portInfo'
 import { useGraphStore } from '@/stores/graph'
 import { Ast } from '@/util/ast'
+import type { AstId } from '@/util/ast/abstract.ts'
 import { Vec2 } from '@/util/data/vec2'
 import { toast } from 'react-toastify'
-import { isUuid, type ExprId } from 'shared/yjsModel.ts'
+import { isUuid } from 'shared/yjsModel.ts'
 
 const graph = useGraphStore()
 const selection = injectGraphSelection(true)
 const interaction = injectInteractionHandler()
 
 const emits = defineEmits<{
-  createNodeFromEdge: [source: ExprId, position: Vec2]
+  createNodeFromEdge: [source: AstId, position: Vec2]
 }>()
 
 const editingEdge: Interaction = {
@@ -55,15 +56,15 @@ function disconnectEdge(target: PortId) {
       const targetStr: string = target
       if (isUuid(targetStr)) {
         console.warn(`Failed to disconnect edge from port ${target}, falling back to direct edit.`)
-        edit.replaceRef(Ast.asNodeId(targetStr as ExprId), Ast.Wildcard.new())
+        edit.replaceRef(targetStr as AstId, Ast.Wildcard.new())
       } else {
         console.error(`Failed to disconnect edge from port ${target}, no fallback possible.`)
       }
     }
   })
 }
 
-function createEdge(source: ExprId, target: PortId) {
+function createEdge(source: AstId, target: PortId) {
   const ident = graph.db.getOutputPortIdentifier(source)
   if (ident == null) return
   const identAst = Ast.parse(ident)
@@ -84,7 +85,7 @@ function createEdge(source: ExprId, target: PortId) {
     if (!graph.updatePortValue(edit, target, identAst)) {
       if (isUuid(target)) {
         console.warn(`Failed to connect edge to port ${target}, falling back to direct edit.`)
-        edit.replaceValue(Ast.asNodeId(target), identAst)
+        edit.replaceValue(Ast.asAstId(target), identAst)
         graph.commitEdit(edit)
       } else {
         console.error(`Failed to connect edge to port ${target}, no fallback possible.`)