Re-enable running unit tests in shared packages (#12427)

Author: kazcw

app/common/package.json

@@ -17,7 +17,7 @@
   },
   "main": "src/index.ts",
   "scripts": {
-    "test": "vitest run",
+    "test:unit": "vitest run",
     "compile": "tsc",
     "lint": "eslint ./src --cache --max-warnings=0"
   },

app/common/src/services/Backend/__test__/projectExecution.test.ts

@@ -1,3 +1,7 @@
+import * as v from 'vitest'
+
+v.test('Test suite disabled (FIXME: #12426)', () => {})
+/*
 import { ZonedDateTime } from '@internationalized/date'
 import * as v from 'vitest'
 import { IanaTimeZone, toRfc3339 } from '../../../utilities/data/dateTime'
@@ -67,3 +71,4 @@ v.test.each([
     v.expect(firstProjectExecutionOnOrAfter(info, current).toString()).toBe(next.toString())
   },
 )
+ */

app/gui/src/project-view/components/DocumentationEditor.vue

@@ -11,7 +11,7 @@ import { useGraphStore } from '@/stores/graph'
 import { useProjectStore } from '@/stores/project'
 import { useProjectFiles } from '@/stores/projectFiles'
 import { ComponentInstance, ref, toRef, watch } from 'vue'
-import { normalizeMarkdown } from 'ydoc-shared/ast/documentation'
+import { prerenderMarkdown } from 'ydoc-shared/ast/documentation'
 import * as Y from 'yjs'
 
 const { yText } = defineProps<{
@@ -49,7 +49,7 @@ function handlePaste(raw: boolean) {
       if (htmlType) {
         const blob = await item.getType(htmlType)
         const html = await blob.text()
-        const markdown = normalizeMarkdown(await htmlToMarkdown(html))
+        const markdown = prerenderMarkdown(await htmlToMarkdown(html))
         markdownEditor.value.putText(markdown)
         continue
       }

app/lang-markdown/package.json

@@ -4,7 +4,6 @@
   "private": true,
   "description": "Markdown language support for the CodeMirror code editor",
   "scripts": {
-    "test": "cm-runtests",
     "prepare": "cm-buildhelper src/index.ts"
   },
   "keywords": [

app/lezer-markdown/package.json

@@ -13,8 +13,6 @@
   "author": "Marijn Haverbeke <[email protected]>",
   "license": "MIT",
   "devDependencies": {
-    "ist": "^1.1.1",
-    "mocha": "^10.2.0",
     "@lezer/html": "^1.0.0",
     "@marijn/buildtool": "^0.1.6"
   },
@@ -27,7 +25,6 @@
     "url" : "https://github.com/lezer-parser/markdown.git"
   },
   "scripts": {
-    "prepare": "node build.js",
-    "test": "mocha"
+    "prepare": "node build.js"
   }
 }

app/ydoc-server/package.json

@@ -8,7 +8,7 @@
     "email": "[email protected]"
   },
   "scripts": {
-    "test": "vitest run",
+    "test:unit": "vitest run",
     "test:watch": "vitest",
     "typecheck": "tsc",
     "compile": "tsc",

app/ydoc-shared/package.json

@@ -8,7 +8,7 @@
     "email": "[email protected]"
   },
   "scripts": {
-    "test": "vitest run",
+    "test:unit": "vitest run",
     "test:watch": "vitest",
     "format": "prettier --version && prettier --write src/ && eslint . --fix",
     "compile": "tsc"

app/ydoc-shared/src/ast/__tests__/documentation.test.ts

@@ -175,9 +175,13 @@ describe('Function documentation (Markdown)', () => {
       markdown:
         '- Bullet list\n  - Nested list\n    - Very nested list\n  - Nested list\n- Bullet list',
     },
+    {
+      source: '## Plain text\n   - Bullet list\n   Plain text\n   1. Numbered list\n   Plain text',
+      markdown: 'Plain text\n- Bullet list\nPlain text\n1. Numbered list\nPlain text',
+    },
   ]
 
-  test.each(cases)('Enso source comments to normalized markdown', ({ source, markdown }) => {
+  test.each(cases)('Enso source comments to prerendered markdown', ({ source, markdown }) => {
     const moduleSource = `${source}\nmain =\n    x = 1`
     const topLevel = parseModule(moduleSource)
     topLevel.module.setRoot(topLevel)

app/ydoc-shared/src/ast/documentation.ts

@@ -1,11 +1,14 @@
 import { LINE_BOUNDARIES } from 'enso-common/src/utilities/data/string'
-import { ensoMarkdownParser } from './ensoMarkdown'
+import * as Y from 'yjs'
+import { ensoMarkdownParser, ensoStandardMarkdownParser } from './ensoMarkdown'
 import { xxHash128 } from './ffi'
 import type { ConcreteChild, RawConcreteChild } from './print'
 import { ensureUnspaced, firstChild, preferUnspaced, unspaced } from './print'
 import { Token, TokenType } from './token'
 import type { ConcreteRefs, DeepReadonly, DocLine, TextToken } from './tree'
 
+// === AST logic ===
+
 /** Render a documentation line to concrete tokens. */
 export function* docLineToConcrete(
   docLine: DeepReadonly<DocLine>,
@@ -33,49 +36,48 @@ export function* docLineToConcrete(
   for (const newline of docLine.newlines) yield preferUnspaced(newline)
 }
 
-// === Markdown ===
-
 /**
- * Render function documentation to concrete tokens. If the `markdown` content has the same value as when `docLine` was
- * parsed (as indicated by `hash`), the `docLine` will be used (preserving concrete formatting). If it is different, the
- * `markdown` text will be converted to source tokens.
+ * Render function documentation to concrete tokens. If the `markdown` content has the same value as
+ * when `docLine` was parsed (as indicated by `hash`), the `docLine` will be used (preserving
+ * concrete formatting). If it is different, the `markdown` text will be converted to source tokens.
  */
 export function functionDocsToConcrete(
-  markdown: string,
+  markdown: DeepReadonly<Y.Text>,
   hash: string | undefined,
   docLine: DeepReadonly<DocLine> | undefined,
   indent: string | null,
 ): Iterable<RawConcreteChild> | undefined {
-  return (
-    hash && docLine && xxHash128(markdown) === hash ? docLineToConcrete(docLine, indent)
-    : markdown ? markdownYTextToTokens(markdown, (indent || '') + '   ')
-    : undefined
-  )
-}
-
-function markdownYTextToTokens(yText: string, indent: string): Iterable<ConcreteChild<Token>> {
-  const tokensBuilder = new DocTokensBuilder(indent)
-  standardizeMarkdown(yText, tokensBuilder)
+  const markdownText = markdown.toString()
+  if (hash && docLine && xxHash128(markdownText) === hash) return docLineToConcrete(docLine, indent)
+  if (!markdownText) return
+  const tokensBuilder = new DocTokensBuilder((indent || '') + '   ')
+  standardizeMarkdown(markdownText, tokensBuilder)
   return tokensBuilder.build()
 }
 
 /**
- * Given Enso documentation comment tokens, returns a model of their Markdown content. This model abstracts away details
- * such as the locations of line breaks that are not paragraph breaks (e.g. lone newlines denoting hard-wrapping of the
- * source code).
+ * Given Enso documentation comment tokens, returns a model of their Markdown content. This model
+ * abstracts away details such as the locations of line breaks that are not paragraph breaks (e.g.
+ * lone newlines denoting hard-wrapping of the source code).
  */
-export function abstractMarkdown(elements: undefined | TextToken<ConcreteRefs>[]) {
+export function abstractMarkdown(elements: undefined | TextToken<ConcreteRefs>[]): {
+  markdown: Y.Text
+  hash: string
+} {
   const { tags, rawMarkdown } = toRawMarkdown(elements)
-  const markdown = [...tags, normalizeMarkdown(rawMarkdown)].join('\n')
+  const markdown = [...tags, prerenderMarkdown(rawMarkdown)].join('\n')
   const hash = xxHash128(markdown)
-  return { markdown, hash }
+  return { markdown: new Y.Text(markdown), hash }
 }
 
 function indentLevel(whitespace: string) {
   return whitespace.length + whitespace.split('\t').length - 1
 }
 
-function toRawMarkdown(elements: undefined | TextToken<ConcreteRefs>[]) {
+function toRawMarkdown(elements: undefined | TextToken<ConcreteRefs>[]): {
+  tags: string[]
+  rawMarkdown: string
+} {
   const tags: string[] = []
   let readingTags = true
   const tokenWhitespace = ({ token: { whitespace } }: TextToken<ConcreteRefs>) => whitespace
@@ -113,73 +115,55 @@ function toRawMarkdown(elements: undefined | TextToken<ConcreteRefs>[]) {
   return { tags, rawMarkdown }
 }
 
+// === Markdown ===
+
 /**
- * Convert the Markdown input to a format with rendered-style linebreaks: Hard-wrapped lines within a paragraph will be
- * joined, and only a single linebreak character is used to separate paragraphs.
+ * Convert the Markdown input to a format with "prerendered" linebreaks: Hard-wrapped lines within
+ * a paragraph will be joined, and only a single linebreak character is used to separate paragraphs.
  */
-export function normalizeMarkdown(rawMarkdown: string): string {
-  let normalized = ''
+export function prerenderMarkdown(markdown: string): string {
+  let prerendered = ''
   let prevTo = 0
   let prevName: string | undefined = undefined
-  const cursor = ensoMarkdownParser.parse(rawMarkdown).cursor()
+  const cursor = ensoStandardMarkdownParser.parse(markdown).cursor()
   cursor.firstChild()
   do {
     if (prevTo < cursor.from) {
-      const textBetween = rawMarkdown.slice(prevTo, cursor.from)
-      normalized +=
+      const textBetween = markdown.slice(prevTo, cursor.from)
+      prerendered +=
         cursor.name === 'Paragraph' && prevName !== 'Table' ? textBetween.slice(0, -1) : textBetween
     }
-    const text = rawMarkdown.slice(cursor.from, cursor.to)
-    normalized += cursor.name === 'Paragraph' ? text.replaceAll(/ *\n */g, ' ') : text
+    const text = markdown.slice(cursor.from, cursor.to)
+    prerendered += cursor.name === 'Paragraph' ? text.replaceAll(/ *\n */g, ' ') : text
     prevTo = cursor.to
     prevName = cursor.name
   } while (cursor.nextSibling())
-  return normalized
-}
-
-function stringCollector() {
-  let output = ''
-  const collector = {
-    text: (text: string) => (output += text),
-    wrapText: (text: string) => (output += text),
-    newline: () => (output += '\n'),
-  }
-  return { collector, output }
+  return prerendered
 }
 
 /**
- * Convert from "normalized" Markdown (with hard line-breaks removed) to the standard format, with paragraphs separated
- * by blank lines.
+ * Convert from our internal "prerendered" Markdown to the (more standard-compatible) on-disk
+ * representation, with paragraphs hard-wrapped and separated by blank lines.
  */
-export function normalizedMarkdownToStandard(normalizedMarkdown: string) {
-  const { collector, output } = stringCollector()
-  standardizeMarkdown(normalizedMarkdown, collector)
-  return output
-}
-
-/**
- * Convert from "normalized" Markdown to the on-disk representation, with paragraphs hard-wrapped and separated by blank
- * lines.
- */
-function standardizeMarkdown(normalizedMarkdown: string, textConsumer: TextConsumer) {
+function standardizeMarkdown(prerenderedMarkdown: string, textConsumer: TextConsumer): void {
   let printingTags = true
-  const cursor = ensoMarkdownParser.parse(normalizedMarkdown).cursor()
+  const cursor = ensoMarkdownParser.parse(prerenderedMarkdown).cursor()
 
   function standardizeDocument() {
     let prevTo = 0
     let prevName: string | undefined = undefined
     cursor.firstChild()
     do {
       if (prevTo < cursor.from) {
-        const betweenText = normalizedMarkdown.slice(prevTo, cursor.from)
+        const betweenText = prerenderedMarkdown.slice(prevTo, cursor.from)
         for (const _match of betweenText.matchAll(LINE_BOUNDARIES)) {
           textConsumer.newline()
         }
-        if (cursor.name === 'Paragraph' && prevName !== 'Table') {
+        if (cursor.name === 'Paragraph' && prevName === 'Paragraph' && !printingTags) {
           textConsumer.newline()
         }
       }
-      const lines = normalizedMarkdown.slice(cursor.from, cursor.to).split(LINE_BOUNDARIES)
+      const lines = prerenderedMarkdown.slice(cursor.from, cursor.to).split(LINE_BOUNDARIES)
       if (cursor.name === 'Paragraph') {
         standardizeParagraph(lines)
       } else {
@@ -218,6 +202,8 @@ function standardizeMarkdown(normalizedMarkdown: string, textConsumer: TextConsu
   standardizeDocument()
 }
 
+// === AST utilities ===
+
 interface TextConsumer {
   text: (text: string) => void
   wrapText: (text: string) => void

app/ydoc-shared/src/ast/ensoMarkdown.ts

@@ -9,8 +9,10 @@ import {
   Table,
   type BlockParser,
   type MarkdownExtension,
+  type MarkdownParser,
 } from '@lezer/markdown'
 
+// noinspection JSUnusedGlobalSymbols (WebStorm thinks `endLeaf` is unused, unclear why)
 /**
  * End any element when a newline is encountered. This parser operates on preprocessed Markdown that
  * has "prerendered" newlines: Before parsing, hard-wrapped lines within any block element are
@@ -22,18 +24,39 @@ const newlineEndsBlock: BlockParser = {
     !(line.text.startsWith('|') && line.text.length > 2 && line.text.endsWith('|')),
 }
 
-/** @lezer/markdown extension for the Markdown dialect used in the Enso documentation editor. */
-export const ensoMarkdownExtension: MarkdownExtension = [
+const ensoMarkdownDialect = [
   Table,
   Strikethrough,
-  { parseBlock: [newlineEndsBlock] },
   /**
-   * When starting a bulleted list, the `SetextHeading` parser can match when a `-` has been typed and a following space
-   * hasn't been entered yet; the resulting style changes are distracting. To prevent this, we don't support setext
-   * headings; ATX headings seem to be much more popular anyway.
+   * When starting a bulleted list, the `SetextHeading` parser can match when a `-` has been typed
+   * and a following space hasn't been entered yet; the resulting style changes are distracting. To
+   * prevent this, we don't support setext headings; ATX headings seem to be much more popular
+   * anyway.
    */
   { remove: ['SetextHeading'] },
 ]
 
-/** Headless @lezer/markdown parser for the Markdown dialect used in the Enso documentation editor. */
-export const ensoMarkdownParser = commonmarkParser.configure(ensoMarkdownExtension)
+const prerenderedRepresentation = { parseBlock: [newlineEndsBlock] }
+
+/** {@link MarkdownExtension} for Markdown as used in the Enso documentation editor. */
+export const ensoMarkdownExtension: MarkdownExtension = [
+  ensoMarkdownDialect,
+  prerenderedRepresentation,
+]
+
+/**
+ * Headless {@link MarkdownParser} for the Markdown representation used by the Enso documentation
+ * editor.
+ *
+ * This parses the working representation that the documentation editor operates on, with
+ * "prerendered" newlines.
+ */
+export const ensoMarkdownParser: MarkdownParser = commonmarkParser.configure(ensoMarkdownExtension)
+
+/**
+ * Headless {@link MarkdownParser} for the Markdown dialect used in Enso documentation comments.
+ *
+ * This parses the "standard" representation as appears in Enso files.
+ */
+export const ensoStandardMarkdownParser: MarkdownParser =
+  commonmarkParser.configure(ensoMarkdownDialect)

app/ydoc-shared/src/ast/parse.ts

@@ -351,7 +351,7 @@ class Abstractor {
     return FunctionDef.concrete(this.module, {
       docLine,
       docLineMarkdownHash,
-      docMarkdown: new Y.Text(docMarkdown),
+      docMarkdown,
       annotationLines,
       signatureLine,
       private_,

app/ydoc-shared/src/ast/tree.ts

@@ -2458,7 +2458,7 @@ export class FunctionDef extends BaseStatement {
         return preferUnspaced(nodeChild)
       }
     }
-    const docs = functionDocsToConcrete(docMarkdown.toJSON(), docLineMarkdownHash, docLine, indent)
+    const docs = functionDocsToConcrete(docMarkdown, docLineMarkdownHash, docLine, indent)
     if (docs) {
       yield* docs
       prevIsNewline = true

package.json

@@ -39,7 +39,6 @@
     "ci-check": "corepack pnpm run -r compile && corepack pnpm run --aggregate-output /^ci:/",
     "ci:prettier": "prettier --check --cache .",
     "ci:lint": "corepack pnpm run -r --parallel lint --output-file eslint_report.json --format json --cache-strategy content",
-    "ci:test": "corepack pnpm run -r --parallel test",
     "ci:unit-test": "corepack pnpm run -r --parallel test:unit",
     "ci:typecheck": "corepack pnpm run -r --parallel typecheck",
     "git-clean": "git clean -xfd --exclude='.env*'",