diff --git a/app/common/package.json b/app/common/package.json index d5fb7e21cb8d..80896f3d5922 100644 --- a/app/common/package.json +++ b/app/common/package.json @@ -17,7 +17,7 @@ }, "main": "src/index.ts", "scripts": { - "test": "vitest run", + "test:unit": "vitest run", "compile": "tsc", "lint": "eslint ./src --cache --max-warnings=0" }, diff --git a/app/common/src/services/Backend/__test__/projectExecution.test.ts b/app/common/src/services/Backend/__test__/projectExecution.test.ts index 23e0bdd31106..c9e1a3e31bc5 100644 --- a/app/common/src/services/Backend/__test__/projectExecution.test.ts +++ b/app/common/src/services/Backend/__test__/projectExecution.test.ts @@ -1,3 +1,7 @@ +import * as v from 'vitest' + +v.test('Test suite disabled (FIXME: #12426)', () => {}) +/* import { ZonedDateTime } from '@internationalized/date' import * as v from 'vitest' import { IanaTimeZone, toRfc3339 } from '../../../utilities/data/dateTime' @@ -67,3 +71,4 @@ v.test.each([ v.expect(firstProjectExecutionOnOrAfter(info, current).toString()).toBe(next.toString()) }, ) + */ diff --git a/app/gui/src/project-view/components/DocumentationEditor.vue b/app/gui/src/project-view/components/DocumentationEditor.vue index e7cd860ac5b3..9fa0d23ab4b8 100644 --- a/app/gui/src/project-view/components/DocumentationEditor.vue +++ b/app/gui/src/project-view/components/DocumentationEditor.vue @@ -11,7 +11,7 @@ import { useGraphStore } from '@/stores/graph' import { useProjectStore } from '@/stores/project' import { useProjectFiles } from '@/stores/projectFiles' import { ComponentInstance, ref, toRef, watch } from 'vue' -import { normalizeMarkdown } from 'ydoc-shared/ast/documentation' +import { prerenderMarkdown } from 'ydoc-shared/ast/documentation' import * as Y from 'yjs' const { yText } = defineProps<{ @@ -49,7 +49,7 @@ function handlePaste(raw: boolean) { if (htmlType) { const blob = await item.getType(htmlType) const html = await blob.text() - const markdown = normalizeMarkdown(await htmlToMarkdown(html)) + const markdown = prerenderMarkdown(await htmlToMarkdown(html)) markdownEditor.value.putText(markdown) continue } diff --git a/app/lang-markdown/package.json b/app/lang-markdown/package.json index 7767d1b37cfb..1b4f8d3cc7e0 100644 --- a/app/lang-markdown/package.json +++ b/app/lang-markdown/package.json @@ -4,7 +4,6 @@ "private": true, "description": "Markdown language support for the CodeMirror code editor", "scripts": { - "test": "cm-runtests", "prepare": "cm-buildhelper src/index.ts" }, "keywords": [ diff --git a/app/lezer-markdown/package.json b/app/lezer-markdown/package.json index 2839a5c1782c..a84e9f4cb402 100644 --- a/app/lezer-markdown/package.json +++ b/app/lezer-markdown/package.json @@ -13,8 +13,6 @@ "author": "Marijn Haverbeke ", "license": "MIT", "devDependencies": { - "ist": "^1.1.1", - "mocha": "^10.2.0", "@lezer/html": "^1.0.0", "@marijn/buildtool": "^0.1.6" }, @@ -27,7 +25,6 @@ "url" : "https://github.com/lezer-parser/markdown.git" }, "scripts": { - "prepare": "node build.js", - "test": "mocha" + "prepare": "node build.js" } } diff --git a/app/ydoc-server/package.json b/app/ydoc-server/package.json index 6ba62e1c15b1..e6f342c77259 100644 --- a/app/ydoc-server/package.json +++ b/app/ydoc-server/package.json @@ -8,7 +8,7 @@ "email": "contact@enso.org" }, "scripts": { - "test": "vitest run", + "test:unit": "vitest run", "test:watch": "vitest", "typecheck": "tsc", "compile": "tsc", diff --git a/app/ydoc-shared/package.json b/app/ydoc-shared/package.json index 8c11a6571e63..691aa0289349 100644 --- a/app/ydoc-shared/package.json +++ b/app/ydoc-shared/package.json @@ -8,7 +8,7 @@ "email": "contact@enso.org" }, "scripts": { - "test": "vitest run", + "test:unit": "vitest run", "test:watch": "vitest", "format": "prettier --version && prettier --write src/ && eslint . --fix", "compile": "tsc" diff --git a/app/ydoc-shared/src/ast/__tests__/documentation.test.ts b/app/ydoc-shared/src/ast/__tests__/documentation.test.ts index 255526f14693..38718ac3e588 100644 --- a/app/ydoc-shared/src/ast/__tests__/documentation.test.ts +++ b/app/ydoc-shared/src/ast/__tests__/documentation.test.ts @@ -175,9 +175,13 @@ describe('Function documentation (Markdown)', () => { markdown: '- Bullet list\n - Nested list\n - Very nested list\n - Nested list\n- Bullet list', }, + { + source: '## Plain text\n - Bullet list\n Plain text\n 1. Numbered list\n Plain text', + markdown: 'Plain text\n- Bullet list\nPlain text\n1. Numbered list\nPlain text', + }, ] - test.each(cases)('Enso source comments to normalized markdown', ({ source, markdown }) => { + test.each(cases)('Enso source comments to prerendered markdown', ({ source, markdown }) => { const moduleSource = `${source}\nmain =\n x = 1` const topLevel = parseModule(moduleSource) topLevel.module.setRoot(topLevel) diff --git a/app/ydoc-shared/src/ast/documentation.ts b/app/ydoc-shared/src/ast/documentation.ts index 7fe044228888..7a2f91c62ccd 100644 --- a/app/ydoc-shared/src/ast/documentation.ts +++ b/app/ydoc-shared/src/ast/documentation.ts @@ -1,11 +1,14 @@ import { LINE_BOUNDARIES } from 'enso-common/src/utilities/data/string' -import { ensoMarkdownParser } from './ensoMarkdown' +import * as Y from 'yjs' +import { ensoMarkdownParser, ensoStandardMarkdownParser } from './ensoMarkdown' import { xxHash128 } from './ffi' import type { ConcreteChild, RawConcreteChild } from './print' import { ensureUnspaced, firstChild, preferUnspaced, unspaced } from './print' import { Token, TokenType } from './token' import type { ConcreteRefs, DeepReadonly, DocLine, TextToken } from './tree' +// === AST logic === + /** Render a documentation line to concrete tokens. */ export function* docLineToConcrete( docLine: DeepReadonly, @@ -33,49 +36,48 @@ export function* docLineToConcrete( for (const newline of docLine.newlines) yield preferUnspaced(newline) } -// === Markdown === - /** - * Render function documentation to concrete tokens. If the `markdown` content has the same value as when `docLine` was - * parsed (as indicated by `hash`), the `docLine` will be used (preserving concrete formatting). If it is different, the - * `markdown` text will be converted to source tokens. + * Render function documentation to concrete tokens. If the `markdown` content has the same value as + * when `docLine` was parsed (as indicated by `hash`), the `docLine` will be used (preserving + * concrete formatting). If it is different, the `markdown` text will be converted to source tokens. */ export function functionDocsToConcrete( - markdown: string, + markdown: DeepReadonly, hash: string | undefined, docLine: DeepReadonly | undefined, indent: string | null, ): Iterable | undefined { - return ( - hash && docLine && xxHash128(markdown) === hash ? docLineToConcrete(docLine, indent) - : markdown ? markdownYTextToTokens(markdown, (indent || '') + ' ') - : undefined - ) -} - -function markdownYTextToTokens(yText: string, indent: string): Iterable> { - const tokensBuilder = new DocTokensBuilder(indent) - standardizeMarkdown(yText, tokensBuilder) + const markdownText = markdown.toString() + if (hash && docLine && xxHash128(markdownText) === hash) return docLineToConcrete(docLine, indent) + if (!markdownText) return + const tokensBuilder = new DocTokensBuilder((indent || '') + ' ') + standardizeMarkdown(markdownText, tokensBuilder) return tokensBuilder.build() } /** - * Given Enso documentation comment tokens, returns a model of their Markdown content. This model abstracts away details - * such as the locations of line breaks that are not paragraph breaks (e.g. lone newlines denoting hard-wrapping of the - * source code). + * Given Enso documentation comment tokens, returns a model of their Markdown content. This model + * abstracts away details such as the locations of line breaks that are not paragraph breaks (e.g. + * lone newlines denoting hard-wrapping of the source code). */ -export function abstractMarkdown(elements: undefined | TextToken[]) { +export function abstractMarkdown(elements: undefined | TextToken[]): { + markdown: Y.Text + hash: string +} { const { tags, rawMarkdown } = toRawMarkdown(elements) - const markdown = [...tags, normalizeMarkdown(rawMarkdown)].join('\n') + const markdown = [...tags, prerenderMarkdown(rawMarkdown)].join('\n') const hash = xxHash128(markdown) - return { markdown, hash } + return { markdown: new Y.Text(markdown), hash } } function indentLevel(whitespace: string) { return whitespace.length + whitespace.split('\t').length - 1 } -function toRawMarkdown(elements: undefined | TextToken[]) { +function toRawMarkdown(elements: undefined | TextToken[]): { + tags: string[] + rawMarkdown: string +} { const tags: string[] = [] let readingTags = true const tokenWhitespace = ({ token: { whitespace } }: TextToken) => whitespace @@ -113,57 +115,39 @@ function toRawMarkdown(elements: undefined | TextToken[]) { return { tags, rawMarkdown } } +// === Markdown === + /** - * Convert the Markdown input to a format with rendered-style linebreaks: Hard-wrapped lines within a paragraph will be - * joined, and only a single linebreak character is used to separate paragraphs. + * Convert the Markdown input to a format with "prerendered" linebreaks: Hard-wrapped lines within + * a paragraph will be joined, and only a single linebreak character is used to separate paragraphs. */ -export function normalizeMarkdown(rawMarkdown: string): string { - let normalized = '' +export function prerenderMarkdown(markdown: string): string { + let prerendered = '' let prevTo = 0 let prevName: string | undefined = undefined - const cursor = ensoMarkdownParser.parse(rawMarkdown).cursor() + const cursor = ensoStandardMarkdownParser.parse(markdown).cursor() cursor.firstChild() do { if (prevTo < cursor.from) { - const textBetween = rawMarkdown.slice(prevTo, cursor.from) - normalized += + const textBetween = markdown.slice(prevTo, cursor.from) + prerendered += cursor.name === 'Paragraph' && prevName !== 'Table' ? textBetween.slice(0, -1) : textBetween } - const text = rawMarkdown.slice(cursor.from, cursor.to) - normalized += cursor.name === 'Paragraph' ? text.replaceAll(/ *\n */g, ' ') : text + const text = markdown.slice(cursor.from, cursor.to) + prerendered += cursor.name === 'Paragraph' ? text.replaceAll(/ *\n */g, ' ') : text prevTo = cursor.to prevName = cursor.name } while (cursor.nextSibling()) - return normalized -} - -function stringCollector() { - let output = '' - const collector = { - text: (text: string) => (output += text), - wrapText: (text: string) => (output += text), - newline: () => (output += '\n'), - } - return { collector, output } + return prerendered } /** - * Convert from "normalized" Markdown (with hard line-breaks removed) to the standard format, with paragraphs separated - * by blank lines. + * Convert from our internal "prerendered" Markdown to the (more standard-compatible) on-disk + * representation, with paragraphs hard-wrapped and separated by blank lines. */ -export function normalizedMarkdownToStandard(normalizedMarkdown: string) { - const { collector, output } = stringCollector() - standardizeMarkdown(normalizedMarkdown, collector) - return output -} - -/** - * Convert from "normalized" Markdown to the on-disk representation, with paragraphs hard-wrapped and separated by blank - * lines. - */ -function standardizeMarkdown(normalizedMarkdown: string, textConsumer: TextConsumer) { +function standardizeMarkdown(prerenderedMarkdown: string, textConsumer: TextConsumer): void { let printingTags = true - const cursor = ensoMarkdownParser.parse(normalizedMarkdown).cursor() + const cursor = ensoMarkdownParser.parse(prerenderedMarkdown).cursor() function standardizeDocument() { let prevTo = 0 @@ -171,15 +155,15 @@ function standardizeMarkdown(normalizedMarkdown: string, textConsumer: TextConsu cursor.firstChild() do { if (prevTo < cursor.from) { - const betweenText = normalizedMarkdown.slice(prevTo, cursor.from) + const betweenText = prerenderedMarkdown.slice(prevTo, cursor.from) for (const _match of betweenText.matchAll(LINE_BOUNDARIES)) { textConsumer.newline() } - if (cursor.name === 'Paragraph' && prevName !== 'Table') { + if (cursor.name === 'Paragraph' && prevName === 'Paragraph' && !printingTags) { textConsumer.newline() } } - const lines = normalizedMarkdown.slice(cursor.from, cursor.to).split(LINE_BOUNDARIES) + const lines = prerenderedMarkdown.slice(cursor.from, cursor.to).split(LINE_BOUNDARIES) if (cursor.name === 'Paragraph') { standardizeParagraph(lines) } else { @@ -218,6 +202,8 @@ function standardizeMarkdown(normalizedMarkdown: string, textConsumer: TextConsu standardizeDocument() } +// === AST utilities === + interface TextConsumer { text: (text: string) => void wrapText: (text: string) => void diff --git a/app/ydoc-shared/src/ast/ensoMarkdown.ts b/app/ydoc-shared/src/ast/ensoMarkdown.ts index a210cfa48f0c..40d7c67b7d42 100644 --- a/app/ydoc-shared/src/ast/ensoMarkdown.ts +++ b/app/ydoc-shared/src/ast/ensoMarkdown.ts @@ -9,8 +9,10 @@ import { Table, type BlockParser, type MarkdownExtension, + type MarkdownParser, } from '@lezer/markdown' +// noinspection JSUnusedGlobalSymbols (WebStorm thinks `endLeaf` is unused, unclear why) /** * End any element when a newline is encountered. This parser operates on preprocessed Markdown that * has "prerendered" newlines: Before parsing, hard-wrapped lines within any block element are @@ -22,18 +24,39 @@ const newlineEndsBlock: BlockParser = { !(line.text.startsWith('|') && line.text.length > 2 && line.text.endsWith('|')), } -/** @lezer/markdown extension for the Markdown dialect used in the Enso documentation editor. */ -export const ensoMarkdownExtension: MarkdownExtension = [ +const ensoMarkdownDialect = [ Table, Strikethrough, - { parseBlock: [newlineEndsBlock] }, /** - * When starting a bulleted list, the `SetextHeading` parser can match when a `-` has been typed and a following space - * hasn't been entered yet; the resulting style changes are distracting. To prevent this, we don't support setext - * headings; ATX headings seem to be much more popular anyway. + * When starting a bulleted list, the `SetextHeading` parser can match when a `-` has been typed + * and a following space hasn't been entered yet; the resulting style changes are distracting. To + * prevent this, we don't support setext headings; ATX headings seem to be much more popular + * anyway. */ { remove: ['SetextHeading'] }, ] -/** Headless @lezer/markdown parser for the Markdown dialect used in the Enso documentation editor. */ -export const ensoMarkdownParser = commonmarkParser.configure(ensoMarkdownExtension) +const prerenderedRepresentation = { parseBlock: [newlineEndsBlock] } + +/** {@link MarkdownExtension} for Markdown as used in the Enso documentation editor. */ +export const ensoMarkdownExtension: MarkdownExtension = [ + ensoMarkdownDialect, + prerenderedRepresentation, +] + +/** + * Headless {@link MarkdownParser} for the Markdown representation used by the Enso documentation + * editor. + * + * This parses the working representation that the documentation editor operates on, with + * "prerendered" newlines. + */ +export const ensoMarkdownParser: MarkdownParser = commonmarkParser.configure(ensoMarkdownExtension) + +/** + * Headless {@link MarkdownParser} for the Markdown dialect used in Enso documentation comments. + * + * This parses the "standard" representation as appears in Enso files. + */ +export const ensoStandardMarkdownParser: MarkdownParser = + commonmarkParser.configure(ensoMarkdownDialect) diff --git a/app/ydoc-shared/src/ast/parse.ts b/app/ydoc-shared/src/ast/parse.ts index 6829a9766b06..4437bd0271d8 100644 --- a/app/ydoc-shared/src/ast/parse.ts +++ b/app/ydoc-shared/src/ast/parse.ts @@ -351,7 +351,7 @@ class Abstractor { return FunctionDef.concrete(this.module, { docLine, docLineMarkdownHash, - docMarkdown: new Y.Text(docMarkdown), + docMarkdown, annotationLines, signatureLine, private_, diff --git a/app/ydoc-shared/src/ast/tree.ts b/app/ydoc-shared/src/ast/tree.ts index eab7fcace3b0..8710ba0d08e7 100644 --- a/app/ydoc-shared/src/ast/tree.ts +++ b/app/ydoc-shared/src/ast/tree.ts @@ -2458,7 +2458,7 @@ export class FunctionDef extends BaseStatement { return preferUnspaced(nodeChild) } } - const docs = functionDocsToConcrete(docMarkdown.toJSON(), docLineMarkdownHash, docLine, indent) + const docs = functionDocsToConcrete(docMarkdown, docLineMarkdownHash, docLine, indent) if (docs) { yield* docs prevIsNewline = true diff --git a/package.json b/package.json index 6a1beb26238b..01466768c8ad 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,6 @@ "ci-check": "corepack pnpm run -r compile && corepack pnpm run --aggregate-output /^ci:/", "ci:prettier": "prettier --check --cache .", "ci:lint": "corepack pnpm run -r --parallel lint --output-file eslint_report.json --format json --cache-strategy content", - "ci:test": "corepack pnpm run -r --parallel test", "ci:unit-test": "corepack pnpm run -r --parallel test:unit", "ci:typecheck": "corepack pnpm run -r --parallel typecheck", "git-clean": "git clean -xfd --exclude='.env*'", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e0954f1599bb..3aef41ebe9b8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -767,12 +767,6 @@ importers: '@marijn/buildtool': specifier: ^0.1.6 version: 0.1.6 - ist: - specifier: ^1.1.1 - version: 1.1.7 - mocha: - specifier: ^10.2.0 - version: 10.8.2 app/rust-ffi: {}