Integrate @lezer/markdown and codemirror/lang-markdown as subtrees (#12371)

Integrate @lezer/markdown and codemirror/lang-markdown with our repo; port our  customizations to direct modifications to the implementation; reorganize and extend the `ensoMarkdown` tests covering @lezer/markdown customizations.

Author: kazcw

.bazelignore

@@ -13,6 +13,8 @@ app/ide-desktop/lib/esbuild-plugin-copy-directories/node_modules
 app/ide-desktop/lib/icons/node_modules
 app/ide-desktop/lib/ts-plugin-namespace-auto-import/node_modules
 app/ide-desktop/node_modules
+app/lang-markdown/node_modules
+app/lezer-markdown/node_modules
 app/rust-ffi/node_modules
 app/ydoc-server/node_modules
 app/ydoc-server-nodejs/node_modules
@@ -33,4 +35,4 @@ build-cache
 .dist
 target
 build
-.git
+.git

app/gui/package.json

@@ -53,7 +53,7 @@
     "@aws-amplify/core": "5.8.5",
     "@babel/parser": "^7.26.3",
     "@codemirror/commands": "^6.7.1",
-    "@codemirror/lang-markdown": "^6.3.1",
+    "@codemirror/lang-markdown": "workspace:*",
     "@codemirror/language": "^6.10.8",
     "@codemirror/lint": "^6.8.4",
     "@codemirror/search": "^6.5.8",

app/gui/src/project-view/components/MarkdownEditor/codemirror/formatting/block.ts

@@ -275,7 +275,7 @@ function toggleQuoteInner(
 function findQuoteMark(node: SyntaxNode): { from: number; to: number } | null {
   const cursor = node.cursor()
   do {
-    if (cursor.type.name === 'EnsoBlockquote') {
+    if (cursor.type.name === 'Blockquote') {
       const quoteMark = cursor.node.getChild('QuoteMark')
       if (quoteMark == null) return null
       return { from: quoteMark.from, to: quoteMark.to }

app/gui/src/project-view/components/MarkdownEditor/markdown/syntax.ts

@@ -1,76 +1,7 @@
-import { markdown as markdownExtension } from '@codemirror/lang-markdown'
-import {
-  defineLanguageFacet,
-  foldNodeProp,
-  foldService,
-  indentNodeProp,
-  Language,
-  languageDataProp,
-  syntaxTree,
-} from '@codemirror/language'
+import { markdown } from '@codemirror/lang-markdown'
+import { foldNodeProp } from '@codemirror/language'
 import { type Extension } from '@codemirror/state'
-import { NodeProp, type NodeType, type Parser, type SyntaxNode } from '@lezer/common'
-import { ensoMarkdownParser } from 'ydoc-shared/ast/ensoMarkdown'
-
-function mkLang(parser: Parser) {
-  return new Language(data, parser, [headerIndent], 'markdown')
-}
-
-const data = defineLanguageFacet({ commentTokens: { block: { open: '<!--', close: '-->' } } })
-
-const headingProp = new NodeProp<number>()
-
-const commonmarkCodemirrorLanguageExtension = {
-  props: [
-    foldNodeProp.add((type) => {
-      return !type.is('Block') || type.is('Document') || isHeading(type) != null || isList(type) ?
-          undefined
-        : (tree, state) => ({ from: state.doc.lineAt(tree.from).to, to: tree.to })
-    }),
-    headingProp.add(isHeading),
-    indentNodeProp.add({
-      Document: () => null,
-    }),
-    languageDataProp.add({
-      Document: data,
-    }),
-  ],
-}
-
-function isHeading(type: NodeType) {
-  const match = /^(?:ATX|Setext)Heading(\d)$/.exec(type.name)
-  return match ? +match[1]! : undefined
-}
-
-function isList(type: NodeType) {
-  return type.name == 'OrderedList' || type.name == 'BulletList'
-}
-
-function findSectionEnd(headerNode: SyntaxNode, level: number) {
-  let last = headerNode
-  for (;;) {
-    const next = last.nextSibling
-    let heading
-    if (!next || ((heading = isHeading(next.type)) != null && heading <= level)) break
-    last = next
-  }
-  return last.to
-}
-
-const headerIndent = foldService.of((state, start, end) => {
-  for (
-    let node: SyntaxNode | null = syntaxTree(state).resolveInner(end, -1);
-    node;
-    node = node.parent
-  ) {
-    if (node.from < start) break
-    const heading = node.type.prop(headingProp)
-    if (heading == null) continue
-    const upto = findSectionEnd(node, heading)
-    if (upto > end) return { from: end, to: upto }
-  }
-  return null
-})
+import { ensoMarkdownExtension } from 'ydoc-shared/ast/ensoMarkdown'
 
 const tableCodemirrorLanguageExtension = {
   props: [
@@ -80,13 +11,8 @@ const tableCodemirrorLanguageExtension = {
   ],
 }
 
-const extension = markdownExtension({
-  base: mkLang(
-    ensoMarkdownParser.configure([
-      commonmarkCodemirrorLanguageExtension,
-      tableCodemirrorLanguageExtension,
-    ]),
-  ),
+const extension = markdown({
+  extensions: [ensoMarkdownExtension, tableCodemirrorLanguageExtension],
 })
 
 export const ensoMarkdownSyntax = (): Extension => extension

app/lang-markdown/BUILD.bazel

@@ -0,0 +1,34 @@
+load("@aspect_rules_js//npm:defs.bzl", "npm_package")
+load("@aspect_rules_ts//ts:defs.bzl", "ts_config", "ts_project")
+load("@npm//:defs.bzl", "npm_link_all_packages", "npm_link_targets")
+
+npm_link_all_packages(name = "node_modules")
+
+ts_config(
+    name = "tsconfig",
+    src = "tsconfig.json",
+    deps = ["//:tsconfig"],
+)
+
+ts_project(
+    name = "tsc",
+    srcs = glob(["src/*.ts"]),
+    composite = True,
+    out_dir = "dist",
+    root_dir = "src",
+    tsconfig = ":tsconfig",
+    validate = select({
+        "@platforms//os:windows": False,
+        "//conditions:default": True,
+    }),
+    deps = npm_link_targets(),
+)
+
+npm_package(
+    name = "pkg",
+    srcs = [
+        "package.json",
+        ":tsc",
+    ],
+    visibility = ["//visibility:public"],
+)

app/lang-markdown/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@codemirror/lang-markdown",
   "version": "6.3.2",
+  "private": true,
   "description": "Markdown language support for the CodeMirror code editor",
   "scripts": {
     "test": "cm-runtests",
@@ -31,7 +32,7 @@
     "@codemirror/language": "^6.3.0",
     "@codemirror/state": "^6.0.0",
     "@codemirror/view": "^6.0.0",
-    "@lezer/markdown": "^1.0.0",
+    "@lezer/markdown": "workspace:*",
     "@lezer/common": "^1.2.1"
   },
   "devDependencies": {

app/lang-markdown/src/markdown.ts

@@ -1,7 +1,7 @@
 import {Language, defineLanguageFacet, languageDataProp, foldNodeProp, indentNodeProp, foldService,
         syntaxTree, LanguageDescription, ParseContext} from "@codemirror/language"
 import {parser as baseParser, MarkdownParser, GFM, Subscript, Superscript, Emoji} from "@lezer/markdown"
-import {SyntaxNode, NodeType, NodeProp} from "@lezer/common"
+import {SyntaxNode, NodeType, NodeProp, Parser} from "@lezer/common"
 
 const data = defineLanguageFacet({commentTokens: {block: {open: "<!--", close: "-->"}}})
 
@@ -25,7 +25,7 @@ const commonmark = baseParser.configure({
 
 function isHeading(type: NodeType) {
   let match = /^(?:ATX|Setext)Heading(\d)$/.exec(type.name)
-  return match ? +match[1] : undefined
+  return match ? +match[1]! : undefined
 }
 
 function isList(type: NodeType) {
@@ -75,7 +75,7 @@ export const markdownLanguage = mkLang(extended)
 export function getCodeParser(
   languages: readonly LanguageDescription[] | ((info: string) => Language | LanguageDescription | null) | undefined,
   defaultLanguage?: Language
-) {
+): (info: string) => Parser | null {
   return (info: string) => {
     if (info && languages) {
       let found = null

app/lang-markdown/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "lib": ["es2017"],
+    "noImplicitReturns": true,
+    "noUnusedLocals": true,
+    "strict": true,
+    "target": "es6",
+    "composite": true,
+    "module": "es2015",
+    "newLine": "lf",
+    "stripInternal": true,
+    "moduleResolution": "node"
+  },
+  "include": ["src/*.ts"]
+}

app/lezer-markdown/BUILD.bazel

@@ -0,0 +1,34 @@
+load("@aspect_rules_js//npm:defs.bzl", "npm_package")
+load("@aspect_rules_ts//ts:defs.bzl", "ts_config", "ts_project")
+load("@npm//:defs.bzl", "npm_link_all_packages", "npm_link_targets")
+
+npm_link_all_packages(name = "node_modules")
+
+ts_config(
+    name = "tsconfig",
+    src = "tsconfig.json",
+    deps = ["//:tsconfig"],
+)
+
+ts_project(
+    name = "tsc",
+    srcs = glob(["src/*.ts"]),
+    composite = True,
+    out_dir = "dist",
+    root_dir = "src",
+    tsconfig = ":tsconfig",
+    validate = select({
+        "@platforms//os:windows": False,
+        "//conditions:default": True,
+    }),
+    deps = npm_link_targets(),
+)
+
+npm_package(
+    name = "pkg",
+    srcs = [
+        "package.json",
+        ":tsc",
+    ],
+    visibility = ["//visibility:public"],
+)

app/lezer-markdown/package.json

@@ -16,8 +16,6 @@
     "ist": "^1.1.1",
     "mocha": "^10.2.0",
     "@lezer/html": "^1.0.0",
-    "getdocs-ts": "^0.1.0",
-    "builddocs": "^1.0.0",
     "@marijn/buildtool": "^0.1.6"
   },
   "dependencies": {
@@ -29,9 +27,7 @@
     "url" : "https://github.com/lezer-parser/markdown.git"
   },
   "scripts": {
-    "watch": "node build.js --watch",
     "prepare": "node build.js",
-    "test": "mocha",
-    "build-readme": "node bin/build-readme.cjs"
+    "test": "mocha"
   }
 }