Skip to content

Commit 725e6b2

Browse files
authored
Integrate @lezer/markdown and codemirror/lang-markdown as subtrees (#12371)
Integrate @lezer/markdown and codemirror/lang-markdown with our repo; port our customizations to direct modifications to the implementation; reorganize and extend the `ensoMarkdown` tests covering @lezer/markdown customizations.
1 parent 48f23a1 commit 725e6b2

File tree

20 files changed

+935
-878
lines changed

20 files changed

+935
-878
lines changed

.bazelignore

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ app/ide-desktop/lib/esbuild-plugin-copy-directories/node_modules
1313
app/ide-desktop/lib/icons/node_modules
1414
app/ide-desktop/lib/ts-plugin-namespace-auto-import/node_modules
1515
app/ide-desktop/node_modules
16+
app/lang-markdown/node_modules
17+
app/lezer-markdown/node_modules
1618
app/rust-ffi/node_modules
1719
app/ydoc-server/node_modules
1820
app/ydoc-server-nodejs/node_modules
@@ -33,4 +35,4 @@ build-cache
3335
.dist
3436
target
3537
build
36-
.git
38+
.git

app/gui/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353
"@aws-amplify/core": "5.8.5",
5454
"@babel/parser": "^7.26.3",
5555
"@codemirror/commands": "^6.7.1",
56-
"@codemirror/lang-markdown": "^6.3.1",
56+
"@codemirror/lang-markdown": "workspace:*",
5757
"@codemirror/language": "^6.10.8",
5858
"@codemirror/lint": "^6.8.4",
5959
"@codemirror/search": "^6.5.8",

app/gui/src/project-view/components/MarkdownEditor/codemirror/formatting/block.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@ function toggleQuoteInner(
275275
function findQuoteMark(node: SyntaxNode): { from: number; to: number } | null {
276276
const cursor = node.cursor()
277277
do {
278-
if (cursor.type.name === 'EnsoBlockquote') {
278+
if (cursor.type.name === 'Blockquote') {
279279
const quoteMark = cursor.node.getChild('QuoteMark')
280280
if (quoteMark == null) return null
281281
return { from: quoteMark.from, to: quoteMark.to }
Lines changed: 5 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -1,76 +1,7 @@
1-
import { markdown as markdownExtension } from '@codemirror/lang-markdown'
2-
import {
3-
defineLanguageFacet,
4-
foldNodeProp,
5-
foldService,
6-
indentNodeProp,
7-
Language,
8-
languageDataProp,
9-
syntaxTree,
10-
} from '@codemirror/language'
1+
import { markdown } from '@codemirror/lang-markdown'
2+
import { foldNodeProp } from '@codemirror/language'
113
import { type Extension } from '@codemirror/state'
12-
import { NodeProp, type NodeType, type Parser, type SyntaxNode } from '@lezer/common'
13-
import { ensoMarkdownParser } from 'ydoc-shared/ast/ensoMarkdown'
14-
15-
function mkLang(parser: Parser) {
16-
return new Language(data, parser, [headerIndent], 'markdown')
17-
}
18-
19-
const data = defineLanguageFacet({ commentTokens: { block: { open: '<!--', close: '-->' } } })
20-
21-
const headingProp = new NodeProp<number>()
22-
23-
const commonmarkCodemirrorLanguageExtension = {
24-
props: [
25-
foldNodeProp.add((type) => {
26-
return !type.is('Block') || type.is('Document') || isHeading(type) != null || isList(type) ?
27-
undefined
28-
: (tree, state) => ({ from: state.doc.lineAt(tree.from).to, to: tree.to })
29-
}),
30-
headingProp.add(isHeading),
31-
indentNodeProp.add({
32-
Document: () => null,
33-
}),
34-
languageDataProp.add({
35-
Document: data,
36-
}),
37-
],
38-
}
39-
40-
function isHeading(type: NodeType) {
41-
const match = /^(?:ATX|Setext)Heading(\d)$/.exec(type.name)
42-
return match ? +match[1]! : undefined
43-
}
44-
45-
function isList(type: NodeType) {
46-
return type.name == 'OrderedList' || type.name == 'BulletList'
47-
}
48-
49-
function findSectionEnd(headerNode: SyntaxNode, level: number) {
50-
let last = headerNode
51-
for (;;) {
52-
const next = last.nextSibling
53-
let heading
54-
if (!next || ((heading = isHeading(next.type)) != null && heading <= level)) break
55-
last = next
56-
}
57-
return last.to
58-
}
59-
60-
const headerIndent = foldService.of((state, start, end) => {
61-
for (
62-
let node: SyntaxNode | null = syntaxTree(state).resolveInner(end, -1);
63-
node;
64-
node = node.parent
65-
) {
66-
if (node.from < start) break
67-
const heading = node.type.prop(headingProp)
68-
if (heading == null) continue
69-
const upto = findSectionEnd(node, heading)
70-
if (upto > end) return { from: end, to: upto }
71-
}
72-
return null
73-
})
4+
import { ensoMarkdownExtension } from 'ydoc-shared/ast/ensoMarkdown'
745

756
const tableCodemirrorLanguageExtension = {
767
props: [
@@ -80,13 +11,8 @@ const tableCodemirrorLanguageExtension = {
8011
],
8112
}
8213

83-
const extension = markdownExtension({
84-
base: mkLang(
85-
ensoMarkdownParser.configure([
86-
commonmarkCodemirrorLanguageExtension,
87-
tableCodemirrorLanguageExtension,
88-
]),
89-
),
14+
const extension = markdown({
15+
extensions: [ensoMarkdownExtension, tableCodemirrorLanguageExtension],
9016
})
9117

9218
export const ensoMarkdownSyntax = (): Extension => extension

app/lang-markdown/BUILD.bazel

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
load("@aspect_rules_js//npm:defs.bzl", "npm_package")
2+
load("@aspect_rules_ts//ts:defs.bzl", "ts_config", "ts_project")
3+
load("@npm//:defs.bzl", "npm_link_all_packages", "npm_link_targets")
4+
5+
npm_link_all_packages(name = "node_modules")
6+
7+
ts_config(
8+
name = "tsconfig",
9+
src = "tsconfig.json",
10+
deps = ["//:tsconfig"],
11+
)
12+
13+
ts_project(
14+
name = "tsc",
15+
srcs = glob(["src/*.ts"]),
16+
composite = True,
17+
out_dir = "dist",
18+
root_dir = "src",
19+
tsconfig = ":tsconfig",
20+
validate = select({
21+
"@platforms//os:windows": False,
22+
"//conditions:default": True,
23+
}),
24+
deps = npm_link_targets(),
25+
)
26+
27+
npm_package(
28+
name = "pkg",
29+
srcs = [
30+
"package.json",
31+
":tsc",
32+
],
33+
visibility = ["//visibility:public"],
34+
)

app/lang-markdown/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"name": "@codemirror/lang-markdown",
33
"version": "6.3.2",
4+
"private": true,
45
"description": "Markdown language support for the CodeMirror code editor",
56
"scripts": {
67
"test": "cm-runtests",
@@ -31,7 +32,7 @@
3132
"@codemirror/language": "^6.3.0",
3233
"@codemirror/state": "^6.0.0",
3334
"@codemirror/view": "^6.0.0",
34-
"@lezer/markdown": "^1.0.0",
35+
"@lezer/markdown": "workspace:*",
3536
"@lezer/common": "^1.2.1"
3637
},
3738
"devDependencies": {

app/lang-markdown/src/markdown.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import {Language, defineLanguageFacet, languageDataProp, foldNodeProp, indentNodeProp, foldService,
22
syntaxTree, LanguageDescription, ParseContext} from "@codemirror/language"
33
import {parser as baseParser, MarkdownParser, GFM, Subscript, Superscript, Emoji} from "@lezer/markdown"
4-
import {SyntaxNode, NodeType, NodeProp} from "@lezer/common"
4+
import {SyntaxNode, NodeType, NodeProp, Parser} from "@lezer/common"
55

66
const data = defineLanguageFacet({commentTokens: {block: {open: "<!--", close: "-->"}}})
77

@@ -25,7 +25,7 @@ const commonmark = baseParser.configure({
2525

2626
function isHeading(type: NodeType) {
2727
let match = /^(?:ATX|Setext)Heading(\d)$/.exec(type.name)
28-
return match ? +match[1] : undefined
28+
return match ? +match[1]! : undefined
2929
}
3030

3131
function isList(type: NodeType) {
@@ -75,7 +75,7 @@ export const markdownLanguage = mkLang(extended)
7575
export function getCodeParser(
7676
languages: readonly LanguageDescription[] | ((info: string) => Language | LanguageDescription | null) | undefined,
7777
defaultLanguage?: Language
78-
) {
78+
): (info: string) => Parser | null {
7979
return (info: string) => {
8080
if (info && languages) {
8181
let found = null

app/lang-markdown/tsconfig.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"compilerOptions": {
3+
"lib": ["es2017"],
4+
"noImplicitReturns": true,
5+
"noUnusedLocals": true,
6+
"strict": true,
7+
"target": "es6",
8+
"composite": true,
9+
"module": "es2015",
10+
"newLine": "lf",
11+
"stripInternal": true,
12+
"moduleResolution": "node"
13+
},
14+
"include": ["src/*.ts"]
15+
}

app/lezer-markdown/BUILD.bazel

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
load("@aspect_rules_js//npm:defs.bzl", "npm_package")
2+
load("@aspect_rules_ts//ts:defs.bzl", "ts_config", "ts_project")
3+
load("@npm//:defs.bzl", "npm_link_all_packages", "npm_link_targets")
4+
5+
npm_link_all_packages(name = "node_modules")
6+
7+
ts_config(
8+
name = "tsconfig",
9+
src = "tsconfig.json",
10+
deps = ["//:tsconfig"],
11+
)
12+
13+
ts_project(
14+
name = "tsc",
15+
srcs = glob(["src/*.ts"]),
16+
composite = True,
17+
out_dir = "dist",
18+
root_dir = "src",
19+
tsconfig = ":tsconfig",
20+
validate = select({
21+
"@platforms//os:windows": False,
22+
"//conditions:default": True,
23+
}),
24+
deps = npm_link_targets(),
25+
)
26+
27+
npm_package(
28+
name = "pkg",
29+
srcs = [
30+
"package.json",
31+
":tsc",
32+
],
33+
visibility = ["//visibility:public"],
34+
)

app/lezer-markdown/package.json

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@
1616
"ist": "^1.1.1",
1717
"mocha": "^10.2.0",
1818
"@lezer/html": "^1.0.0",
19-
"getdocs-ts": "^0.1.0",
20-
"builddocs": "^1.0.0",
2119
"@marijn/buildtool": "^0.1.6"
2220
},
2321
"dependencies": {
@@ -29,9 +27,7 @@
2927
"url" : "https://github.com/lezer-parser/markdown.git"
3028
},
3129
"scripts": {
32-
"watch": "node build.js --watch",
3330
"prepare": "node build.js",
34-
"test": "mocha",
35-
"build-readme": "node bin/build-readme.cjs"
31+
"test": "mocha"
3632
}
3733
}

app/lezer-markdown/src/extension.ts

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@ import {InlineContext, BlockContext, MarkdownConfig,
22
LeafBlockParser, LeafBlock, Line, Element, space, Punctuation} from "./markdown"
33
import {tags as t} from "@lezer/highlight"
44

5+
const allowEmptyFormatNodes = true
6+
const allowSpaceBeforeCloseDelimiter = true
7+
58
const StrikethroughDelim = {resolve: "Strikethrough", mark: "StrikethroughMark"}
69

710
/// An extension that implements
@@ -18,13 +21,16 @@ export const Strikethrough: MarkdownConfig = {
1821
parseInline: [{
1922
name: "Strikethrough",
2023
parse(cx, next, pos) {
21-
if (next != 126 /* '~' */ || cx.char(pos + 1) != 126 || cx.char(pos + 2) == 126) return -1
22-
let before = cx.slice(pos - 1, pos), after = cx.slice(pos + 2, pos + 3)
23-
let sBefore = /\s|^$/.test(before), sAfter = /\s|^$/.test(after)
24+
if (next != 126 /* '~' */ || cx.char(pos + 1) != 126 ||
25+
(!allowEmptyFormatNodes && cx.char(pos + 2) == 126)) return -1
26+
let before = cx.slice(pos - 1, pos), after = cx.slice(pos + 2, cx.end)
27+
let sBefore = /\s|^$/.test(before), sAfter = /^(?:\s|$)/.test(after)
2428
let pBefore = Punctuation.test(before), pAfter = Punctuation.test(after)
29+
let wAfter = /^[\p{S}|\p{P}]*\p{L}/u.test(after)
2530
return cx.addDelimiter(StrikethroughDelim, pos, pos + 2,
2631
!sAfter && (!pAfter || sBefore || pBefore),
27-
!sBefore && (!pBefore || sAfter || pAfter))
32+
(!sBefore && (!pBefore || sAfter || pAfter)) ||
33+
(allowSpaceBeforeCloseDelimiter && sBefore && !wAfter))
2834
},
2935
after: "Emphasis"
3036
}]

0 commit comments

Comments
 (0)