Skip to content

Commit 128058e

Browse files
hecrjhecrj
committed
Draft incremental markdown parsing
Specially useful when dealing with long Markdown streams, like LLMs.
1 parent 6aab76e commit 128058e

File tree

3 files changed

+168
-29
lines changed

3 files changed

+168
-29
lines changed

‎examples/markdown/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,6 @@ publish = false
77

88
[dependencies]
99
iced.workspace = true
10-
iced.features = ["markdown", "highlighter", "debug"]
10+
iced.features = ["markdown", "highlighter", "tokio", "debug"]
1111

1212
open = "5.3"

‎examples/markdown/src/main.rs‎

Lines changed: 85 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,37 @@
11
use iced::highlighter;
2-
use iced::widget::{self, markdown, row, scrollable, text_editor};
3-
use iced::{Element, Fill, Font, Task, Theme};
2+
use iced::time::{self, milliseconds};
3+
use iced::widget::{
4+
self, hover, markdown, right, row, scrollable, text_editor, toggler,
5+
};
6+
use iced::{Element, Fill, Font, Subscription, Task, Theme};
47

58
pub fn main() -> iced::Result {
69
iced::application("Markdown - Iced", Markdown::update, Markdown::view)
10+
.subscription(Markdown::subscription)
711
.theme(Markdown::theme)
812
.run_with(Markdown::new)
913
}
1014

1115
struct Markdown {
1216
content: text_editor::Content,
13-
items: Vec<markdown::Item>,
17+
mode: Mode,
1418
theme: Theme,
1519
}
1620

21+
enum Mode {
22+
Oneshot(Vec<markdown::Item>),
23+
Stream {
24+
pending: String,
25+
parsed: markdown::Content,
26+
},
27+
}
28+
1729
#[derive(Debug, Clone)]
1830
enum Message {
1931
Edit(text_editor::Action),
2032
LinkClicked(markdown::Url),
33+
ToggleStream(bool),
34+
NextToken,
2135
}
2236

2337
impl Markdown {
@@ -29,7 +43,7 @@ impl Markdown {
2943
(
3044
Self {
3145
content: text_editor::Content::with_text(INITIAL_CONTENT),
32-
items: markdown::parse(INITIAL_CONTENT).collect(),
46+
mode: Mode::Oneshot(markdown::parse(INITIAL_CONTENT).collect()),
3347
theme,
3448
},
3549
widget::focus_next(),
@@ -44,13 +58,48 @@ impl Markdown {
4458
self.content.perform(action);
4559

4660
if is_edit {
47-
self.items =
48-
markdown::parse(&self.content.text()).collect();
61+
self.mode = match self.mode {
62+
Mode::Oneshot(_) => Mode::Oneshot(
63+
markdown::parse(&self.content.text()).collect(),
64+
),
65+
Mode::Stream { .. } => Mode::Stream {
66+
pending: self.content.text(),
67+
parsed: markdown::Content::parse(""),
68+
},
69+
}
4970
}
5071
}
5172
Message::LinkClicked(link) => {
5273
let _ = open::that_in_background(link.to_string());
5374
}
75+
Message::ToggleStream(enable_stream) => {
76+
self.mode = if enable_stream {
77+
Mode::Stream {
78+
pending: self.content.text(),
79+
parsed: markdown::Content::parse(""),
80+
}
81+
} else {
82+
Mode::Oneshot(
83+
markdown::parse(&self.content.text()).collect(),
84+
)
85+
};
86+
}
87+
Message::NextToken => match &mut self.mode {
88+
Mode::Oneshot(_) => {}
89+
Mode::Stream { pending, parsed } => {
90+
if pending.is_empty() {
91+
self.mode = Mode::Oneshot(parsed.items().to_vec());
92+
} else {
93+
let mut tokens = pending.split(' ');
94+
95+
if let Some(token) = tokens.next() {
96+
parsed.push_str(&format!("{token} "));
97+
}
98+
99+
*pending = tokens.collect::<Vec<_>>().join(" ");
100+
}
101+
}
102+
},
54103
}
55104
}
56105

@@ -63,20 +112,45 @@ impl Markdown {
63112
.font(Font::MONOSPACE)
64113
.highlight("markdown", highlighter::Theme::Base16Ocean);
65114

115+
let items = match &self.mode {
116+
Mode::Oneshot(items) => items.as_slice(),
117+
Mode::Stream { parsed, .. } => parsed.items(),
118+
};
119+
66120
let preview = markdown(
67-
&self.items,
121+
items,
68122
markdown::Settings::default(),
69123
markdown::Style::from_palette(self.theme.palette()),
70124
)
71125
.map(Message::LinkClicked);
72126

73-
row![editor, scrollable(preview).spacing(10).height(Fill)]
74-
.spacing(10)
75-
.padding(10)
76-
.into()
127+
row![
128+
editor,
129+
hover(
130+
scrollable(preview).spacing(10).width(Fill).height(Fill),
131+
right(
132+
toggler(matches!(self.mode, Mode::Stream { .. }))
133+
.label("Stream")
134+
.on_toggle(Message::ToggleStream)
135+
)
136+
.padding([0, 20])
137+
)
138+
]
139+
.spacing(10)
140+
.padding(10)
141+
.into()
77142
}
78143

79144
fn theme(&self) -> Theme {
80145
self.theme.clone()
81146
}
147+
148+
fn subscription(&self) -> Subscription<Message> {
149+
match self.mode {
150+
Mode::Oneshot(_) => Subscription::none(),
151+
Mode::Stream { .. } => {
152+
time::every(milliseconds(20)).map(|_| Message::NextToken)
153+
}
154+
}
155+
}
82156
}

‎widget/src/markdown.rs‎

Lines changed: 82 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
//! }
4848
//! }
4949
//! ```
50+
#![allow(missing_docs)]
5051
use crate::core::border;
5152
use crate::core::font::{self, Font};
5253
use crate::core::padding;
@@ -57,12 +58,47 @@ use crate::core::{
5758
use crate::{column, container, rich_text, row, scrollable, span, text};
5859

5960
use std::cell::{Cell, RefCell};
61+
use std::ops::Range;
6062
use std::sync::Arc;
6163

6264
pub use core::text::Highlight;
6365
pub use pulldown_cmark::HeadingLevel;
6466
pub use url::Url;
6567

68+
#[derive(Debug, Clone)]
69+
pub struct Content {
70+
items: Vec<Item>,
71+
state: State,
72+
}
73+
74+
impl Content {
75+
pub fn parse(markdown: &str) -> Self {
76+
let mut state = State::default();
77+
let items = parse_with(&mut state, markdown).collect();
78+
79+
Self { items, state }
80+
}
81+
82+
pub fn push_str(&mut self, markdown: &str) {
83+
// Append to last leftover text
84+
let mut leftover = std::mem::take(&mut self.state.leftover);
85+
leftover.push_str(markdown);
86+
87+
// Pop the last item
88+
let _ = self.items.pop();
89+
90+
// Re-parse last item and new text
91+
let new_items = parse_with(&mut self.state, &leftover);
92+
self.items.extend(new_items);
93+
94+
dbg!(&self.state);
95+
}
96+
97+
pub fn items(&self) -> &[Item] {
98+
&self.items
99+
}
100+
}
101+
66102
/// A Markdown item.
67103
#[derive(Debug, Clone)]
68104
pub enum Item {
@@ -232,6 +268,24 @@ impl Span {
232268
/// }
233269
/// ```
234270
pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
271+
parse_with(State::default(), markdown)
272+
}
273+
274+
#[derive(Debug, Clone, Default)]
275+
pub struct State {
276+
leftover: String,
277+
}
278+
279+
impl AsMut<Self> for State {
280+
fn as_mut(&mut self) -> &mut Self {
281+
self
282+
}
283+
}
284+
285+
fn parse_with<'a>(
286+
mut state: impl AsMut<State> + 'a,
287+
markdown: &'a str,
288+
) -> impl Iterator<Item = Item> + 'a {
235289
struct List {
236290
start: Option<u64>,
237291
items: Vec<Vec<Item>>,
@@ -255,27 +309,31 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
255309
| pulldown_cmark::Options::ENABLE_PLUSES_DELIMITED_METADATA_BLOCKS
256310
| pulldown_cmark::Options::ENABLE_TABLES
257311
| pulldown_cmark::Options::ENABLE_STRIKETHROUGH,
258-
);
259-
260-
let produce = |lists: &mut Vec<List>, item| {
261-
if lists.is_empty() {
262-
Some(item)
263-
} else {
264-
lists
265-
.last_mut()
266-
.expect("list context")
267-
.items
268-
.last_mut()
269-
.expect("item context")
270-
.push(item);
312+
)
313+
.into_offset_iter();
271314

272-
None
273-
}
274-
};
315+
let mut produce =
316+
move |lists: &mut Vec<List>, item, source: Range<usize>| {
317+
if lists.is_empty() {
318+
state.as_mut().leftover = markdown[source.start..].to_owned();
319+
320+
Some(item)
321+
} else {
322+
lists
323+
.last_mut()
324+
.expect("list context")
325+
.items
326+
.last_mut()
327+
.expect("item context")
328+
.push(item);
329+
330+
None
331+
}
332+
};
275333

276334
// We want to keep the `spans` capacity
277335
#[allow(clippy::drain_collect)]
278-
parser.filter_map(move |event| match event {
336+
parser.filter_map(move |(event, source)| match event {
279337
pulldown_cmark::Event::Start(tag) => match tag {
280338
pulldown_cmark::Tag::Strong if !metadata && !table => {
281339
strong = true;
@@ -311,6 +369,7 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
311369
produce(
312370
&mut lists,
313371
Item::Paragraph(Text::new(spans.drain(..).collect())),
372+
source,
314373
)
315374
};
316375

@@ -350,6 +409,7 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
350409
produce(
351410
&mut lists,
352411
Item::Paragraph(Text::new(spans.drain(..).collect())),
412+
source,
353413
)
354414
};
355415

@@ -370,6 +430,7 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
370430
produce(
371431
&mut lists,
372432
Item::Heading(level, Text::new(spans.drain(..).collect())),
433+
source,
373434
)
374435
}
375436
pulldown_cmark::TagEnd::Strong if !metadata && !table => {
@@ -392,6 +453,7 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
392453
produce(
393454
&mut lists,
394455
Item::Paragraph(Text::new(spans.drain(..).collect())),
456+
source,
395457
)
396458
}
397459
pulldown_cmark::TagEnd::Item if !metadata && !table => {
@@ -401,6 +463,7 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
401463
produce(
402464
&mut lists,
403465
Item::Paragraph(Text::new(spans.drain(..).collect())),
466+
source,
404467
)
405468
}
406469
}
@@ -413,6 +476,7 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
413476
start: list.start,
414477
items: list.items,
415478
},
479+
source,
416480
)
417481
}
418482
pulldown_cmark::TagEnd::CodeBlock if !metadata && !table => {
@@ -424,6 +488,7 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
424488
produce(
425489
&mut lists,
426490
Item::CodeBlock(Text::new(spans.drain(..).collect())),
491+
source,
427492
)
428493
}
429494
pulldown_cmark::TagEnd::MetadataBlock(_) => {

0 commit comments

Comments
 (0)