Skip to content

Commit

Permalink
Draft incremental markdown parsing
Browse files Browse the repository at this point in the history 
Specially useful when dealing with long Markdown
streams, like LLMs.
  • Loading branch information
@hecrj
hecrj committed Jan 31, 2025
1 parent 6aab76e commit 128058e
Show file tree
Hide file tree
Showing 3 changed files with 168 additions and 29 deletions.
2 changes: 1 addition & 1 deletion examples/markdown/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ publish = false

[dependencies]
iced.workspace = true
iced.features = ["markdown", "highlighter", "debug"]
iced.features = ["markdown", "highlighter", "tokio", "debug"]

open = "5.3"
96 changes: 85 additions & 11 deletions examples/markdown/src/main.rs
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,37 @@
use iced::highlighter;
use iced::widget::{self, markdown, row, scrollable, text_editor};
use iced::{Element, Fill, Font, Task, Theme};
use iced::time::{self, milliseconds};
use iced::widget::{
self, hover, markdown, right, row, scrollable, text_editor, toggler,
};
use iced::{Element, Fill, Font, Subscription, Task, Theme};

pub fn main() -> iced::Result {
iced::application("Markdown - Iced", Markdown::update, Markdown::view)
.subscription(Markdown::subscription)
.theme(Markdown::theme)
.run_with(Markdown::new)
}

struct Markdown {
content: text_editor::Content,
items: Vec<markdown::Item>,
mode: Mode,
theme: Theme,
}

enum Mode {
Oneshot(Vec<markdown::Item>),
Stream {
pending: String,
parsed: markdown::Content,
},
}

#[derive(Debug, Clone)]
enum Message {
Edit(text_editor::Action),
LinkClicked(markdown::Url),
ToggleStream(bool),
NextToken,
}

impl Markdown {
Expand All @@ -29,7 +43,7 @@ impl Markdown {
(
Self {
content: text_editor::Content::with_text(INITIAL_CONTENT),
items: markdown::parse(INITIAL_CONTENT).collect(),
mode: Mode::Oneshot(markdown::parse(INITIAL_CONTENT).collect()),
theme,
},
widget::focus_next(),
Expand All @@ -44,13 +58,48 @@ impl Markdown {
self.content.perform(action);

if is_edit {
self.items =
markdown::parse(&self.content.text()).collect();
self.mode = match self.mode {
Mode::Oneshot(_) => Mode::Oneshot(
markdown::parse(&self.content.text()).collect(),
),
Mode::Stream { .. } => Mode::Stream {
pending: self.content.text(),
parsed: markdown::Content::parse(""),
},
}
}
}
Message::LinkClicked(link) => {
let _ = open::that_in_background(link.to_string());
}
Message::ToggleStream(enable_stream) => {
self.mode = if enable_stream {
Mode::Stream {
pending: self.content.text(),
parsed: markdown::Content::parse(""),
}
} else {
Mode::Oneshot(
markdown::parse(&self.content.text()).collect(),
)
};
}
Message::NextToken => match &mut self.mode {
Mode::Oneshot(_) => {}
Mode::Stream { pending, parsed } => {
if pending.is_empty() {
self.mode = Mode::Oneshot(parsed.items().to_vec());
} else {
let mut tokens = pending.split(' ');

if let Some(token) = tokens.next() {
parsed.push_str(&format!("{token} "));
}

*pending = tokens.collect::<Vec<_>>().join(" ");
}
}
},
}
}

Expand All @@ -63,20 +112,45 @@ impl Markdown {
.font(Font::MONOSPACE)
.highlight("markdown", highlighter::Theme::Base16Ocean);

let items = match &self.mode {
Mode::Oneshot(items) => items.as_slice(),
Mode::Stream { parsed, .. } => parsed.items(),
};

let preview = markdown(
&self.items,
items,
markdown::Settings::default(),
markdown::Style::from_palette(self.theme.palette()),
)
.map(Message::LinkClicked);

row![editor, scrollable(preview).spacing(10).height(Fill)]
.spacing(10)
.padding(10)
.into()
row![
editor,
hover(
scrollable(preview).spacing(10).width(Fill).height(Fill),
right(
toggler(matches!(self.mode, Mode::Stream { .. }))
.label("Stream")
.on_toggle(Message::ToggleStream)
)
.padding([0, 20])
)
]
.spacing(10)
.padding(10)
.into()
}

fn theme(&self) -> Theme {
self.theme.clone()
}

fn subscription(&self) -> Subscription<Message> {
match self.mode {
Mode::Oneshot(_) => Subscription::none(),
Mode::Stream { .. } => {
time::every(milliseconds(20)).map(|_| Message::NextToken)
}
}
}
}
99 changes: 82 additions & 17 deletions widget/src/markdown.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
//! }
//! }
//! ```
#![allow(missing_docs)]
use crate::core::border;
use crate::core::font::{self, Font};
use crate::core::padding;
Expand All @@ -57,12 +58,47 @@ use crate::core::{
use crate::{column, container, rich_text, row, scrollable, span, text};

use std::cell::{Cell, RefCell};
use std::ops::Range;
use std::sync::Arc;

pub use core::text::Highlight;
pub use pulldown_cmark::HeadingLevel;
pub use url::Url;

#[derive(Debug, Clone)]
pub struct Content {
items: Vec<Item>,
state: State,
}

impl Content {
pub fn parse(markdown: &str) -> Self {
let mut state = State::default();
let items = parse_with(&mut state, markdown).collect();

Self { items, state }
}

pub fn push_str(&mut self, markdown: &str) {
// Append to last leftover text
let mut leftover = std::mem::take(&mut self.state.leftover);
leftover.push_str(markdown);

// Pop the last item
let _ = self.items.pop();

// Re-parse last item and new text
let new_items = parse_with(&mut self.state, &leftover);
self.items.extend(new_items);

dbg!(&self.state);
}

pub fn items(&self) -> &[Item] {
&self.items
}
}

/// A Markdown item.
#[derive(Debug, Clone)]
pub enum Item {
Expand Down Expand Up @@ -232,6 +268,24 @@ impl Span {
/// }
/// ```
pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
parse_with(State::default(), markdown)
}

#[derive(Debug, Clone, Default)]
pub struct State {
leftover: String,
}

impl AsMut<Self> for State {
fn as_mut(&mut self) -> &mut Self {
self
}
}

fn parse_with<'a>(
mut state: impl AsMut<State> + 'a,
markdown: &'a str,
) -> impl Iterator<Item = Item> + 'a {
struct List {
start: Option<u64>,
items: Vec<Vec<Item>>,
Expand All @@ -255,27 +309,31 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
| pulldown_cmark::Options::ENABLE_PLUSES_DELIMITED_METADATA_BLOCKS
| pulldown_cmark::Options::ENABLE_TABLES
| pulldown_cmark::Options::ENABLE_STRIKETHROUGH,
);

let produce = |lists: &mut Vec<List>, item| {
if lists.is_empty() {
Some(item)
} else {
lists
.last_mut()
.expect("list context")
.items
.last_mut()
.expect("item context")
.push(item);
)
.into_offset_iter();

None
}
};
let mut produce =
move |lists: &mut Vec<List>, item, source: Range<usize>| {
if lists.is_empty() {
state.as_mut().leftover = markdown[source.start..].to_owned();

Some(item)
} else {
lists
.last_mut()
.expect("list context")
.items
.last_mut()
.expect("item context")
.push(item);

None
}
};

// We want to keep the `spans` capacity
#[allow(clippy::drain_collect)]
parser.filter_map(move |event| match event {
parser.filter_map(move |(event, source)| match event {
pulldown_cmark::Event::Start(tag) => match tag {
pulldown_cmark::Tag::Strong if !metadata && !table => {
strong = true;
Expand Down Expand Up @@ -311,6 +369,7 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
produce(
&mut lists,
Item::Paragraph(Text::new(spans.drain(..).collect())),
source,
)
};

Expand Down Expand Up @@ -350,6 +409,7 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
produce(
&mut lists,
Item::Paragraph(Text::new(spans.drain(..).collect())),
source,
)
};

Expand All @@ -370,6 +430,7 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
produce(
&mut lists,
Item::Heading(level, Text::new(spans.drain(..).collect())),
source,
)
}
pulldown_cmark::TagEnd::Strong if !metadata && !table => {
Expand All @@ -392,6 +453,7 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
produce(
&mut lists,
Item::Paragraph(Text::new(spans.drain(..).collect())),
source,
)
}
pulldown_cmark::TagEnd::Item if !metadata && !table => {
Expand All @@ -401,6 +463,7 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
produce(
&mut lists,
Item::Paragraph(Text::new(spans.drain(..).collect())),
source,
)
}
}
Expand All @@ -413,6 +476,7 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
start: list.start,
items: list.items,
},
source,
)
}
pulldown_cmark::TagEnd::CodeBlock if !metadata && !table => {
Expand All @@ -424,6 +488,7 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
produce(
&mut lists,
Item::CodeBlock(Text::new(spans.drain(..).collect())),
source,
)
}
pulldown_cmark::TagEnd::MetadataBlock(_) => {
Expand Down

0 comments on commit 128058e

Please sign in to comment.