parser WIP

Author: willmcgugan

src/textual/css/tokenize.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import re
-from typing import TYPE_CHECKING, Iterable
+from typing import TYPE_CHECKING, ClassVar, Iterable
 
 from textual.css.tokenizer import Expect, Token, Tokenizer
 
@@ -176,6 +176,43 @@
 
 
 class TokenizerState:
+    EXPECT: ClassVar[Expect] = expect_root_scope
+    STATE_MAP: ClassVar[dict[str, Expect]] = {}
+    STATE_PUSH: ClassVar[dict[str, Expect]] = {}
+    STATE_POP: ClassVar[dict[str, str]] = {}
+
+    def __call__(self, code: str, read_from: CSSLocation) -> Iterable[Token]:
+        tokenizer = Tokenizer(code, read_from=read_from)
+        expect = self.EXPECT
+        get_token = tokenizer.get_token
+        get_state = self.STATE_MAP.get
+        state_stack: list[Expect] = []
+
+        skip_get_token = False
+        while True:
+            if not skip_get_token:
+                token = get_token(expect)
+            skip_get_token = False
+            name = token.name
+            if name in self.STATE_MAP:
+                expect = get_state(token.name, expect)
+            elif name in self.STATE_PUSH:
+                expect = self.STATE_PUSH[name]
+                state_stack.append(expect)
+            elif name in self.STATE_POP:
+                expect_pop = self.STATE_POP[name]
+                if state_stack:
+                    expect = state_stack.pop()
+                else:
+                    expect = self.EXPECT
+                    skip_get_token = True
+
+            yield token
+            if name == "eof":
+                break
+
+
+class TCSSTokenizerState:
     """State machine for the tokenizer.
 
     Attributes:
@@ -232,19 +269,19 @@ def __call__(self, code: str, read_from: CSSLocation) -> Iterable[Token]:
             yield token
 
 
-class DeclarationTokenizerState(TokenizerState):
+class DeclarationTokenizerState(TCSSTokenizerState):
     EXPECT = expect_declaration_solo
     STATE_MAP = {
         "declaration_name": expect_declaration_content,
         "declaration_end": expect_declaration_solo,
     }
 
 
-class ValueTokenizerState(TokenizerState):
+class ValueTokenizerState(TCSSTokenizerState):
     EXPECT = expect_declaration_content_solo
 
 
-class StyleTokenizerState(TokenizerState):
+class StyleTokenizerState(TCSSTokenizerState):
     EXPECT = (
         Expect(
             "style token",
@@ -262,7 +299,7 @@ class StyleTokenizerState(TokenizerState):
     )
 
 
-tokenize = TokenizerState()
+tokenize = TCSSTokenizerState()
 tokenize_declarations = DeclarationTokenizerState()
 tokenize_value = ValueTokenizerState()
 tokenize_style = StyleTokenizerState()
@@ -283,3 +320,25 @@ def tokenize_values(values: dict[str, str]) -> dict[str, list[Token]]:
         for name, value in values.items()
     }
     return value_tokens
+
+
+if __name__ == "__main__":
+    text = "[@click=app.notify(['foo', 500])] Click me! [/] :-)"
+
+    # text = "[@click=hello]Click"
+    from rich.console import Console
+
+    c = Console(markup=False)
+
+    from textual._profile import timer
+
+    with timer("tokenize"):
+        list(tokenize_markup(text, read_from=("", "")))
+
+    from textual.markup import _parse
+
+    with timer("_parse"):
+        list(_parse(text))
+
+    for token in tokenize_markup(text, read_from=("", "")):
+        c.print(repr(token))

src/textual/css/tokenizer.py

@@ -128,17 +128,22 @@ def __init__(self, description: str, **tokens: str) -> None:
         self.search = self._regex.search
         self._expect_eof = False
         self._expect_semicolon = True
+        self._extract_text = False
 
-    def expect_eof(self, eof: bool) -> Expect:
+    def expect_eof(self, eof: bool = True) -> Expect:
         """Expect an end of file."""
         self._expect_eof = eof
         return self
 
-    def expect_semicolon(self, semicolon: bool) -> Expect:
+    def expect_semicolon(self, semicolon: bool = True) -> Expect:
         """Tokenizer expects text to be terminated with a semi-colon."""
         self._expect_semicolon = semicolon
         return self
 
+    def extract_text(self, extract: bool = True) -> Expect:
+        self._extract_text = extract
+        return self
+
     def __rich_repr__(self) -> rich.repr.Result:
         yield from zip(self.names, self.regexes)
 
@@ -253,7 +258,30 @@ def get_token(self, expect: Expect) -> Token:
                     "Unexpected end of file; did you forget a '}' ?",
                 )
         line = self.lines[line_no]
-        match = expect.match(line, col_no)
+        preceding_text: str = ""
+        if expect._extract_text:
+            match = expect.search(line, col_no)
+            if match is None:
+                preceding_text = line[self.col_no :]
+                self.line_no += 1
+            else:
+                col_no = match.start()
+                preceding_text = line[self.col_no : col_no]
+            if preceding_text:
+                token = Token(
+                    "text",
+                    preceding_text,
+                    self.read_from,
+                    self.code,
+                    (line_no, col_no),
+                    referenced_by=None,
+                )
+                self.col_no = col_no
+                return token
+
+        else:
+            match = expect.match(line, col_no)
+
         if match is None:
             error_line = line[col_no:].rstrip()
             error_message = (

src/textual/markup.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+__all__ = ["MarkupError", "escape", "to_content"]
+
 import re
 from ast import literal_eval
 from operator import attrgetter
@@ -15,6 +17,14 @@
     Union,
 )
 
+from textual.css.tokenize import (
+    COLOR,
+    PERCENT,
+    TOKEN,
+    VARIABLE_REF,
+    Expect,
+    TokenizerState,
+)
 from textual.style import Style
 
 if TYPE_CHECKING:
@@ -25,7 +35,63 @@ class MarkupError(Exception):
     """An error occurred parsing Textual markup."""
 
 
-__all__ = ["MarkupError", "escape", "to_content"]
+expect_markup_tag = Expect(
+    "style token",
+    end_tag=r"(?<!\\)\]",
+    key=r"[@a-zA-Z_-][a-zA-Z0-9_-]*=",
+    percent=PERCENT,
+    color=COLOR,
+    token=TOKEN,
+    variable_ref=VARIABLE_REF,
+    whitespace=r"\s+",
+)
+
+expect_markup = Expect(
+    "markup token",
+    open_closing_tag=r"(?<!\\)\[/",
+    open_tag=r"(?<!\\)\[",
+    end_tag=r"(?<!\\)\]",
+).extract_text()
+
+expect_markup_expression = Expect(
+    "markup",
+    end_tag=r"(?<!\\)\]",
+    word=r"\w+",
+    period=r"\.",
+    round_start=r"\(",
+    round_end=r"\)",
+    square_start=r"\[",
+    square_end=r"\]",
+    curly_start=r"\{",
+    curly_end=r"\}",
+    comma=",",
+    whitespace=r"\s+",
+    double_string=r"\".*?\"",
+    single_string=r"'.*?'",
+)
+
+
+class MarkupTokenizer(TokenizerState):
+    """Tokenizes Textual markup."""
+
+    EXPECT = expect_markup.expect_eof(True)
+    STATE_MAP = {
+        "open_tag": expect_markup_tag,
+        "open_closing_tag": expect_markup_tag,
+        "end_tag": expect_markup,
+        "key": expect_markup_expression,
+    }
+    STATE_PUSH = {
+        "round_start": expect_markup_expression,
+        "square_start": expect_markup_expression,
+        "curly_start": expect_markup_expression,
+    }
+    STATE_POP = {
+        "round_end": "round_start",
+        "square_end": "square_start",
+        "curly_end": "curly_start",
+    }
+
 
 RE_TAGS = re.compile(
     r"""((\\*)\[([\$a-z#/@][^[]*?)])""",
@@ -248,6 +314,68 @@ def pop_style(style_name: str) -> Tuple[int, Tag]:
     return content
 
 
+def to_content(markup: str, style: str | Style = "") -> Content:
+
+    from textual.content import Content, Span
+
+    tokenizer = MarkupTokenizer()
+    text: list[str] = []
+    iter_tokens = iter(tokenizer(markup, ("inline", "")))
+
+    style_stack: list[tuple[int, str]] = []
+
+    spans: list[Span] = []
+
+    position = 0
+    tag_text: list[str]
+    for token in iter_tokens:
+        print(repr(token))
+        token_name = token.name
+        if token_name == "text":
+            text.append(token.value)
+            position += len(token.value)
+        elif token_name == "open_tag":
+            tag_text = []
+            print("open")
+            for token in iter_tokens:
+                print("  ", repr(token))
+                if token.name == "end_tag":
+                    break
+                tag_text.append(token.value)
+            opening_tag = "".join(tag_text)
+            style_stack.append((position, opening_tag))
+
+        elif token_name == "open_closing_tag":
+            tag_text = []
+            print("closing")
+            for token in iter_tokens:
+                print("  ", repr(token))
+                if token.name == "end_tag":
+                    break
+                tag_text.append(token.value)
+            closing_tag = "".join(tag_text)
+            if closing_tag:
+                for index, (tag_position, tag_body) in enumerate(reversed(style_stack)):
+                    if tag_body == closing_tag:
+                        style_stack.pop(-index)
+                        spans.append(Span(tag_position, position, tag_body))
+                        break
+
+            else:
+                open_position, tag = style_stack.pop()
+                spans.append(Span(open_position, position, tag))
+
+    content_text = "".join(text)
+    text_length = len(content_text)
+    while style_stack:
+        position, tag = style_stack.pop()
+        spans.append(Span(position, text_length, tag))
+
+    content = Content(content_text, spans)
+    print(repr(content))
+    return content
+
+
 if __name__ == "__main__":  # pragma: no cover
     from rich.highlighter import ReprHighlighter
 
@@ -296,7 +424,10 @@ def on_markup_changed(self, event: TextArea.Changed) -> None:
                 results.update(event.text_area.text)
             except Exception as error:
                 highlight = ReprHighlighter()
-                results.update(highlight(str(error)))
+                # results.update(highlight(str(error)))
+                from rich.traceback import Traceback
+
+                results.update(Traceback())
                 self.query_one("#results-container").add_class("-error")
             else:
                 self.query_one("#results-container").remove_class("-error")