Change HTMLText token to split into blanks and words. (#54)

Author: ota-meshi

src/parser/converts/text.ts

@@ -13,7 +13,20 @@ export function convertText(
         parent,
         ...ctx.getConvertLocation(node),
     }
-    ctx.addToken("HTMLText", node)
+    let start = node.start
+    let word = false
+    for (let index = node.start; index < node.end; index++) {
+        if (word !== Boolean(ctx.code[index].trim())) {
+            if (start < index) {
+                ctx.addToken("HTMLText", { start, end: index })
+            }
+            word = !word
+            start = index
+        }
+    }
+    if (start < node.end) {
+        ctx.addToken("HTMLText", { start, end: node.end })
+    }
     return text
 }
 
@@ -29,6 +42,19 @@ export function convertTextToLiteral(
         parent,
         ...ctx.getConvertLocation(node),
     }
-    ctx.addToken("HTMLText", node)
+    let start = node.start
+    let word = false
+    for (let index = node.start; index < node.end; index++) {
+        if (word !== Boolean(ctx.code[index].trim())) {
+            if (start < index) {
+                ctx.addToken("HTMLText", { start, end: index })
+            }
+            word = !word
+            start = index
+        }
+    }
+    if (start < node.end) {
+        ctx.addToken("HTMLText", { start, end: node.end })
+    }
     return text
 }

tests/fixtures/parser/ast/blog/write-less-code01-output.json

@@ -1916,16 +1916,52 @@
     },
     {
       "type": "HTMLText",
-      "value": " + ",
+      "value": " ",
       "range": [
         125,
-        128
+        126
       ],
       "loc": {
         "start": {
           "line": 9,
           "column": 6
         },
+        "end": {
+          "line": 9,
+          "column": 7
+        }
+      }
+    },
+    {
+      "type": "HTMLText",
+      "value": "+",
+      "range": [
+        126,
+        127
+      ],
+      "loc": {
+        "start": {
+          "line": 9,
+          "column": 7
+        },
+        "end": {
+          "line": 9,
+          "column": 8
+        }
+      }
+    },
+    {
+      "type": "HTMLText",
+      "value": " ",
+      "range": [
+        127,
+        128
+      ],
+      "loc": {
+        "start": {
+          "line": 9,
+          "column": 8
+        },
         "end": {
           "line": 9,
           "column": 9
@@ -1988,16 +2024,52 @@
     },
     {
       "type": "HTMLText",
-      "value": " = ",
+      "value": " ",
       "range": [
         131,
-        134
+        132
       ],
       "loc": {
         "start": {
           "line": 9,
           "column": 12
         },
+        "end": {
+          "line": 9,
+          "column": 13
+        }
+      }
+    },
+    {
+      "type": "HTMLText",
+      "value": "=",
+      "range": [
+        132,
+        133
+      ],
+      "loc": {
+        "start": {
+          "line": 9,
+          "column": 13
+        },
+        "end": {
+          "line": 9,
+          "column": 14
+        }
+      }
+    },
+    {
+      "type": "HTMLText",
+      "value": " ",
+      "range": [
+        133,
+        134
+      ],
+      "loc": {
+        "start": {
+          "line": 9,
+          "column": 14
+        },
         "end": {
           "line": 9,
           "column": 15

tests/fixtures/parser/ast/components02-output.json

@@ -642,16 +642,34 @@
     },
     {
       "type": "HTMLText",
-      "value": "\n\tcontents",
+      "value": "\n\t",
       "range": [
         82,
-        92
+        84
       ],
       "loc": {
         "start": {
           "line": 5,
           "column": 13
         },
+        "end": {
+          "line": 6,
+          "column": 1
+        }
+      }
+    },
+    {
+      "type": "HTMLText",
+      "value": "contents",
+      "range": [
+        84,
+        92
+      ],
+      "loc": {
+        "start": {
+          "line": 6,
+          "column": 1
+        },
         "end": {
           "line": 6,
           "column": 9

tests/fixtures/parser/ast/components03-output.json

@@ -985,16 +985,34 @@
     },
     {
       "type": "HTMLText",
-      "value": "\n\tcontents",
+      "value": "\n\t",
       "range": [
         95,
-        105
+        97
       ],
       "loc": {
         "start": {
           "line": 5,
           "column": 26
         },
+        "end": {
+          "line": 6,
+          "column": 1
+        }
+      }
+    },
+    {
+      "type": "HTMLText",
+      "value": "contents",
+      "range": [
+        97,
+        105
+      ],
+      "loc": {
+        "start": {
+          "line": 6,
+          "column": 1
+        },
         "end": {
           "line": 6,
           "column": 9
@@ -1327,16 +1345,34 @@
     },
     {
       "type": "HTMLText",
-      "value": "\n\tcontents",
+      "value": "\n\t",
       "range": [
         177,
-        187
+        179
       ],
       "loc": {
         "start": {
           "line": 8,
           "column": 32
         },
+        "end": {
+          "line": 9,
+          "column": 1
+        }
+      }
+    },
+    {
+      "type": "HTMLText",
+      "value": "contents",
+      "range": [
+        179,
+        187
+      ],
+      "loc": {
+        "start": {
+          "line": 9,
+          "column": 1
+        },
         "end": {
           "line": 9,
           "column": 9