|
| 1 | +const assert = require('assert') |
| 2 | + |
| 3 | +module.exports = class CustomTokenizer { |
| 4 | + constructor (text, code, { startingLine, startingColumn }) { |
| 5 | + // set initial tokenizer states used by the parser |
| 6 | + this.expressionEnabled = true |
| 7 | + this.namespace = "http://www.w3.org/1999/xhtml" |
| 8 | + this.text = code |
| 9 | + // ignore actual input and just hardcode tokens |
| 10 | + assert.equal(text, `\n A-totally[:made="up"] {{ templating + language }} <=== comment ===>\n`) |
| 11 | + assert.equal(startingLine, 1) |
| 12 | + assert.equal(startingColumn, 28) |
| 13 | + this.line = startingLine |
| 14 | + this.column = startingColumn |
| 15 | + this.offset = 28 |
| 16 | + const tokens = [ |
| 17 | + this.generateToken("\n ", { |
| 18 | + type: "CustomWhitespace" |
| 19 | + }), |
| 20 | + this.generateToken("A-totally", { |
| 21 | + type: "CustomTagOpen", |
| 22 | + value: "A-totally" |
| 23 | + }), |
| 24 | + this.generateToken(":made", { |
| 25 | + type: "CustomIdentifier" |
| 26 | + }), |
| 27 | + this.generateToken("=", { |
| 28 | + type: "CustomAssociation", |
| 29 | + value: "" |
| 30 | + }), |
| 31 | + this.generateToken("\"up\"", { |
| 32 | + type: "CustomLiteral", |
| 33 | + value: "up" |
| 34 | + }), |
| 35 | + this.generateToken("]", { |
| 36 | + type: "CustomTagClose", |
| 37 | + value: "" |
| 38 | + }), |
| 39 | + this.generateToken("{{", { |
| 40 | + type: "VExpressionStart" |
| 41 | + }), |
| 42 | + this.generateToken(" templating + language ", { |
| 43 | + type: "CustomText" |
| 44 | + }), |
| 45 | + this.generateToken("}}", { |
| 46 | + type: "VExpressionEnd" |
| 47 | + }), |
| 48 | + this.generateToken("", { |
| 49 | + type: "CustomEndTag" |
| 50 | + }), |
| 51 | + this.generateToken("<=== comment ===>", { |
| 52 | + type: "CustomComment", |
| 53 | + value: "comment" |
| 54 | + }) |
| 55 | + ] |
| 56 | + this.comments = tokens.filter(token => token.type === "CustomComment") |
| 57 | + this.tokens = tokens.filter(token => !["CustomEndTag", "CustomComment"].includes(token.type)) |
| 58 | + |
| 59 | + this.errors = [{ |
| 60 | + "message": "totally-made-up-error", |
| 61 | + "index": 9001, |
| 62 | + "lineNumber": 15, |
| 63 | + "column": 8 |
| 64 | + }] |
| 65 | + |
| 66 | + const attribute = { |
| 67 | + type: "VAttribute", |
| 68 | + parent: {}, |
| 69 | + directive: false, |
| 70 | + range: [tokens[2].range[0], tokens[4].range[1]], |
| 71 | + loc: { |
| 72 | + start: tokens[2].start, |
| 73 | + end: tokens[4].end |
| 74 | + } |
| 75 | + } |
| 76 | + |
| 77 | + attribute.key = { |
| 78 | + type: "VIdentifier", |
| 79 | + parent: attribute, |
| 80 | + name: ":made", |
| 81 | + rawName: ":made", |
| 82 | + range: tokens[2].range, |
| 83 | + loc: tokens[2].loc |
| 84 | + } |
| 85 | + |
| 86 | + attribute.value = { |
| 87 | + type: "VLiteral", |
| 88 | + parent: attribute, |
| 89 | + value: "up", |
| 90 | + range: tokens[4].range, |
| 91 | + loc: tokens[4].loc |
| 92 | + } |
| 93 | + |
| 94 | + // these tokens get returned by nextToken |
| 95 | + const intermediateTokens = [{ |
| 96 | + type: "StartTag", |
| 97 | + name: "a-totally", |
| 98 | + rawName: "A-totally", |
| 99 | + range: [tokens[1].range[0], tokens[5].range[1]], |
| 100 | + loc: { |
| 101 | + start: tokens[1].loc.start, |
| 102 | + end: tokens[5].loc.end |
| 103 | + }, |
| 104 | + selfClosing: false, |
| 105 | + attributes: [attribute] |
| 106 | + }, { |
| 107 | + type: "Mustache", |
| 108 | + value: " templating + language ", |
| 109 | + range: [tokens[6].range[0], tokens[8].range[1]], |
| 110 | + loc: { |
| 111 | + start: tokens[6].loc.start, |
| 112 | + end: tokens[8].loc.end |
| 113 | + }, |
| 114 | + startToken: tokens[6], |
| 115 | + endToken: tokens[8] |
| 116 | + }, { |
| 117 | + type: "EndTag", |
| 118 | + name: "a-totally", |
| 119 | + range: tokens[9].range, |
| 120 | + loc: tokens[9].loc, |
| 121 | + }] |
| 122 | + this.tokenIterator = intermediateTokens[Symbol.iterator]() |
| 123 | + } |
| 124 | + |
| 125 | + nextToken () { |
| 126 | + return this.tokenIterator.next().value |
| 127 | + } |
| 128 | + |
| 129 | + // set range and loc based on text length and current offset |
| 130 | + generateToken (text, data) { |
| 131 | + const skip = this.text.indexOf(text, this.offset) - this.offset |
| 132 | + const range = [this.offset + skip, this.offset + skip + text.length] |
| 133 | + this.offset = range[1] |
| 134 | + const loc = { |
| 135 | + start: { |
| 136 | + line: this.line, |
| 137 | + column: this.column + skip |
| 138 | + } |
| 139 | + } |
| 140 | + this.line += text.split('\n').length - 1 |
| 141 | + this.column = text.split('\n').length - 1 ? text.length - text.lastIndexOf('\n') - 1 : this.column + skip + text.length |
| 142 | + loc.end = { |
| 143 | + line: this.line, |
| 144 | + column: this.column |
| 145 | + } |
| 146 | + return { |
| 147 | + range, |
| 148 | + loc, |
| 149 | + value: text, |
| 150 | + ...data |
| 151 | + } |
| 152 | + } |
| 153 | +} |
0 commit comments