Parse pug with Tokenizer

Author: Shinigami92

Diff for: .gitignore

@@ -7,3 +7,6 @@
 /npm-debug.log
 /test.js
 /test/fixtures/espree-v8/node_modules
+.yarn
+.yarnrc.yml
+yarn.lock

Diff for: package.json

@@ -10,7 +10,13 @@
     "index.*"
   ],
   "peerDependencies": {
-    "eslint": ">=5.0.0"
+    "eslint": ">=5.0.0",
+    "pug-lexer": ">=5.0.0"
+  },
+  "peerDependenciesMeta": {
+    "pug-lexer": {
+      "optional": true
+    }
   },
   "dependencies": {
     "debug": "^4.1.1",
@@ -52,8 +58,10 @@
     "nyc": "^14.0.0",
     "opener": "^1.5.1",
     "prettier": "^2.3.1",
+    "pug-lexer": "^5.0.1",
     "rimraf": "^2.6.3",
     "rollup": "^1.1.2",
+    "rollup-plugin-commonjs": "^10.1.0",
     "rollup-plugin-node-resolve": "^4.0.0",
     "rollup-plugin-sourcemaps": "^0.4.2",
     "ts-node": "^8.1.0",

Diff for: rollup.config.js

@@ -5,6 +5,7 @@
  */
 import resolve from "rollup-plugin-node-resolve"
 import sourcemaps from "rollup-plugin-sourcemaps"
+import commonjs from "rollup-plugin-commonjs"
 
 const pkg = require("./package.json")
 const deps = new Set(
@@ -23,6 +24,6 @@ export default {
  * See LICENSE file in root directory for full license.
  */`,
     },
-    plugins: [sourcemaps(), resolve()],
+    plugins: [sourcemaps(), resolve(), commonjs()],
     external: id => deps.has(id) || id.startsWith("lodash"),
 }

Diff for: src/index.ts

@@ -7,6 +7,7 @@ import * as path from "path"
 import * as AST from "./ast"
 import { LocationCalculatorForHtml } from "./common/location-calculator"
 import { HTMLParser, HTMLTokenizer } from "./html"
+import { PugParser, PugTokenizer } from "./pug"
 import { parseScript, parseScriptElement } from "./script"
 import * as services from "./parser-services"
 import type { ParserOptions } from "./common/parser-options"
@@ -74,9 +75,15 @@ export function parseForESLint(
             ecmaVersion: options.ecmaVersion || DEFAULT_ECMA_VERSION,
         }
         const skipParsingScript = options.parser === false
-        const tokenizer = new HTMLTokenizer(code, optionsForTemplate)
-        const rootAST = new HTMLParser(tokenizer, optionsForTemplate).parse()
-
+        let tokenizer: HTMLTokenizer | PugTokenizer
+        let rootAST: AST.VDocumentFragment
+        if (/^\<template\s+lang\=["']pug["']\>/i.test(code)) {
+            tokenizer = new PugTokenizer(code, optionsForTemplate)
+            rootAST = new PugParser(tokenizer, optionsForTemplate).parse()
+        } else {
+            tokenizer = new HTMLTokenizer(code, optionsForTemplate)
+            rootAST = new HTMLParser(tokenizer as HTMLTokenizer, optionsForTemplate).parse()
+        }
         locationCalculator = new LocationCalculatorForHtml(
             tokenizer.gaps,
             tokenizer.lineTerminators,
@@ -90,7 +97,7 @@ export function parseForESLint(
             errors: rootAST.errors,
         }
         const templateBody =
-            template != null && templateLang === "html"
+            template != null && (templateLang === "html" || templateLang === "pug")
                 ? Object.assign(template, concreteInfo)
                 : undefined
 

Diff for: src/pug/index.ts

@@ -0,0 +1,2 @@
+export { Parser as PugParser } from "./parser"
+export { Tokenizer as PugTokenizer } from "./tokenizer"

Diff for: src/pug/parser.ts

@@ -0,0 +1,64 @@
+import type { Token } from "pug-lexer";
+import type { VDocumentFragment } from "../ast";
+import { ParseError } from "../ast";
+import type { ParserOptions } from "../common/parser-options";
+import { isSFCFile } from "../common/parser-options";
+import type { Tokenizer } from "./tokenizer";
+
+/**
+ * Parser for pug.
+ */
+export class Parser {
+  private tokenizer: Tokenizer;
+  // @ts-expect-error: should be used later
+  private baseParserOptions: ParserOptions;
+  // @ts-expect-error: should be used later
+  private isSFC: boolean;
+  private document: VDocumentFragment;
+
+  /**
+   * The tokens.
+   */
+  private get tokens(): Token[] {
+    return this.tokenizer.tokens;
+  }
+
+  /**
+   * The comments.
+   */
+  private get comments(): Token[] {
+    return [];
+    // return this.tokenizer.comments;
+  }
+
+  /**
+   * The syntax errors which are found in this parsing.
+   */
+  private get errors(): ParseError[] {
+    return [];
+    // return this.tokenizer.errors;
+  }
+
+  public constructor(tokenizer: Tokenizer, parserOptions: ParserOptions) {
+    this.tokenizer = tokenizer;
+    this.baseParserOptions = parserOptions;
+    this.isSFC = isSFCFile(parserOptions);
+    this.document = {
+      type: "VDocumentFragment",
+      range: [0, 0],
+      loc: {
+        start: { line: 1, column: 0 },
+        end: { line: 1, column: 0 },
+      },
+      parent: null,
+      children: [],
+      tokens: this.tokens as any[],
+      comments: this.comments as any[],
+      errors: this.errors,
+    };
+  }
+
+  public parse(): VDocumentFragment {
+    return this.document;
+  }
+}

Diff for: src/pug/tokenizer.ts

@@ -0,0 +1,32 @@
+import type { Token } from "pug-lexer";
+import lex from "pug-lexer";
+import { debug } from "../common/debug";
+import type { ParserOptions } from "../common/parser-options";
+
+/**
+ * Tokenizer for pug.
+ */
+export class Tokenizer {
+  // Reading
+  public readonly text: string;
+  public readonly gaps: number[];
+  public readonly lineTerminators: number[];
+  public readonly tokens: Token[];
+  // @ts-expect-error: should be used later
+  private readonly parserOptions: ParserOptions;
+
+  /**
+   * Initialize this tokenizer.
+   * @param text The source code to tokenize.
+   * @param parserOptions The parser options.
+   */
+  public constructor(text: string, parserOptions?: ParserOptions) {
+    debug("[pug] the source code length: %d", text.length);
+    this.text = text;
+    this.gaps = [];
+    this.lineTerminators = [];
+    this.parserOptions = parserOptions || {};
+
+    this.tokens = lex(text);
+  }
+}