feat: support custom params when run benchmark test method

Author: jialan

Diff for: benchmark/data/flinksql/create.sql renamed to benchmark/data/flink/create.sql

@@ -0,0 +1,12 @@
+{
+    "create": {
+        "validate": ["$sql"],
+        "getAllTokens": ["$sql"],
+        "getAllEntities": ["$sql", { "lineNumber": 8, "column": 1 }]
+    },
+    "select": {
+        "validate": ["$sql"],
+        "getAllTokens": ["$sql"],
+        "getAllEntities": ["$sql", { "lineNumber": 8, "column": 1 }]
+    }
+}

Diff for: benchmark/data/flinksql/select.sql renamed to benchmark/data/flink/select.sql

@@ -1,44 +1,51 @@
 import path from 'path';
 import argsParser from 'yargs-parser';
 import fs from 'fs';
-import SqlBenchmark, { languageNameMap } from './sqlBenchmark';
+import SqlBenchmark, { languageNameMap, languages } from './sqlBenchmark';
+import type { Language } from './sqlBenchmark';
 import inquirer from 'inquirer';
 import chalk from 'chalk';
 import { Table } from 'console-table-printer';
 
-const languages: string[] = fs.readdirSync(path.resolve(__dirname, '../src/grammar'));
 const argv = argsParser(process.argv.slice(2));
 const isChooseAll = argv.lang === 'all';
 
 type TestFile = {
     /** Benchmark Name */
     name: string;
-    /** Test sql path */
-    path: string;
+    /** Test sql name */
+    sqlFileName: string;
     /** Test run times */
     loopTimes?: number;
     /** Test method name of parser */
     testTypes: string[];
     /** Exclude languages */
-    excludes?: string[];
+    excludes?: Language[];
     /** Include languages */
-    includes?: string[];
+    includes?: Language[];
 };
 
 const testFiles: TestFile[] = [
     {
         name: 'Query Collection (100 Rows)',
-        path: 'select.sql',
+        sqlFileName: 'select.sql',
         loopTimes: 3,
         testTypes: ['validate', 'getAllTokens'],
-        excludes: ['pgsql', 'mysql'],
+        excludes: ['plsql', 'postgresql'],
+    },
+    {
+        name: 'Select All Entities',
+        sqlFileName: 'select.sql',
+        loopTimes: 1,
+        testTypes: ['getAllEntities'],
+        includes: ['hive'],
     },
     {
         name: 'Create Table (100 Rows)',
-        path: 'create.sql',
+        sqlFileName: 'create.sql',
         loopTimes: 3,
-        testTypes: ['validate'],
-        excludes: ['pgsql', 'mysql'],
+        testTypes: ['validate', 'getAllTokens'],
+        excludes: ['plsql', 'postgresql'],
     },
 ];
 
@@ -50,6 +57,18 @@ const readSql = (fileName: string, lang: string) => {
     return fs.readFileSync(sqlPath, 'utf-8');
 };
 
+const readParams = (lang: string) => {
+    const paramsPath = path.join(__dirname, `./data/${lang}/params.json`);
+    if (!fs.existsSync(paramsPath)) return null;
+    return fs.readFileSync(paramsPath, 'utf-8');
+};
+
+const getParams = (originalParams: any, sqlFileName: string, methodType: string): any[] => {
+    if (!originalParams) return ['$sql'];
+    const fileName = sqlFileName.substring(0, sqlFileName.lastIndexOf('.sql') + 1);
+    return originalParams[fileName]?.[methodType] || ['$sql'];
+};
+
 const askForSaveResult = () => {
     inquirer
         .prompt([
@@ -125,13 +144,22 @@ const printSummaryReport = () => {
     table.printTable();
 };
 
-const benchmark = (lang: string) => {
+const benchmark = (lang: Language) => {
     const sqlBenchmark = new SqlBenchmark(lang);
+    const originalParams = readParams(lang);
+
     testFiles.forEach((testInfo) => {
-        const { name, path, testTypes, loopTimes, excludes, includes } = testInfo;
+        const { name, sqlFileName, testTypes, loopTimes, excludes, includes } = testInfo;
         if (excludes?.includes(lang) || (includes?.length && !includes.includes(lang))) return;
-        const sql = readSql(path, lang);
-        testTypes.forEach((type) => sqlBenchmark.run(type, name, sql, loopTimes));
+        const sql = readSql(sqlFileName, lang);
+        const sqlRows = sql.split('\n').length;
+        testTypes.forEach((type) => {
+            const params = getParams(originalParams, sqlFileName, type);
+            const sqlParamIndex = params.findIndex((param) => param === '$sql');
+            // replace with real sql text
+            if (sqlParamIndex !== -1) params.splice(sqlParamIndex, 1, sql);
+            sqlBenchmark.run(type, name, params, sqlRows, loopTimes);
+        });
     });
     sqlBenchmark.printResults();
     benchmarkResults.push(sqlBenchmark);

Diff for: benchmark/data/hive/params.json

@@ -2,7 +2,7 @@ import path from 'path';
 import { MarkdownWritter } from './markdownWritter';
 import envinfo from 'envinfo';
 import { Table } from 'console-table-printer';
-import BasicParser from '../src/parser/common/basicParser';
+import { BasicSQL } from '../src/parser/common/basicSQL';
 import fs from 'fs';
 
 export type BenchmarkResult = {
@@ -38,25 +38,32 @@ const tableColumns = [
     },
     {
         name: 'loopTimes',
-        title: 'Loop',
+        title: 'Loops',
     },
     {
         name: 'type',
         title: 'Type',
     },
 ];
 
+/**
+ * Key is sql directory name, value is module export name.
+ */
 export const languageNameMap = {
     hive: 'HiveSQL',
     mysql: 'MySQL',
     plsql: 'PLSQL',
-    flinksql: 'FlinkSQL',
+    flink: 'FlinkSQL',
     spark: 'SparkSQL',
-    pgsql: 'PostgresSQL',
-    trinosql: 'TrinoSQL',
+    postgresql: 'PostgreSQL',
+    trino: 'TrinoSQL',
     impala: 'ImpalaSQL',
 };
 
+export type Language = keyof typeof languageNameMap;
+
+export const languages = Object.keys(languageNameMap) as Language[];
+
 class SqlBenchmark {
     constructor(language: string) {
         this.language = language;
@@ -81,11 +88,11 @@ class SqlBenchmark {
      *
      * Due to the presence of ATN cache in antlr4, we will clear the module cache to ensure that each parser is brand new and with no cache.
      */
-    getSqlParser(): BasicParser {
+    getSqlParser(): BasicSQL {
         const caches = Object.keys(require.cache);
         const cacheModules = [
-            path.join(__dirname, `../src/parser/${this.language}.ts`),
             path.join(__dirname, `../src/parser/common/`),
+            path.join(__dirname, `../src/parser/${this.language}/`),
             path.join(__dirname, `../src/lib/${this.language}/`),
             path.normalize(require.resolve('antlr4ng')),
         ];
@@ -94,28 +101,34 @@ class SqlBenchmark {
                 cacheModules.some((cacheModuleName) => moduleName.includes(cacheModuleName))
             )
             .forEach((moduleName) => delete require.cache[moduleName]);
-        const Parser = require(path.join(__dirname, `../src/parser/${this.language}.ts`)).default;
+        const Parser = require(path.join(__dirname, `../src/parser/${this.language}/index.ts`))[
+            languageNameMap[this.language]
+        ];
         return new Parser();
     }
 
     /**
      * @param type Which parser method you want to run
      * @param name Benchmark name
-     * @param sql SQL string
+     * @param params Parser method parameters
+     * @param params Rows count of sql
      * @param loopTimes Loop times, default run 3 times
      */
-    run(type: string, name: string, sql: string, loopTimes: number = this._DEFAULT_LOOP_TIMES) {
+    run(
+        type: string,
+        name: string,
+        params: any[],
+        sqlRows: number,
+        loopTimes: number = this._DEFAULT_LOOP_TIMES
+    ) {
         const costTimes: number[] = [];
         const lastResult =
             this._lastResultsCache?.find((item) => item.type === type && item.name === name) ?? {};
-        const rows = sql.split('\n').length;
-
         for (let i = 0; i < loopTimes; i++) {
             const parser = this.getSqlParser();
             if (!parser[type] || typeof parser[type] !== 'function') return;
-
             const startTime = performance.now();
-            parser[type](sql);
+            parser[type](...params);
             const costTime = performance.now() - startTime;
 
             costTimes.push(Math.round(costTime));
@@ -128,7 +141,7 @@ class SqlBenchmark {
             lastCostTime: lastResult['avgTime'],
             costTimes,
             loopTimes,
-            rows,
+            rows: sqlRows,
             type,
         };
 

Diff for: benchmark/data/trinosql/create.sql renamed to benchmark/data/trino/create.sql

@@ -5,9 +5,10 @@ const argv = require('yargs-parser')(process.argv.slice(2));
 const inquirer = require('inquirer');
 const chalk = require('chalk');
 
-const outputPath = path.resolve(__dirname, '../src/grammar');
-
-const languageEntries = fs.readdirSync(outputPath);
+const outputPath = path.resolve(__dirname, '../src/lib');
+const languages = fs.readdirSync(outputPath).filter((item) => {
+    return fs.statSync(path.join(outputPath, item)).isDirectory();
+});
 
 const cmd = 'tsx';
 
@@ -28,7 +29,7 @@ function prompt() {
                 type: 'list',
                 name: 'language',
                 message: 'Which language you want to run benchmark (or all languages)',
-                choices: ['All Languages', ...languageEntries],
+                choices: ['All Languages', ...languages],
                 loop: true,
             },
         ])
@@ -39,7 +40,7 @@ function prompt() {
 
 function main() {
     if (argv.lang) {
-        const supportedLanguage = languageEntries.some((language) => language === argv.lang);
+        const supportedLanguage = languages.some((language) => language === argv.lang);
         if (supportedLanguage || argv.lang === 'All Languages') {
             runBenchmark(argv.lang);
         } else {