Skip to content

Commit

Permalink
Merge pull request #283 from codefori/fix/parser_conditionals
Browse files Browse the repository at this point in the history 
Improved support for parser conditional and additional statement types
  • Loading branch information
@worksofliam
worksofliam authored Oct 3, 2024
2 parents 00428ed + 435166d commit f810185
Show file tree
Hide file tree
Showing 6 changed files with 391 additions and 185 deletions.
25 changes: 16 additions & 9 deletions src/language/sql/document.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ export default class Document {
let statementStart = 0;

for (let i = 0; i < tokens.length; i++) {
const upperValue = tokens[i].value?.toUpperCase();
switch (tokens[i].type) {
case `semicolon`:
const statementTokens = tokens.slice(statementStart, i);
Expand All @@ -45,19 +46,25 @@ export default class Document {
break;

case `statementType`:
currentStatementType = StatementTypeWord[tokens[i].value?.toUpperCase()];
currentStatementType = StatementTypeWord[upperValue];
break;

case `keyword`:
switch (tokens[i].value?.toUpperCase()) {
switch (upperValue) {
case `LOOP`:
// This handles the case that 'END LOOP' is supported.
if (currentStatementType === StatementType.End) {
break;
}
case `THEN`:
case `BEGIN`:
case `DO`:
case `THEN`:
// This handles the case that 'END LOOP' is supported.
if (upperValue === `LOOP` && currentStatementType === StatementType.End) {
break;
}

// Support for THEN in conditionals
if (upperValue === `THEN` && !Statement.typeIsConditional(currentStatementType)) {
break;
}

// We include BEGIN in the current statement
// then the next statement beings
const statementTokens = tokens.slice(statementStart, i+1);
Expand Down Expand Up @@ -102,7 +109,7 @@ export default class Document {
let depth = 0;

for (const statement of this.statements) {
if (statement.isBlockEnder()) {
if (statement.isCompoundEnd()) {
if (depth > 0) {
currentGroup.push(statement);

Expand All @@ -118,7 +125,7 @@ export default class Document {
currentGroup = [];
}
} else
if (statement.isBlockOpener()) {
if (statement.isCompoundStart()) {
if (depth > 0) {
currentGroup.push(statement);
} else {
Expand Down
84 changes: 58 additions & 26 deletions src/language/sql/statement.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ const tokenIs = (token: Token|undefined, type: string, value?: string) => {

export default class Statement {
public type: StatementType = StatementType.Unknown;
private label: string|undefined;

constructor(public tokens: Token[], public range: IRange) {
this.tokens = this.tokens.filter(newToken => newToken.type !== `newline`);
Expand All @@ -19,11 +20,15 @@ export default class Statement {
first = this.tokens[2];
}

if (tokenIs(first, `statementType`) || tokenIs(first, `keyword`, `END`) || tokenIs(first, `keyword`, `BEGIN`)) {
const wordValue = first.value?.toUpperCase();

this.type = StatementTypeWord[wordValue];
if (tokenIs(first, `word`) && tokenIs(this.tokens[1], `colon`)) {
// Possible label?
this.label = first.value;
first = this.tokens[2];
}

const wordValue = first.value?.toUpperCase();

this.type = StatementTypeWord[wordValue] || StatementType.Unknown;

switch (this.type) {
case StatementType.Create:
Expand All @@ -35,7 +40,7 @@ export default class Statement {
}
}

isBlockOpener() {
isCompoundStart() {
if (this.tokens.length === 1 && tokenIs(this.tokens[0], `keyword`, `BEGIN`)) {
return true;
}
Expand All @@ -51,7 +56,19 @@ export default class Statement {
return false;
}

isBlockEnder() {
static typeIsConditional(type: StatementType) {
return [StatementType.If, StatementType.While, StatementType.Loop, StatementType.For].includes(type);
}

isConditionStart() {
return Statement.typeIsConditional(this.type);
}

isConditionEnd() {
return this.type === StatementType.End && this.tokens.length > 1;
}

isCompoundEnd() {
return this.type === StatementType.End && this.tokens.length === 1;
}

Expand Down Expand Up @@ -314,17 +331,21 @@ export default class Statement {
}

const basicQueryFinder = (startIndex: number): void => {
let currentClause: undefined|"select"|"from";
for (let i = startIndex; i < this.tokens.length; i++) {
if (tokenIs(this.tokens[i], `clause`, `FROM`)) {
inFromClause = true;
} else if (inFromClause && tokenIs(this.tokens[i], `clause`) || tokenIs(this.tokens[i], `join`) || tokenIs(this.tokens[i], `closebracket`)) {
inFromClause = false;
currentClause = `from`;
}
else if (tokenIs(this.tokens[i], `statementType`, `SELECT`)) {
currentClause = `select`;
} else if (currentClause === `from` && tokenIs(this.tokens[i], `clause`) || tokenIs(this.tokens[i], `join`) || tokenIs(this.tokens[i], `closebracket`)) {
currentClause = undefined;
}

if (tokenIs(this.tokens[i], `clause`, `FROM`) ||
(this.type !== StatementType.Select && tokenIs(this.tokens[i], `clause`, `INTO`)) ||
tokenIs(this.tokens[i], `join`) ||
(inFromClause && tokenIs(this.tokens[i], `comma`)
(currentClause === `from` && tokenIs(this.tokens[i], `comma`)
)) {
const sqlObj = this.getRefAtToken(i+1);
if (sqlObj) {
Expand All @@ -334,6 +355,15 @@ export default class Statement {
i += 3; //For the brackets
}
}
} else if (currentClause === `select` && tokenIs(this.tokens[i], `function`)) {
const sqlObj = this.getRefAtToken(i);
if (sqlObj) {
doAdd(sqlObj);
i += sqlObj.tokens.length;
if (sqlObj.isUDTF || sqlObj.fromLateral) {
i += 3; //For the brackets
}
}
}
}
}
Expand Down Expand Up @@ -592,7 +622,7 @@ export default class Statement {
}

if (options.withSystemName) {
if (tokenIs(this.tokens[endIndex+1], `keyword`, `FOR`) && tokenIs(this.tokens[endIndex+2], `word`, `SYSTEM`) && tokenIs(this.tokens[endIndex+3], `word`, `NAME`)) {
if (tokenIs(this.tokens[endIndex+1], `statementType`, `FOR`) && tokenIs(this.tokens[endIndex+2], `word`, `SYSTEM`) && tokenIs(this.tokens[endIndex+3], `word`, `NAME`)) {
if (this.tokens[endIndex+4] && NameTypes.includes(this.tokens[endIndex+4].type)) {
sqlObj.object.system = this.tokens[endIndex+4].value;
}
Expand Down Expand Up @@ -624,10 +654,25 @@ export default class Statement {

switch (currentToken.type) {
case `statementType`:
if (declareStmt) continue;
const currentValue = currentToken.value.toLowerCase();
if (declareStmt) {
if (currentValue === `for`) {
ranges.push({
type: `remove`,
range: {
start: declareStmt.range.start,
end: currentToken.range.end
}
});

declareStmt = undefined;
}

continue;
};

// If we're in a DECLARE, it's likely a cursor definition
if (currentToken.value.toLowerCase() === `declare`) {
if (currentValue === `declare`) {
declareStmt = currentToken;
}
break;
Expand Down Expand Up @@ -716,19 +761,6 @@ export default class Statement {
}
});
}
} else
if (declareStmt && tokenIs(currentToken, `keyword`, `FOR`)) {
// If we're a DECLARE, and we found the FOR keyword, the next
// set of tokens should be the select.
ranges.push({
type: `remove`,
range: {
start: declareStmt.range.start,
end: currentToken.range.end
}
});

declareStmt = undefined;
}
break;
}
Expand Down
78 changes: 58 additions & 20 deletions src/language/sql/tests/blocks.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@

import { assert, describe, expect, test } from 'vitest'
import { describe, expect, test } from 'vitest'
import Document from '../document';
import { StatementType } from '../types';

describe(`Block statement tests`, () => {
const parserScenarios = describe.each([
{newDoc: (content: string) => new Document(content), isFormatted: false},
]);

parserScenarios(`Block statement tests`, ({newDoc, isFormatted}) => {
test('Block start tests', () => {
const lines = [
`CREATE ALIAS "TestDelimiters"."Delimited Alias" FOR "TestDelimiters"."Delimited Table";`,
Expand All @@ -15,19 +18,19 @@ describe(`Block statement tests`, () => {
`LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`,
].join(`\n`);

const doc = new Document(lines);
const doc = newDoc(lines);

// CREATE, CREATE, RETURN, END, CREATE, SET, END
expect(doc.statements.length).toBe(7);

const aliasDef = doc.statements[0];
expect(aliasDef.isBlockOpener()).toBeFalsy();
expect(aliasDef.isCompoundStart()).toBeFalsy();

const functionDef = doc.statements[1];
expect(functionDef.isBlockOpener()).toBeTruthy();
expect(functionDef.isCompoundStart()).toBeTruthy();

const procedureDef = doc.statements[4];
expect(procedureDef.isBlockOpener()).toBeTruthy();
expect(procedureDef.isCompoundStart()).toBeTruthy();
});

test('Compound statement test', () => {
Expand All @@ -53,21 +56,21 @@ describe(`Block statement tests`, () => {
`LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`,
].join(`\n`);

const doc = new Document(lines);
const doc = newDoc(lines);

const t = doc.statements.length;

const aliasDef = doc.statements[0];
expect(aliasDef.isBlockOpener()).toBeFalsy();
expect(aliasDef.isCompoundStart()).toBeFalsy();

const functionDef = doc.statements[1];
expect(functionDef.isBlockOpener()).toBeTruthy();
expect(functionDef.isCompoundStart()).toBeTruthy();

const functionEnd = doc.statements[3];
expect(functionEnd.isBlockEnder()).toBeTruthy();
expect(functionEnd.isCompoundEnd()).toBeTruthy();

const beginBlock = doc.statements[4];
expect(beginBlock.isBlockOpener()).toBeTruthy();
expect(beginBlock.isCompoundStart()).toBeTruthy();
});

test('Statement groups', () => {
Expand Down Expand Up @@ -96,23 +99,58 @@ describe(`Block statement tests`, () => {
`LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`,
].join(`\r\n`);

const doc = new Document(lines);
const doc = newDoc(lines);

const groups = doc.getStatementGroups();

expect(groups.length).toBe(4);

const aliasStatement = groups[0];
const aliasSubstring = lines.substring(aliasStatement.range.start, aliasStatement.range.end);
const aliasSubstring = doc.content.substring(aliasStatement.range.start, aliasStatement.range.end);
expect(aliasSubstring).toBe(`CREATE ALIAS "TestDelimiters"."Delimited Alias" FOR "TestDelimiters"."Delimited Table"`);

const functionStatement = groups[1];
const functionSubstring = doc.content.substring(functionStatement.range.start, functionStatement.range.end);

if (isFormatted) {
expect(functionSubstring).toBe([
`CREATE FUNCTION "TestDelimiters"."Delimited Function"(`,
` "Delimited Parameter" INTEGER`,
`) RETURNS INTEGER LANGUAGE SQL BEGIN`,
` RETURN "Delimited Parameter";`,
`END`,
].join(`\r\n`));
} else {
expect(functionSubstring).toBe([
`CREATE FUNCTION "TestDelimiters"."Delimited Function" ("Delimited Parameter" INTEGER) `,
`RETURNS INTEGER LANGUAGE SQL BEGIN RETURN "Delimited Parameter"; END`
].join(`\r\n`))
}
const beginStatement = groups[2];
const compoundSubstring = lines.substring(beginStatement.range.start, beginStatement.range.end);
expect(compoundSubstring).toBe(compoundStatement);
expect(beginStatement.statements.length).toBe(9);
const compoundSubstring = doc.content.substring(beginStatement.range.start, beginStatement.range.end);

if (isFormatted) {
expect(compoundSubstring).toBe([
`BEGIN`,
` DECLARE already_exists SMALLINT DEFAULT 0;`,
` DECLARE dup_object_hdlr CONDITION FOR SQLSTATE '42710';`,
` DECLARE CONTINUE HANDLER FOR dup_object_hdlr SET already_exists = 1;`,
` CREATE TABLE table1(`,
` col1 INT`,
` );`,
` IF already_exists > 0 THEN;`,
` DELETE FROM table1;`,
` END IF;`,
`END`,
].join(`\r\n`));
} else {
expect(compoundSubstring).toBe(compoundStatement);
}
});
});

describe(`Definition tests`, () => {
parserScenarios(`Definition tests`, ({newDoc}) => {
test(`Alias, function, procedure`, () => {
const lines = [
`CREATE ALIAS "TestDelimiters"."Delimited Alias" FOR "TestDelimiters"."Delimited Table";`,
Expand All @@ -124,7 +162,7 @@ describe(`Definition tests`, () => {
`LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`,
].join(`\n`);

const doc = new Document(lines);
const doc = newDoc(lines);

const defs = doc.getDefinitions();

Expand Down Expand Up @@ -161,7 +199,7 @@ describe(`Definition tests`, () => {
`END;`,
].join(`\r\n`);

const doc = new Document(lines);
const doc = newDoc(lines);

const defs = doc.getDefinitions();

Expand Down Expand Up @@ -245,7 +283,7 @@ describe(`Definition tests`, () => {
`END ; `,
].join(`\n`);

const doc = new Document(lines);
const doc = newDoc(lines);

const groups = doc.getStatementGroups();
expect(groups.length).toBe(1);
Expand Down
Loading

0 comments on commit f810185

Please sign in to comment.