Add improvements to PQL Parser for full PQL grammar parsing (#230)

pengnam · web-flow · commit 74d1205398ab · 2020-10-03T15:16:31.000+08:00
* Add sanitize in PQlLexer

* Fix test

* Add semantic check of assignment of attribute name to proper synonym

* Add check to throw exception if raw refs not of same type

* Format
diff --git a/Team20/Code20/IntegrationTesting/TestPQLParserLexerIntegration.cpp b/Team20/Code20/IntegrationTesting/TestPQLParserLexerIntegration.cpp
@@ -78,24 +78,44 @@ TEST_METHOD(TestLexAndParse_SynonymDeclaredWithKeyword) {
 
 TEST_METHOD(TestLexAndParse_TupleResultWithWithClause) {
   const std::string input =
-      "assign a; variable v;\n\nSelect <a, v.stmt#> such that Uses "
-      "(a, v) pattern a (v, _) with a.procName  = v.procName";
+      "print p; variable v;\n\nSelect <p.stmt#, v> such that Uses "
+      "(p, v) with p.varName = v.varName";
   const auto actualResult = Pql::parse(Pql::lex(input));
-  const DECLARATIONS expectedDeclarations{{"a", TokenType::ASSIGN},
+  const DECLARATIONS expectedDeclarations{{"p", TokenType::PRINT},
                                           {"v", TokenType::VARIABLE}};
   const RESULTS expectedResults{PqlResultType::Tuple,
                                 {
-                                    {"a", AttributeRefType::NONE},
-                                    {"v", AttributeRefType::STATEMENT_NUM},
+                                    {"p", AttributeRefType::STATEMENT_NUM},
+                                    {"v", AttributeRefType::NONE},
                                 }};
   const RELATIONSHIPS expectedRelationships{
-      {TokenType::USES, {TokenType::ASSIGN, "a"}, {TokenType::VARIABLE, "v"}}};
+      {TokenType::USES, {TokenType::PRINT, "p"}, {TokenType::VARIABLE, "v"}}};
+  const PATTERNS expectedPatterns{};
+  const WITHS expectedWiths = {
+      {Reference{Element{"p", AttributeRefType::VARNAME}},
+       Reference{Element{"v", AttributeRefType::VARNAME}}}};
+  Assert::IsTrue(actualResult.declarations == expectedDeclarations);
+  Assert::IsTrue(actualResult.results == expectedResults);
+  Assert::IsTrue(actualResult.relationships == expectedRelationships);
+  Assert::IsTrue(actualResult.patterns == expectedPatterns);
+  Assert::IsTrue(actualResult.withs == expectedWiths);
+} // namespace UnitTesting
+TEST_METHOD(TestLexAndParse_BooleanWithProgline) {
+  const std::string input =
+      "prog_line pl;assign a; variable v; \n\nSelect BOOLEAN such that Uses "
+      "(pl, v) pattern a (v, _)";
+  const auto actualResult = Pql::parse(Pql::lex(input));
+  const DECLARATIONS expectedDeclarations{{"a", TokenType::ASSIGN},
+                                          {"v", TokenType::VARIABLE},
+                                          {"pl", TokenType::PROG_LINE}};
+  const RESULTS expectedResults{PqlResultType::Boolean};
+  const RELATIONSHIPS expectedRelationships{{TokenType::USES,
+                                             {TokenType::PROG_LINE, "pl"},
+                                             {TokenType::VARIABLE, "v"}}};
   const PATTERNS expectedPatterns{{{TokenType::ASSIGN, "a"},
                                    {TokenType::VARIABLE, "v"},
                                    {PatternMatchType::Any}}};
-  const WITHS expectedWiths = {
-      {Reference{Element{"a", AttributeRefType::PROCNAME}},
-       Reference{Element{"v", AttributeRefType::PROCNAME}}}};
+  const WITHS expectedWiths = {};
   Assert::IsTrue(actualResult.declarations == expectedDeclarations);
   Assert::IsTrue(actualResult.results == expectedResults);
   Assert::IsTrue(actualResult.relationships == expectedRelationships);
diff --git a/Team20/Code20/UnitTesting/TestPQLParser.cpp b/Team20/Code20/UnitTesting/TestPQLParser.cpp
@@ -100,6 +100,16 @@ TEST_METHOD(TestLex_BracketWithinString) {
 
   Assert::IsTrue(expectedTokens == actualTokens);
 } // namespace UnitTesting
+TEST_METHOD(TestLex_SanitizeProgLine) {
+  const auto actualTokens = Pql::lex("prog_line p,q;");
+  const std::vector<PqlToken> expectedTokens = {{TokenType::PROG_LINE},
+                                                {TokenType::SYNONYM, "p"},
+                                                {TokenType::COMMA},
+                                                {TokenType::SYNONYM, "q"},
+                                                {TokenType::SEMICOLON}};
+
+  Assert::IsTrue(expectedTokens == actualTokens);
+} // namespace UnitTesting
 }
 ;
 TEST_CLASS(TestPqlParser){
@@ -473,6 +483,18 @@ TEST_METHOD(TestParse_WithClause) {
        Reference{PqlToken{TokenType::NUMBER, "34"}}}};
   Assert::IsTrue(actual == expected);
 }
+TEST_METHOD(TestParse_WithClauseRawRefsNotMatching_ThrowsException) {
+  const std::vector<PqlToken> input = {
+      {TokenType::IF},           {TokenType::SYNONYM, "i"},
+      {TokenType::SEMICOLON},    {TokenType::VARIABLE},
+      {TokenType::SYNONYM, "v"}, {TokenType::SEMICOLON},
+      {TokenType::SELECT},       {TokenType::SYNONYM, "v"},
+      {TokenType::WITH},         {TokenType::STRING, "i"},
+      {TokenType::EQUALS},       {TokenType::NUMBER, "34"},
+  };
+
+  Assert::ExpectException<const char *>([input] { Pql::parse(input); });
+} // namespace UnitTesting
 }
 ;
 }
diff --git a/Team20/Code20/source/PQL.cpp b/Team20/Code20/source/PQL.cpp
@@ -40,7 +40,8 @@ std::unordered_map<std::string, TokenType> stringTokenMap = {
     {"with", TokenType::WITH},
     {"and", TokenType::AND},
     {"=", TokenType::EQUALS},
-    {"BOOLEAN", TokenType::BOOLEAN}};
+    {"BOOLEAN", TokenType::BOOLEAN},
+    {"progline", TokenType::PROG_LINE}};
 
 ParsedQuery Pql::parse(std::vector<PqlToken> query) {
   auto parser = PqlParser(query);
diff --git a/Team20/Code20/source/PQLLexer.cpp b/Team20/Code20/source/PQLLexer.cpp
@@ -39,6 +39,7 @@ bool isStringToken(std::string token) {
 PqlLexer::PqlLexer(std::string query) { this->query = query; }
 
 std::vector<PqlToken> PqlLexer::lex() {
+  sanitize(query);
   std::vector<std::string> rawTokens = delimit(query);
   std::vector<PqlToken> result;
   for (const auto token : rawTokens) {
@@ -75,6 +76,16 @@ std::vector<std::string> PqlLexer::split(const std::string &s, char delim) {
   return result;
 }
 
+void PqlLexer::sanitize(std::string &s) {
+  size_t pos = 0;
+  std::string search = "prog_line";
+  std::string replace = "progline";
+  while ((pos = s.find(search, pos)) != std::string::npos) {
+    s.replace(pos, search.length(), replace);
+    pos += replace.length();
+  }
+}
+
 std::vector<std::string> PqlLexer::delimit(std::string s) {
   std::vector<char> result;
   bool isWithinStringLiterals = false;
@@ -127,6 +138,7 @@ std::vector<std::string> PqlLexer::delimit(std::string s) {
       }
       break;
     // 3B. Characters that won't appear within string literals
+    case ';':
     case '_':
     case ',':
     case '<':
diff --git a/Team20/Code20/source/PQLLexer.h b/Team20/Code20/source/PQLLexer.h
@@ -14,4 +14,5 @@ class PqlLexer {
   std::string query;
   std::vector<std::string> split(const std::string &s, char delim);
   std::vector<std::string> delimit(std::string s);
+  void sanitize(std::string &s);
 };
diff --git a/Team20/Code20/source/PQLParser.cpp b/Team20/Code20/source/PQLParser.cpp
@@ -14,11 +14,10 @@
 #include "SimpleTokenizer.h"
 
 std::unordered_set<TokenType> entities = {
-    TokenType::STMT,      TokenType::READ,     TokenType::PRINT,
-    TokenType::CALL,      TokenType::WHILE,    TokenType::IF,
-    TokenType::ASSIGN,    TokenType::VARIABLE, TokenType::CONSTANT,
-    TokenType::PROCEDURE,
-};
+    TokenType::STMT,     TokenType::PROG_LINE, TokenType::READ,
+    TokenType::PRINT,    TokenType::CALL,      TokenType::WHILE,
+    TokenType::IF,       TokenType::ASSIGN,    TokenType::VARIABLE,
+    TokenType::CONSTANT, TokenType::PROCEDURE};
 
 std::unordered_set<TokenType> attributeNames = {
     TokenType::PROCNAME, TokenType::VARNAME, TokenType::VALUE,
@@ -132,6 +131,16 @@ std::unordered_map<TokenType, std::vector<std::unordered_set<TokenType>>>
          }
 
         }};
+std::unordered_map<AttributeRefType, std::unordered_set<TokenType>> attributes =
+    {
+        {AttributeRefType::PROCNAME, {TokenType::PROCEDURE, TokenType::CALL}},
+        {AttributeRefType::VARNAME,
+         {TokenType::VARIABLE, TokenType::READ, TokenType::PRINT}},
+        {AttributeRefType::VALUE, {TokenType::CONSTANT}},
+        {AttributeRefType::STATEMENT_NUM,
+         {TokenType::STMT, TokenType::READ, TokenType::PRINT, TokenType::CALL,
+          TokenType::WHILE, TokenType::IF, TokenType::ASSIGN}},
+};
 
 template <class T> bool contains(std::unordered_set<T> set, T item) {
   return set.find(item) != set.end();
@@ -384,20 +393,23 @@ std::unordered_map<TokenType, AttributeRefType> tokenTypeToAttributeRefType{
     {TokenType::STATEMENT_NUM, AttributeRefType::STATEMENT_NUM}};
 
 Element PqlParser::getElem() {
-  auto nextToken = getNextExpectedToken(TokenType::SYNONYM);
-  getDeclarationForSynonym(nextToken);
-  Element result{nextToken.value, AttributeRefType::NONE};
-  if (it != end && it->type == TokenType::DOT) {
-    getNextExpectedToken(TokenType::DOT);
-    const auto nextToken = getNextToken();
-    // TODO: store attribute names
-    if (!contains(attributeNames, nextToken.type)) {
-      throw "ERROR: Expected next token to be an attribute name but attribute "
-            "name not found";
-    }
-    result.refType = tokenTypeToAttributeRefType[nextToken.type];
+  auto synonym = getNextExpectedToken(TokenType::SYNONYM);
+  const auto declaration = getDeclarationForSynonym(synonym);
+  if (it == end || it->type != TokenType::DOT) {
+    return Element{synonym.value, AttributeRefType::NONE};
+  }
+  getNextExpectedToken(TokenType::DOT);
+  const auto nextToken = getNextToken();
+  if (!contains(attributeNames, nextToken.type)) {
+    throw "ERROR: Expected next token to be an attribute name but attribute "
+          "name not found";
   }
-  return result;
+  const auto refType = tokenTypeToAttributeRefType[nextToken.type];
+  const auto acceptableEntityTypes = attributes[refType];
+  if (acceptableEntityTypes.find(declaration) == acceptableEntityTypes.end()) {
+    throw "ERROR: Declaration not found in acceptable entity types";
+  }
+  return Element{synonym.value, refType};
 }
 
 void PqlParser::parseWithClause() {
@@ -415,7 +427,13 @@ void PqlParser::parseAttributeCompare() {
   const auto secondRef = getRef();
   // TODO: Perform check on attribute compare to check if ref is valid, and to
   // check if at least one is an elem
-  // TODO: Check if reference can be of type null
+  if (firstRef.referenceType == ReferenceType::RAW_VALUE &&
+      secondRef.referenceType == ReferenceType::RAW_VALUE) {
+    if (firstRef.pqlToken.type != secondRef.pqlToken.type) {
+      throw "ERROR: Expecting string to compare with string or number to "
+            "compare with number";
+    }
+  }
   pq.withs.push_back({firstRef, secondRef});
 }
 

Original file line number	Diff line number	Diff line change
`@@ -100,6 +100,16 @@ TEST_METHOD(TestLex_BracketWithinString) {`
`100`	`100`
`101`	`101`	`Assert::IsTrue(expectedTokens == actualTokens);`
`102`	`102`	`} // namespace UnitTesting`
	`103`	`+TEST_METHOD(TestLex_SanitizeProgLine) {`
	`104`	`+ const auto actualTokens = Pql::lex("prog_line p,q;");`
	`105`	`+ const std::vector<PqlToken> expectedTokens = {{TokenType::PROG_LINE},`
	`106`	`+ {TokenType::SYNONYM, "p"},`
	`107`	`+ {TokenType::COMMA},`
	`108`	`+ {TokenType::SYNONYM, "q"},`
	`109`	`+ {TokenType::SEMICOLON}};`
	`110`	`+`
	`111`	`+ Assert::IsTrue(expectedTokens == actualTokens);`
	`112`	`+} // namespace UnitTesting`
`103`	`113`	`}`
`104`	`114`	`;`
`105`	`115`	`TEST_CLASS(TestPqlParser){`
`@@ -473,6 +483,18 @@ TEST_METHOD(TestParse_WithClause) {`
`473`	`483`	`Reference{PqlToken{TokenType::NUMBER, "34"}}}};`
`474`	`484`	`Assert::IsTrue(actual == expected);`
`475`	`485`	`}`
	`486`	`+TEST_METHOD(TestParse_WithClauseRawRefsNotMatching_ThrowsException) {`
	`487`	`+ const std::vector<PqlToken> input = {`
	`488`	`+ {TokenType::IF}, {TokenType::SYNONYM, "i"},`
	`489`	`+ {TokenType::SEMICOLON}, {TokenType::VARIABLE},`
	`490`	`+ {TokenType::SYNONYM, "v"}, {TokenType::SEMICOLON},`
	`491`	`+ {TokenType::SELECT}, {TokenType::SYNONYM, "v"},`
	`492`	`+ {TokenType::WITH}, {TokenType::STRING, "i"},`
	`493`	`+ {TokenType::EQUALS}, {TokenType::NUMBER, "34"},`
	`494`	`+ };`
	`495`	`+`
	`496`	`+ Assert::ExpectException<const char *>([input] { Pql::parse(input); });`
	`497`	`+} // namespace UnitTesting`
`476`	`498`	`}`
`477`	`499`	`;`
`478`	`500`	`}`