From 624f7851bd549f695feb59f6b0583670742b05a1 Mon Sep 17 00:00:00 2001 From: HeronErin Date: Fri, 19 Apr 2024 19:54:22 -0400 Subject: [PATCH] phaseOne and phaseTwo complete --- source/parsing/treegen/astTypes.d | 77 ++++++++++++++------ source/parsing/treegen/expressionParser.d | 81 +++++++++++++++------ source/parsing/treegen/tokenRelationships.d | 1 + source/parsing/treegen/treeGenUtils.d | 4 +- 4 files changed, 115 insertions(+), 48 deletions(-) diff --git a/source/parsing/treegen/astTypes.d b/source/parsing/treegen/astTypes.d index 27bc065..e5a4b5c 100644 --- a/source/parsing/treegen/astTypes.d +++ b/source/parsing/treegen/astTypes.d @@ -108,14 +108,14 @@ enum OperationVariety struct SingleArgumentOperationNodeData { OperationVariety pperationVariety; - AstNode* value; + AstNode value; } struct DoubleArgumentOperationNodeData { OperationVariety pperationVariety; - AstNode* left; - AstNode* right; + AstNode left; + AstNode right; } struct ExpressionNodeData @@ -128,10 +128,10 @@ struct ExpressionNodeData struct CallNodeData { NameUnit func; - AstNode* args; + AstNode args; } -struct AstNode +class AstNode { AstAction action; union @@ -159,26 +159,57 @@ struct AstNode sink("{"); switch (action) { - case AstAction.Keyword: - sink(keywordNodeData.to!string); - break; - case AstAction.TokenHolder: - sink(tokenBeingHeld.to!string); - break; - case AstAction.Expression: - sink(expressionNodeData.components.to!string); - break; - default: break; + case AstAction.Keyword: + sink(keywordNodeData.to!string); + break; + case AstAction.TokenHolder: + sink(tokenBeingHeld.to!string); + break; + case AstAction.Expression: + sink(expressionNodeData.components.to!string); + break; + case AstAction.NamedUnit: + sink(namedUnit.names.to!string); + break; + case AstAction.Call: + sink(callNodeData.func.names.to!string); + sink("(\n"); + sink(callNodeData.args.to!string); + sink("\n)"); + break; + case AstAction.LiteralUnit: + sink(literalUnitCompenents.to!string); + break; + default: + break; } sink("}"); } } -struct ScopeParsingMode{ - bool allowDefiningObjects; - bool allowDefiningFunctions; - bool allowVariableDefinitions; - bool allowInlineVariableAssignments; - bool hasProperties; - bool isCommaSeperated; -} \ No newline at end of file +// struct ScopeParsingMode{ +// bool allowDefiningObjects; +// bool allowDefiningFunctions; +// bool allowVariableDefinitions; +// bool allowInlineVariableAssignments; +// bool hasProperties; +// bool isCommaSeperated; +// } +import std.container.array; + +Nullable!AstNode nextNonWhiteNode(Array!AstNode nodes, ref size_t index) +{ + Nullable!AstNode found; + while (nodes.length > index) + { + import parsing.tokenizer.tokens; + AstNode node = nodes[index++]; + if (node.action == AstAction.TokenHolder && + (node.tokenBeingHeld.tokenVariety == TokenType.WhiteSpace + || node.tokenBeingHeld.tokenVariety == TokenType.Comment)) + continue; + found = node; + break; + } + return found; +} diff --git a/source/parsing/treegen/expressionParser.d b/source/parsing/treegen/expressionParser.d index 03d17f6..19d1f47 100644 --- a/source/parsing/treegen/expressionParser.d +++ b/source/parsing/treegen/expressionParser.d @@ -1,23 +1,26 @@ module parsing.treegen.expressionParser; +import tern.typecons.common : Nullable, nullable; import parsing.treegen.astTypes; import parsing.tokenizer.tokens; import parsing.treegen.tokenRelationships; import errors; import std.stdio; +import std.container.array; -// First step of the AST gen process. Puts the tokens into -// AstNode objects and extracts parenthesis into deeper -// levels of nesting so that later they can be recursivly parsed -AstNode[] parenthesisExtract(Token[] tokens) +// Group letters.letters.letters into NamedUnit s +// Group Parenthesis into AstNode.Expression s to be parsed speratly +private AstNode[] phaseOne(Token[] tokens) { AstNode[] ret; AstNode[] parenthesisStack; - foreach (Token token; tokens) + bool isLastTokenWhite = false; + for (size_t index = 0; index < tokens.length; index++) { + Token token = tokens[index]; if (token.tokenVariety == TokenType.OpenBraces) { - AstNode newExpression; + AstNode newExpression = new AstNode(); newExpression.action = AstAction.Expression; newExpression.expressionNodeData = ExpressionNodeData( token.value[0], @@ -46,10 +49,24 @@ AstNode[] parenthesisExtract(Token[] tokens) parenthesisStack[$ - 1].expressionNodeData.components ~= node; continue; } + AstNode tokenToBeParsedLater = new AstNode(); + if (token.tokenVariety == TokenType.Letter){ + tokenToBeParsedLater.action = AstAction.NamedUnit; + tokenToBeParsedLater.namedUnit = tokens.genNameUnit(index); + index--; + }else if(token.tokenVariety == TokenType.Number){ + tokenToBeParsedLater.action = AstAction.LiteralUnit; + tokenToBeParsedLater.literalUnitCompenents = [token]; + } + else if(token.tokenVariety != TokenType.Comment){ + bool isWhite = token.tokenVariety == TokenType.WhiteSpace; + if (isWhite && isLastTokenWhite) continue; + isLastTokenWhite = isWhite; + + tokenToBeParsedLater.action = AstAction.TokenHolder; + tokenToBeParsedLater.tokenBeingHeld = token; + } - AstNode tokenToBeParsedLater; - tokenToBeParsedLater.action = AstAction.TokenHolder; - tokenToBeParsedLater.tokenBeingHeld = token; if (parenthesisStack.length == 0) ret ~= tokenToBeParsedLater; else @@ -57,30 +74,48 @@ AstNode[] parenthesisExtract(Token[] tokens) } return ret; } -void parseExpression(AstNode[] nodes){ - for(size_t index = 0; index < nodes.length; index++){ +// Handle function calls +private void phaseTwo(Array!AstNode nodes){ + for (size_t index = 0; index < nodes.length; index++){ AstNode node = nodes[index]; - if (index != 0 && node.action == AstAction.Expression && node.expressionNodeData.opener == '(' - && nodes[index-1].action == AstAction.TokenHolder - && nodes[index-1].tokenBeingHeld.tokenVariety == TokenType.Letter){ - AstNode functionCall; + if (node.action == AstAction.NamedUnit && index+1 < nodes.length && nodes[index+1].action == AstAction.Expression){ + AstNode functionCall = new AstNode(); + AstNode args = nodes[index+1]; + + Array!AstNode components; + components~=args.expressionNodeData.components; + phaseTwo(components); + args.expressionNodeData.components.length = components.data.length; + args.expressionNodeData.components[0..$] = components.data[0..$]; + + functionCall.action = AstAction.Call; + functionCall.callNodeData = CallNodeData( + node.namedUnit, + args + ); + nodes[index] = functionCall; + nodes.linearRemove(nodes[index+1..index+2]); } + // else if(node.action == AstAction.TokenHolder && node.tokenBeingHeld.tokenVariety == TokenType.Number){ + + // } } } -void parseExpression(Token[] tokens) -{ - parseExpression(parenthesisExtract(tokens)); - // tokens[0].tokenVariety - - -} +import parsing.treegen.treeGenUtils; unittest { import parsing.tokenizer.make_tokens; - parseExpression("sqrt(8*9+5*2 / (6+10*2))".tokenizeText); + AstNode[] phaseOneNodes = phaseOne("math.sqrt(P(1) + 1 + 2 + 3)".tokenizeText); + + Array!AstNode nodes; + nodes~=phaseOneNodes; + phaseTwo(nodes); + nodes.writeln; + ";".writeln; + } \ No newline at end of file diff --git a/source/parsing/treegen/tokenRelationships.d b/source/parsing/treegen/tokenRelationships.d index 1cfb869..8b08e65 100644 --- a/source/parsing/treegen/tokenRelationships.d +++ b/source/parsing/treegen/tokenRelationships.d @@ -432,3 +432,4 @@ const OperatorPrecedenceLayer[] operatorPrecedence = [ ]) ]; + diff --git a/source/parsing/treegen/treeGenUtils.d b/source/parsing/treegen/treeGenUtils.d index 13b6e10..dcb2a93 100644 --- a/source/parsing/treegen/treeGenUtils.d +++ b/source/parsing/treegen/treeGenUtils.d @@ -16,10 +16,10 @@ NameUnit genNameUnit(Token[] tokens, ref size_t index) return ret; token = tokenNullable; - while (token.tokenVariety == TokenType.Letter || token.tokenVariety == TokenType.Period) + while (token.tokenVariety == TokenType.Letter || token.tokenVariety == TokenType.Number || token.tokenVariety == TokenType.Period) { - if (token.tokenVariety == TokenType.Letter) + if (token.tokenVariety != TokenType.Period) ret.names ~= token.value; tokenNullable = tokens.nextToken(index);