phaseOne and phaseTwo complete

fern-pl · Apr 19, 2024 · 624f785 · 624f785
1 parent e769204
commit 624f785
Show file tree

Hide file tree

Showing 4 changed files with 115 additions and 48 deletions.
diff --git a/source/parsing/treegen/astTypes.d b/source/parsing/treegen/astTypes.d
@@ -108,14 +108,14 @@ enum OperationVariety
 struct SingleArgumentOperationNodeData
 {
     OperationVariety pperationVariety;
-    AstNode* value;
+    AstNode value;
 }
 
 struct DoubleArgumentOperationNodeData
 {
     OperationVariety pperationVariety;
-    AstNode* left;
-    AstNode* right;
+    AstNode left;
+    AstNode right;
 }
 
 struct ExpressionNodeData
@@ -128,10 +128,10 @@ struct ExpressionNodeData
 struct CallNodeData
 {
     NameUnit func;
-    AstNode* args;
+    AstNode args;
 }
 
-struct AstNode
+class AstNode
 {
     AstAction action;
     union
@@ -159,26 +159,57 @@ struct AstNode
         sink("{");
         switch (action)
         {
-            case AstAction.Keyword:
-                sink(keywordNodeData.to!string);
-                break;
-            case AstAction.TokenHolder:
-                sink(tokenBeingHeld.to!string);
-                break;
-            case AstAction.Expression:
-                sink(expressionNodeData.components.to!string);
-                break;
-            default: break;
+        case AstAction.Keyword:
+            sink(keywordNodeData.to!string);
+            break;
+        case AstAction.TokenHolder:
+            sink(tokenBeingHeld.to!string);
+            break;
+        case AstAction.Expression:
+            sink(expressionNodeData.components.to!string);
+            break;
+        case AstAction.NamedUnit:
+            sink(namedUnit.names.to!string);
+            break;
+        case AstAction.Call:
+            sink(callNodeData.func.names.to!string);
+            sink("(\n");
+            sink(callNodeData.args.to!string);
+            sink("\n)");
+            break;
+        case AstAction.LiteralUnit:
+            sink(literalUnitCompenents.to!string);
+            break;
+        default:
+            break;
         }
         sink("}");
     }
 }
 
-struct ScopeParsingMode{
-    bool allowDefiningObjects;
-    bool allowDefiningFunctions;
-    bool allowVariableDefinitions;
-    bool allowInlineVariableAssignments;
-    bool hasProperties;
-    bool isCommaSeperated;
-}
+// struct ScopeParsingMode{
+//     bool allowDefiningObjects;
+//     bool allowDefiningFunctions;
+//     bool allowVariableDefinitions;
+//     bool allowInlineVariableAssignments;
+//     bool hasProperties;
+//     bool isCommaSeperated;
+// }
+import std.container.array;
+
+Nullable!AstNode nextNonWhiteNode(Array!AstNode nodes, ref size_t index)
+{
+    Nullable!AstNode found;
+    while (nodes.length > index)
+    {
+        import parsing.tokenizer.tokens;
+        AstNode node = nodes[index++];
+        if (node.action == AstAction.TokenHolder &&
+            (node.tokenBeingHeld.tokenVariety == TokenType.WhiteSpace
+                || node.tokenBeingHeld.tokenVariety == TokenType.Comment))
+            continue;
+        found = node;
+        break;
+    }
+    return found;
+}
diff --git a/source/parsing/treegen/expressionParser.d b/source/parsing/treegen/expressionParser.d
@@ -1,23 +1,26 @@
 module parsing.treegen.expressionParser;
 
+import tern.typecons.common : Nullable, nullable;
 import parsing.treegen.astTypes;
 import parsing.tokenizer.tokens;
 import parsing.treegen.tokenRelationships;
 import errors;
 import std.stdio;
+import std.container.array;
 
-// First step of the AST gen process. Puts the tokens into
-// AstNode objects and extracts parenthesis into deeper
-// levels of nesting so that later they can be recursivly parsed
-AstNode[] parenthesisExtract(Token[] tokens)
+// Group letters.letters.letters into NamedUnit s
+// Group Parenthesis into AstNode.Expression s to be parsed speratly
+private AstNode[] phaseOne(Token[] tokens)
 {
     AstNode[] ret;
     AstNode[] parenthesisStack;
-    foreach (Token token; tokens)
+    bool isLastTokenWhite = false;
+    for (size_t index = 0; index < tokens.length; index++)
     {
+        Token token = tokens[index];
         if (token.tokenVariety == TokenType.OpenBraces)
         {
-            AstNode newExpression;
+            AstNode newExpression = new AstNode();
             newExpression.action = AstAction.Expression;
             newExpression.expressionNodeData = ExpressionNodeData(
                 token.value[0],
@@ -46,41 +49,73 @@ AstNode[] parenthesisExtract(Token[] tokens)
                 parenthesisStack[$ - 1].expressionNodeData.components ~= node;
             continue;
         }
+        AstNode tokenToBeParsedLater = new AstNode();
+        if (token.tokenVariety == TokenType.Letter){
+            tokenToBeParsedLater.action = AstAction.NamedUnit;
+            tokenToBeParsedLater.namedUnit = tokens.genNameUnit(index);
+            index--;
+        }else if(token.tokenVariety == TokenType.Number){
+            tokenToBeParsedLater.action = AstAction.LiteralUnit;
+            tokenToBeParsedLater.literalUnitCompenents = [token];
+        }
+        else if(token.tokenVariety != TokenType.Comment){
+            bool isWhite = token.tokenVariety == TokenType.WhiteSpace;
+            if (isWhite && isLastTokenWhite) continue;
+            isLastTokenWhite = isWhite;
+
+            tokenToBeParsedLater.action = AstAction.TokenHolder;
+            tokenToBeParsedLater.tokenBeingHeld = token;
+        }        
 
-        AstNode tokenToBeParsedLater;
-        tokenToBeParsedLater.action = AstAction.TokenHolder;
-        tokenToBeParsedLater.tokenBeingHeld = token;
         if (parenthesisStack.length == 0)
             ret ~= tokenToBeParsedLater;
         else
             parenthesisStack[$ - 1].expressionNodeData.components ~= tokenToBeParsedLater;
     }
     return ret;
 }
-void parseExpression(AstNode[] nodes){
-    for(size_t index = 0; index < nodes.length; index++){
+// Handle function calls
+private void phaseTwo(Array!AstNode nodes){
+    for (size_t index = 0; index < nodes.length; index++){
         AstNode node = nodes[index];
-        if (index != 0 && node.action == AstAction.Expression && node.expressionNodeData.opener == '('
-                && nodes[index-1].action == AstAction.TokenHolder 
-                && nodes[index-1].tokenBeingHeld.tokenVariety == TokenType.Letter){
-            AstNode functionCall;
+        if (node.action == AstAction.NamedUnit && index+1 < nodes.length && nodes[index+1].action == AstAction.Expression){
+            AstNode functionCall = new AstNode();
+            AstNode args = nodes[index+1];
+
+            Array!AstNode components;
+            components~=args.expressionNodeData.components;
+            phaseTwo(components);
+            args.expressionNodeData.components.length = components.data.length;
+            args.expressionNodeData.components[0..$] = components.data[0..$];
+
+
             functionCall.action = AstAction.Call;
+            functionCall.callNodeData = CallNodeData(
+                node.namedUnit,
+                args
+            );
+            nodes[index] = functionCall;
+            nodes.linearRemove(nodes[index+1..index+2]);   
         }
+        // else if(node.action == AstAction.TokenHolder && node.tokenBeingHeld.tokenVariety == TokenType.Number){
+
+        // }
     }
 }
 
-void parseExpression(Token[] tokens)
-{
-    parseExpression(parenthesisExtract(tokens));
-    // tokens[0].tokenVariety
-
-
-}
+import parsing.treegen.treeGenUtils;
 
 
 unittest
 {
 
     import parsing.tokenizer.make_tokens;
-    parseExpression("sqrt(8*9+5*2 / (6+10*2))".tokenizeText);
+    AstNode[] phaseOneNodes =  phaseOne("math.sqrt(P(1) + 1 + 2 + 3)".tokenizeText);
+
+    Array!AstNode nodes;
+    nodes~=phaseOneNodes;
+    phaseTwo(nodes);
+    nodes.writeln;
+    ";".writeln;
+
 }
diff --git a/source/parsing/treegen/tokenRelationships.d b/source/parsing/treegen/tokenRelationships.d
@@ -432,3 +432,4 @@ const OperatorPrecedenceLayer[] operatorPrecedence = [
         ])
 
 ];
+
diff --git a/source/parsing/treegen/treeGenUtils.d b/source/parsing/treegen/treeGenUtils.d
@@ -16,10 +16,10 @@ NameUnit genNameUnit(Token[] tokens, ref size_t index)
         return ret;
     token = tokenNullable;
 
-    while (token.tokenVariety == TokenType.Letter || token.tokenVariety == TokenType.Period)
+    while (token.tokenVariety == TokenType.Letter || token.tokenVariety == TokenType.Number || token.tokenVariety == TokenType.Period)
     {
 
-        if (token.tokenVariety == TokenType.Letter)
+        if (token.tokenVariety != TokenType.Period)
             ret.names ~= token.value;
 
         tokenNullable = tokens.nextToken(index);
Original file line number	Diff line number	Diff line change
Expand Up		@@ -432,3 +432,4 @@ const OperatorPrecedenceLayer[] operatorPrecedence = [
		])

		];