Skip to content

Commit

Permalink
Operation pairing is working, and tree() function for debugging AST n…
Browse files Browse the repository at this point in the history 
…odes
  • Loading branch information
@PsychedelicPalimpsest
PsychedelicPalimpsest committed Apr 21, 2024
1 parent a912b8d commit 2de62dd
Show file tree
Hide file tree
Showing 3 changed files with 127 additions and 75 deletions.
44 changes: 42 additions & 2 deletions source/parsing/treegen/astTypes.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,13 +107,13 @@ enum OperationVariety

struct SingleArgumentOperationNodeData
{
OperationVariety pperationVariety;
OperationVariety operationVariety;
AstNode value;
}

struct DoubleArgumentOperationNodeData
{
OperationVariety pperationVariety;
OperationVariety operationVariety;
AstNode left;
AstNode right;
}
Expand Down Expand Up @@ -180,11 +180,50 @@ class AstNode
case AstAction.LiteralUnit:
sink(literalUnitCompenents.to!string);
break;
case AstAction.DoubleArgumentOperation:
sink(doubleArgumentOperationNodeData.operationVariety.to!string);
sink(", ");
sink(doubleArgumentOperationNodeData.left.to!string);
sink(", ");
sink(doubleArgumentOperationNodeData.right.to!string);
break;
default:
break;
}
sink("}");
}

void tree(size_t tabCount)
{
import std.stdio;
import std.conv;

foreach (i; 0 .. tabCount)
write("| ");

switch (action)
{
case AstAction.Call:
writeln(callNodeData.func.to!string ~ ":");
callNodeData.args.tree(tabCount + 1);
break;
case AstAction.DoubleArgumentOperation:
writeln(doubleArgumentOperationNodeData.operationVariety.to!string ~ ":");
doubleArgumentOperationNodeData.left.tree(tabCount + 1);
doubleArgumentOperationNodeData.right.tree(tabCount + 1);
break;
case AstAction.Expression:
writeln("Result of expression with " ~ expressionNodeData.components.length.to!string ~ " components:");
foreach (subnode; expressionNodeData.components)
{
subnode.tree(tabCount + 1);
}
break;
default:
writeln(this.to!string);
break;
}
}
}

// struct ScopeParsingMode{
Expand All @@ -203,6 +242,7 @@ Nullable!AstNode nextNonWhiteNode(Array!AstNode nodes, ref size_t index)
while (nodes.length > index)
{
import parsing.tokenizer.tokens;

AstNode node = nodes[index++];
if (node.action == AstAction.TokenHolder &&
(node.tokenBeingHeld.tokenVariety == TokenType.WhiteSpace
Expand Down
10 changes: 5 additions & 5 deletions source/parsing/treegen/expressionParser.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ private void phaseTwo(Array!AstNode nodes){
Array!AstNode components;
components~=args.expressionNodeData.components;
phaseTwo(components);
operatorPairingPhase(components);
scanAndMergeOperators(components);
args.expressionNodeData.components.length = components.data.length;
args.expressionNodeData.components[0..$] = components.data[0..$];

Expand All @@ -108,7 +108,7 @@ private void phaseTwo(Array!AstNode nodes){
Array!AstNode components;
components~=node.expressionNodeData.components;
phaseTwo(components);
operatorPairingPhase(components);
scanAndMergeOperators(components);
node.expressionNodeData.components.length = components.data.length;
node.expressionNodeData.components[0..$] = components.data[0..$];
}
Expand All @@ -122,12 +122,12 @@ unittest
{

import parsing.tokenizer.make_tokens;
AstNode[] phaseOneNodes = phaseOne("3*5+6*7/2".tokenizeText);
AstNode[] phaseOneNodes = phaseOne("math.sqrt(3*5+6*7/2)*3".tokenizeText);

Array!AstNode nodes;
nodes~=phaseOneNodes;
phaseTwo(nodes);
nodes.scanForOperators;
nodes.writeln;
scanAndMergeOperators(nodes);
nodes[0].tree(0);

}
148 changes: 80 additions & 68 deletions source/parsing/treegen/tokenRelationships.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,8 +272,8 @@ struct OperationPrecedenceEntry
}

private Token OPR(dchar o)
{
return Token(o != '=' ?TokenType.Operator : TokenType.Equals, [o]);
{
return Token(o != '=' ? TokenType.Operator : TokenType.Equals, [o]);
}

// https://en.cppreference.com/w/c/language/operator_precedence
Expand Down Expand Up @@ -433,94 +433,106 @@ const OperatorPrecedenceLayer[] operatorPrecedence = [

];
import std.container.array;
private bool testAstEntry(const(OperationPrecedenceEntry) entry, AstNode[] nodes){

private bool testAstEntry(const(OperationPrecedenceEntry) entry, AstNode[] nodes)
{
if (entry.tokens.length > nodes.length)
return false;
for(size_t index = 0; index < entry.tokens.length; index++){
switch (entry.tokens[index].tokenVariety){
case TokenType.Filler:
AstNode node = nodes[index];
if (node.action == AstAction.TokenHolder || node.action == AstAction.Keyword || node.action == AstAction.Scope)
return false;
break;
case TokenType.Operator:
AstNode node = nodes[index];
if (node.action != AstAction.TokenHolder)
return false;
Token token = node.tokenBeingHeld;
if (token.tokenVariety != TokenType.Equals && token.tokenVariety != TokenType.Operator)
return false;
if (token.value != entry.tokens[index].value)
return false;
break;
default:
// entry.tokens[index].writeln;
assert(0);

for (size_t index = 0; index < entry.tokens.length; index++)
{
switch (entry.tokens[index].tokenVariety)
{
case TokenType.Filler:
AstNode node = nodes[index];
if (node.action == AstAction.TokenHolder || node.action == AstAction.Keyword || node.action == AstAction
.Scope)
return false;
break;
case TokenType.Operator:
AstNode node = nodes[index];
if (node.action != AstAction.TokenHolder)
return false;
Token token = node.tokenBeingHeld;
if (token.tokenVariety != TokenType.Equals && token.tokenVariety != TokenType.Operator)
return false;
if (token.value != entry.tokens[index].value)
return false;
break;
default:
// entry.tokens[index].writeln;
assert(0);

}
}
return true;
}
private void merge(const(OperationPrecedenceEntry) entry, Array!AstNode nodes, size_t startIndex){
AstNode[] nodeData;

for(size_t index = 0; index < entry.tokens.length; index++){
switch (entry.tokens[index].tokenVariety){
case TokenType.Filler:
nodeData ~= nodes[startIndex + index];
break;
case TokenType.Operator:
break;
default:
assert(0);
private void merge(const(OperationPrecedenceEntry) entry, ref Array!AstNode nodes, size_t startIndex)
{
AstNode[] nodeData;
for (size_t index = 0; index < entry.tokens.length; index++)
{
switch (entry.tokens[index].tokenVariety)
{
case TokenType.Filler:
nodeData ~= nodes[startIndex + index];
break;
case TokenType.Operator:
break;
default:
assert(0);
}
}
AstNode oprNode = new AstNode();
oprNode.action = AstAction.DoubleArgumentOperation;
oprNode.doubleArgumentOperationNodeData = DoubleArgumentOperationNodeData(
entry.operation,
nodeData[0],
nodeData[1]
);
if (nodeData.length == 0)
assert(0);
if (nodeData.length == 1)
{
oprNode.action = AstAction.SingleArgumentOperation;
oprNode.singleArgumentOperationNodeData = SingleArgumentOperationNodeData(
entry.operation,
nodeData[0],
);
}
if (nodeData.length == 2)
oprNode.doubleArgumentOperationNodeData = DoubleArgumentOperationNodeData(
entry.operation,
nodeData[0],
nodeData[1]
);

nodes[startIndex] = oprNode;
nodes.linearRemove(nodes[startIndex+1..startIndex+entry.tokens.length]);
nodes.linearRemove(nodes[startIndex + 1 .. startIndex + entry.tokens.length]);

}

void scanForOperators(Array!AstNode nodes){
void scanAndMergeOperators(Array!AstNode nodes)
{
// OperatorOrder order;
auto data = nodes.data;
static foreach (layer; operatorPrecedence)
{
static if (layer.order == OperatorOrder.LeftToRight){
for (size_t index = 0; index < nodes.length; index++){
// layer.layer.writeln;
// nodes.data.writeln;
static if (layer.order == OperatorOrder.LeftToRight)
{
for (size_t index = 0; index < nodes.length; index++)
{
foreach (entry; layer.layer)
{
// entry.writeln;
// data[index..$].writeln;
bool isEntry = entry.testAstEntry(data[index..$]);
if (isEntry) entry.merge(nodes, index);
if (entry.testAstEntry(data[index .. $]))
entry.merge(nodes, index);
}

}
}
static if (layer.order == OperatorOrder.RightToLeft){
for (size_t index = nodes.length; index != -1; index--){
foreach (entry; layer.layer)
{
if (entry.testAstEntry(data[index .. $]))
entry.merge(nodes, index);
}

}

}
// static if (layer.order == OperatorOrder.RightToLeft){
// "right".writeln;
// for (size_t index = nodes.length; index != -1; index--){
// foreach (entry; layer.layer)
// {
// bool isEntry = entry.testAstEntry(nodes.data[index..$]);
// if (isEntry) entry.operation.writeln;
// }
// }
// }
// order = layer.order;
// foreach (entry; layer.layer)
// {

// }
}
}
}

0 comments on commit 2de62dd

Please sign in to comment.