Skip to content

Commit

Permalink
Function parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@PsychedelicPalimpsest
PsychedelicPalimpsest committed Apr 25, 2024
1 parent 328a09f commit 3cee025
Show file tree
Hide file tree
Showing 4 changed files with 125 additions and 48 deletions.
31 changes: 15 additions & 16 deletions source/main.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,19 @@ void main()
{
size_t index = 0;

auto newScope = parseMultilineScope(FUNCTION_SCOPE_PARSE, "
string axolotl, frog = `Hello world` * 2 + 1;
int constant = 69 /* nice!!! */ ;
".tokenizeText(), index, nullable!ScopeData(null));
foreach (instruction; newScope.instructions)
{
instruction.tree;
}
"\n\nDeclared variables: ".writeln;
foreach (var; newScope.declaredVariables)
{
var.writeln;
}
auto newScope = parseMultilineScope(GLOBAL_SCOPE_PARSE, "
int main(){
int x = 4;
}
".tokenizeText, index, nullable!ScopeData(null));
newScope.writeln;
// foreach (instruction; newScope.instructions)
// {
// instruction.tree;
// }
// "\n\nDeclared variables: ".writeln;
// foreach (var; newScope.declaredVariables)
// {
// var.writeln;
// }
}
16 changes: 0 additions & 16 deletions source/parsing/treegen/astTypes.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ enum AstAction
{
Keyword, // Standalong keywords Ex: import std.std.io;
Scope,
DefineFunction,
DefineVariable, // Ex: int x;
AssignVariable, // Ex: x = 5;
IndexInto, // X[...]

Expand Down Expand Up @@ -41,20 +39,7 @@ struct KeywordNodeData
Token[] keywardArgs;
}

struct DefineFunctionNodeData
{
dchar[][] precedingKeywords;
dchar[][] suffixKeywords;
NameUnit returnType;
AstNode functionScope;
}

struct DefineVariableNodeData
{
dchar[][] precedingKeywords;
NameUnit returnType;
AstNode[] functionScope;
}

struct AssignVariableNodeData
{
Expand Down Expand Up @@ -150,7 +135,6 @@ class AstNode
union
{
KeywordNodeData keywordNodeData; // Keyword
DefineVariableNodeData defineVariableNodeData; // DefineVariable
AssignVariableNodeData assignVariableNodeData; // AssignVariable

SingleArgumentOperationNodeData singleArgumentOperationNodeData; // SingleArgumentOperation
Expand Down
47 changes: 41 additions & 6 deletions source/parsing/treegen/scopeParser.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,15 @@ struct ImportStatement
NameUnit nameUnit;
NameUnit[] importSelection; // empty for importing everything
}

struct DeclaredFunction
{
dchar[][] precedingKeywords;
dchar[][] suffixKeywords;
NameUnit returnType;
NameUnit name;
// TODO: Args
ScopeData functionScope;
}
struct DeclaredVariable
{
NameUnit name;
Expand All @@ -31,6 +39,7 @@ class ScopeData
Nullable!NameUnit moduleName;
ImportStatement[] imports;

DeclaredFunction[] declaredFunctions;
DeclaredVariable[] declaredVariables;
Array!AstNode instructions;
}
Expand All @@ -50,10 +59,11 @@ LineVarietyTestResult getLineVarietyTestResult(
foreach (method; scopeParseMethod)
{
Nullable!(TokenGrepResult[]) grepResults = method.test.matchesToken(tokens, temp_index);
if (null != grepResults)
if (null != grepResults){
return LineVarietyTestResult(
method.variety, temp_index - index, grepResults.value
);
}
temp_index = index;
}

Expand Down Expand Up @@ -119,12 +129,13 @@ LineVarietyTestResult parseLine(const(VarietyTestPair[]) scopeParseMethod, Token

auto statement = ImportStatement(
keywords,
lineVariety.tokenMatches[IMPORT_PACKAGE_NAME].name,
lineVariety.tokenMatches[IMPORT_PACKAGE_NAME].assertAs(TokenGrepMethod.NameUnit).name,
[]
);

statement.importSelection ~= lineVariety
.tokenMatches[SELECTIVE_IMPORT_SELECTIONS]
.assertAs(TokenGrepMethod.PossibleCommaSeperated)
.commaSeperated
.collectNameUnits();

Expand All @@ -136,8 +147,10 @@ LineVarietyTestResult parseLine(const(VarietyTestPair[]) scopeParseMethod, Token
scope (exit)
index = endingIndex;

NameUnit declarationType = lineVariety.tokenMatches[DECLARATION_TYPE].name;
NameUnit[] declarationNames = lineVariety.tokenMatches[DECLARATION_VARS].commaSeperated.collectNameUnits();
NameUnit declarationType = lineVariety.tokenMatches[DECLARATION_TYPE].assertAs(TokenGrepMethod.NameUnit).name;
NameUnit[] declarationNames = lineVariety.tokenMatches[DECLARATION_VARS]
.assertAs(TokenGrepMethod.PossibleCommaSeperated)
.commaSeperated.collectNameUnits();
AstNode[] nameNodes;
foreach (NameUnit name; declarationNames)
{
Expand All @@ -151,7 +164,9 @@ LineVarietyTestResult parseLine(const(VarietyTestPair[]) scopeParseMethod, Token
if (lineVariety.lineVariety == LineVariety.DeclarationLine)
break;

auto nodes = lineVariety.tokenMatches[DECLARATION_EXPRESSION].tokens.expressionNodeFromTokens();
auto nodes = lineVariety.tokenMatches[DECLARATION_EXPRESSION]
.assertAs(TokenGrepMethod.Glob)
.tokens.expressionNodeFromTokens();

if (nodes.length != 1)
throw new SyntaxError(
Expand All @@ -164,6 +179,26 @@ LineVarietyTestResult parseLine(const(VarietyTestPair[]) scopeParseMethod, Token

parent.instructions ~= assignment;

break;
case LineVariety.FunctionDeclaration:
size_t endingIndex = index + lineVariety.length;
scope (exit)
index = endingIndex;
size_t temp;
parent.declaredFunctions ~= DeclaredFunction(
keywords,
[],
lineVariety.tokenMatches[FUNCTION_NAME].assertAs(TokenGrepMethod.NameUnit).name,
lineVariety.tokenMatches[FUNCTION_RETURN_TYPE].assertAs(TokenGrepMethod.NameUnit).name,
parseMultilineScope(
FUNCTION_SCOPE_PARSE,
lineVariety.tokenMatches[FUNCTION_SCOPE].assertAs(TokenGrepMethod.Glob).tokens,
temp,
nullable!ScopeData(parent)
)
);

// assert(0);
break;
case LineVariety.SimpleExpression:
size_t expression_end = tokens.findNearestSemiColon(index);
Expand Down
79 changes: 69 additions & 10 deletions source/parsing/treegen/tokenRelationships.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ struct TokenGrepResult
NameUnit name;

}
pragma(always_inline)
TokenGrepResult assertAs(TokenGrepMethod test){
debug assert(this.method == test);
return this;
}
}

TokenGrepPacket TokenGrepPacketToken(TokenGrepMethod method, Token[] list)
Expand Down Expand Up @@ -103,8 +108,6 @@ const TokenGrepPacket[] IfStatementWithScope = [
// This is all unmantainable as FUCK, and confusing to read.
// But this works, and is quite convinent.

// TODO: Refactor ALL of this:

const size_t DECLARATION_TYPE = 0;
const size_t DECLARATION_VARS = 1;
// int x, y, z;
Expand Down Expand Up @@ -176,11 +179,62 @@ const TokenGrepPacket[] ModuleDeclaration = [
Token(TokenType.Semicolon, [])
])
];

const FUNCTION_RETURN_TYPE = 0;
const FUNCTION_NAME = 1;
const FUNCTION_ARGS = 2;
const FUNCTION_SCOPE = 3;

// void main();
const TokenGrepPacket[] AbstractFunctionDeclaration = [
TokenGrepPacketToken(TokenGrepMethod.NameUnit, []),
TokenGrepPacketToken(TokenGrepMethod.NameUnit, []),

TokenGrepPacketToken(TokenGrepMethod.MatchesTokens, [
Token(TokenType.OpenBraces, ['('])
]),
// Parameters
TokenGrepPacketToken(TokenGrepMethod.Glob, []),
TokenGrepPacketToken(TokenGrepMethod.MatchesTokens, [
Token(TokenType.CloseBraces, [')'])
]),
// Prepended attributes
TokenGrepPacketToken(TokenGrepMethod.Glob, []),
TokenGrepPacketToken(TokenGrepMethod.MatchesTokenType, [
Token(TokenType.Semicolon, [])
])
];
const TokenGrepPacket[] FunctionDeclaration = [
TokenGrepPacketToken(TokenGrepMethod.NameUnit, []),
TokenGrepPacketToken(TokenGrepMethod.NameUnit, []),

TokenGrepPacketToken(TokenGrepMethod.MatchesTokens, [
Token(TokenType.OpenBraces, ['('])
]),
// Parameters
TokenGrepPacketToken(TokenGrepMethod.Glob, []),
TokenGrepPacketToken(TokenGrepMethod.MatchesTokens, [
Token(TokenType.CloseBraces, [')'])
]),
// Body
TokenGrepPacketToken(TokenGrepMethod.MatchesTokens, [
Token(TokenType.OpenBraces, ['{'])
]),
// Parameters
TokenGrepPacketToken(TokenGrepMethod.Glob, []),
TokenGrepPacketToken(TokenGrepMethod.MatchesTokens, [
Token(TokenType.CloseBraces, ['}'])
]),
];



enum LineVariety
{
TotalImport,
SelectiveImport,
ModuleDeclaration,
FunctionDeclaration,

SimpleExpression,
IfStatementWithScope,
Expand All @@ -202,7 +256,8 @@ const VarietyTestPair[] ABSTRACT_SCOPE_PARSE = [
VarietyTestPair(LineVariety.DeclarationAndAssignment, DeclarationAndAssignment),
];
const VarietyTestPair[] GLOBAL_SCOPE_PARSE = [
VarietyTestPair(LineVariety.ModuleDeclaration, ModuleDeclaration)
VarietyTestPair(LineVariety.ModuleDeclaration, ModuleDeclaration),
VarietyTestPair(LineVariety.FunctionDeclaration, FunctionDeclaration)
] ~ ABSTRACT_SCOPE_PARSE;

const VarietyTestPair[] FUNCTION_SCOPE_PARSE = [
Expand Down Expand Up @@ -304,13 +359,16 @@ Nullable!(TokenGrepResult[]) matchesToken(in TokenGrepPacket[] testWith, Token[]
break;

case TokenGrepMethod.Glob:
auto firstGlob = testWith[testIndex + 1 .. $].matchesToken(tokens[index .. $]);
size_t temp_index;
auto firstGlob = testWith[testIndex + 1 .. $].matchesToken(tokens[index .. $], temp_index);

TokenGrepResult globResult;
globResult.method = TokenGrepMethod.Glob;
globResult.tokens = [];

if (firstGlob.ptr)
if (firstGlob.ptr){
index+=temp_index;
return tokenGrepBox(returnVal ~ globResult ~ firstGlob.value);
}

int braceDeph = 0;
size_t startingIndex = index;
Expand All @@ -325,17 +383,18 @@ Nullable!(TokenGrepResult[]) matchesToken(in TokenGrepPacket[] testWith, Token[]

if (token.tokenVariety == TokenType.OpenBraces)
braceDeph += 1;
else if (token.tokenVariety == TokenType.CloseBraces && braceDeph != 0)
else if (token.tokenVariety == TokenType.CloseBraces){
braceDeph -= 1;
else if (braceDeph == 0)
if (braceDeph == -1)
return tokenGrepBox(null);
}else if (braceDeph == 0)
{
size_t index_inc;
size_t index_inc = 0;
auto res = grepMatchGroup.matchesToken(tokens[index .. $], index_inc);
if (res.ptr)
{

globResult.tokens = tokens[startingIndex .. index];

index += index_inc;
return tokenGrepBox(returnVal ~ globResult ~ res.value);
}
Expand Down

0 comments on commit 3cee025

Please sign in to comment.