Update

Author: elbakramer

main.py

@@ -3,6 +3,23 @@
 from pynescript.ast.types import AST
 
 
+def _dump_value_impl(value, indent: int = 0, depth: int = 0):
+    indent_step = " " * indent
+    if isinstance(value, AST):
+        return _dump_impl(value, indent=indent, depth=depth)
+    elif isinstance(value, list) and len(value) > 0:
+        lines = []
+        lines.append("[")
+        for subvalue in value:
+            lines.append(
+                f"{indent_step * (depth + 1)}{_dump_value_impl(subvalue, indent=indent, depth=depth + 1)},"
+            )
+        lines.append(f"{indent_step * depth}]")
+        return "\n".join(lines)
+    else:
+        return f"{value!r}"
+
+
 def _dump_impl(node: AST, indent: int = 0, depth: int = 0):
     class_name = node.__class__.__name__
     class_params_except = [
@@ -20,27 +37,11 @@ def _dump_impl(node: AST, indent: int = 0, depth: int = 0):
     }
     if indent > 0 and len(class_params) > 0:
         indent_step = " " * indent
-
-        def _dump_value(value, indent=0, depth=0):
-            if isinstance(value, AST):
-                return _dump_impl(value, indent=indent, depth=depth)
-            elif isinstance(value, list) and len(value) > 0:
-                lines = []
-                lines.append("[")
-                for subvalue in value:
-                    lines.append(
-                        f"{indent_step * (depth + 1)}{_dump_value(subvalue, indent=indent, depth=depth + 1)},"
-                    )
-                lines.append(f"{indent_step * depth}]")
-                return "\n".join(lines)
-            else:
-                return f"{value!r}"
-
         lines = []
         lines.append(f"{class_name}(")
         for name, value in class_params.items():
             lines.append(
-                f"{indent_step * (depth + 1)}{name}={_dump_value(value, indent=indent, depth=depth + 1)},"
+                f"{indent_step * (depth + 1)}{name}={_dump_value_impl(value, indent=indent, depth=depth + 1)},"
             )
         lines.append(f"{indent_step * depth})")
         return "\n".join(lines)

pynescript/ast/helpers.py

@@ -5,7 +5,9 @@
 if debug:
     pyparsing.enable_all_warnings()
     pyparsing.enable_diag(pyparsing.Diagnostics.enable_debug_on_named_expressions)
-    pyparsing.disable_diag(pyparsing.Diagnostics.warn_name_set_on_empty_Forward)
+    pyparsing.disable_diag(
+        pyparsing.Diagnostics.warn_ungrouped_named_tokens_in_collection
+    )
 
 pyparsing.ParserElement.enable_left_recursion()
 
@@ -30,9 +32,7 @@
 
 assignment_grammars.structure <<= structure_grammars.structure
 assignment_grammars.expression <<= expression_grammars.expression
-assignment_grammars.atomic_type_name <<= common_grammars.atomic_type_name
-assignment_grammars.collection_type_name <<= common_grammars.collection_type_name
-assignment_grammars.type_argument <<= common_grammars.type_argument
+assignment_grammars.type_specifier <<= common_grammars.type_specifier
 assignment_grammars.identifier_declarator <<= common_grammars.identifier_declarator
 assignment_grammars.tuple_declarator <<= common_grammars.tuple_declarator
 assignment_grammars.function_call <<= common_grammars.function_call

pynescript/ast/parser/grammars/__init__.py

@@ -1,20 +1,17 @@
 from pyparsing import (
     Forward,
-    Suppress,
-    Combine,
     Opt,
 )
 
 from pynescript import ast
 from pynescript.ast.parser.parser_elements import (
+    ResultNameableForward as Forward,
     ConvertToNode,
 )
 from pynescript.ast.parser.tokens import (
     IDENTIFIER,
     VAR,
     VARIP,
-    LBRACKET,
-    RBRACKET,
     ASSIGN,
     COLON_ASSIGN,
     MUL_ASSIGN,
@@ -27,35 +24,13 @@
 structure = Forward()
 expression = Forward()
 
-atomic_type_name = Forward()
-collection_type_name = Forward()
-type_argument = Forward()
+type_specifier = Forward()
 identifier_declarator = Forward()
 tuple_declarator = Forward()
 function_call = Forward()
 
-
 declaration_mode = ConvertToNode(ast.Var)(VAR) | ConvertToNode(ast.VarIp)(VARIP)
-
-
-def handle_array_type_specifier(argument):
-    node = ast.TypeReference(ast.Subscript(argument))
-    return node
-
-
-array_type_specifier = ConvertToNode(handle_array_type_specifier)(
-    Combine(atomic_type_name("value") + Suppress(Combine(LBRACKET + RBRACKET)))
-)
-
-collection_type_specifier = ConvertToNode(ast.TypeReference)(
-    collection_type_name("name") + type_argument("argument")
-)
-
-atomic_type_specifier = ConvertToNode(ast.TypeReference)(atomic_type_name("name"))
-
-type_specifier = (
-    collection_type_specifier | array_type_specifier | atomic_type_specifier
-)
+declaration_mode.set_name("declaration_mode")
 
 assignment_operator = ConvertToNode(ast.Assign)(ASSIGN)
 
@@ -66,14 +41,17 @@ def handle_array_type_specifier(argument):
     + assignment_operator("operator")
     + (structure | expression)("value")
 )
+identifier_declarator_declaration.set_name("identifier_declarator_declaration")
 
 tuple_declarator_declaration = ConvertToNode(ast.Assignment)(
     tuple_declarator("target")
     + assignment_operator("operator")
     + (structure | function_call)("value")
 )
+tuple_declarator_declaration.set_name("tuple_declarator_declaration")
 
 variable_declaration = identifier_declarator_declaration | tuple_declarator_declaration
+variable_declaration.set_name("variable_declaration")
 
 reassignment_operator = (
     ConvertToNode(ast.ColonAssign)(COLON_ASSIGN)
@@ -89,5 +67,7 @@ def handle_array_type_specifier(argument):
     + reassignment_operator("operator")
     + (structure | expression)("value")
 )
+variable_reassignment.set_name("variable_reassignment")
 
 assignment = variable_declaration | variable_reassignment
+assignment.set_name("assignment")

pynescript/ast/parser/grammars/assignment.py

@@ -1,6 +1,7 @@
 from pyparsing import (
     Forward,
     Suppress,
+    Combine,
     Group,
     Opt,
     Empty,
@@ -10,6 +11,8 @@
 
 from pynescript import ast
 from pynescript.ast.parser.parser_elements import (
+    ResultNameableForward as Forward,
+    IndentedBlockWithTabs as IndentedBlock,
     ConvertToNode,
 )
 from pynescript.ast.parser.tokens import (
@@ -40,6 +43,9 @@
 expression = Forward()
 local_statement = Forward()
 
+type_specifier = Forward()
+type_specifier.set_name("type_specifier")
+
 atomic_type_name = (
     ConvertToNode(ast.Int)(INT)
     | ConvertToNode(ast.Float)(FLOAT)
@@ -53,18 +59,74 @@
     | ConvertToNode(ast.Table)(TABLE)
 )
 
+atomic_type_name.set_name("atomic_type_name")
+
 array_type_name = ConvertToNode(ast.Array)(ARRAY)
+array_type_name.set_name("array_type_name")
+
 matrix_type_name = ConvertToNode(ast.Matrix)(MATRIX)
+matrix_type_name.set_name("matrix_type_name")
 
 collection_type_name = array_type_name | matrix_type_name
+collection_type_name.set_name("collection_type_name")
+
+type_name = collection_type_name | atomic_type_name
+type_name.set_name("type_name")
+
+type_argument = Suppress(LCHEVRON) + type_specifier + Suppress(RCHEVRON)
+type_argument.set_name("type_argument")
+
+collection_type_specifier = ConvertToNode(ast.CollectionType)(
+    collection_type_name("type_name") + type_argument("type_argument")
+)
+collection_type_specifier.set_name("collection_type_specifier")
+
+array_type_suffix = Combine(LBRACKET + RBRACKET)
+array_type_suffix.set_name("array_type_suffix")
+
+array_type_specifier = ConvertToNode(ast.ArrayType)(
+    Combine(type_specifier("element_type") + Suppress(array_type_suffix))
+)
+array_type_specifier.set_name("array_type_specifier")
+
+atomic_type_specifier = atomic_type_name
+atomic_type_specifier.set_name("atomic_type_specifier")
+
+# this gives "ParseException raised: Forward recursion without base case, found '?' ..." error
+type_specifier <<= (
+    collection_type_specifier | array_type_specifier | atomic_type_specifier
+)
+
+# this solves the recursion problem above, but requires extra effort on parser action
+type_specifier_trailing = Forward()
+type_specifier_trailing <<= Opt(
+    (array_type_suffix | type_argument) + type_specifier_trailing
+)
+
+
+def handle_type_specifier(results):
+    last_token = results[-1]
+    if len(results) == 1:
+        return last_token
+    else:
+        results_except_last_token = results[:-1]
+        if last_token == "[]":
+            element_type_token = handle_type_specifier(results_except_last_token)
+            return ast.ArrayType(element_type_token)
+        else:
+            type_name_token = handle_type_specifier(results_except_last_token)
+            type_argument_token = last_token
+            return ast.CollectionType(type_name_token, type_argument_token)
+
+
+type_specifier <<= ConvertToNode(handle_type_specifier)(
+    Group(type_name + type_specifier_trailing)
+)
 
-type_argument = Suppress(LCHEVRON) + atomic_type_name + Suppress(RCHEVRON)
 
 name = ConvertToNode(ast.Name)(IDENTIFIER)
-attributed_name = Empty() + delimited_list(name, DOT).leave_whitespace()
 
 
-@attributed_name.set_parse_action
 def handle_attributed_name(results):
     node = results[0]
     for attr in results[1:]:
@@ -82,6 +144,11 @@ def handle_attributed_name(results):
     return node
 
 
+attributed_name = ConvertToNode(handle_attributed_name)(
+    Group(Empty() + delimited_list(name, DOT).leave_whitespace())
+)
+
+
 full_argument = IDENTIFIER("name") + Suppress(ASSIGN) + expression("value")
 value_only_argument = expression("value")
 
@@ -91,7 +158,7 @@ def handle_attributed_name(results):
 
 function_call = ConvertToNode(ast.Call)(
     attributed_name("name")
-    + Opt(type_argument)
+    + Opt(type_argument("type_argument"))
     + Suppress(LPAREN)
     + Opt(argument_list("arguments"))
     + Suppress(RPAREN)

pynescript/ast/parser/grammars/common.py

@@ -7,6 +7,7 @@
 
 from pynescript import ast
 from pynescript.ast.parser.parser_elements import (
+    ResultNameableForward as Forward,
     ConvertToNode,
 )
 from pynescript.ast.parser.tokens import (

pynescript/ast/parser/grammars/expression.py

@@ -2,7 +2,9 @@
     Forward,
     Literal,
     Keyword,
+    SkipTo,
     FollowedBy,
+    Suppress,
     LineEnd,
     Group,
     ZeroOrMore,
@@ -11,12 +13,15 @@
 
 from pynescript import ast
 from pynescript.ast.parser.parser_elements import (
+    ResultNameableForward as Forward,
     ConvertToNode,
 )
 
 global_statement = Forward()
 
-comment = Literal("//") + ... + FollowedBy(LineEnd())
+comment = Literal("//") + SkipTo(LineEnd())("comment")
+
+comment_suppressed = Suppress(comment)
 
 version_comment = (
     Literal("//")
@@ -28,5 +33,3 @@
 )
 
 script = ConvertToNode(ast.Script)(Group(ZeroOrMore(global_statement)))
-
-script.ignore(comment)

pynescript/ast/parser/grammars/module.py

@@ -20,6 +20,7 @@
     ASSIGN,
 )
 from pynescript.ast.parser.parser_elements import (
+    ResultNameableForward as Forward,
     ConvertToNode,
 )
 
@@ -29,6 +30,7 @@
 local_body = Forward()
 
 common_statement = assignment | structure | expression
+common_statement.set_name("common_statement")
 
 import_statement = (
     Suppress(IMPORT)
@@ -40,6 +42,7 @@
     + Suppress(AS)
     + IDENTIFIER
 )
+import_statement.set_name("import_statement")
 
 default_value = expression
 
@@ -65,6 +68,7 @@
     + Suppress(RIGHT_DOUBLE_ARROW)
     + local_body("body")
 )
+function_declaration.set_name("function_declaration")
 
 global_only_statement = import_statement | function_declaration
 
@@ -78,9 +82,15 @@
 global_atomic_statement = global_only_statement | common_statement
 local_atomic_statement = local_only_statement | common_statement
 
+global_atomic_statement.set_name("global_atomic_statement")
+local_atomic_statement.set_name("local_atomic_statement")
+
 global_statement = delimited_list(global_atomic_statement, COMMA)
 local_statement = delimited_list(local_atomic_statement, COMMA)
 
+global_statement.set_name("global_statement")
+local_statement.set_name("local_statement")
+
 statement = (
     break_statement
     | continue_statement
@@ -90,3 +100,4 @@
     | structure
     | expression
 )
+statement.set_name("statement")