switch from thonny ast tokenization to asttokens lib

- requires a 'module wrapper' that inherits first and last tokens from children
- some more tests for added robustness
- simplify some tests + switch to pytest for some

Author: Filip Schouwenaars

.gitignore

@@ -73,4 +73,7 @@ target/
 tests/.DS_Store
 
 # pytest
-.pytest_cache/
+.pytest_cache/
+
+# datasets
+*.csv

pythonwhat/Feedback.py

@@ -1,112 +1,18 @@
 import re
-from pythonwhat import utils
-from pythonwhat import utils_ast
-import _ast
 
 class Feedback(object):
 
     def __init__(self, message, astobj = None):
         self.message = message
         self.line_info = {}
         try:
-            if astobj is not None:
-                if issubclass(type(astobj), (_ast.Module, _ast.Expression)):
-                    astobj = astobj.body
-                if isinstance(astobj, list) and len(astobj) > 0:
-                    start = astobj[0]
-                    end = astobj[-1]
-                else:
-                    start = astobj
-                    end = astobj
-                if  hasattr(start, "lineno") and \
-                    hasattr(start, "col_offset") and \
-                    hasattr(end, "end_lineno") and \
-                    hasattr(end, "end_col_offset"):
-                    self.line_info["line_start"] = start.lineno
-                    self.line_info["column_start"] = start.col_offset
-                    self.line_info["line_end"] = end.end_lineno
-                    self.line_info["column_end"] = end.end_col_offset
+            if astobj is not None and \
+                hasattr(astobj, "first_token") and \
+                hasattr(astobj, "last_token"):
+                self.line_info["line_start"] = astobj.first_token.start[0]
+                self.line_info["column_start"] = astobj.first_token.start[1]
+                self.line_info["line_end"] = astobj.last_token.end[0]
+                self.line_info["column_end"] = astobj.last_token.end[1]
         except:
             pass
 
-# TODO FILIP: No used for now, come back to this later.
-class FeedbackMessage(object):
-    """Generate feedback.
-
-    Don't use this yet!
-
-    This class will hold all functionality which is related to feedback messaging.
-    At the moment it is NOT used, feedback generation is still HIGLY interwoven with
-    test_... files. Should be decoupled.
-
-    Class should be refactored to use .format() instead.
-
-    Will be documented when it's refactored.
-    """
-    def __init__(self, message_string):
-        self.set(message_string)
-        self.information = {}
-
-    def add_information(self, key, value):
-        if (not(key in self.information)):
-            self.set_information(key, value)
-
-    def set_information(self, key, value):
-        self.information[key] = utils.shorten_str(str(value))
-
-    def remove_information(self, key):
-        if (key in self.information):
-            self.information.pop(key)
-
-    def set(self, message_string):
-        self.message_string = str(message_string)
-
-    def append(self, message_string):
-        self.message_string += str(message_string)
-
-    def cond_append(self, cond, message_string):
-        self.message_string += "${{" + \
-            str(cond) + " ? " + str(message_string) + "}}"
-
-    def generateString(self):
-        generated_string = FeedbackMessage.replaceRegularTags(
-            self.message_string, self.information)
-        generated_string = FeedbackMessage.replaceConditionalTags(
-            generated_string, self.information)
-        return(generated_string)
-
-    def replaceRegularTags(message_string, information):
-        generated_string = message_string
-
-        pattern = "\${([a-zA-Z]*?)}"
-
-        keywords = re.findall(pattern, generated_string)
-        for keyword in keywords:
-            replace = "\${" + keyword + "}"
-            if (keyword in information):
-                generated_string = re.sub(
-                    replace, information[keyword], generated_string)
-            else:
-                generated_string = re.sub(replace, "", generated_string)
-
-        return(generated_string)
-
-    def replaceConditionalTags(message_string, information):
-        generated_string = message_string.replace("\n", "\\\\n")
-        pattern = "\${{([a-zA-Z]*?) \? (.*?)}}"
-
-        cond_keywords = re.findall(pattern, generated_string)
-        for (keyword, k_string) in cond_keywords:
-            replace = "\${{" + keyword + " \? " + re.escape(k_string) + "}}"
-            if (keyword in information):
-                generated_string = re.sub(
-                    replace,
-                    " " +
-                    FeedbackMessage.replaceRegularTags(
-                        k_string,
-                        information),
-                    generated_string)
-            else:
-                generated_string = re.sub(replace, "", generated_string)
-
-        return(generated_string)

pythonwhat/Reporter.py

@@ -29,7 +29,6 @@ def do_test(self, testobj, prepend_on_fail="", fallback_ast=None):
         Execute a given test, unless some previous test has failed. If the test has failed,
         the state of the reporter changes and the feedback is kept.
         """
-
         if prepend_on_fail: self.failure_msg = prepend_on_fail
         if fallback_ast: self.fallback_ast = fallback_ast
 
@@ -40,7 +39,7 @@ def do_test(self, testobj, prepend_on_fail="", fallback_ast=None):
                 self.failed_test = True
                 self.feedback = testobj.get_feedback()
                 self.feedback.message = self.failure_msg + self.feedback.message
-                if not self.feedback.line_info and self.fallback_ast: 
+                if not self.feedback.line_info and self.fallback_ast:
                     self.feedback = Feedback(self.feedback.message, self.fallback_ast)
                 raise TestFail
 

pythonwhat/State.py

@@ -6,12 +6,13 @@
 from pythonwhat.parsing import TargetVars, FunctionParser, ObjectAccessParser, parser_dict
 from pythonwhat.Reporter import Reporter
 from pythonwhat.Feedback import Feedback
-from pythonwhat import utils_ast
 from pythonwhat import signatures
 from pythonwhat.converters import get_manual_converters
 from collections.abc import Mapping
 from itertools import chain
 from jinja2 import Template
+import asttokens
+from pythonwhat.utils_ast import wrap_in_module
 
 class Context(Mapping):
     def __init__(self, context=None, prev=None):
@@ -88,13 +89,13 @@ def __init__(self,
 
         # parse code if didn't happen yet
         if not hasattr(self, 'student_tree'):
-            self.student_tree = State.parse_ext(self.student_code)
+            self.student_tree_tokens, self.student_tree = State.parse_external(self.student_code)
 
         if not hasattr(self, 'solution_tree'):
-            self.solution_tree = State.parse_int(self.solution_code)
+            _, self.solution_tree = State.parse_internal(self.solution_code)
 
         if not hasattr(self, 'pre_exercise_tree'):
-            self.pre_exercise_tree = State.parse_int(self.pre_exercise_code)
+            _, self.pre_exercise_tree = State.parse_internal(self.pre_exercise_code)
 
         if not hasattr(self, 'parent_state'):
             self.parent_state = None
@@ -170,9 +171,9 @@ def to_child_state(self, student_subtree, solution_subtree,
         """
 
         if isinstance(student_subtree, list):
-            student_subtree = ast.Module(student_subtree)
+            student_subtree = wrap_in_module(student_subtree)
         if isinstance(solution_subtree, list):
-            solution_subtree = ast.Module(solution_subtree)
+            solution_subtree = wrap_in_module(solution_subtree)
 
         # get new contexts
         if solution_context is not None: 
@@ -196,8 +197,9 @@ def to_child_state(self, student_subtree, solution_subtree,
                                highlight = highlight, messages = messages)
 
         klass = State if not node_name else self.SUBCLASSES[node_name]
-        child = klass(student_code = utils_ast.extract_text_from_node(self.full_student_code, student_subtree),
+        child = klass(student_code = self.student_tree_tokens.get_text(student_subtree),
                       full_student_code = self.full_student_code,
+                      student_tree_tokens = self.student_tree_tokens,
                       pre_exercise_code = self.pre_exercise_code,
                       student_context = student_context,
                       solution_context  = solution_context,
@@ -222,14 +224,13 @@ def update(self, **kwargs):
         return child
 
     @staticmethod
-    def parse_ext(x):
+    def parse_external(x):
         rep = Reporter.active_reporter
 
-        res = None
+        res = (None, None)
         try:
-            res = ast.parse(x)
-            # enrich tree with end lines and end columns
-            utils_ast.mark_text_ranges(res, x + '\n')
+            res = asttokens.ASTTokens(x, parse = True)
+            return(res, res._tree)
 
         except IndentationError as e:
             e.filename = "script.py"
@@ -249,26 +250,19 @@ def parse_ext(x):
             rep.feedback.message = "Something went wrong while parsing your code."
             rep.failed_test = True
 
-        finally:
-            if (res is None):
-                res = False
-
         return(res)
 
     @staticmethod
-    def parse_int(x):
-        res = None
+    def parse_internal(x):
+        res = (None, None)
         try:
-            res = ast.parse(x)
-            utils_ast.mark_text_ranges(res, x + '\n')
+            res = asttokens.ASTTokens(x, parse = True)
+            return(res, res._tree)
 
         except SyntaxError as e:
             raise SyntaxError(str(e))
         except TypeError as e:
             raise TypeError(str(e))
-        finally:
-            if (res is None):
-                res = False
 
         return(res)
 

pythonwhat/check_funcs.py

@@ -30,7 +30,6 @@ def check_part(name, part_msg, state=None, missing_msg="Are you sure it's define
     append_message = {'msg': expand_msg, 'kwargs': {'part': part_msg,}}
 
     has_part(name, missing_msg, state, append_message['kwargs'])
-
     stu_part = state.student_parts[name]
     sol_part = state.solution_parts[name]
 
@@ -231,7 +230,7 @@ def with_context(*args, state=None):
     solution_res = setUpNewEnvInProcess(process = state.solution_process,
                                         context = state.solution_parts['with_items'])
     if isinstance(solution_res, Exception):
-        raise Exception("error in the solution, running test_with() on with %d: %s" % (index - 1, str(solution_res)))
+        raise Exception("error in the solution, running test_with(): %s" % str(solution_res))
 
     student_res = setUpNewEnvInProcess(process = state.student_process,
                                        context = state.student_parts['with_items'])

pythonwhat/parsing.py

@@ -1,4 +1,5 @@
 import ast
+from pythonwhat.utils_ast import wrap_in_module
 from collections.abc import Sequence, Mapping
 from collections import OrderedDict
 from contextlib import ExitStack
@@ -472,7 +473,6 @@ def get_part(name_node, ass_node=None):
         name = getattr(name_node, 'id', name_node)
         load_name = ast.Name(id=name, ctx=ast.Load())
         ast.fix_missing_locations(load_name)
-        # 
         return {'name': name,
                 'node': load_name,
                 'highlight': ass_node or name_node,
@@ -610,7 +610,9 @@ def parse_node(cls, node):
         all_args = [*args, varargs, *kw_args, kwargs]
 
         if isinstance(node, ast.Lambda): body_node = node.body
-        else: body_node = FunctionBodyTransformer().visit(ast.Module(node.body))
+        else:
+            bodyMod = wrap_in_module(node.body)
+            body_node = FunctionBodyTransformer().visit(bodyMod)
 
         return {
             "node": node,
@@ -757,13 +759,10 @@ def visit_Return(self, node):
         new_node = ast.copy_location(ast.Expr(value = node.value), node)
         return FunctionBodyTransformer.decorate(new_node, node)
 
+    @staticmethod
     def decorate(new_node, node):
-        try:
-            # only possible on the student side!
-            new_node.end_lineno = node.end_lineno
-            new_node.end_col_offset = node.end_col_offset
-        except:
-            pass
+        new_node.first_token = node.first_token
+        new_node.last_token = node.last_token
         return new_node
 
 class WithParser(Parser):

pythonwhat/test_funcs/test_for_loop.py

@@ -62,7 +62,6 @@ def test_for_loop(index=1,
         :code:`test_exression_output()` will pass on the body code.
     """
     rep = Reporter.active_reporter
-
     state = check_node('for_loops', index-1, "`for` loops", MSG_MISSING, MSG_PREPEND, state=state)
 
     # TODO for_iter is a level up, so shouldn't have targets set, but this is done is check_node

pythonwhat/test_funcs/test_function.py

@@ -5,7 +5,6 @@
 from pythonwhat.Reporter import Reporter
 from pythonwhat.Feedback import Feedback
 from pythonwhat.utils import get_ord, get_num
-from pythonwhat.utils_ast import extract_text_from_node
 from pythonwhat.tasks import getResultInProcess, getSignatureInProcess, ReprFail
 from .test_or import test_or
 

pythonwhat/test_funcs/test_if_else.py

@@ -70,7 +70,6 @@ def test_if_else(index=1,
     """
     rep = Reporter.active_reporter
 
-
     # get state with specific if block
     node_name = 'if_exps' if use_if_exp else 'if_elses'
     # TODO original typestr for check_node used if rather than `if`