Autogen Bug fix for imported facts

Author: ashwinprasadme

autogen/generate_typeevalpy_dataset.py

@@ -2,7 +2,7 @@
 import os
 import shutil
 from pathlib import Path
-import tqdm
+from tqdm import tqdm
 import time
 import json
 import datetime
@@ -17,6 +17,7 @@
 )
 error_folder = f"{ROOT_DIR}/.scrapy/error"
 benchmark_dir = f"{ROOT_DIR}/micro-benchmark-autogen-templates"
+# benchmark_dir = f"{ROOT_DIR}/.scrapy/micro-benchmark-autogen-templates"
 shutil.rmtree(output_folder, ignore_errors=True)
 shutil.rmtree(error_folder, ignore_errors=True)
 
@@ -27,25 +28,30 @@
 last_folder = ""
 start_time = time.time()
 total_start_time = time.time()
-for file in tqdm.tqdm(python_files, desc="Processing files"):
+for file in tqdm(python_files, desc="Processing files"):
     try:
         # print the folder path if its not the same as the last one
         if str(file.parent.parent.name) != last_folder:
             if last_folder:
-                print(
+                tqdm.write(
                     f"Time taken for {last_folder}: {time.time() - start_time} seconds"
                 )
-            print(
+            tqdm.write(
                 f"##################\nProcessing: {file.parent.parent.name}\n##################"
             )
             last_folder = str(file.parent.parent.name)
             start_time = time.time()
 
         # ignore if not main.py
         if file.name != "main.py":
-            print(f">> Ignoring: {file}")
+            tqdm.write(f">> Ignoring: {file}")
             continue
 
+        # debug point
+
+        if file.parent.name == "parent_import":
+            tqdm.write(f">> Debug point: {file}")
+
         template_data = read_template(file)
         if template_data["replacement_mode"] == "Imports":
             process_import_case(
@@ -68,7 +74,7 @@
             )
 
     except Exception as e:
-        print(e)
+        tqdm.write(str(e))
         pass
 
 

autogen/helpers.py

@@ -6,9 +6,12 @@
 import shutil
 import subprocess
 import sys
+from tqdm import tqdm
 
 from pathlib import Path
 
+random.seed(42)
+
 SCRIPT_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 
 
@@ -96,12 +99,11 @@ def is_variable_fact(item):
     return True if "variable" in item else False
 
 
-def replace_placeholders_and_generate_json(code, json_template_str, data_type_mapping):
+def replace_placeholders_and_generate_json(
+    code, json_template_str, data_type_mapping, placeholder_values
+):
     """Replace placeholders with values for their respective types and update JSON"""
-    placeholder_values = {}
-    for placeholder, data_type in data_type_mapping.items():
-        value, _ = generate_value_for_type(data_type)
-        placeholder_values[placeholder] = value
+    for placeholder, value in placeholder_values.items():
         code = code.replace(placeholder, str(value))
 
     json_template = json.loads(json_template_str)
@@ -166,12 +168,12 @@ def save_files(
 
     with open(code_file_path, "w") as file:
         file.write(code)
-    # print(f"Saved Python code to {code_file_path}")
+    # tqdm.write(f"Saved Python code to {code_file_path}")
 
     if json_data:
         with open(json_file_path, "w") as file:
             json.dump(json_data["ground_truth"], file, indent=4)
-    # print(f"Saved JSON ground truth to {json_file_path}")
+    # tqdm.write(f"Saved JSON ground truth to {json_file_path}")
 
     return code_file_path, json_file_path
 
@@ -214,11 +216,11 @@ def run_python_script(script):
 
         # Check if the script ran successfully
         if result.returncode != 0:
-            print("Script failed to execute" + result.stderr)
+            tqdm.write("Script failed to execute" + result.stderr)
             return False
         return True
     except Exception as e:
-        print("Script failed to execute" + result.stderr)
+        tqdm.write("Script failed to execute" + result.stderr)
         return False
 
 
@@ -231,11 +233,11 @@ def run_python_script_from_path(script_path):
 
         # Check if the script ran successfully
         if result.returncode != 0:
-            print("Script failed to execute" + result.stderr)
+            tqdm.write("Script failed to execute" + result.stderr)
             return False
         return True
     except Exception as e:
-        print("Script failed to execute" + result.stderr)
+        tqdm.write("Script failed to execute" + result.stderr)
         return False
 
 
@@ -247,7 +249,7 @@ def process_file(
     file_path,
     output_folder,
 ):
-    print(f"Processing file {file_path}")
+    tqdm.write(f"Processing file {file_path}")
     case_number = 1
     total_cases = 1
     error_count = 0
@@ -257,8 +259,14 @@ def process_file(
     for data_type_combo in generate_data_type_permutations(placeholders, data_types):
         data_type_mapping = {ph: dt for ph, dt in zip(placeholders, data_type_combo)}
         type_name = "_".join(data_type_combo)
+
+        placeholder_values = {}
+        for placeholder, data_type in data_type_mapping.items():
+            value, _ = generate_value_for_type(data_type)
+            placeholder_values[placeholder] = value
+
         replaced_code, json_data = replace_placeholders_and_generate_json(
-            code_template, json_template, data_type_mapping
+            code_template, json_template, data_type_mapping, placeholder_values
         )
         try:
             # result = exec(replaced_code)
@@ -268,13 +276,13 @@ def process_file(
 
         except Exception as e:
             if "exception" in type_name:
-                print(
+                tqdm.write(
                     "Skipping errror report as the data_type is exception for"
                     f" '{name}_{type_name}'"
                 )
             else:
-                print(f"\tError executing script '{name}_{type_name}'")
-                print(f"\tError : {e}\n")
+                tqdm.write(f"\tError executing script '{name}_{type_name}'")
+                tqdm.write(f"\tError : {e}\n")
                 # Save error files separately
                 save_files(
                     replaced_code,
@@ -312,7 +320,7 @@ def process_import_case(
     file_parent,
     output_folder,
 ):
-    print(f"Processing file {file_path}")
+    tqdm.write(f"Processing file {file_path}")
 
     #  Load json_template
 
@@ -335,14 +343,22 @@ def process_import_case(
         data_type_mapping = {ph: dt for ph, dt in zip(placeholders, data_type_combo)}
         type_name = "_".join(data_type_combo)
 
+        placeholder_values = {}
+        for placeholder, data_type in data_type_mapping.items():
+            value, _ = generate_value_for_type(data_type)
+            placeholder_values[placeholder] = value
+
         for imported_python_file in template_data["imports"]:
             # Load the imported python file
             imported_python_file_path = os.path.join(file_parent, imported_python_file)
             with open(imported_python_file_path, "r") as file:
                 imported_code_template = file.read()
 
             replaced_code, _ = replace_placeholders_and_generate_json(
-                imported_code_template, json_template, data_type_mapping
+                imported_code_template,
+                json_template,
+                data_type_mapping,
+                placeholder_values,
             )
 
             save_files(
@@ -358,7 +374,7 @@ def process_import_case(
 
         # Save main file and test file
         replaced_code, json_data = replace_placeholders_and_generate_json(
-            code_template, json_template, data_type_mapping
+            code_template, json_template, data_type_mapping, placeholder_values
         )
 
         code_file_path, json_file_path = save_files(
@@ -379,13 +395,13 @@ def process_import_case(
 
         except Exception as e:
             if "exception" in type_name:
-                print(
+                tqdm.write(
                     "Skipping errror report as the data_type is exception for"
                     f" '{name}_{type_name}'"
                 )
             else:
-                print(f"\tError executing script '{name}_{type_name}'")
-                print(f"\tError : {e}\n")
+                tqdm.write(f"\tError executing script '{name}_{type_name}'")
+                tqdm.write(f"\tError : {e}\n")
                 # remove created folder if error
                 shutil.rmtree(
                     os.path.join(output_folder, name + "_" + type_name),

autogen_typeevalpy_benchmark/python_features/args/assigned_call_1_1_int/main.py

@@ -2,7 +2,7 @@
 # A function `func` is defined which takes as a parameter a variable which has a function 'param_func' assigned to it which it later calls.
 # The 'param_func' function returns a string value.
 def param_func():
-    return 5
+    return 82
 
 
 def func(a):

autogen_typeevalpy_benchmark/python_features/args/assigned_call_1_2_float/main.py

@@ -2,7 +2,7 @@
 # A function `func` is defined which takes as a parameter a variable which has a function 'param_func' assigned to it which it later calls.
 # The 'param_func' function returns a string value.
 def param_func():
-    return 38.63
+    return 12.02
 
 
 def func(a):

autogen_typeevalpy_benchmark/python_features/args/assigned_call_1_3_str/main.py

@@ -2,7 +2,7 @@
 # A function `func` is defined which takes as a parameter a variable which has a function 'param_func' assigned to it which it later calls.
 # The 'param_func' function returns a string value.
 def param_func():
-    return 'mlczq'
+    return 'tgdct'
 
 
 def func(a):

autogen_typeevalpy_benchmark/python_features/args/assigned_call_1_5_list/main.py

@@ -2,7 +2,7 @@
 # A function `func` is defined which takes as a parameter a variable which has a function 'param_func' assigned to it which it later calls.
 # The 'param_func' function returns a string value.
 def param_func():
-    return [65, 8, 63]
+    return [76, 55, 5]
 
 
 def func(a):

autogen_typeevalpy_benchmark/python_features/args/assigned_call_1_6_dict/main.py

@@ -2,7 +2,7 @@
 # A function `func` is defined which takes as a parameter a variable which has a function 'param_func' assigned to it which it later calls.
 # The 'param_func' function returns a string value.
 def param_func():
-    return {'yalzb': 27, 'ufptv': 58, 'katmx': 65}
+    return {'afnaf': 98, 'qofpv': 44, 'ausie': 14}
 
 
 def func(a):

autogen_typeevalpy_benchmark/python_features/args/assigned_call_1_6_dict/main_gt.json

@@ -49,7 +49,7 @@
         "file": "main.py",
         "line_number": 13,
         "col_offset": 1,
-        "variable": "c['yalzb']",
+        "variable": "c['afnaf']",
         "type": [
             "int"
         ]
@@ -58,7 +58,7 @@
         "file": "main.py",
         "line_number": 13,
         "col_offset": 1,
-        "variable": "c['ufptv']",
+        "variable": "c['qofpv']",
         "type": [
             "int"
         ]
@@ -67,7 +67,7 @@
         "file": "main.py",
         "line_number": 13,
         "col_offset": 1,
-        "variable": "c['katmx']",
+        "variable": "c['ausie']",
         "type": [
             "int"
         ]

autogen_typeevalpy_benchmark/python_features/args/assigned_call_1_7_tuple/main.py

@@ -2,7 +2,7 @@
 # A function `func` is defined which takes as a parameter a variable which has a function 'param_func' assigned to it which it later calls.
 # The 'param_func' function returns a string value.
 def param_func():
-    return (44, 12, 11)
+    return (12, 49, 13)
 
 
 def func(a):

autogen_typeevalpy_benchmark/python_features/args/call_1_1_int/main.py

@@ -1,7 +1,7 @@
 # A function func is defined which takes as a parameter a function which it later calls.
 # The 'param_func' function returns a string value.
 def param_func():
-    return 49
+    return 46
 
 
 def func(a):

autogen_typeevalpy_benchmark/python_features/args/call_1_2_float/main.py

@@ -1,7 +1,7 @@
 # A function func is defined which takes as a parameter a function which it later calls.
 # The 'param_func' function returns a string value.
 def param_func():
-    return 80.45
+    return 84.9
 
 
 def func(a):

autogen_typeevalpy_benchmark/python_features/args/call_1_3_str/main.py

@@ -1,7 +1,7 @@
 # A function func is defined which takes as a parameter a function which it later calls.
 # The 'param_func' function returns a string value.
 def param_func():
-    return 'nwhkv'
+    return 'pusnz'
 
 
 def func(a):

autogen_typeevalpy_benchmark/python_features/args/call_1_5_list/main.py

@@ -1,7 +1,7 @@
 # A function func is defined which takes as a parameter a function which it later calls.
 # The 'param_func' function returns a string value.
 def param_func():
-    return [62, 93, 91]
+    return [11, 71, 38]
 
 
 def func(a):

autogen_typeevalpy_benchmark/python_features/args/call_1_6_dict/main.py

@@ -1,7 +1,7 @@
 # A function func is defined which takes as a parameter a function which it later calls.
 # The 'param_func' function returns a string value.
 def param_func():
-    return {'xattr': 14, 'ugpvw': 39, 'mnhuy': 19}
+    return {'vqwps': 27, 'bfhcg': 86, 'chqjj': 35}
 
 
 def func(a):

autogen_typeevalpy_benchmark/python_features/args/call_1_6_dict/main_gt.json

@@ -40,7 +40,7 @@
         "file": "main.py",
         "line_number": 11,
         "col_offset": 1,
-        "variable": "b['xattr']",
+        "variable": "b['vqwps']",
         "type": [
             "int"
         ]
@@ -49,7 +49,7 @@
         "file": "main.py",
         "line_number": 11,
         "col_offset": 1,
-        "variable": "b['ugpvw']",
+        "variable": "b['bfhcg']",
         "type": [
             "int"
         ]
@@ -58,7 +58,7 @@
         "file": "main.py",
         "line_number": 11,
         "col_offset": 1,
-        "variable": "b['mnhuy']",
+        "variable": "b['chqjj']",
         "type": [
             "int"
         ]

autogen_typeevalpy_benchmark/python_features/args/call_1_7_tuple/main.py

@@ -1,7 +1,7 @@
 # A function func is defined which takes as a parameter a function which it later calls.
 # The 'param_func' function returns a string value.
 def param_func():
-    return (70, 82, 28)
+    return (90, 88, 83)
 
 
 def func(a):