Run on custom benchmark

Author: ashwinprasadme

.gitignore

@@ -178,3 +178,6 @@ src/config.yaml
 src/target_tools/ollama/src/fine_tuning/auto_generation/generated_dataset/*
 src/target_tools/ollama/src/fine_tuning/wandb/*
 src/target_tools/ollama/src/fine_tuning/outputs/*
+
+# Ignore autogen files
+autogen/data

src/main_runner.py

@@ -77,6 +77,8 @@ def get_args():
         "--nocache", action="store_true", help="Do not use docker image cache."
     )
 
+    parser.add_argument("--custom_benchmark_dir", default=None, type=str)
+
     return parser.parse_args()
 
 
@@ -89,35 +91,68 @@ def main():
     available_runners = {
         "headergen": (
             HeaderGenRunner,
-            {"debug": args.debug, "nocache": args.nocache},
+            {
+                "debug": args.debug,
+                "nocache": args.nocache,
+                "custom_benchmark_dir": args.custom_benchmark_dir,
+            },
         ),
         "pyright": (
             PyrightRunner,
-            {"debug": False, "nocache": args.nocache},
+            {
+                "debug": False,
+                "nocache": args.nocache,
+                "custom_benchmark_dir": args.custom_benchmark_dir,
+            },
         ),
         "scalpel": (
             ScalpelRunner,
-            {"debug": False, "nocache": args.nocache},
+            {
+                "debug": False,
+                "nocache": args.nocache,
+                "custom_benchmark_dir": args.custom_benchmark_dir,
+            },
         ),
         "hityper": (
             HityperRunner,
-            {"debug": False, "nocache": args.nocache},
+            {
+                "debug": False,
+                "nocache": args.nocache,
+                "custom_benchmark_dir": args.custom_benchmark_dir,
+            },
         ),
         "type4py": (
             Type4pyRunner,
-            {"debug": False, "nocache": args.nocache},
+            {
+                "debug": False,
+                "nocache": args.nocache,
+                "custom_benchmark_dir": args.custom_benchmark_dir,
+            },
         ),
         "hityperdl": (
             HityperDLRunner,
-            {"debug": False, "nocache": args.nocache},
+            {
+                "debug": False,
+                "nocache": args.nocache,
+                "custom_benchmark_dir": args.custom_benchmark_dir,
+            },
         ),
         "jedi": (
             JediRunner,
-            {"debug": args.debug, "nocache": args.nocache},
+            {
+                "debug": args.debug,
+                "nocache": args.nocache,
+                "custom_benchmark_dir": args.custom_benchmark_dir,
+            },
         ),
         "ollama": (
             OllamaRunner,
-            {"debug": args.debug, "nocache": args.nocache, "config": config},
+            {
+                "debug": args.debug,
+                "nocache": args.nocache,
+                "custom_benchmark_dir": args.custom_benchmark_dir,
+                "config": config,
+            },
         ),
         # PySonar2Runner,
         # PytypeRunner,

src/result_analyzer/analysis_tables.py

@@ -475,6 +475,9 @@ def create_comparison_table(stats):
 
 def analysis_sensitivities_table(stats):
     stats = utils.sort_stats(stats)
+    # check if sensitivity_sound_data is present in the stats
+    if not stats[list(stats.keys())[0]].get("sensitivity_sound_data"):
+        return
 
     with open(f"tools_sensitivities_data.csv", "w", newline="") as csvfile:
         fieldnames = [

src/runner_class.py

@@ -21,13 +21,20 @@ def __init__(
         dockerfile_name="Dockerfile",
         volumes={},
         nocache=False,
+        custom_benchmark_dir=None,
     ):
         self.docker_client = docker.from_env()
         self.tool_name = tool_name
         self.dockerfile_path = dockerfile_path
         self.dockerfile_name = dockerfile_name
         self.test_runner_script_path = f"/tmp/src/runner.py"
         self.benchmark_path = "/tmp/micro-benchmark"
+
+        if custom_benchmark_dir:
+            self.local_benchmark_path = custom_benchmark_dir
+        else:
+            self.local_benchmark_path = os.path.abspath("../micro-benchmark")
+
         self.host_results_path = host_results_path
         self.volumes = volumes
         self.nocache = nocache
@@ -79,7 +86,7 @@ def run_tool_test(self):
         self._build_docker_image()
         self.container = self.spawn_docker_instance()
 
-        src = "../micro-benchmark"
+        src = self.local_benchmark_path
         dst = "/tmp"
         self.file_handler.copy_files_to_container(self.container, src, dst)
 
@@ -103,35 +110,81 @@ def run_tool_test(self):
 
 
 class ScalpelRunner(TypeEvalPyRunner):
-    def __init__(self, host_results_path, debug=False, nocache=False):
+    def __init__(
+        self,
+        host_results_path,
+        debug=False,
+        nocache=False,
+        custom_benchmark_dir=None,
+    ):
         super().__init__(
-            "scalpel", "./target_tools/scalpel", host_results_path, nocache=nocache
+            "scalpel",
+            "./target_tools/scalpel",
+            host_results_path,
+            nocache=nocache,
+            custom_benchmark_dir=custom_benchmark_dir,
         )
 
 
 class PyreRunner(TypeEvalPyRunner):
-    def __init__(self, host_results_path, debug=False, nocache=False):
+    def __init__(
+        self,
+        host_results_path,
+        debug=False,
+        nocache=False,
+        custom_benchmark_dir=None,
+    ):
         super().__init__(
-            "pyre", "./target_tools/pyre", host_results_path, nocache=nocache
+            "pyre",
+            "./target_tools/pyre",
+            host_results_path,
+            nocache=nocache,
+            custom_benchmark_dir=custom_benchmark_dir,
         )
 
 
 class PyrightRunner(TypeEvalPyRunner):
-    def __init__(self, host_results_path, debug=False, nocache=False):
+    def __init__(
+        self,
+        host_results_path,
+        debug=False,
+        nocache=False,
+        custom_benchmark_dir=None,
+    ):
         super().__init__(
-            "pyright", "./target_tools/pyright", host_results_path, nocache=nocache
+            "pyright",
+            "./target_tools/pyright",
+            host_results_path,
+            nocache=nocache,
+            custom_benchmark_dir=custom_benchmark_dir,
         )
 
 
 class PytypeRunner(TypeEvalPyRunner):
-    def __init__(self, host_results_path, debug=False, nocache=False):
+    def __init__(
+        self,
+        host_results_path,
+        debug=False,
+        nocache=False,
+        custom_benchmark_dir=None,
+    ):
         super().__init__(
-            "pytype", "./target_tools/pytype", host_results_path, nocache=nocache
+            "pytype",
+            "./target_tools/pytype",
+            host_results_path,
+            nocache=nocache,
+            custom_benchmark_dir=custom_benchmark_dir,
         )
 
 
 class JediRunner(TypeEvalPyRunner):
-    def __init__(self, host_results_path, debug=False, nocache=False):
+    def __init__(
+        self,
+        host_results_path,
+        debug=False,
+        nocache=False,
+        custom_benchmark_dir=None,
+    ):
         if debug:
             super().__init__(
                 "jedi",
@@ -145,24 +198,49 @@ def __init__(self, host_results_path, debug=False, nocache=False):
                     }
                 },
                 nocache=nocache,
+                custom_benchmark_dir=custom_benchmark_dir,
             )
         else:
             super().__init__(
-                "jedi", "./target_tools/jedi", host_results_path, nocache=nocache
+                "jedi",
+                "./target_tools/jedi",
+                host_results_path,
+                nocache=nocache,
+                custom_benchmark_dir=custom_benchmark_dir,
             )
 
 
 class HityperRunner(TypeEvalPyRunner):
-    def __init__(self, host_results_path, debug=False, nocache=False):
+    def __init__(
+        self,
+        host_results_path,
+        debug=False,
+        nocache=False,
+        custom_benchmark_dir=None,
+    ):
         super().__init__(
-            "hityper", "./target_tools/hityper", host_results_path, nocache=nocache
+            "hityper",
+            "./target_tools/hityper",
+            host_results_path,
+            nocache=nocache,
+            custom_benchmark_dir=custom_benchmark_dir,
         )
 
 
 class HityperDLRunner(TypeEvalPyRunner):
-    def __init__(self, host_results_path, debug=False, nocache=False):
+    def __init__(
+        self,
+        host_results_path,
+        debug=False,
+        nocache=False,
+        custom_benchmark_dir=None,
+    ):
         super().__init__(
-            "hityperdl", "./target_tools/hityperdl", host_results_path, nocache=nocache
+            "hityperdl",
+            "./target_tools/hityperdl",
+            host_results_path,
+            nocache=nocache,
+            custom_benchmark_dir=custom_benchmark_dir,
         )
 
     def spawn_docker_instance(self):
@@ -179,7 +257,13 @@ def spawn_docker_instance(self):
 
 
 class HeaderGenRunner(TypeEvalPyRunner):
-    def __init__(self, host_results_path, debug=False, nocache=False):
+    def __init__(
+        self,
+        host_results_path,
+        debug=False,
+        nocache=False,
+        custom_benchmark_dir=None,
+    ):
         if debug:
             super().__init__(
                 "headergen",
@@ -199,20 +283,41 @@ def __init__(self, host_results_path, debug=False, nocache=False):
                 "./target_tools/headergen",
                 host_results_path,
                 nocache=nocache,
+                custom_benchmark_dir=custom_benchmark_dir,
             )
 
 
 class PySonar2Runner(TypeEvalPyRunner):
-    def __init__(self, host_results_path, debug=False, nocache=False):
+    def __init__(
+        self,
+        host_results_path,
+        debug=False,
+        nocache=False,
+        custom_benchmark_dir=None,
+    ):
         super().__init__(
-            "pysonar2", "./target_tools/pysonar2", host_results_path, nocache=nocache
+            "pysonar2",
+            "./target_tools/pysonar2",
+            host_results_path,
+            nocache=nocache,
+            custom_benchmark_dir=custom_benchmark_dir,
         )
 
 
 class Type4pyRunner(TypeEvalPyRunner):
-    def __init__(self, host_results_path, debug=False, nocache=False):
+    def __init__(
+        self,
+        host_results_path,
+        debug=False,
+        nocache=False,
+        custom_benchmark_dir=None,
+    ):
         super().__init__(
-            "type4py", "./target_tools/type4py", host_results_path, nocache=nocache
+            "type4py",
+            "./target_tools/type4py",
+            host_results_path,
+            nocache=nocache,
+            custom_benchmark_dir=custom_benchmark_dir,
         )
 
     def spawn_docker_instance(self):
@@ -229,9 +334,20 @@ def spawn_docker_instance(self):
 
 
 class OllamaRunner(TypeEvalPyRunner):
-    def __init__(self, host_results_path, config, debug=False, nocache=False):
+    def __init__(
+        self,
+        host_results_path,
+        config,
+        debug=False,
+        nocache=False,
+        custom_benchmark_dir=None,
+    ):
         super().__init__(
-            "ollama", "./target_tools/ollama", host_results_path, nocache=nocache
+            "ollama",
+            "./target_tools/ollama",
+            host_results_path,
+            nocache=nocache,
+            custom_benchmark_dir=custom_benchmark_dir,
         )
         self.config = config
 
@@ -266,9 +382,20 @@ def copy_results_from_container(self):
 
 
 class OllamaRunner(TypeEvalPyRunner):
-    def __init__(self, host_results_path, config, debug=False, nocache=False):
+    def __init__(
+        self,
+        host_results_path,
+        config,
+        debug=False,
+        nocache=False,
+        custom_benchmark_dir=None,
+    ):
         super().__init__(
-            "ollama", "./target_tools/ollama", host_results_path, nocache=nocache
+            "ollama",
+            "./target_tools/ollama",
+            host_results_path,
+            nocache=nocache,
+            custom_benchmark_dir=custom_benchmark_dir,
         )
         self.config = config
 

src/utils.py

@@ -8,7 +8,7 @@ def copy_files_to_container(self, container, src, dst):
         # Create tar of micro-bench folder
         temp_path = "/tmp/temp.tar"
         with tarfile.open(temp_path, "w:gz") as tar:
-            base_folder = os.path.basename(src)
+            base_folder = "micro-benchmark"
             tar.add(src, arcname=base_folder)
 
         with open(temp_path, "rb") as file: