Fixed results scripts

Author: ashwinprasadme

.vscode/launch_example.json

@@ -5,15 +5,15 @@
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "Python: Current File",
+            "name": "Ollama Runner",
             "type": "python",
             "request": "launch",
-            "program": "./TypeEvalPy_LLM/src/target_tools/ollama/src/runner.py",
+            "program": "./src/target_tools/ollama/src/runner.py",
             "console": "integratedTerminal",
             "justMyCode": true,
             "args": [
                 "--bechmark_path",
-                "./TypeEvalPy_LLM/.scrapy/test",
+                "./micro-benchmark",
                 "--ollama_models",
                 "codellama:34b-instruct",
                 "codellama:34b-python",
@@ -22,8 +22,10 @@
                 "--ollama_url",
                 "",
                 "--prompt_id",
-                "json_based_1"
+                "questions_based_2",
+                "--results_dir",
+                ""
             ]
         }
     ]
-}
+}

src/main_analyze_results.py

@@ -1,3 +1,4 @@
+import argparse
 import json
 import logging
 import os
@@ -896,10 +897,8 @@ def generate_top_n_performance(test_suite_dir, tool_name=None):
     return results_cat
 
 
-def run_results_analyzer():
-    results_dir = None
-    # results_dir = Path("../results/results_<>")
-    if results_dir is None:
+def run_results_analyzer(args):
+    if args.results_dir is None:
         dir_path = Path(SCRIPT_DIR) / "../results"
         directories = [
             f
@@ -909,11 +908,13 @@ def run_results_analyzer():
         directories.sort(key=lambda x: x.stat().st_mtime, reverse=True)
         # Get the latest directory
         results_dir = directories[0] if directories else None
+    else:
+        results_dir = Path(args.results_dir)
 
     tools_results = {}
 
     for item in results_dir.glob("*"):
-        if item.is_file():
+        if item.is_file() or item.name == "analysis_results":
             # ignore
             pass
         elif item.is_dir():
@@ -965,7 +966,7 @@ def run_results_analyzer():
     tools_list = utils.ML_TOOLS + utils.STANDARD_TOOLS
 
     if len(tools_results) > 1:
-        analysis_tables.create_comparison_table(tools_results, tools_list)
+        analysis_tables.create_comparison_table(tools_results)
 
     os.makedirs(results_dir / "analysis_results", exist_ok=True)
     results_dir = results_dir / "analysis_results"
@@ -1045,4 +1046,12 @@ def run_results_analyzer():
 
 
 if __name__ == "__main__":
-    run_results_analyzer()
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--results_dir",
+        help="Specify the results path",
+        default=None,
+    )
+    args = parser.parse_args()
+
+    run_results_analyzer(args)

src/result_analyzer/analysis_tables.py

@@ -409,13 +409,13 @@ def exact_match_category_table(stats):
         writer.writerows(rows)
 
 
-def create_comparison_table(stats, tools):
+def create_comparison_table(stats):
     # Sort stats based on total_caught
     stats = utils.sort_stats(stats)
 
     headers = ["Tool Name"]
     stats = utils.sort_stats(stats)
-    tool_names = [tool for tool in stats.keys() if tool in tools]
+    tool_names = [tool for tool in stats.keys()]
     categories = list(stats[tool_names[0]]["exact_match_category"].keys())
     type_categories = list(
         list(stats[tool_names[0]]["exact_match_category"].values())[0].keys()

src/target_tools/ollama/src/runner.py

@@ -180,22 +180,27 @@ def process_file(file_path, llm, openai_llm, prompt_id):
 
 
 def main_runner(args):
-    error_count = 0
-    timeout_count = 0
-    json_count = 0
     model_name = "text-davinci-003"
     temperature = 0.0
     openai_llm = OpenAI(
         model_name=model_name, temperature=temperature, openai_api_key=args.openai_key
     )
 
     for model in args.ollama_models:
+        error_count = 0
+        timeout_count = 0
+        json_count = 0
         files_analyzed = 0
 
         # Create result folder for model specific results
         bechmark_path = Path(args.bechmark_path)
         results_src = bechmark_path
-        results_dst = bechmark_path.parent / model / bechmark_path.name
+        if args.results_dir is None:
+            results_dst = bechmark_path.parent / model / bechmark_path.name
+        else:
+            results_dst = Path(args.results_dir) / model / bechmark_path.name
+            os.makedirs(results_dst, exist_ok=True)
+
         utils.copy_folder(results_src, results_dst)
 
         python_files = list_python_files(results_dst)
@@ -267,6 +272,12 @@ def main_runner(args):
         default="/tmp/micro-benchmark",
     )
 
+    parser.add_argument(
+        "--results_dir",
+        help="Specify the benchmark path",
+        default=None,
+    )
+
     parser.add_argument(
         "--ollama_url", help="Specify the ollama server url", required=True
     )