Merge branch 'master' into anandhu-eng-patch-4

Author: arjunsuresh

automotive/3d-object-detection/README.md

@@ -18,16 +18,23 @@ You can also do `pip install mlc-scripts` and then use `mlcr` commands for downl
 > By default, the waymo dataset is downloaded from the mlcommons official drive. One has to accept the [MLCommons Waymo Open Dataset EULA](https://waymo.mlcommons.org/) to access the dataset files.
 
 ```
-mlcr get,ml-model,pointpainting --outdirname=<path_to_download> -j
+mlcr get,ml-model,pointpainting,_r2-downloader,_mlc --outdirname=<path_to_download> -j
 ```
 
 ### Download dataset through MLCFlow Automation
 
 > [!Note]
 > By default, the waymo dataset is downloaded from the mlcommons official drive. One has to accept the [MLCommons Waymo Open Dataset EULA](https://waymo.mlcommons.org/) to access the dataset files.
 
+**Includes validation and calibration dataset**
 ```
-mlcr get,dataset,waymo --outdirname=<path_to_download> -j
+mlcr get,dataset,waymo,_r2-downloader,_mlc --outdirname=<path_to_download> -j
+```
+
+**Includes only calibration dataset**
+
+```
+mlcr get,dataset,waymo,calibration,_r2-downloader,_mlc --outdirname=<path_to_download> -j
 ```
 
 ## Downloading the dataset and model checkpoints
@@ -106,4 +113,4 @@ python accuracy_waymo.py --mlperf-accuracy-file <path to accuracy file>/mlperf_l
 
 ## Automated command for submission generation via MLCFlow
 
-Please see the [new docs site](https://docs.mlcommons.org/inference/submission/) for an automated way to generate submission through MLCFlow. 
+Please see the [new docs site](https://docs.mlcommons.org/inference/submission/) for an automated way to generate submission through MLCFlow. 

compliance/TEST01/run_verification.py

@@ -76,51 +76,68 @@ def main():
     output_dir = os.path.join(args.output_dir, "TEST01")
     unixmode = ""
     if args.unixmode:
-        unixmode = " --unixmode"
-        for binary in ["wc", "md5sum", "grep", "awk", "sed", "head", "tail"]:
+        if os.name != "posix":
+            print(
+                "Warning: --unixmode not supported on this OS. Using Python fallback...")
+            unixmode = ""
+        else:
+            unixmode = " --unixmode"
             missing_binary = False
-            if shutil.which(binary) is None:
-                print(
-                    "Error: This script requires the {:} commandline utility".format(
-                        binary
+            for binary in ["wc", "md5sum", "grep",
+                           "awk", "sed", "head", "tail"]:
+                if shutil.which(binary) is None:
+                    print(
+                        "Error: This script requires the {:} commandline utility".format(
+                            binary
+                        )
                     )
-                )
-                missing_binary = True
-        if missing_binary:
-            exit()
+                    missing_binary = True
+            if missing_binary:
+                exit()
 
     dtype = args.dtype
 
     verify_accuracy_binary = os.path.join(
         os.path.dirname(__file__), "verify_accuracy.py"
     )
+
+    unixmode_str = unixmode if unixmode == "" else unixmode + " "
+
     # run verify accuracy
     verify_accuracy_command = (
-        "python3 "
+        sys.executable + " "
         + verify_accuracy_binary
         + " --dtype "
         + args.dtype
-        + unixmode
+        + unixmode_str
         + " -r "
-        + results_dir
-        + "/accuracy/mlperf_log_accuracy.json"
+        + os.path.join(results_dir, "accuracy", "mlperf_log_accuracy.json")
         + " -t "
-        + compliance_dir
-        + "/mlperf_log_accuracy.json | tee verify_accuracy.txt"
+        + os.path.join(compliance_dir, "mlperf_log_accuracy.json")
     )
     try:
-        os.system(verify_accuracy_command)
+        with open("verify_accuracy.txt", "w") as f:
+            process = subprocess.Popen(
+                verify_accuracy_command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                shell=True,
+                text=True
+            )
+            # Write output to both console and file
+            for line in process.stdout:
+                print(line, end="")
+                f.write(line)
+            process.wait()
     except Exception:
         print(
             "Exception occurred trying to execute:\n  " +
             verify_accuracy_command)
     # check if verify accuracy script passes
 
-    accuracy_pass_command = "grep PASS verify_accuracy.txt"
     try:
-        accuracy_pass = "TEST PASS" in subprocess.check_output(
-            accuracy_pass_command, shell=True
-        ).decode("utf-8")
+        with open("verify_accuracy.txt", "r") as file:
+            accuracy_pass = "TEST PASS" in file.read()
     except Exception:
         accuracy_pass = False
 
@@ -129,28 +146,38 @@ def main():
         os.path.dirname(__file__), "verify_performance.py"
     )
     verify_performance_command = (
-        "python3 "
+        sys.executable + " "
         + verify_performance_binary
-        + " -r "
-        + results_dir
-        + "/performance/run_1/mlperf_log_detail.txt"
-        + " -t "
-        + compliance_dir
-        + "/mlperf_log_detail.txt | tee verify_performance.txt"
+        + " -r"
+        + os.path.join(results_dir, "performance",
+                       "run_1", "mlperf_log_detail.txt")
+        + " -t"
+        + os.path.join(compliance_dir, "mlperf_log_detail.txt")
     )
+
     try:
-        os.system(verify_performance_command)
+        with open("verify_performance.txt", "w") as f:
+            process = subprocess.Popen(
+                verify_performance_command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                shell=True,
+            )
+            # Write output to both console and file
+            for line in process.stdout:
+                print(line, end="")
+                f.write(line)
+            process.wait()
     except Exception:
         print(
             "Exception occurred trying to execute:\n  " +
             verify_performance_command)
 
     # check if verify performance script passes
-    performance_pass_command = "grep PASS verify_performance.txt"
     try:
-        performance_pass = "TEST PASS" in subprocess.check_output(
-            performance_pass_command, shell=True
-        ).decode("utf-8")
+        with open("verify_performance.txt", "r") as file:
+            performance_pass = "TEST PASS" in file.read()
     except Exception:
         performance_pass = False
 

compliance/TEST01/verify_accuracy.py

@@ -20,6 +20,8 @@
 import subprocess
 import sys
 import shutil
+import hashlib
+import re
 
 sys.path.append(os.getcwd())
 
@@ -161,15 +163,11 @@ def main():
         print("Error: This script requires Python v3.3 or later")
         exit()
 
-    get_perf_lines_cmd = "wc -l " + perf_log + "| awk '{print $1}'"
-    num_perf_lines = int(
-        subprocess.check_output(get_perf_lines_cmd, shell=True).decode("utf-8")
-    )
+    with open(perf_log, "r") as file:
+        num_perf_lines = sum(1 for _ in file)
 
-    get_acc_lines_cmd = "wc -l " + acc_log + "| awk '{print $1}'"
-    num_acc_lines = int(
-        subprocess.check_output(get_acc_lines_cmd, shell=True).decode("utf-8")
-    )
+    with open(acc_log, "r") as file:
+        num_acc_lines = sum(1 for _ in file)
 
     num_acc_log_entries = num_acc_lines - 2
     num_perf_log_entries = num_perf_lines - 2
@@ -189,42 +187,38 @@ def main():
             continue
 
         # calculate md5sum of line in perf mode accuracy_log
-        perf_md5sum_cmd = (
-            "head -n "
-            + str(perf_line + 1)
-            + " "
-            + perf_log
-            + "| tail -n 1| sed -r 's/,//g' | sed -r 's/\"seq_id\" : \\S+//g' | md5sum"
-        )
-        # print(perf_md5sum_cmd)
-        perf_md5sum = subprocess.check_output(perf_md5sum_cmd, shell=True).decode(
-            "utf-8"
-        )
-
-        # get qsl idx
-        get_qsl_idx_cmd = (
-            "head -n "
-            + str(perf_line + 1)
-            + " "
-            + perf_log
-            + "| tail -n 1| awk -F\": |,\" '{print $4}'"
-        )
-        qsl_idx = (
-            subprocess.check_output(get_qsl_idx_cmd, shell=True)
-            .decode("utf-8")
-            .rstrip()
-        )
+        # read the specific line
+        with open(perf_log, "r") as f:
+            for i, line in enumerate(f):
+                if i == perf_line:
+                    line_content = line.strip()
+                    break
+
+        # remove commas and remove 'seq_id' key-value
+        clean_line = line_content.replace(",", "")
+        clean_line = re.sub(r'"seq_id"\s*:\s*\S+', '', clean_line)
+
+        # calculate md5sum
+        perf_md5sum = hashlib.md5(clean_line.encode("utf-8")).hexdigest()
+
+        # extract qsl idx
+        fields = re.split(r": |,", line_content)
+        qsl_idx = fields[3].strip()
 
         # calculate md5sum of line in acc mode accuracy_log
-        acc_md5sum_cmd = (
-            'grep "qsl_idx\\" : '
-            + qsl_idx
-            + '," '
-            + acc_log
-            + "| sed -r 's/,//g' | sed -r 's/\"seq_id\" : \\S+//g' | md5sum"
-        )
-        acc_md5sum = subprocess.check_output(
-            acc_md5sum_cmd, shell=True).decode("utf-8")
+        acc_matches = []
+        with open(acc_log, "r") as f:
+            for line in f:
+                if f'"qsl_idx" : {qsl_idx},' in line:
+                    acc_matches.append(line.strip())
+
+        # join all matching lines together
+        acc_line = "\n".join(acc_matches)
+
+        acc_line = acc_line.replace(",", "")
+        acc_line = re.sub(r'"seq_id"\s*:\s*\S+', '', acc_line)
+
+        acc_md5sum = hashlib.md5(acc_line.encode("utf-8")).hexdigest()
 
         if perf_md5sum != acc_md5sum:
             num_perf_log_data_mismatch += 1

compliance/TEST04/run_verification.py

@@ -58,28 +58,38 @@ def main():
         os.path.dirname(__file__), "verify_performance.py"
     )
     verify_performance_command = (
-        "python3 "
+        sys.executable + " "
         + verify_performance_binary
-        + " -r "
-        + results_dir
-        + "/performance/run_1/mlperf_log_summary.txt"
-        + " -t "
-        + compliance_dir
-        + "/mlperf_log_summary.txt | tee verify_performance.txt"
+        + " -r"
+        + os.path.join(results_dir, "performance",
+                       "run_1", "mlperf_log_summary.txt")
+        + " -t"
+        + os.path.join(compliance_dir, "mlperf_log_summary.txt")
     )
+
     try:
-        os.system(verify_performance_command)
+        with open("verify_performance.txt", "w") as f:
+            process = subprocess.Popen(
+                verify_performance_command,
+                stdout=subprocess.PIPE,  # capture output
+                stderr=subprocess.STDOUT,
+                text=True,  # decode output as text
+                shell=True,
+            )
+            # Write output to both console and file
+            for line in process.stdout:
+                print(line, end="")  # console
+                f.write(line)        # file
+            process.wait()
     except Exception:
         print(
             "Exception occurred trying to execute:\n  " +
             verify_performance_command)
 
     # check if verify performance script passes
-    performance_pass_command = "grep PASS verify_performance.txt"
     try:
-        performance_pass = "TEST PASS" in subprocess.check_output(
-            performance_pass_command, shell=True
-        ).decode("utf-8")
+        with open("verify_performance.txt", "r") as file:
+            performance_pass = "TEST PASS" in file.read()
     except Exception:
         performance_pass = False
 

docs/benchmarks/automotive/3d_object_detection/get-pointpainting-data.md

@@ -13,16 +13,16 @@ The benchmark implementation run command will automatically download the preproc
 
 === "Validation"
 
-    ### Get Validation Dataset
+    ### Get Validation and Calibration Dataset
     ```
-    mlcr get,dataset,waymo -j
+    mlcr get,dataset,waymo,_r2-downloader,_mlc -j
     ```
 
 === "Calibration"
 
-    ### Get Calibration Dataset
+    ### Get Calibration Dataset only
     ```
-    mlcr get,dataset,waymo,calibration -j
+    mlcr get,dataset,waymo,calibration,_r2-downloader,_mlc -j
     ```
 
 - `--outdirname=<PATH_TO_DOWNLOAD_WAYMO_DATASET>` could be provided to download the dataset to a specific location.
@@ -33,7 +33,7 @@ The benchmark implementation run command will automatically download the preproc
 The benchmark implementation run command will automatically download the model. In case you want to download only the PointPainting model, you can use the below command.
 
 ```bash
-mlcr get,ml-model,pointpainting -j
+mlcr get,ml-model,pointpainting,_r2-downloader,_mlc -j
 ```
 
 - `--outdirname=<PATH_TO_DOWNLOAD_POINTPAINTING_MODEL>` could be provided to download the model files to a specific location.

docs/benchmarks/graph/get-rgat-data.md

@@ -46,7 +46,7 @@ Get the Official MLPerf R-GAT Model
 
     ### PyTorch
     ```
-    mlcr get,ml-model,rgat -j
+    mlcr get,ml-model,rgat,_r2-downloader,_mlcommons -j
     ```
 
 - `--outdirname=<PATH_TO_DOWNLOAD_RGAT_MODEL>` could be provided to download the model to a specific location.

docs/benchmarks/image_classification/get-resnet50-data.md

@@ -15,7 +15,7 @@ The benchmark implementation run command will automatically download the validat
 
         ### Get Validation Dataset
         ```
-        mlcr get,dataset,imagenet,validation -j
+        mlcr get,dataset,imagenet,validation,_full -j
         ```
     === "Calibration"
         ResNet50 calibration dataset consist of 500 images selected from the Imagenet 2012 validation dataset. There are 2 alternative options for the calibration dataset.
@@ -32,7 +32,7 @@ The benchmark implementation run command will automatically download the validat
     ### Get ResNet50 preprocessed dataset
 
     ```
-    mlcr get,dataset,image-classification,imagenet,preprocessed,_pytorch -j
+    mlcr get,dataset,image-classification,imagenet,preprocessed,_pytorch,_full-j
     ```
 
 - `--outdirname=<PATH_TO_DOWNLOAD_IMAGENET_DATASET>` could be provided to download the dataset to a specific location.
@@ -52,7 +52,7 @@ Get the Official MLPerf ResNet50 Model
 
     ### Onnx
     ```
-    mlcr get,ml-model,resnet50,_onnx -j
+    mlcr get,ml-model,resnet50,image-classification,_onnx -j
     ```
 
 - `--outdirname=<PATH_TO_DOWNLOAD_RESNET50_MODEL>` could be provided to download the model to a specific location.