Feature/276 new script file for testing framework

.github/coverage/cpp.develop.coverage_value.txt

@@ -1 +1 @@
-64.2822
+64.2075

.github/scripts/run_coverage_cpp.sh

@@ -2,18 +2,19 @@
 
 cd /vdms/tests
 
-# Run Local C++ PMGD Based Tests
-chmod +x run_tests.sh
-echo 'Running run_tests.sh script'
-./run_tests.sh
-
 # Run S3 C++ PMGD Based Tests
 echo 'Checking for the available disk space due MinIO requires at least 1gb free'
 df -h
-chmod +x run_aws_tests.sh
-echo 'Running run_aws_tests.sh script'
-./run_aws_tests.sh -u ${AWS_ACCESS_KEY_ID} -p ${AWS_SECRET_ACCESS_KEY}
 
+chmod +x ./run_all_tests.py
+
+echo 'Running run_all_tests.py script for remote C++ tests (-t=ru)'
+python ./run_all_tests.py -t=ru -k -u ${AWS_ACCESS_KEY_ID} -p ${AWS_SECRET_ACCESS_KEY}
+
+echo 'Running run_all_tests script for C++ tests (-t=ut)'
+python ./run_all_tests.py -t=ut -k
+
+echo 'Getting the coverage for C++'
 # Obtain Coverage
 gcovr --root /vdms \
     -e /vdms/src/pmgd -e /vdms/build -e /vdms/distributed -e /vdms/tests \

.github/scripts/run_coverage_py.sh

@@ -1,11 +1,28 @@
 #!/bin/bash -e
 
-cd /vdms/tests/python
+cd /vdms/tests
+chmod +x ./run_all_tests.py
+chmod +x ./TestScript.py
 
-./run_python_tests.sh
-./run_python_aws_tests.sh -u ${AWS_ACCESS_KEY_ID} -p ${AWS_SECRET_ACCESS_KEY}
-python -m coverage report -m 2>&1 | tee /vdms/tests/coverage_report/python.new.coverage_report.txt
-python -m coverage xml -o /vdms/tests/coverage_report/python.new.coverage_report.xml
+# Check .coverage file doesn't exist
+if [ -f './.coverage' ]; then
+    echo ".coverage exists."
+    rm ./.coverage
+    echo ".coverage deleted."
+fi
+
+echo 'Running run_all_tests script for Python tests (-t=pt)'
+python3 ./run_all_tests.py -t=pt -k
+
+echo 'Running run_all_tests script for Remote Python tests (-t=rp)'
+python3 ./run_all_tests.py -t=rp -k -u=${AWS_ACCESS_KEY_ID} -p=${AWS_SECRET_ACCESS_KEY}
+
+echo 'Running the tests included in TestScript.py file'
+# Append the coverage results to the ones obtained from running run_all_tests.py
+python3 -m coverage run -a --omit="./run_all_tests.py,./TestScript.py" -m unittest discover --pattern=TestScript.py -v
+
+python3 -m coverage report -m 2>&1 | tee /vdms/tests/coverage_report/python.new.coverage_report.txt
+python3 -m coverage xml -o /vdms/tests/coverage_report/python.new.coverage_report.xml
 
 echo "DONE"
 

.github/workflows/_CI_coverage.yml

@@ -159,16 +159,16 @@ jobs:
           echo "VDMS_TAG=${VDMS_TAG_STR}" >> $GITHUB_OUTPUT
 
           # Commands for neo4j tests
-          COMMON_ARGS="-n ${{ secrets.NEO4J_USER }} -w ${{ secrets.NEO4J_PASS }} -v ${{ env.NEO_TEST_PORT}}"
-          S3_ARGS="-a ${{ env.AWS_API_PORT }} -c ${{ env.AWS_CONSOLE_PORT}} -u ${{ secrets.AWS_ACCESS_KEY_ID }} -p ${{ secrets.AWS_SECRET_ACCESS_KEY }}"
+          COMMON_ARGS="--neo4j_username ${{ secrets.NEO4J_USER }} --neo4j_password ${{ secrets.NEO4J_PASS }} --neo4j_port ${{ env.NEO_TEST_PORT }}"
+          S3_ARGS="--minio_port ${{ env.AWS_API_PORT }} --minio_console_port ${{ env.AWS_CONSOLE_PORT }} -u ${{ secrets.AWS_ACCESS_KEY_ID }} -p ${{ secrets.AWS_SECRET_ACCESS_KEY }}"
 
-          CMD_STR_e2e_str="./run_neo4j_tests.sh -t Neo4JE2ETest ${S3_ARGS} ${COMMON_ARGS} -e neo4j://neo4j:${{ env.NEO_TEST_PORT}}"
+          CMD_STR_e2e_str="./run_all_tests.py --type_of_test neo --test_name 'Neo4JE2ETest.*' ${S3_ARGS} ${COMMON_ARGS} --neo4j_endpoint neo4j://neo4j:${{ env.NEO_TEST_PORT}}"
           echo "CMD_STR_e2e=${CMD_STR_e2e_str}" >> $GITHUB_OUTPUT
 
-          CMD_STR_OpsIO_str="./run_neo4j_tests.sh -t OpsIOCoordinatorTest ${S3_ARGS} ${COMMON_ARGS} -e neo4j://neo4j:${{ env.NEO_TEST_PORT}}"
+          CMD_STR_OpsIO_str="./run_all_tests.py --type_of_test neo --test_name 'OpsIOCoordinatorTest.*' ${S3_ARGS} ${COMMON_ARGS} --neo4j_endpoint neo4j://neo4j:${{ env.NEO_TEST_PORT }}"
           echo "CMD_STR_OpsIO=${CMD_STR_OpsIO_str}" >> $GITHUB_OUTPUT
 
-          CMD_STR_bkend_str="./run_neo4j_tests.sh -t Neo4jBackendTest ${COMMON_ARGS} -e neo4j://neo4j:${{ env.NEO_TEST_PORT}}"
+          CMD_STR_bkend_str="./run_all_tests.py --type_of_test neo --test_name 'Neo4jBackendTest.*' ${COMMON_ARGS} --neo4j_endpoint neo4j://neo4j:${{ env.NEO_TEST_PORT }}"
           echo "CMD_STR_bkend=${CMD_STR_bkend_str}" >> $GITHUB_OUTPUT
 
       - name: Run Neo4J Tests
@@ -183,7 +183,7 @@ jobs:
 
           # E2E Test
           echo "${{ env.CMD_STR_e2e }}"
-          docker exec -w /vdms/tests ${{ env.SOURCE_CONTAINER_NAME }} bash -c "${{ env.CMD_STR_e2e }}"
+          docker exec -w /vdms/tests ${{ env.SOURCE_CONTAINER_NAME }} bash -c "python ${{ env.CMD_STR_e2e }}"
           docker kill ${{ env.NEO4J_CONTAINER_NAME }} || true
           sleep 15
 
@@ -203,7 +203,7 @@ jobs:
 
           # Ops IO Test
           echo "${{ env.CMD_STR_OpsIO }}"
-          docker exec -w /vdms/tests ${{ env.SOURCE_CONTAINER_NAME }} bash -c "${{ env.CMD_STR_OpsIO }}"
+          docker exec -w /vdms/tests ${{ env.SOURCE_CONTAINER_NAME }} bash -c "python ${{ env.CMD_STR_OpsIO }}"
           docker kill ${{ env.NEO4J_CONTAINER_NAME }} || true
           sleep 15
 
@@ -226,7 +226,43 @@ jobs:
           docker cp ${{ env.SOURCE_CONTAINER_NAME }}:/vdms/tests/coverage_report/python.new.coverage_value.txt ${GITHUB_WORKSPACE}/.github/coverage/python.new.coverage_value.txt || true
           echo "coverage_value_py=$(cat ${GITHUB_WORKSPACE}/.github/coverage/python.new.coverage_value.txt)" >> $GITHUB_ENV
           echo "pr_dev_value_py=$(cat ${GITHUB_WORKSPACE}/.github/coverage/python.old-develop.coverage_value.txt)" >> $GITHUB_ENV
-
+      - name: Print out the logs
+        shell: bash
+        id: print_out_logs
+        if: ${{ always() }}
+        run: |
+          echo "print pip version"
+          docker exec ${{ env.SOURCE_CONTAINER_NAME }} pip -V || true
+          echo "Python version"
+          docker exec ${{ env.SOURCE_CONTAINER_NAME }} python --version || true
+          echo "Python3 version"
+          docker exec ${{ env.SOURCE_CONTAINER_NAME }} python3 --version || true
+          echo "Print out the logs"
+          echo "tests_stderr_log:"
+          docker exec ${{ env.SOURCE_CONTAINER_NAME }} cat /vdms/tests/tests_output_dir/tests_stderr_log.log || true
+          echo "tests_stdout_log:"
+          docker exec ${{ env.SOURCE_CONTAINER_NAME }} cat /vdms/tests/tests_output_dir/tests_stdout_log.log || true
+          echo "tls_stderr_log:"
+          docker exec ${{ env.SOURCE_CONTAINER_NAME }} cat /vdms/tests/tests_output_dir/tls_stderr_log.log || true
+          echo "tls_stdout_log:"
+          docker exec ${{ env.SOURCE_CONTAINER_NAME }} cat /vdms/tests/tests_output_dir/tls_stdout_log.log || true
+          echo "udf_local_stderr_log:"
+          docker exec ${{ env.SOURCE_CONTAINER_NAME }} cat /vdms/tests/tests_output_dir/udf_local_stderr_log.log || true
+          echo "udf_local_stdout_log:"
+          docker exec ${{ env.SOURCE_CONTAINER_NAME }} cat /vdms/tests/tests_output_dir/udf_local_stdout_log.log || true
+          echo "udf_server_stderr_log:"
+          docker exec ${{ env.SOURCE_CONTAINER_NAME }} cat /vdms/tests/tests_output_dir/udf_server_stderr_log.log || true
+          echo "udf_server_stdout_log:"
+          docker exec ${{ env.SOURCE_CONTAINER_NAME }} cat /vdms/tests/tests_output_dir/udf_server_stdout_log.log || true
+          echo "vdms_stderr_log:"
+          docker exec ${{ env.SOURCE_CONTAINER_NAME }} cat /vdms/tests/tests_output_dir/vdms_stderr_log.log || true
+          echo "vdms_stdout_log:"
+          docker exec ${{ env.SOURCE_CONTAINER_NAME }} cat /vdms/tests/tests_output_dir/vdms_stdout_log.log || true
+          echo "End of Print out the logs..."
+      - name: Docker Compose down
+        shell: bash
+        id: docker_compose_down
+        run: |
           # Stops containers and removes containers, networks, volumes, and images
           cd ${GITHUB_WORKSPACE}/.github/scripts
           docker compose down

include/vcl/utils.h

@@ -5,7 +5,7 @@
  *
  * The MIT License
  *
- * @copyright Copyright (c) 2017 Intel Corporation
+ * @copyright Copyright (c) 2024 Intel Corporation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -31,6 +31,8 @@
 
 #include <time.h>
 
+#include <cstdint>
+#include <string>
 #include <vector>
 
 namespace VCL {

remote_function/functions/caption.py

@@ -1,29 +1,46 @@
 import cv2
-import skvideo.io
 import uuid
+import os
+import sys
 
+DEBUG_MODE = False
 
-def run(ipfilename, format, options):
-    opfilename = "tmpfile" + uuid.uuid1().hex + "." + str(format)
-    print(opfilename)
+
+def run(ipfilename, format, options, tmp_dir_path, functions_path):
+    if DEBUG_MODE:
+        print("Temporary path:", tmp_dir_path, file=sys.stderr)
+        print("Functions path:", functions_path, file=sys.stderr)
+        print("options:", options, file=sys.stderr)
+        print("format:", format, file=sys.stderr)
+        print("ipfilename", ipfilename, file=sys.stderr)
+    opfilename = os.path.join(
+        tmp_dir_path, "tmpfile" + uuid.uuid1().hex + "." + str(format)
+    )
+    if DEBUG_MODE:
+        print("opfilename:", opfilename, file=sys.stderr)
     vs = cv2.VideoCapture(ipfilename)
+    frame_width = int(vs.get(3))
+    frame_height = int(vs.get(4))
+
+    video = cv2.VideoWriter(
+        opfilename, cv2.VideoWriter_fourcc(*"XVID"), 30, (frame_width, frame_height)
+    )
 
-    video = skvideo.io.FFmpegWriter(opfilename, {"-pix_fmt": "bgr24"})
-    print(options)
+    if DEBUG_MODE:
+        print(options, file=sys.stderr)
 
     while True:
         (grabbed, frame) = vs.read()
         if not grabbed:
             print("[INFO] no frame read from stream - exiting")
-            video.close()
-            # sys.exit(0)
             break
 
         label = options["text"]
         cv2.putText(
             frame, label, (10, 25), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 255), 2
         )
 
-        video.writeFrame(frame)
+        video.write(frame)
+    video.release()
 
-    return opfilename
+    return opfilename, None

remote_function/functions/facedetect.py

@@ -1,14 +1,37 @@
 import cv2
+import os
+import sys
 
-face_cascade = cv2.CascadeClassifier(
-    # This file is available from OpenCV 'data' directory at
-    # https://github.com/opencv/opencv/blob/4.x/data/haarcascades/haarcascade_frontalface_default.xml
-    "functions/files/haarcascade_frontalface_default.xml"
-)
+DEBUG_MODE = False
 
 
-def run(ipfilename, format, options):
-    global face_cascade
+def run(ipfilename, format, options, tmp_dir_path, functions_path):
+    if DEBUG_MODE:
+        print("Temporary path:", tmp_dir_path, file=sys.stderr)
+        print("Functions path:", functions_path, file=sys.stderr)
+        print("options:", options, file=sys.stderr)
+        print("format:", format, file=sys.stderr)
+        print("ipfilename", ipfilename, file=sys.stderr)
+
+    haarcascade_frontalface_default_path = os.path.join(
+        functions_path, "files/haarcascade_frontalface_default.xml"
+    )
+
+    if not os.path.exists(haarcascade_frontalface_default_path):
+        raise Exception(f"{haarcascade_frontalface_default_path}: path is invalid")
+
+    face_cascade = cv2.CascadeClassifier(
+        # This file is available from OpenCV 'data' directory at
+        # https://github.com/opencv/opencv/blob/4.x/data/haarcascades/haarcascade_frontalface_default.xml
+        haarcascade_frontalface_default_path
+    )
+
+    if DEBUG_MODE:
+        print("Facedetect: ipfilename", ipfilename)
+    if not os.path.exists(ipfilename):
+        raise Exception(
+            f"Facedetect error: File ipfilename: {ipfilename} does not exist"
+        )
 
     img = cv2.imread(ipfilename)
     gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
@@ -17,4 +40,4 @@ def run(ipfilename, format, options):
     for x, y, w, h in faces:
         cv2.rectangle(img, (x, y), (x + w, y + h), (255, 0, 0), 2)
 
-    return img
+    return img, None

remote_function/functions/metadata.py

@@ -1,12 +1,28 @@
 import cv2
 import uuid
 import json
+import os
+import sys
 
-face_cascade = cv2.CascadeClassifier(
-    # This file is available from OpenCV 'data' directory at
-    # https://github.com/opencv/opencv/blob/4.x/data/haarcascades/haarcascade_frontalface_default.xml
-    "functions/files/haarcascade_frontalface_default.xml"
-)
+DEBUG_MODE = False
+
+face_cascade = None
+
+
+def set_face_cascade(functions_path):
+    global face_cascade
+    haarcascade_frontalface_default_path = os.path.join(
+        functions_path, "files/haarcascade_frontalface_default.xml"
+    )
+
+    if not os.path.exists(haarcascade_frontalface_default_path):
+        raise Exception(f"{haarcascade_frontalface_default_path}: path is invalid")
+
+    face_cascade = cv2.CascadeClassifier(
+        # This file is available from OpenCV 'data' directory at
+        # https://github.com/opencv/opencv/blob/4.x/data/haarcascades/haarcascade_frontalface_default.xml
+        haarcascade_frontalface_default_path
+    )
 
 
 def facedetectbbox(frame):
@@ -16,7 +32,16 @@ def facedetectbbox(frame):
     return faces
 
 
-def run(ipfilename, format, options):
+def run(ipfilename, format, options, tmp_dir_path, functions_path):
+    if DEBUG_MODE:
+        print("Temporary path:", tmp_dir_path, file=sys.stderr)
+        print("Functions path:", functions_path, file=sys.stderr)
+        print("options:", options, file=sys.stderr)
+        print("format:", format, file=sys.stderr)
+        print("ipfilename:", ipfilename, file=sys.stderr)
+
+    set_face_cascade(functions_path)
+
     # Extract metadata for video files
     if options["media_type"] == "video":
         vs = cv2.VideoCapture(ipfilename)
@@ -69,13 +94,20 @@ def run(ipfilename, format, options):
 
         response = {"opFile": ipfilename, "metadata": metadata}
 
-        jsonfile = "jsonfile" + uuid.uuid1().hex + ".json"
+        jsonfile = os.path.join(tmp_dir_path, "jsonfile" + uuid.uuid1().hex + ".json")
         with open(jsonfile, "w") as f:
             json.dump(response, f, indent=4)
         return ipfilename, jsonfile
     # Extract metadata for image files
     else:
         tdict = {}
+        if DEBUG_MODE:
+            print("Metadata: ipfilename=", ipfilename, file=sys.stderr)
+        if not os.path.exists(ipfilename):
+            raise Exception(
+                f"Metadata error: File ipfilename {ipfilename} does not exist"
+            )
+
         img = cv2.imread(ipfilename)
         if options["otype"] == "face":
             faces = facedetectbbox(img)