Limit combinations of backends and targets in demos and benchmark (#145)

* limit backend and target combination in demos and benchmark

* simpler version checking

Author: fengyuentau

benchmark/benchmark.py

@@ -8,9 +8,31 @@
 from models import MODELS
 from utils import METRICS, DATALOADERS
 
+# Check OpenCV version
+assert cv.__version__ >= "4.7.0", \
+       "Please install latest opencv-python for benchmark: python3 -m pip install --upgrade opencv-python"
+
+# Valid combinations of backends and targets
+backend_target_pairs = [
+    [cv.dnn.DNN_BACKEND_OPENCV, cv.dnn.DNN_TARGET_CPU],
+    [cv.dnn.DNN_BACKEND_CUDA,   cv.dnn.DNN_TARGET_CUDA],
+    [cv.dnn.DNN_BACKEND_CUDA,   cv.dnn.DNN_TARGET_CUDA_FP16],
+    [cv.dnn.DNN_BACKEND_TIMVX,  cv.dnn.DNN_TARGET_NPU],
+    [cv.dnn.DNN_BACKEND_CANN,   cv.dnn.DNN_TARGET_NPU]
+]
+
 parser = argparse.ArgumentParser("Benchmarks for OpenCV Zoo.")
 parser.add_argument('--cfg', '-c', type=str,
                     help='Benchmarking on the given config.')
+parser.add_argument('--cfg_overwrite_backend_target', type=int, default=-1,
+                    help='''Choose one of the backend-target pair to run this demo:
+                        others: (default) use the one from config,
+                        {:d}: OpenCV implementation + CPU,
+                        {:d}: CUDA + GPU (CUDA),
+                        {:d}: CUDA + GPU (CUDA FP16),
+                        {:d}: TIM-VX + NPU,
+                        {:d}: CANN + NPU
+                    '''.format(*[x for x in range(len(backend_target_pairs))]))
 parser.add_argument("--fp32", action="store_true", help="Runs models of float32 precision only.")
 parser.add_argument("--fp16", action="store_true", help="Runs models of float16 precision only.")
 parser.add_argument("--int8", action="store_true", help="Runs models of int8 precision only.")
@@ -56,6 +78,8 @@ def __init__(self, **kwargs):
             opencv=cv.dnn.DNN_BACKEND_OPENCV,
             # vkcom=cv.dnn.DNN_BACKEND_VKCOM,
             cuda=cv.dnn.DNN_BACKEND_CUDA,
+            timvx=cv.dnn.DNN_BACKEND_TIMVX,
+            cann=cv.dnn.DNN_BACKEND_CANN,
         )
 
         target_id = kwargs.pop('target', 'cpu')
@@ -69,28 +93,20 @@ def __init__(self, **kwargs):
             cuda=cv.dnn.DNN_TARGET_CUDA,
             cuda_fp16=cv.dnn.DNN_TARGET_CUDA_FP16,
             # hddl=cv.dnn.DNN_TARGET_HDDL,
+            npu=cv.dnn.DNN_TARGET_NPU,
         )
 
-        # add extra backends & targets
-        try:
-            available_backends['timvx'] = cv.dnn.DNN_BACKEND_TIMVX
-            available_targets['npu'] = cv.dnn.DNN_TARGET_NPU
-        except:
-            print('OpenCV is not compiled with TIM-VX backend enbaled. See https://github.com/opencv/opencv/wiki/TIM-VX-Backend-For-Running-OpenCV-On-NPU for more details on how to enable TIM-VX backend.')
-        try:
-            available_backends['cann'] = cv.dnn.DNN_BACKEND_CANN
-            available_targets['npu'] = cv.dnn.DNN_TARGET_NPU
-        except:
-            print('OpenCV is not compiled with CANN backend enabled. See https://github.com/opencv/opencv/wiki/Huawei-CANN-Backend for more details on how to enable CANN backend.')
-
         self._backend = available_backends[backend_id]
         self._target = available_targets[target_id]
 
         self._benchmark_results = dict()
 
+    def setBackendAndTarget(self, backend_id, target_id):
+        self._backend = backend_id
+        self._target = target_id
+
     def run(self, model):
-        model.setBackend(self._backend)
-        model.setTarget(self._target)
+        model.setBackendAndTarget(self._backend, self._target)
 
         for idx, data in enumerate(self._dataloader):
             filename, input_data = data[:2]
@@ -118,6 +134,11 @@ def printResults(self):
     # Instantiate benchmark
     benchmark = Benchmark(**cfg['Benchmark'])
 
+    if args.cfg_overwrite_backend_target >= 0:
+        backend_id = backend_target_pairs[args.backend_target][0]
+        target_id = backend_target_pairs[args.backend_target][1]
+        benchmark.setBackendAndTarget(backend_id, target_id)
+
     # Instantiate model
     model_config = cfg['Model']
     model_handler, model_paths = MODELS.get(model_config.pop('name'))

models/face_detection_yunet/demo.py

@@ -11,36 +11,42 @@
 
 from yunet import YuNet
 
-def str2bool(v):
-    if v.lower() in ['on', 'yes', 'true', 'y', 't']:
-        return True
-    elif v.lower() in ['off', 'no', 'false', 'n', 'f']:
-        return False
-    else:
-        raise NotImplementedError
-
-backends = [cv.dnn.DNN_BACKEND_OPENCV, cv.dnn.DNN_BACKEND_CUDA]
-targets = [cv.dnn.DNN_TARGET_CPU, cv.dnn.DNN_TARGET_CUDA, cv.dnn.DNN_TARGET_CUDA_FP16]
-help_msg_backends = "Choose one of the computation backends: {:d}: OpenCV implementation (default); {:d}: CUDA"
-help_msg_targets = "Choose one of the target computation devices: {:d}: CPU (default); {:d}: CUDA; {:d}: CUDA fp16"
-try:
-    backends += [cv.dnn.DNN_BACKEND_TIMVX]
-    targets += [cv.dnn.DNN_TARGET_NPU]
-    help_msg_backends += "; {:d}: TIMVX"
-    help_msg_targets += "; {:d}: NPU"
-except:
-    print('This version of OpenCV does not support TIM-VX and NPU. Visit https://github.com/opencv/opencv/wiki/TIM-VX-Backend-For-Running-OpenCV-On-NPU for more information.')
+# Check OpenCV version
+assert cv.__version__ >= "4.7.0", \
+       "Please install latest opencv-python to try this demo: python3 -m pip install --upgrade opencv-python"
+
+# Valid combinations of backends and targets
+backend_target_pairs = [
+    [cv.dnn.DNN_BACKEND_OPENCV, cv.dnn.DNN_TARGET_CPU],
+    [cv.dnn.DNN_BACKEND_CUDA,   cv.dnn.DNN_TARGET_CUDA],
+    [cv.dnn.DNN_BACKEND_CUDA,   cv.dnn.DNN_TARGET_CUDA_FP16],
+    [cv.dnn.DNN_BACKEND_TIMVX,  cv.dnn.DNN_TARGET_NPU],
+    [cv.dnn.DNN_BACKEND_CANN,   cv.dnn.DNN_TARGET_NPU]
+]
 
 parser = argparse.ArgumentParser(description='YuNet: A Fast and Accurate CNN-based Face Detector (https://github.com/ShiqiYu/libfacedetection).')
-parser.add_argument('--input', '-i', type=str, help='Usage: Set input to a certain image, omit if using camera.')
-parser.add_argument('--model', '-m', type=str, default='face_detection_yunet_2022mar.onnx', help="Usage: Set model type, defaults to 'face_detection_yunet_2022mar.onnx'.")
-parser.add_argument('--backend', '-b', type=int, default=backends[0], help=help_msg_backends.format(*backends))
-parser.add_argument('--target', '-t', type=int, default=targets[0], help=help_msg_targets.format(*targets))
-parser.add_argument('--conf_threshold', type=float, default=0.9, help='Usage: Set the minimum needed confidence for the model to identify a face, defauts to 0.9. Smaller values may result in faster detection, but will limit accuracy. Filter out faces of confidence < conf_threshold.')
-parser.add_argument('--nms_threshold', type=float, default=0.3, help='Usage: Suppress bounding boxes of iou >= nms_threshold. Default = 0.3.')
-parser.add_argument('--top_k', type=int, default=5000, help='Usage: Keep top_k bounding boxes before NMS.')
-parser.add_argument('--save', '-s', type=str, default=False, help='Usage: Set “True” to save file with results (i.e. bounding box, confidence level). Invalid in case of camera input. Default will be set to “False”.')
-parser.add_argument('--vis', '-v', type=str2bool, default=True, help='Usage: Default will be set to “True” and will open a new window to show results. Set to “False” to stop visualizations from being shown. Invalid in case of camera input.')
+parser.add_argument('--input', '-i', type=str,
+                    help='Usage: Set input to a certain image, omit if using camera.')
+parser.add_argument('--model', '-m', type=str, default='face_detection_yunet_2022mar.onnx',
+                    help="Usage: Set model type, defaults to 'face_detection_yunet_2022mar.onnx'.")
+parser.add_argument('--backend_target', '-bt', type=int, default=0,
+                    help='''Choose one of the backend-target pair to run this demo:
+                        {:d}: (default) OpenCV implementation + CPU,
+                        {:d}: CUDA + GPU (CUDA),
+                        {:d}: CUDA + GPU (CUDA FP16),
+                        {:d}: TIM-VX + NPU,
+                        {:d}: CANN + NPU
+                    '''.format(*[x for x in range(len(backend_target_pairs))]))
+parser.add_argument('--conf_threshold', type=float, default=0.9,
+                    help='Usage: Set the minimum needed confidence for the model to identify a face, defauts to 0.9. Smaller values may result in faster detection, but will limit accuracy. Filter out faces of confidence < conf_threshold.')
+parser.add_argument('--nms_threshold', type=float, default=0.3,
+                    help='Usage: Suppress bounding boxes of iou >= nms_threshold. Default = 0.3.')
+parser.add_argument('--top_k', type=int, default=5000,
+                    help='Usage: Keep top_k bounding boxes before NMS.')
+parser.add_argument('--save', '-s', action='store_true',
+                    help='Usage: Specify to save file with results (i.e. bounding box, confidence level). Invalid in case of camera input.')
+parser.add_argument('--vis', '-v', action='store_true',
+                    help='Usage: Specify to open a new window to show results. Invalid in case of camera input.')
 args = parser.parse_args()
 
 def visualize(image, results, box_color=(0, 255, 0), text_color=(0, 0, 255), fps=None):
@@ -70,14 +76,17 @@ def visualize(image, results, box_color=(0, 255, 0), text_color=(0, 0, 255), fps
     return output
 
 if __name__ == '__main__':
+    backend_id = backend_target_pairs[args.backend_target][0]
+    target_id = backend_target_pairs[args.backend_target][1]
+
     # Instantiate YuNet
     model = YuNet(modelPath=args.model,
                   inputSize=[320, 320],
                   confThreshold=args.conf_threshold,
                   nmsThreshold=args.nms_threshold,
                   topK=args.top_k,
-                  backendId=args.backend,
-                  targetId=args.target)
+                  backendId=backend_id,
+                  targetId=target_id)
 
     # If input is an image
     if args.input is not None:
@@ -134,4 +143,3 @@ def visualize(image, results, box_color=(0, 255, 0), text_color=(0, 0, 255), fps
             cv.imshow('YuNet Demo', frame)
 
             tm.reset()
-

models/face_detection_yunet/yunet.py

@@ -33,19 +33,8 @@ def __init__(self, modelPath, inputSize=[320, 320], confThreshold=0.6, nmsThresh
     def name(self):
         return self.__class__.__name__
 
-    def setBackend(self, backendId):
+    def setBackendAndTarget(self, backendId, targetId):
         self._backendId = backendId
-        self._model = cv.FaceDetectorYN.create(
-            model=self._modelPath,
-            config="",
-            input_size=self._inputSize,
-            score_threshold=self._confThreshold,
-            nms_threshold=self._nmsThreshold,
-            top_k=self._topK,
-            backend_id=self._backendId,
-            target_id=self._targetId)
-
-    def setTarget(self, targetId):
         self._targetId = targetId
         self._model = cv.FaceDetectorYN.create(
             model=self._modelPath,
@@ -64,4 +53,3 @@ def infer(self, image):
         # Forward
         faces = self._model.detect(image)
         return faces[1]
-

models/face_recognition_sface/demo.py

@@ -15,49 +15,55 @@
 sys.path.append('../face_detection_yunet')
 from yunet import YuNet
 
-def str2bool(v):
-    if v.lower() in ['on', 'yes', 'true', 'y', 't']:
-        return True
-    elif v.lower() in ['off', 'no', 'false', 'n', 'f']:
-        return False
-    else:
-        raise NotImplementedError
+# Check OpenCV version
+assert cv.__version__ >= "4.7.0", \
+       "Please install latest opencv-python to try this demo: python3 -m pip install --upgrade opencv-python"
 
-backends = [cv.dnn.DNN_BACKEND_OPENCV, cv.dnn.DNN_BACKEND_CUDA]
-targets = [cv.dnn.DNN_TARGET_CPU, cv.dnn.DNN_TARGET_CUDA, cv.dnn.DNN_TARGET_CUDA_FP16]
-help_msg_backends = "Choose one of the computation backends: {:d}: OpenCV implementation (default); {:d}: CUDA \n Usage: Set backend DNN model, defaults to cv.dnn.DNN_BACKEND_OPENCV (int = 0). Based on your OpenCV version, it may or may not support cv.dnn.DNN_BACKEND_TIMVX. More details: [https://gist.github.com/fengyuentau/5a7a5ba36328f2b763aea026c43fa45f]"
-help_msg_targets = "Chose one of the target computation devices: {:d}: CPU (default); {:d}: CUDA; {:d}: CUDA fp16"
-try:
-    backends += [cv.dnn.DNN_BACKEND_TIMVX]
-    targets += [cv.dnn.DNN_TARGET_NPU]
-    help_msg_backends += "; {:d}: TIMVX"
-    help_msg_targets += "; {:d}: NPU"
-except:
-    print('This version of OpenCV does not support TIM-VX and NPU. Visit https://github.com/opencv/opencv/wiki/TIM-VX-Backend-For-Running-OpenCV-On-NPU for more information.')
+# Valid combinations of backends and targets
+backend_target_pairs = [
+    [cv.dnn.DNN_BACKEND_OPENCV, cv.dnn.DNN_TARGET_CPU],
+    [cv.dnn.DNN_BACKEND_CUDA,   cv.dnn.DNN_TARGET_CUDA],
+    [cv.dnn.DNN_BACKEND_CUDA,   cv.dnn.DNN_TARGET_CUDA_FP16],
+    [cv.dnn.DNN_BACKEND_TIMVX,  cv.dnn.DNN_TARGET_NPU],
+    [cv.dnn.DNN_BACKEND_CANN,   cv.dnn.DNN_TARGET_NPU]
+]
 
 parser = argparse.ArgumentParser(
     description="SFace: Sigmoid-Constrained Hypersphere Loss for Robust Face Recognition (https://ieeexplore.ieee.org/document/9318547)")
-parser.add_argument('--input1', '-i1', type=str, help='Usage: Set path to the input image 1 (original face).')
-parser.add_argument('--input2', '-i2', type=str, help='Usage: Set path to the input image 2 (comparison face).')
-parser.add_argument('--model', '-m', type=str, default='face_recognition_sface_2021dec.onnx', help='Usage: Set model path, defaults to face_recognition_sface_2021dec.onnx.')
-parser.add_argument('--backend', '-b', type=int, default=backends[0], help=help_msg_backends.format(*backends))
-parser.add_argument('--target', '-t', type=int, default=targets[0], help=help_msg_targets.format(*targets))
-parser.add_argument('--dis_type', type=int, choices=[0, 1], default=0, help='Usage: Distance type. \'0\': cosine, \'1\': norm_l1. Defaults to \'0\'')
-parser.add_argument('--save', '-s', type=str, default=False, help='Usage: Set “True” to save file with results (i.e. bounding box, confidence level). Invalid in case of camera input. Default will be set to “False”.')
-parser.add_argument('--vis', '-v', type=str2bool, default=True, help='Usage: Default will be set to “True” and will open a new window to show results. Set to “False” to stop visualizations from being shown. Invalid in case of camera input.')
+parser.add_argument('--input1', '-i1', type=str,
+                    help='Usage: Set path to the input image 1 (original face).')
+parser.add_argument('--input2', '-i2', type=str,
+                    help='Usage: Set path to the input image 2 (comparison face).')
+parser.add_argument('--model', '-m', type=str, default='face_recognition_sface_2021dec.onnx',
+                    help='Usage: Set model path, defaults to face_recognition_sface_2021dec.onnx.')
+parser.add_argument('--backend_target', '-bt', type=int, default=0,
+                    help='''Choose one of the backend-target pair to run this demo:
+                        {:d}: (default) OpenCV implementation + CPU,
+                        {:d}: CUDA + GPU (CUDA),
+                        {:d}: CUDA + GPU (CUDA FP16),
+                        {:d}: TIM-VX + NPU,
+                        {:d}: CANN + NPU
+                    '''.format(*[x for x in range(len(backend_target_pairs))]))
+parser.add_argument('--dis_type', type=int, choices=[0, 1], default=0,
+                    help='Usage: Distance type. \'0\': cosine, \'1\': norm_l1. Defaults to \'0\'')
 args = parser.parse_args()
 
 if __name__ == '__main__':
+    backend_id = backend_target_pairs[args.backend_target][0]
+    target_id = backend_target_pairs[args.backend_target][1]
     # Instantiate SFace for face recognition
-    recognizer = SFace(modelPath=args.model, disType=args.dis_type, backendId=args.backend, targetId=args.target)
+    recognizer = SFace(modelPath=args.model,
+                       disType=args.dis_type,
+                       backendId=backend_id,
+                       targetId=target_id)
     # Instantiate YuNet for face detection
     detector = YuNet(modelPath='../face_detection_yunet/face_detection_yunet_2022mar.onnx',
                      inputSize=[320, 320],
                      confThreshold=0.9,
                      nmsThreshold=0.3,
                      topK=5000,
-                     backendId=args.backend,
-                     targetId=args.target)
+                     backendId=backend_id,
+                     targetId=target_id)
 
     img1 = cv.imread(args.input1)
     img2 = cv.imread(args.input2)
@@ -73,4 +79,3 @@ def str2bool(v):
     # Match
     result = recognizer.match(img1, face1[0][:-1], img2, face2[0][:-1])
     print('Result: {}.'.format('same identity' if result else 'different identities'))
-

models/face_recognition_sface/sface.py

@@ -7,8 +7,6 @@
 import numpy as np
 import cv2 as cv
 
-from _testcapi import FLT_MIN
-
 class SFace:
     def __init__(self, modelPath, disType=0, backendId=0, targetId=0):
         self._modelPath = modelPath
@@ -30,15 +28,8 @@ def __init__(self, modelPath, disType=0, backendId=0, targetId=0):
     def name(self):
         return self.__class__.__name__
 
-    def setBackend(self, backendId):
+    def setBackendAndTarget(self, backendId, targetId):
         self._backendId = backendId
-        self._model = cv.FaceRecognizerSF.create(
-            model=self._modelPath,
-            config="",
-            backend_id=self._backendId,
-            target_id=self._targetId)
-
-    def setTarget(self, targetId):
         self._targetId = targetId
         self._model = cv.FaceRecognizerSF.create(
             model=self._modelPath,
@@ -70,4 +61,3 @@ def match(self, image1, face1, image2, face2):
         else: # NORM_L2
             norml2_distance = self._model.match(feature1, feature2, self._disType)
             return 1 if norml2_distance <= self._threshold_norml2 else 0
-