From 8247a8c1dd2706def61f7ede02094e03be7fcf22 Mon Sep 17 00:00:00 2001 From: Charles Zaloom <38677807+czaloom@users.noreply.github.com> Date: Wed, 9 Oct 2024 13:06:12 -0400 Subject: [PATCH] Boundiing Box Examples as dict (#785) --- lite/tests/detection/test_confusion_matrix.py | 241 ++++++++++++------ lite/valor_lite/detection/manager.py | 46 ++-- lite/valor_lite/detection/metric.py | 12 +- 3 files changed, 196 insertions(+), 103 deletions(-) diff --git a/lite/tests/detection/test_confusion_matrix.py b/lite/tests/detection/test_confusion_matrix.py index 4c3344873..ebceb9586 100644 --- a/lite/tests/detection/test_confusion_matrix.py +++ b/lite/tests/detection/test_confusion_matrix.py @@ -444,6 +444,12 @@ def test_confusion_matrix( as_dict=True, ) + rect1_dict = evaluator._convert_example_to_dict(np.array(rect1)) + rect2_dict = evaluator._convert_example_to_dict(np.array(rect2)) + rect3_dict = evaluator._convert_example_to_dict(np.array(rect3)) + rect4_dict = evaluator._convert_example_to_dict(np.array(rect4)) + rect5_dict = evaluator._convert_example_to_dict(np.array(rect5)) + actual_metrics = [m for m in metrics[MetricType.ConfusionMatrix]] expected_metrics = [ { @@ -456,8 +462,8 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "groundtruth": rect1, - "prediction": rect1, + "groundtruth": rect1_dict, + "prediction": rect1_dict, "score": 0.5, } ], @@ -470,7 +476,7 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "prediction": rect5, + "prediction": rect5_dict, "score": 0.30000001192092896, } ], @@ -480,7 +486,7 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "prediction": rect4, + "prediction": rect4_dict, "score": 0.10000000149011612, } ], @@ -490,7 +496,7 @@ def test_confusion_matrix( "examples": [ { "datum": "uid2", - "prediction": rect2, + "prediction": rect2_dict, "score": 0.5, } ], @@ -499,15 +505,21 @@ def test_confusion_matrix( "missing_predictions": { "missed_detection": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect2}], + "examples": [ + {"datum": "uid1", "groundtruth": rect2_dict} + ], }, "v2": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect3}], + "examples": [ + {"datum": "uid1", "groundtruth": rect3_dict} + ], }, "low_iou": { "count": 1, - "examples": [{"datum": "uid2", "groundtruth": rect1}], + "examples": [ + {"datum": "uid2", "groundtruth": rect1_dict} + ], }, }, }, @@ -526,8 +538,8 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "groundtruth": rect1, - "prediction": rect1, + "groundtruth": rect1_dict, + "prediction": rect1_dict, "score": 0.5, } ], @@ -540,7 +552,7 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "prediction": rect5, + "prediction": rect5_dict, "score": 0.30000001192092896, } ], @@ -550,7 +562,7 @@ def test_confusion_matrix( "examples": [ { "datum": "uid2", - "prediction": rect2, + "prediction": rect2_dict, "score": 0.5, } ], @@ -559,15 +571,21 @@ def test_confusion_matrix( "missing_predictions": { "missed_detection": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect2}], + "examples": [ + {"datum": "uid1", "groundtruth": rect2_dict} + ], }, "v2": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect3}], + "examples": [ + {"datum": "uid1", "groundtruth": rect3_dict} + ], }, "low_iou": { "count": 1, - "examples": [{"datum": "uid2", "groundtruth": rect1}], + "examples": [ + {"datum": "uid2", "groundtruth": rect1_dict} + ], }, }, }, @@ -586,8 +604,8 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "groundtruth": rect1, - "prediction": rect1, + "groundtruth": rect1_dict, + "prediction": rect1_dict, "score": 0.5, } ], @@ -600,7 +618,7 @@ def test_confusion_matrix( "examples": [ { "datum": "uid2", - "prediction": rect2, + "prediction": rect2_dict, "score": 0.5, } ], @@ -609,15 +627,21 @@ def test_confusion_matrix( "missing_predictions": { "missed_detection": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect2}], + "examples": [ + {"datum": "uid1", "groundtruth": rect2_dict} + ], }, "v2": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect3}], + "examples": [ + {"datum": "uid1", "groundtruth": rect3_dict} + ], }, "low_iou": { "count": 1, - "examples": [{"datum": "uid2", "groundtruth": rect1}], + "examples": [ + {"datum": "uid2", "groundtruth": rect1_dict} + ], }, }, }, @@ -636,8 +660,8 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "groundtruth": rect1, - "prediction": rect1, + "groundtruth": rect1_dict, + "prediction": rect1_dict, "score": 0.5, } ], @@ -650,7 +674,7 @@ def test_confusion_matrix( "examples": [ { "datum": "uid2", - "prediction": rect2, + "prediction": rect2_dict, "score": 0.5, } ], @@ -659,15 +683,21 @@ def test_confusion_matrix( "missing_predictions": { "missed_detection": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect2}], + "examples": [ + {"datum": "uid1", "groundtruth": rect2_dict} + ], }, "v2": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect3}], + "examples": [ + {"datum": "uid1", "groundtruth": rect3_dict} + ], }, "low_iou": { "count": 1, - "examples": [{"datum": "uid2", "groundtruth": rect1}], + "examples": [ + {"datum": "uid2", "groundtruth": rect1_dict} + ], }, }, }, @@ -684,19 +714,27 @@ def test_confusion_matrix( "missing_predictions": { "v1": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect1}], + "examples": [ + {"datum": "uid1", "groundtruth": rect1_dict} + ], }, "missed_detection": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect2}], + "examples": [ + {"datum": "uid1", "groundtruth": rect2_dict} + ], }, "v2": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect3}], + "examples": [ + {"datum": "uid1", "groundtruth": rect3_dict} + ], }, "low_iou": { "count": 1, - "examples": [{"datum": "uid2", "groundtruth": rect1}], + "examples": [ + {"datum": "uid2", "groundtruth": rect1_dict} + ], }, }, }, @@ -713,19 +751,27 @@ def test_confusion_matrix( "missing_predictions": { "v1": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect1}], + "examples": [ + {"datum": "uid1", "groundtruth": rect1_dict} + ], }, "missed_detection": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect2}], + "examples": [ + {"datum": "uid1", "groundtruth": rect2_dict} + ], }, "v2": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect3}], + "examples": [ + {"datum": "uid1", "groundtruth": rect3_dict} + ], }, "low_iou": { "count": 1, - "examples": [{"datum": "uid2", "groundtruth": rect1}], + "examples": [ + {"datum": "uid2", "groundtruth": rect1_dict} + ], }, }, }, @@ -767,8 +813,8 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "groundtruth": rect1, - "prediction": rect1, + "groundtruth": rect1_dict, + "prediction": rect1_dict, "score": 0.5, } ], @@ -780,8 +826,8 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "groundtruth": rect3, - "prediction": rect5, + "groundtruth": rect3_dict, + "prediction": rect5_dict, "score": 0.30000001192092896, } ], @@ -794,7 +840,7 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "prediction": rect4, + "prediction": rect4_dict, "score": 0.10000000149011612, } ], @@ -804,7 +850,7 @@ def test_confusion_matrix( "examples": [ { "datum": "uid2", - "prediction": rect2, + "prediction": rect2_dict, "score": 0.5, } ], @@ -813,14 +859,16 @@ def test_confusion_matrix( "missing_predictions": { "missed_detection": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect2}], + "examples": [ + {"datum": "uid1", "groundtruth": rect2_dict} + ], }, "low_iou": { "count": 1, "examples": [ { "datum": "uid2", - "groundtruth": rect1, + "groundtruth": rect1_dict, } ], }, @@ -841,8 +889,8 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "groundtruth": rect1, - "prediction": rect1, + "groundtruth": rect1_dict, + "prediction": rect1_dict, "score": 0.5, } ], @@ -854,8 +902,8 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "groundtruth": rect3, - "prediction": rect5, + "groundtruth": rect3_dict, + "prediction": rect5_dict, "score": 0.30000001192092896, } ], @@ -868,7 +916,7 @@ def test_confusion_matrix( "examples": [ { "datum": "uid2", - "prediction": rect2, + "prediction": rect2_dict, "score": 0.5, } ], @@ -877,14 +925,16 @@ def test_confusion_matrix( "missing_predictions": { "missed_detection": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect2}], + "examples": [ + {"datum": "uid1", "groundtruth": rect2_dict} + ], }, "low_iou": { "count": 1, "examples": [ { "datum": "uid2", - "groundtruth": rect1, + "groundtruth": rect1_dict, } ], }, @@ -905,8 +955,8 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "groundtruth": rect1, - "prediction": rect1, + "groundtruth": rect1_dict, + "prediction": rect1_dict, "score": 0.5, } ], @@ -919,7 +969,7 @@ def test_confusion_matrix( "examples": [ { "datum": "uid2", - "prediction": rect2, + "prediction": rect2_dict, "score": 0.5, } ], @@ -928,18 +978,22 @@ def test_confusion_matrix( "missing_predictions": { "missed_detection": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect2}], + "examples": [ + {"datum": "uid1", "groundtruth": rect2_dict} + ], }, "v2": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect3}], + "examples": [ + {"datum": "uid1", "groundtruth": rect3_dict} + ], }, "low_iou": { "count": 1, "examples": [ { "datum": "uid2", - "groundtruth": rect1, + "groundtruth": rect1_dict, } ], }, @@ -960,8 +1014,8 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "groundtruth": rect1, - "prediction": rect1, + "groundtruth": rect1_dict, + "prediction": rect1_dict, "score": 0.5, } ], @@ -974,7 +1028,7 @@ def test_confusion_matrix( "examples": [ { "datum": "uid2", - "prediction": rect2, + "prediction": rect2_dict, "score": 0.5, } ], @@ -983,18 +1037,22 @@ def test_confusion_matrix( "missing_predictions": { "missed_detection": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect2}], + "examples": [ + {"datum": "uid1", "groundtruth": rect2_dict} + ], }, "v2": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect3}], + "examples": [ + {"datum": "uid1", "groundtruth": rect3_dict} + ], }, "low_iou": { "count": 1, "examples": [ { "datum": "uid2", - "groundtruth": rect1, + "groundtruth": rect1_dict, } ], }, @@ -1016,24 +1074,28 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "groundtruth": rect1, + "groundtruth": rect1_dict, } ], }, "missed_detection": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect2}], + "examples": [ + {"datum": "uid1", "groundtruth": rect2_dict} + ], }, "v2": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect3}], + "examples": [ + {"datum": "uid1", "groundtruth": rect3_dict} + ], }, "low_iou": { "count": 1, "examples": [ { "datum": "uid2", - "groundtruth": rect1, + "groundtruth": rect1_dict, } ], }, @@ -1055,24 +1117,28 @@ def test_confusion_matrix( "examples": [ { "datum": "uid1", - "groundtruth": rect1, + "groundtruth": rect1_dict, } ], }, "missed_detection": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect2}], + "examples": [ + {"datum": "uid1", "groundtruth": rect2_dict} + ], }, "v2": { "count": 1, - "examples": [{"datum": "uid1", "groundtruth": rect3}], + "examples": [ + {"datum": "uid1", "groundtruth": rect3_dict} + ], }, "low_iou": { "count": 1, "examples": [ { "datum": "uid2", - "groundtruth": rect1, + "groundtruth": rect1_dict, } ], }, @@ -1532,8 +1598,18 @@ def test_confusion_matrix_fp_hallucination_edge_case( "examples": [ { "datum": "uid1", - "groundtruth": (0.0, 5.0, 0.0, 5.0), - "prediction": (0.0, 5.0, 0.0, 5.0), + "groundtruth": { + "xmin": 0.0, + "xmax": 5.0, + "ymin": 0.0, + "ymax": 5.0, + }, + "prediction": { + "xmin": 0.0, + "xmax": 5.0, + "ymin": 0.0, + "ymax": 5.0, + }, "score": 0.800000011920929, } ], @@ -1546,7 +1622,12 @@ def test_confusion_matrix_fp_hallucination_edge_case( "examples": [ { "datum": "uid2", - "prediction": (10.0, 20.0, 10.0, 20.0), + "prediction": { + "xmin": 10.0, + "xmax": 20.0, + "ymin": 10.0, + "ymax": 20.0, + }, "score": 0.800000011920929, } ], @@ -1558,7 +1639,12 @@ def test_confusion_matrix_fp_hallucination_edge_case( "examples": [ { "datum": "uid2", - "groundtruth": (0.0, 5.0, 0.0, 5.0), + "groundtruth": { + "xmin": 0.0, + "xmax": 5.0, + "ymin": 0.0, + "ymax": 5.0, + }, } ], } @@ -1580,7 +1666,12 @@ def test_confusion_matrix_fp_hallucination_edge_case( "examples": [ { "datum": "uid1", - "groundtruth": (0.0, 5.0, 0.0, 5.0), + "groundtruth": { + "xmin": 0.0, + "xmax": 5.0, + "ymin": 0.0, + "ymax": 5.0, + }, } ], } diff --git a/lite/valor_lite/detection/manager.py b/lite/valor_lite/detection/manager.py index a992f77e4..a835d7827 100644 --- a/lite/valor_lite/detection/manager.py +++ b/lite/valor_lite/detection/manager.py @@ -436,6 +436,19 @@ def evaluate( return metrics + def _convert_example_to_dict( + self, box: NDArray[np.float16] + ) -> dict[str, float]: + """ + Converts a cached bounding box example to dictionary format. + """ + return { + "xmin": box[0], + "xmax": box[1], + "ymin": box[2], + "ymax": box[3], + } + def _unpack_confusion_matrix( self, confusion_matrix: NDArray[np.float64], @@ -451,7 +464,7 @@ def _unpack_confusion_matrix( | list[ dict[ str, - str | float | tuple[float, float, float, float], + str | float | dict[str, float], ] ], ], @@ -507,7 +520,7 @@ def _unpack_confusion_matrix( gt_label_idx, pd_label_idx, example_idx ) ], - "groundtruth": tuple( + "groundtruth": self._convert_example_to_dict( self.groundtruth_examples[ datum_idx( gt_label_idx, @@ -520,9 +533,9 @@ def _unpack_confusion_matrix( pd_label_idx, example_idx, ) - ].tolist() + ] ), - "prediction": tuple( + "prediction": self._convert_example_to_dict( self.prediction_examples[ datum_idx( gt_label_idx, @@ -535,7 +548,7 @@ def _unpack_confusion_matrix( pd_label_idx, example_idx, ) - ].tolist() + ] ), "score": score_idx( gt_label_idx, pd_label_idx, example_idx @@ -560,8 +573,7 @@ def _unpack_hallucinations( str, dict[ str, - int - | list[dict[str, str | float | tuple[float, float, float, float]]], + int | list[dict[str, str | float | dict[str, float]]], ], ]: """ @@ -606,12 +618,10 @@ def _unpack_hallucinations( "datum": self.index_to_uid[ datum_idx(pd_label_idx, example_idx) ], - "prediction": tuple( + "prediction": self._convert_example_to_dict( self.prediction_examples[ datum_idx(pd_label_idx, example_idx) - ][ - prediction_idx(pd_label_idx, example_idx) - ].tolist() + ][prediction_idx(pd_label_idx, example_idx)] ), "score": score_idx(pd_label_idx, example_idx), } @@ -627,13 +637,7 @@ def _unpack_missing_predictions( missing_predictions: NDArray[np.int32], number_of_labels: int, number_of_examples: int, - ) -> dict[ - str, - dict[ - str, - int | list[dict[str, str | tuple[float, float, float, float]]], - ], - ]: + ) -> dict[str, dict[str, int | list[dict[str, str | dict[str, float]]]]]: """ Unpacks a numpy array of missing prediction counts and examples. """ @@ -667,12 +671,10 @@ def _unpack_missing_predictions( "datum": self.index_to_uid[ datum_idx(gt_label_idx, example_idx) ], - "groundtruth": tuple( + "groundtruth": self._convert_example_to_dict( self.groundtruth_examples[ datum_idx(gt_label_idx, example_idx) - ][ - groundtruth_idx(gt_label_idx, example_idx) - ].tolist() + ][groundtruth_idx(gt_label_idx, example_idx)] ), } for example_idx in range(number_of_examples) diff --git a/lite/valor_lite/detection/metric.py b/lite/valor_lite/detection/metric.py index 0d13930b1..587d13cf7 100644 --- a/lite/valor_lite/detection/metric.py +++ b/lite/valor_lite/detection/metric.py @@ -315,8 +315,8 @@ class ConfusionMatrix: dict[ str, # either `datum`, `groundtruth`, `prediction` or score str # datum uid - | tuple[ - float, float, float, float + | dict[ + str, float ] # bounding box (xmin, xmax, ymin, ymax) | float, # prediction score ] @@ -334,8 +334,8 @@ class ConfusionMatrix: str, # either `datum`, `prediction` or score str # datum uid | float # prediction score - | tuple[ - float, float, float, float + | dict[ + str, float ], # bounding box (xmin, xmax, ymin, ymax) ] ], @@ -350,8 +350,8 @@ class ConfusionMatrix: dict[ str, # either `datum` or `groundtruth` str # datum uid - | tuple[ - float, float, float, float + | dict[ + str, float ], # bounding box (xmin, xmax, ymin, ymax) ] ],