Skip to content

Commit 40f7b3b

Browse files
czaloomczaloom
authored
Object Detection Benchmarking (#704)
1 parent 60622a3 commit 40f7b3b

File tree

11 files changed

+6300
-1702
lines changed

11 files changed

+6300
-1702
lines changed

‎api/tests/functional-tests/crud/test_create_delete.py

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1064,15 +1064,11 @@ def test_gt_seg_as_mask_or_polys(
10641064
assert len(segs.annotations) == 2
10651065

10661066
assert segs.annotations[0].raster and segs.annotations[1].raster
1067-
decoded_mask0 = np.array(
1068-
_bytes_to_pil(b64decode(segs.annotations[0].raster.mask))
1069-
)
1067+
decoded_mask0 = segs.annotations[0].raster.array
10701068
assert decoded_mask0.shape == mask.shape
10711069
np.testing.assert_equal(decoded_mask0, mask)
10721070

1073-
decoded_mask1 = np.array(
1074-
_bytes_to_pil(b64decode(segs.annotations[1].raster.mask))
1075-
)
1071+
decoded_mask1 = segs.annotations[1].raster.array
10761072
assert decoded_mask1.shape == mask.shape
10771073
np.testing.assert_equal(decoded_mask1, mask)
10781074

‎api/valor_api/backend/metrics/detection.py

Lines changed: 43 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from typing import Sequence, Tuple
88

99
from geoalchemy2 import functions as gfunc
10-
from sqlalchemy import CTE, and_, func, or_, select
10+
from sqlalchemy import CTE, and_, case, func, or_, select
1111
from sqlalchemy.orm import Session, aliased
1212

1313
from valor_api import enums, schemas
@@ -458,10 +458,14 @@ def _compute_detailed_curves(
458458
if misclassification_detected:
459459
fn["misclassifications"].append(
460460
(dataset_name, datum_uid, gt_geojson)
461+
if gt_geojson is not None
462+
else (dataset_name, datum_uid)
461463
)
462464
else:
463465
fn["no_predictions"].append(
464466
(dataset_name, datum_uid, gt_geojson)
467+
if gt_geojson is not None
468+
else (dataset_name, datum_uid)
465469
)
466470

467471
if label_id in predictions_per_label:
@@ -501,10 +505,14 @@ def _compute_detailed_curves(
501505
if misclassification_detected:
502506
fp["misclassifications"].append(
503507
(dataset_name, datum_uid, pd_geojson)
508+
if pd_geojson is not None
509+
else (dataset_name, datum_uid)
504510
)
505511
elif hallucination_detected:
506512
fp["hallucinations"].append(
507513
(dataset_name, datum_uid, pd_geojson)
514+
if pd_geojson is not None
515+
else (dataset_name, datum_uid)
508516
)
509517

510518
# calculate metrics
@@ -1158,14 +1166,22 @@ def _annotation_type_to_column(
11581166
select(
11591167
gt_pd_counts.c.gt_annotation_id,
11601168
gt_pd_counts.c.pd_annotation_id,
1161-
func.coalesce(
1162-
gt_pd_counts.c.intersection
1163-
/ (
1169+
case(
1170+
(
11641171
gt_pd_counts.c.gt_count
11651172
+ gt_pd_counts.c.pd_count
11661173
- gt_pd_counts.c.intersection
1174+
== 0,
1175+
0,
1176+
),
1177+
else_=(
1178+
gt_pd_counts.c.intersection
1179+
/ (
1180+
gt_pd_counts.c.gt_count
1181+
+ gt_pd_counts.c.pd_count
1182+
- gt_pd_counts.c.intersection
1183+
)
11671184
),
1168-
0,
11691185
).label("iou"),
11701186
)
11711187
.select_from(gt_pd_counts)
@@ -1183,7 +1199,10 @@ def _annotation_type_to_column(
11831199
select(
11841200
gt_pd_pairs.c.gt_annotation_id,
11851201
gt_pd_pairs.c.pd_annotation_id,
1186-
iou_computation.label("iou"),
1202+
case(
1203+
(gfunc.ST_Area(gunion) == 0, 0),
1204+
else_=iou_computation,
1205+
).label("iou"),
11871206
)
11881207
.select_from(gt_pd_pairs)
11891208
.join(
@@ -1209,10 +1228,7 @@ def _annotation_type_to_column(
12091228
gt.c.label_id.label("gt_label_id"),
12101229
pd.c.label_id.label("pd_label_id"),
12111230
pd.c.score.label("score"),
1212-
func.coalesce(
1213-
gt_pd_ious.c.iou,
1214-
0,
1215-
).label("iou"),
1231+
gt_pd_ious.c.iou,
12161232
gt.c.geojson.label("gt_geojson"),
12171233
)
12181234
.select_from(pd)
@@ -1576,14 +1592,22 @@ def _annotation_type_to_column(
15761592
select(
15771593
gt_pd_counts.c.gt_annotation_id,
15781594
gt_pd_counts.c.pd_annotation_id,
1579-
func.coalesce(
1580-
gt_pd_counts.c.intersection
1581-
/ (
1595+
case(
1596+
(
15821597
gt_pd_counts.c.gt_count
15831598
+ gt_pd_counts.c.pd_count
15841599
- gt_pd_counts.c.intersection
1600+
== 0,
1601+
0,
1602+
),
1603+
else_=(
1604+
gt_pd_counts.c.intersection
1605+
/ (
1606+
gt_pd_counts.c.gt_count
1607+
+ gt_pd_counts.c.pd_count
1608+
- gt_pd_counts.c.intersection
1609+
)
15851610
),
1586-
0,
15871611
).label("iou"),
15881612
)
15891613
.select_from(gt_pd_counts)
@@ -1601,7 +1625,10 @@ def _annotation_type_to_column(
16011625
select(
16021626
gt_pd_pairs.c.gt_annotation_id,
16031627
gt_pd_pairs.c.pd_annotation_id,
1604-
iou_computation.label("iou"),
1628+
case(
1629+
(gfunc.ST_Area(gunion) == 0, 0),
1630+
else_=iou_computation,
1631+
).label("iou"),
16051632
)
16061633
.select_from(gt_pd_pairs)
16071634
.join(
@@ -1627,10 +1654,7 @@ def _annotation_type_to_column(
16271654
gt.c.label_id.label("gt_label_id"),
16281655
pd.c.label_id.label("pd_label_id"),
16291656
pd.c.score.label("score"),
1630-
func.coalesce(
1631-
gt_pd_ious.c.iou,
1632-
0,
1633-
).label("iou"),
1657+
gt_pd_ious.c.iou,
16341658
gt.c.geojson.label("gt_geojson"),
16351659
(gt.c.label_id == pd.c.label_id).label("is_match"),
16361660
)

‎api/valor_api/schemas/geometry.py

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
ST_GeomFromText,
1212
ST_MakeEmptyRaster,
1313
ST_MapAlgebra,
14-
ST_SnapToGrid,
1514
)
1615
from pydantic import (
1716
BaseModel,
@@ -1028,8 +1027,8 @@ def to_psql(self) -> ScalarSelect | bytes:
10281027
if self.geometry:
10291028
empty_raster = ST_AddBand(
10301029
ST_MakeEmptyRaster(
1031-
self.width,
1032-
self.height,
1030+
self.width, # width
1031+
self.height, # height
10331032
0, # upperleftx
10341033
0, # upperlefty
10351034
1, # scalex
@@ -1038,23 +1037,18 @@ def to_psql(self) -> ScalarSelect | bytes:
10381037
0, # skewy
10391038
0, # srid
10401039
),
1041-
"1BB",
1042-
)
1043-
geom_raster = ST_AsRaster(
1044-
ST_SnapToGrid(
1045-
ST_GeomFromText(self.geometry.to_wkt()),
1046-
1.0,
1047-
),
1048-
1.0, # scalex
1049-
1.0, # scaley
10501040
"1BB", # pixeltype
1051-
1, # value
1052-
0, # nodataval
10531041
)
10541042
return select(
10551043
ST_MapAlgebra(
10561044
empty_raster,
1057-
geom_raster,
1045+
ST_AsRaster(
1046+
ST_GeomFromText(self.geometry.to_wkt()),
1047+
empty_raster,
1048+
"1BB",
1049+
1,
1050+
0,
1051+
),
10581052
"[rast2]",
10591053
"1BB",
10601054
"UNION",

‎client/unit-tests/schemas/test_geojson.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ def test_multipolygon():
105105
MultiPolygon([coords]) # type: ignore - testing
106106
with pytest.raises(TypeError):
107107
MultiPolygon([[coords], 123]) # type: ignore - testing
108-
with pytest.raises(TypeError):
108+
with pytest.raises(ValueError):
109109
MultiPolygon([[[coords]]]) # type: ignore - testing
110110

111111

‎client/valor/schemas/symbolic/types.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -769,12 +769,14 @@ def __init__(
769769

770770
@classmethod
771771
def __validate__(cls, value: typing.Any):
772-
if not isinstance(value, tuple):
772+
if not isinstance(value, (tuple, list)):
773773
raise TypeError(
774774
f"Expected type 'typing.Tuple[float, float]' received type '{type(value).__name__}'"
775775
)
776776
elif len(value) != 2:
777-
raise ValueError("")
777+
raise ValueError(
778+
"A point should contain only two x-y coordinates."
779+
)
778780
for item in value:
779781
if not isinstance(item, (int, float, np.floating)):
780782
raise TypeError(

0 commit comments

Comments
 (0)