Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@czaloom
czaloom committed Aug 22, 2024
1 parent 070a05e commit 6323876
Show file tree
Hide file tree
Showing 5 changed files with 122 additions and 147 deletions.
216 changes: 93 additions & 123 deletions api/valor_api/backend/metrics/detection.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,16 @@
from valor_api.enums import AnnotationType


def profiler(fn):
def wrapper(*args, **kwargs):
print("ENTERING", fn.__name__)
result = fn(*args, **kwargs)
print("EXITING", fn.__name__)
return result

return wrapper


@dataclass
class RankedPair:
dataset_name: str
Expand Down Expand Up @@ -215,6 +225,7 @@ def _calculate_ap_and_ar(
return ap_metrics, ar_metrics


@profiler
def _compute_curves(
sorted_ranked_pairs: dict[int, list[RankedPair]],
labels: dict[int, tuple[str, str]],
Expand Down Expand Up @@ -331,6 +342,7 @@ def _compute_curves(
]


@profiler
def _compute_detailed_curves(
sorted_ranked_pairs: dict[int, list[RankedPair]],
labels: dict[int, tuple[str, str]],
Expand Down Expand Up @@ -644,6 +656,7 @@ def _compute_detailed_curves(
return output


@profiler
def _compute_detection_metrics_averaged_over_ious_from_aps(
ap_scores: Sequence[schemas.APMetric],
) -> Sequence[schemas.APMetricAveragedOverIOUs]:
Expand Down Expand Up @@ -672,6 +685,7 @@ def _compute_detection_metrics_averaged_over_ious_from_aps(
return ret


@profiler
def _average_ignore_minus_one(a):
"""Average a list of metrics, ignoring values of -1"""
num, denom = 0.0, 0.0
Expand All @@ -684,6 +698,7 @@ def _average_ignore_minus_one(a):
return -1 if div0_flag else num / denom


@profiler
def _compute_mean_ar_metrics(
ar_metrics: Sequence[schemas.ARMetric],
) -> list[schemas.mARMetric]:
Expand Down Expand Up @@ -712,6 +727,7 @@ def _compute_mean_ar_metrics(
return mean_metrics


@profiler
def _compute_mean_detection_metrics_from_aps(
ap_scores: Sequence[schemas.APMetric | schemas.APMetricAveragedOverIOUs],
) -> Sequence[schemas.mAPMetric | schemas.mAPMetricAveragedOverIOUs]:
Expand Down Expand Up @@ -756,6 +772,7 @@ def _compute_mean_detection_metrics_from_aps(
return mean_detection_metrics


@profiler
def _convert_annotations_to_common_type(
db: Session,
datasets: list[models.Dataset],
Expand All @@ -764,6 +781,8 @@ def _convert_annotations_to_common_type(
) -> enums.AnnotationType:
"""Convert all annotations to a common type."""

print("\n\n\n === CONVERSION START === \n\n\n")

if target_type is None:
# find the greatest common type
groundtruth_type = AnnotationType.RASTER
Expand Down Expand Up @@ -826,9 +845,25 @@ def _convert_annotations_to_common_type(
task_type=enums.TaskType.OBJECT_DETECTION,
)

print("\n\n\n === CONVERSION END === \n\n\n")
print(target_type)
return target_type


def _annotation_type_to_column(
annotation_type: AnnotationType,
table,
):
match annotation_type:
case AnnotationType.BOX:
return table.box
case AnnotationType.POLYGON:
return table.polygon
case _:
raise RuntimeError


@profiler
def _annotation_type_to_geojson(
annotation_type: AnnotationType,
):
Expand All @@ -852,6 +887,7 @@ def _annotation_type_to_geojson(
return gfunc.ST_AsGeoJSON(box)


@profiler
def _aggregate_data(
db: Session,
groundtruth_filter: schemas.Filter,
Expand Down Expand Up @@ -988,6 +1024,7 @@ def _aggregate_data(
return (groundtruths_cte, predictions_cte, labels)


@profiler
def _compute_detection_metrics(
db: Session,
parameters: schemas.EvaluationParameters,
Expand Down Expand Up @@ -1027,20 +1064,6 @@ def _compute_detection_metrics(
"""

def _annotation_type_to_column(
annotation_type: AnnotationType,
table,
):
match annotation_type:
case AnnotationType.BOX:
return table.box
case AnnotationType.POLYGON:
return table.polygon
case AnnotationType.RASTER:
return table.raster
case _:
raise RuntimeError

if (
parameters.iou_thresholds_to_return is None
or parameters.iou_thresholds_to_compute is None
Expand All @@ -1067,35 +1090,35 @@ def _annotation_type_to_column(
label_map=parameters.label_map,
)

# Alias the annotation table (required for joining twice)
gt_annotation = aliased(models.Annotation)
pd_annotation = aliased(models.Annotation)

# Get distinct annotations
gt_pd_pairs = (
select(
gt.c.annotation_id.label("gt_annotation_id"),
pd.c.annotation_id.label("pd_annotation_id"),
)
.select_from(pd)
.join(
gt,
and_(
pd.c.datum_id == gt.c.datum_id,
pd.c.label_id == gt.c.label_id,
),
)
.distinct()
.cte()
)

# IOU Computation Block
if target_type == AnnotationType.RASTER:

# Alias the annotation table (required for joining twice)
gt_annotation = aliased(models.Annotation)
pd_annotation = aliased(models.Annotation)

# Alias bitmasks
gt_bitmask = aliased(models.Bitmask)
pd_bitmask = aliased(models.Bitmask)

# Get distinct annotations
gt_pd_pairs = (
select(
gt.c.annotation_id.label("gt_annotation_id"),
pd.c.annotation_id.label("pd_annotation_id"),
)
.select_from(pd)
.join(
gt,
and_(
pd.c.datum_id == gt.c.datum_id,
pd.c.label_id == gt.c.label_id,
),
)
.distinct()
.cte()
)

gt_pd_counts = (
select(
gt_pd_pairs.c.gt_annotation_id,
Expand Down Expand Up @@ -1146,40 +1169,20 @@ def _annotation_type_to_column(
)

else:
# Alias the annotation table (required for joining twice)
gt_annotation = aliased(models.Annotation)
pd_annotation = aliased(models.Annotation)

# Get distinct annotations
gt_pd_pairs = (
select(
gt.c.annotation_id.label("gt_annotation_id"),
pd.c.annotation_id.label("pd_annotation_id"),
)
.select_from(pd)
.join(
gt,
and_(
pd.c.datum_id == gt.c.datum_id,
pd.c.label_id == gt.c.label_id,
),
)
.distinct()
.cte()
)

gt_geom = _annotation_type_to_column(target_type, gt_annotation)
pd_geom = _annotation_type_to_column(target_type, pd_annotation)
gintersection = gfunc.ST_Intersection(gt_geom, pd_geom)
gunion = gfunc.ST_Union(gt_geom, pd_geom)
iou_computation = gfunc.ST_Area(gintersection) / gfunc.ST_Area(gunion)
gintersection = gfunc.ST_Area(gfunc.ST_Intersection(gt_geom, pd_geom))
gunion = (
gfunc.ST_Area(gt_geom) + gfunc.ST_Area(pd_geom) - gintersection
)
iou_computation = gintersection / gunion

gt_pd_ious = (
select(
gt_pd_pairs.c.gt_annotation_id,
gt_pd_pairs.c.pd_annotation_id,
case(
(gfunc.ST_Area(gunion) == 0, 0),
(gunion == 0, 0),
else_=iou_computation,
).label("iou"),
)
Expand Down Expand Up @@ -1228,9 +1231,11 @@ def _annotation_type_to_column(
.subquery()
)

ordered_ious = (
db.query(ious).order_by(-ious.c.score, -ious.c.iou, ious.c.gt_id).all()
)
print(ious)

ordered_ious = profiler(
db.query(ious).order_by(-ious.c.score, -ious.c.iou, ious.c.gt_id).all
)()

matched_pd_set = set()
matched_sorted_ranked_pairs = defaultdict(list)
Expand Down Expand Up @@ -1390,6 +1395,7 @@ def _annotation_type_to_column(
return ap_ar_output + pr_curves


@profiler
def _compute_detection_metrics_with_detailed_precision_recall_curve(
db: Session,
parameters: schemas.EvaluationParameters,
Expand Down Expand Up @@ -1429,20 +1435,6 @@ def _compute_detection_metrics_with_detailed_precision_recall_curve(
"""

def _annotation_type_to_column(
annotation_type: AnnotationType,
table,
):
match annotation_type:
case AnnotationType.BOX:
return table.box
case AnnotationType.POLYGON:
return table.polygon
case AnnotationType.RASTER:
return table.raster
case _:
raise RuntimeError

if (
parameters.iou_thresholds_to_return is None
or parameters.iou_thresholds_to_compute is None
Expand All @@ -1469,35 +1461,35 @@ def _annotation_type_to_column(
label_map=parameters.label_map,
)

# Alias the annotation table (required for joining twice)
gt_annotation = aliased(models.Annotation)
pd_annotation = aliased(models.Annotation)

# Get distinct annotations
gt_pd_pairs = (
select(
gt.c.annotation_id.label("gt_annotation_id"),
pd.c.annotation_id.label("pd_annotation_id"),
)
.select_from(pd)
.join(
gt,
and_(
gt.c.datum_id == pd.c.datum_id,
gt.c.key == pd.c.key,
),
)
.distinct()
.cte()
)

# IOU Computation Block
if target_type == AnnotationType.RASTER:

# Alias the annotation table (required for joining twice)
gt_annotation = aliased(models.Annotation)
pd_annotation = aliased(models.Annotation)

# Alias bitmasks
gt_bitmask = aliased(models.Bitmask)
pd_bitmask = aliased(models.Bitmask)

# Get distinct annotations
gt_pd_pairs = (
select(
gt.c.annotation_id.label("gt_annotation_id"),
pd.c.annotation_id.label("pd_annotation_id"),
)
.select_from(pd)
.join(
gt,
and_(
gt.c.datum_id == pd.c.datum_id,
gt.c.key == pd.c.key,
),
)
.distinct()
.cte()
)

gt_pd_counts = (
select(
gt_pd_pairs.c.gt_annotation_id,
Expand Down Expand Up @@ -1548,28 +1540,6 @@ def _annotation_type_to_column(
)

else:
# Alias the annotation table (required for joining twice)
gt_annotation = aliased(models.Annotation)
pd_annotation = aliased(models.Annotation)

# Get distinct annotations
gt_pd_pairs = (
select(
gt.c.annotation_id.label("gt_annotation_id"),
pd.c.annotation_id.label("pd_annotation_id"),
)
.select_from(pd)
.join(
gt,
and_(
gt.c.datum_id == pd.c.datum_id,
gt.c.key == pd.c.key,
),
)
.distinct()
.cte()
)

gt_geom = _annotation_type_to_column(target_type, gt_annotation)
pd_geom = _annotation_type_to_column(target_type, pd_annotation)
gintersection = gfunc.ST_Intersection(gt_geom, pd_geom)
Expand Down
2 changes: 1 addition & 1 deletion api/valor_api/backend/query/filtering.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ def raise_not_implemented(x):
# 'area' attribute
SupportedSymbol.BOX_AREA: (Annotation, ST_Area(Annotation.box)),
SupportedSymbol.POLYGON_AREA: (Annotation, ST_Area(Annotation.polygon)),
SupportedSymbol.RASTER_AREA: (Annotation, func.bit_count(Bitmask.value)),
SupportedSymbol.RASTER_AREA: (Bitmask, func.bit_count(Bitmask.value)),
# backend use only
SupportedSymbol.DATASET_ID: (Dataset, Dataset.id),
SupportedSymbol.MODEL_ID: (Model, Model.id),
Expand Down
Loading

0 comments on commit 6323876

Please sign in to comment.