notebook/tutorials.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "import cv2\n",
    "import torch\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/100 [00:00<?, ?it/s]"
     ]
    }
   ],
   "source": [
    "gt_path = '../testdata/labels/'\n",
    "preds_path = '../testdata/preds/'\n",
    "\n",
    "# get all imgs'name, remove the extension\n",
    "images_folder1 = [f for f in os.listdir(gt_path) if\n",
    "                  os.path.isfile(os.path.join(gt_path, f)) and f.lower().endswith(\n",
    "                      ('.png', '.jpg', '.jpeg', '.bmp', '.tiff', '.gif'))]\n",
    "images_names = [os.path.splitext(f)[0] for f in images_folder1]\n",
    "tbar = tqdm(images_names)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Note\n",
    "We found that the different data formats made the metrics change, which was not due to an error in our calculations, but was caused by the matching mechanism of the forloop.  \n",
    "The different data formats are converted to a uniform data format within the code, but there may be some reason (guessing memory discontinuity) that makes `skimage.measure.regionprops` return results in a different order. This causes the order in the distances/iou matrix to change, so the `forloops` give different matching results.  \n",
    "We will use a small example to demonstrate this phenomenon, or observe it by setting `debug='True'` and then running the data in a different format."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Definition of the forloops match function.\n",
    "def forloops_match(dis, thr):\n",
    "    num_gt, num_pred = dis.shape\n",
    "    tp = torch.zeros(1)\n",
    "    fp= torch.zeros(1)\n",
    "    fn = torch.zeros(1)\n",
    "    for i in range(num_gt):\n",
    "        for j in range(num_pred):\n",
    "            if dis[i,i]< thr:\n",
    "                tp += 1\n",
    "                dis[:,j]= torch.inf # mark matched\n",
    "                break\n",
    "    fp = num_pred-tp\n",
    "    fn = num_gt-tp\n",
    "    pre = torch.divide(tp,tp+fp)\n",
    "    rec = torch.divide(tp,tp+fn)\n",
    "    return tp, fp, fn, pre, rec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Definition of the distances matrix, rows and columns are presented as groundtruth and prediction respectively.\n",
    "dis =torch.tensor([[0.33, 2],\n",
    "                  [0.2, 10]])\n",
    "thr = 3 # threshold for matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'thr=3,pre=tensor([0.5000]),rec=tensor([0.5000]),tp=tensor([1.]),fp=tensor([1.]),fn=tensor([1.])'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tp, fp, fn, pre, rec = forloops_match(dis,thr)\n",
    "f\"{thr=},{pre=},{rec=},{tp=},{fp=},{fn=}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'thr=3,pre=tensor([1.]),rec=tensor([1.]),tp=tensor([2.]),fp=tensor([0.]),fn=tensor([0.])'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's swap the ordering of gt\n",
    "dis =torch.tensor([[0.2, 10],\n",
    "                    [0.33, 2],\n",
    "                  ])\n",
    "tp, fp, fn, pre, rec = forloops_match(dis,thr)\n",
    "f\"{thr=},{pre=},{rec=},{tp=},{fp=},{fn=}\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All the metrics have changed!  \n",
    "In practice, this may have no effect on the comparison of the algorithms, since in the experiments all the algorithms used `forloops` and the same data format.'  \n",
    "However, we still want to get fully consistent results, so we will improve this in subsequent releases(development has been completed, but it is still being tested).\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Target Level"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Center-Level"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Center PrecisionRecallF1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reading image_name=0506: 100%|██████████| 100/100 [00:17<00:00,  5.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\n",
      "CenterPrecisionRecallF1.update() took 0.00s each time.\n",
      "+---------+-----+-----+-----+------------------+---------------+----------------+\n",
      "| Dis-Thr |  TP |  FP |  FN | target_Precision | target_Recall | target_F1score |\n",
      "+---------+-----+-----+-----+------------------+---------------+----------------+\n",
      "|   1.0   | 148 | 210 | 170 |     0.41341      |    0.46541    |    0.43787     |\n",
      "|   2.0   | 236 | 122 |  82 |     0.65922      |    0.74214    |    0.69822     |\n",
      "|   3.0   | 266 |  92 |  52 |     0.74302      |    0.83648    |    0.78698     |\n",
      "|   4.0   | 276 |  82 |  42 |     0.77095      |    0.86792    |    0.81657     |\n",
      "|   5.0   | 282 |  76 |  36 |     0.78771      |    0.88679    |    0.83432     |\n",
      "|   6.0   | 284 |  74 |  34 |     0.79330      |    0.89308    |    0.84024     |\n",
      "|   7.0   | 286 |  72 |  32 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   8.0   | 286 |  72 |  32 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   9.0   | 286 |  72 |  32 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   10.0  | 286 |  72 |  32 |     0.79888      |    0.89937    |    0.84615     |\n",
      "+---------+-----+-----+-----+------------------+---------------+----------------+\n",
      "CenterPrecisionRecallF1(match_alg=forloop conf_thr=[0.5])\n",
      "Test image of list [hwc, hwc, ...]\n",
      "CenterPrecisionRecallF1.update() took 0.01s each time.\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "| Dis-Thr |  TP |  FP | FN | target_Precision | target_Recall | target_F1score |\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "|   1.0   |  74 | 105 | 85 |     0.41341      |    0.46541    |    0.43787     |\n",
      "|   2.0   | 118 |  61 | 41 |     0.65922      |    0.74214    |    0.69822     |\n",
      "|   3.0   | 133 |  46 | 26 |     0.74302      |    0.83648    |    0.78698     |\n",
      "|   4.0   | 138 |  41 | 21 |     0.77095      |    0.86792    |    0.81657     |\n",
      "|   5.0   | 141 |  38 | 18 |     0.78771      |    0.88679    |    0.83432     |\n",
      "|   6.0   | 142 |  37 | 17 |     0.79330      |    0.89308    |    0.84024     |\n",
      "|   7.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   8.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   9.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   10.0  | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "CenterPrecisionRecallF1(match_alg=forloop conf_thr=[0.5])\n",
      "Test image_path of list, [img_path, img_path, ...]\n",
      "CenterPrecisionRecallF1.update() took 0.01s each time.\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "| Dis-Thr |  TP |  FP | FN | target_Precision | target_Recall | target_F1score |\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "|   1.0   |  74 | 105 | 85 |     0.41341      |    0.46541    |    0.43787     |\n",
      "|   2.0   | 118 |  61 | 41 |     0.65922      |    0.74214    |    0.69822     |\n",
      "|   3.0   | 133 |  46 | 26 |     0.74302      |    0.83648    |    0.78698     |\n",
      "|   4.0   | 138 |  41 | 21 |     0.77095      |    0.86792    |    0.81657     |\n",
      "|   5.0   | 141 |  38 | 18 |     0.78771      |    0.88679    |    0.83432     |\n",
      "|   6.0   | 142 |  37 | 17 |     0.79330      |    0.89308    |    0.84024     |\n",
      "|   7.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   8.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   9.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   10.0  | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "CenterPrecisionRecallF1(match_alg=forloop conf_thr=[0.5])\n",
      "Test image of np.array, bhwc\n",
      "CenterPrecisionRecallF1.update() took 0.01s each time.\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "| Dis-Thr |  TP |  FP | FN | target_Precision | target_Recall | target_F1score |\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "|   1.0   |  74 | 105 | 85 |     0.41341      |    0.46541    |    0.43787     |\n",
      "|   2.0   | 118 |  61 | 41 |     0.65922      |    0.74214    |    0.69822     |\n",
      "|   3.0   | 133 |  46 | 26 |     0.74302      |    0.83648    |    0.78698     |\n",
      "|   4.0   | 138 |  41 | 21 |     0.77095      |    0.86792    |    0.81657     |\n",
      "|   5.0   | 141 |  38 | 18 |     0.78771      |    0.88679    |    0.83432     |\n",
      "|   6.0   | 142 |  37 | 17 |     0.79330      |    0.89308    |    0.84024     |\n",
      "|   7.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   8.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   9.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   10.0  | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "CenterPrecisionRecallF1(match_alg=forloop conf_thr=[0.5])\n",
      "Test image of tensor, bchw\n",
      "CenterPrecisionRecallF1.update() took 0.01s each time.\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "| Dis-Thr |  TP |  FP | FN | target_Precision | target_Recall | target_F1score |\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "|   1.0   |  74 | 105 | 85 |     0.41341      |    0.46541    |    0.43787     |\n",
      "|   2.0   | 118 |  61 | 41 |     0.65922      |    0.74214    |    0.69822     |\n",
      "|   3.0   | 133 |  46 | 26 |     0.74302      |    0.83648    |    0.78698     |\n",
      "|   4.0   | 138 |  41 | 21 |     0.77095      |    0.86792    |    0.81657     |\n",
      "|   5.0   | 141 |  38 | 18 |     0.78771      |    0.88679    |    0.83432     |\n",
      "|   6.0   | 142 |  37 | 17 |     0.79330      |    0.89308    |    0.84024     |\n",
      "|   7.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   8.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   9.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   10.0  | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "CenterPrecisionRecallF1(match_alg=forloop conf_thr=[0.5])\n",
      "Test image of list,  [chw, chw, ...]\n",
      "CenterPrecisionRecallF1.update() took 0.01s each time.\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "| Dis-Thr |  TP |  FP | FN | target_Precision | target_Recall | target_F1score |\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "|   1.0   |  74 | 105 | 85 |     0.41341      |    0.46541    |    0.43787     |\n",
      "|   2.0   | 118 |  61 | 41 |     0.65922      |    0.74214    |    0.69822     |\n",
      "|   3.0   | 133 |  46 | 26 |     0.74302      |    0.83648    |    0.78698     |\n",
      "|   4.0   | 138 |  41 | 21 |     0.77095      |    0.86792    |    0.81657     |\n",
      "|   5.0   | 141 |  38 | 18 |     0.78771      |    0.88679    |    0.83432     |\n",
      "|   6.0   | 142 |  37 | 17 |     0.79330      |    0.89308    |    0.84024     |\n",
      "|   7.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   8.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   9.0   | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "|   10.0  | 143 |  36 | 16 |     0.79888      |    0.89937    |    0.84615     |\n",
      "+---------+-----+-----+----+------------------+---------------+----------------+\n",
      "CenterPrecisionRecallF1(match_alg=forloop conf_thr=[0.5])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([0.41340782, 0.65921788, 0.74301676, 0.77094972, 0.7877095 ,\n",
       "        0.79329609, 0.79888268, 0.79888268, 0.79888268, 0.79888268]),\n",
       " array([0.46540881, 0.74213836, 0.83647799, 0.86792453, 0.88679245,\n",
       "        0.89308176, 0.89937107, 0.89937107, 0.89937107, 0.89937107]),\n",
       " array([0.43786982, 0.69822485, 0.78698225, 0.81656805, 0.83431953,\n",
       "        0.84023669, 0.84615385, 0.84615385, 0.84615385, 0.84615385]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stdeval.metrics import CenterPrecisionRecallF1\n",
    "# For test multiple format of input.\n",
    "gt_img_list = []\n",
    "pred_img_list = [] \n",
    "\n",
    "gt_img_list_chw = []\n",
    "pred_img_list_chw = []\n",
    "\n",
    "gt_img_paths = []\n",
    "pred_img_paths = []\n",
    "\n",
    "Metric = CenterPrecisionRecallF1(\n",
    "    dis_thrs=[1, 10],\n",
    "    conf_thr=0.5,\n",
    "    match_alg='forloop',\n",
    "    # debug = True\n",
    "    )\n",
    "for image_name in tbar:\n",
    "    tbar.set_description(f\"Reading image_name={image_name}\")\n",
    "    gt_image_path = os.path.join(gt_path, f\"{image_name}.png\")  # \n",
    "    pred_image_path = os.path.join(preds_path, f\"{image_name}.png\") \n",
    "\n",
    "    gt_img = cv2.imread(gt_image_path)\n",
    "    pred_img = cv2.imread(pred_image_path)\n",
    "\n",
    "    # for test [chw, chw, ...] format\n",
    "    gt_img_list_chw.append(gt_img.transpose(2,1,0))\n",
    "    pred_img_list_chw.append(pred_img.transpose(2,1,0))\n",
    "\n",
    "    # for test [hwc, hwc] format\n",
    "    gt_img_list.append(gt_img)\n",
    "    pred_img_list.append(pred_img)\n",
    "\n",
    "    # for test [path, path] format\n",
    "\n",
    "    gt_img_paths.append(gt_image_path)\n",
    "    pred_img_paths.append(pred_image_path)\n",
    "\n",
    "    # for test single img or img path\n",
    "    Metric.update(gt_img, pred_img)\n",
    "    Metric.update(gt_image_path, pred_image_path)\n",
    "\n",
    "print(\"Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\")\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of list [hwc, hwc, ...]\")\n",
    "Metric.update(gt_img_list, pred_img_list) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image_path of list, [img_path, img_path, ...]\")\n",
    "Metric.update(gt_img_paths, pred_img_paths)\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of np.array, bhwc\")\n",
    "Metric.update(np.stack(gt_img_list), np.stack(pred_img_list)) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of tensor, bchw\")\n",
    "Metric.update(torch.from_numpy(np.stack(gt_img_list_chw)), torch.from_numpy(np.stack(pred_img_list_chw))) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of list,  [chw, chw, ...]\")\n",
    "Metric.update(gt_img_list_chw, pred_img_list_chw) \n",
    "Metric.get()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dis-Thr</th>\n",
       "      <th>TP</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>105.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>0.413408</td>\n",
       "      <td>0.465409</td>\n",
       "      <td>0.437870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.659218</td>\n",
       "      <td>0.742138</td>\n",
       "      <td>0.698225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.0</td>\n",
       "      <td>133.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0.743017</td>\n",
       "      <td>0.836478</td>\n",
       "      <td>0.786982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>138.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>0.770950</td>\n",
       "      <td>0.867925</td>\n",
       "      <td>0.816568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.787709</td>\n",
       "      <td>0.886792</td>\n",
       "      <td>0.834320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6.0</td>\n",
       "      <td>142.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.793296</td>\n",
       "      <td>0.893082</td>\n",
       "      <td>0.840237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7.0</td>\n",
       "      <td>143.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.798883</td>\n",
       "      <td>0.899371</td>\n",
       "      <td>0.846154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8.0</td>\n",
       "      <td>143.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.798883</td>\n",
       "      <td>0.899371</td>\n",
       "      <td>0.846154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9.0</td>\n",
       "      <td>143.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.798883</td>\n",
       "      <td>0.899371</td>\n",
       "      <td>0.846154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10.0</td>\n",
       "      <td>143.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.798883</td>\n",
       "      <td>0.899371</td>\n",
       "      <td>0.846154</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Dis-Thr     TP     FP    FN  Precision    Recall        F1\n",
       "0      1.0   74.0  105.0  85.0   0.413408  0.465409  0.437870\n",
       "1      2.0  118.0   61.0  41.0   0.659218  0.742138  0.698225\n",
       "2      3.0  133.0   46.0  26.0   0.743017  0.836478  0.786982\n",
       "3      4.0  138.0   41.0  21.0   0.770950  0.867925  0.816568\n",
       "4      5.0  141.0   38.0  18.0   0.787709  0.886792  0.834320\n",
       "5      6.0  142.0   37.0  17.0   0.793296  0.893082  0.840237\n",
       "6      7.0  143.0   36.0  16.0   0.798883  0.899371  0.846154\n",
       "7      8.0  143.0   36.0  16.0   0.798883  0.899371  0.846154\n",
       "8      9.0  143.0   36.0  16.0   0.798883  0.899371  0.846154\n",
       "9     10.0  143.0   36.0  16.0   0.798883  0.899371  0.846154"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Metric.table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Center Average Precision\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\n",
      "CenterAveragePrecision.update() took 0.02s each time.\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "|    AP   | 0.1924 | 0.4892 | 0.6215 | 0.6691 | 0.6985 | 0.6996 | 0.7008 | 0.6921 | 0.6921 | 0.6668 |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterAveragePrecision(match_alg=forloop)\n",
      "Test image of list [hwc, hwc, ...]\n",
      "CenterAveragePrecision.update() took 0.03s each time.\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "|    AP   | 0.1924 | 0.4892 | 0.6215 | 0.6691 | 0.6985 | 0.6996 | 0.7008 | 0.6921 | 0.6921 | 0.6668 |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterAveragePrecision(match_alg=forloop)\n",
      "Test image_path of list, [img_path, img_path, ...]\n",
      "CenterAveragePrecision.update() took 0.04s each time.\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "|    AP   | 0.1924 | 0.4892 | 0.6215 | 0.6691 | 0.6985 | 0.6996 | 0.7008 | 0.6921 | 0.6921 | 0.6668 |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterAveragePrecision(match_alg=forloop)\n",
      "Test image of np.array, bhwc\n",
      "CenterAveragePrecision.update() took 0.05s each time.\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "|    AP   | 0.1924 | 0.4892 | 0.6215 | 0.6691 | 0.6985 | 0.6996 | 0.7008 | 0.6921 | 0.6921 | 0.6668 |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterAveragePrecision(match_alg=forloop)\n",
      "Test image of tensor, bchw\n",
      "CenterAveragePrecision.update() took 0.06s each time.\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "|    AP   | 0.1924 | 0.4892 | 0.6215 | 0.6691 | 0.6985 | 0.6996 | 0.7008 | 0.6921 | 0.6921 | 0.6668 |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterAveragePrecision(match_alg=forloop)\n",
      "Test image of list,  [chw, chw, ...]\n",
      "CenterAveragePrecision.update() took 0.07s each time.\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "|    AP   | 0.1924 | 0.4892 | 0.6215 | 0.6691 | 0.6985 | 0.6996 | 0.7008 | 0.6921 | 0.6921 | 0.6668 |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterAveragePrecision(match_alg=forloop)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.        , 0.41340782, 0.41340782, 0.41340782, 0.41340782,\n",
       "         0.41340782, 0.41340782, 0.41340782, 0.41340782, 0.41340782],\n",
       "        [0.        , 0.65921788, 0.65921788, 0.65921788, 0.65921788,\n",
       "         0.65921788, 0.65921788, 0.65921788, 0.65921788, 0.65921788],\n",
       "        [0.        , 0.74301676, 0.74301676, 0.74301676, 0.74301676,\n",
       "         0.74301676, 0.74301676, 0.74301676, 0.74301676, 0.74301676],\n",
       "        [0.        , 0.77094972, 0.77094972, 0.77094972, 0.77094972,\n",
       "         0.77094972, 0.77094972, 0.77094972, 0.77094972, 0.77094972],\n",
       "        [0.        , 0.7877095 , 0.7877095 , 0.7877095 , 0.7877095 ,\n",
       "         0.7877095 , 0.7877095 , 0.7877095 , 0.7877095 , 0.7877095 ],\n",
       "        [0.01      , 0.79329609, 0.79329609, 0.79329609, 0.79329609,\n",
       "         0.79329609, 0.79329609, 0.79329609, 0.79329609, 0.79329609],\n",
       "        [0.02      , 0.79888268, 0.79888268, 0.79888268, 0.79888268,\n",
       "         0.79888268, 0.79888268, 0.79888268, 0.79888268, 0.79888268],\n",
       "        [0.03      , 0.79888268, 0.79888268, 0.79888268, 0.79888268,\n",
       "         0.79888268, 0.79888268, 0.79888268, 0.79888268, 0.79888268],\n",
       "        [0.03      , 0.79888268, 0.79888268, 0.79888268, 0.79888268,\n",
       "         0.79888268, 0.79888268, 0.79888268, 0.79888268, 0.79888268],\n",
       "        [0.06      , 0.79888268, 0.79888268, 0.79888268, 0.79888268,\n",
       "         0.79888268, 0.79888268, 0.79888268, 0.79888268, 0.79888268]]),\n",
       " array([[0.        , 0.46540881, 0.46540881, 0.46540881, 0.46540881,\n",
       "         0.46540881, 0.46540881, 0.46540881, 0.46540881, 0.46540881],\n",
       "        [0.        , 0.74213836, 0.74213836, 0.74213836, 0.74213836,\n",
       "         0.74213836, 0.74213836, 0.74213836, 0.74213836, 0.74213836],\n",
       "        [0.        , 0.83647799, 0.83647799, 0.83647799, 0.83647799,\n",
       "         0.83647799, 0.83647799, 0.83647799, 0.83647799, 0.83647799],\n",
       "        [0.        , 0.86792453, 0.86792453, 0.86792453, 0.86792453,\n",
       "         0.86792453, 0.86792453, 0.86792453, 0.86792453, 0.86792453],\n",
       "        [0.        , 0.88679245, 0.88679245, 0.88679245, 0.88679245,\n",
       "         0.88679245, 0.88679245, 0.88679245, 0.88679245, 0.88679245],\n",
       "        [0.00628931, 0.89308176, 0.89308176, 0.89308176, 0.89308176,\n",
       "         0.89308176, 0.89308176, 0.89308176, 0.89308176, 0.89308176],\n",
       "        [0.01257862, 0.89937107, 0.89937107, 0.89937107, 0.89937107,\n",
       "         0.89937107, 0.89937107, 0.89937107, 0.89937107, 0.89937107],\n",
       "        [0.01886792, 0.89937107, 0.89937107, 0.89937107, 0.89937107,\n",
       "         0.89937107, 0.89937107, 0.89937107, 0.89937107, 0.89937107],\n",
       "        [0.01886792, 0.89937107, 0.89937107, 0.89937107, 0.89937107,\n",
       "         0.89937107, 0.89937107, 0.89937107, 0.89937107, 0.89937107],\n",
       "        [0.03773585, 0.89937107, 0.89937107, 0.89937107, 0.89937107,\n",
       "         0.89937107, 0.89937107, 0.89937107, 0.89937107, 0.89937107]]),\n",
       " array([[4.19955799e-322, 4.37869822e-001, 4.37869822e-001,\n",
       "         4.37869822e-001, 4.37869822e-001, 4.37869822e-001,\n",
       "         4.37869822e-001, 4.37869822e-001, 4.37869822e-001,\n",
       "         4.37869822e-001],\n",
       "        [4.24896455e-322, 6.98224852e-001, 6.98224852e-001,\n",
       "         6.98224852e-001, 6.98224852e-001, 6.98224852e-001,\n",
       "         6.98224852e-001, 6.98224852e-001, 6.98224852e-001,\n",
       "         6.98224852e-001],\n",
       "        [4.29837112e-322, 7.86982249e-001, 7.86982249e-001,\n",
       "         7.86982249e-001, 7.86982249e-001, 7.86982249e-001,\n",
       "         7.86982249e-001, 7.86982249e-001, 7.86982249e-001,\n",
       "         7.86982249e-001],\n",
       "        [4.34777768e-322, 8.16568047e-001, 8.16568047e-001,\n",
       "         8.16568047e-001, 8.16568047e-001, 8.16568047e-001,\n",
       "         8.16568047e-001, 8.16568047e-001, 8.16568047e-001,\n",
       "         8.16568047e-001],\n",
       "        [4.44659081e-322, 8.34319527e-001, 8.34319527e-001,\n",
       "         8.34319527e-001, 8.34319527e-001, 8.34319527e-001,\n",
       "         8.34319527e-001, 8.34319527e-001, 8.34319527e-001,\n",
       "         8.34319527e-001],\n",
       "        [7.72200772e-003, 8.40236686e-001, 8.40236686e-001,\n",
       "         8.40236686e-001, 8.40236686e-001, 8.40236686e-001,\n",
       "         8.40236686e-001, 8.40236686e-001, 8.40236686e-001,\n",
       "         8.40236686e-001],\n",
       "        [1.54440154e-002, 8.46153846e-001, 8.46153846e-001,\n",
       "         8.46153846e-001, 8.46153846e-001, 8.46153846e-001,\n",
       "         8.46153846e-001, 8.46153846e-001, 8.46153846e-001,\n",
       "         8.46153846e-001],\n",
       "        [2.31660232e-002, 8.46153846e-001, 8.46153846e-001,\n",
       "         8.46153846e-001, 8.46153846e-001, 8.46153846e-001,\n",
       "         8.46153846e-001, 8.46153846e-001, 8.46153846e-001,\n",
       "         8.46153846e-001],\n",
       "        [2.31660232e-002, 8.46153846e-001, 8.46153846e-001,\n",
       "         8.46153846e-001, 8.46153846e-001, 8.46153846e-001,\n",
       "         8.46153846e-001, 8.46153846e-001, 8.46153846e-001,\n",
       "         8.46153846e-001],\n",
       "        [4.63320463e-002, 8.46153846e-001, 8.46153846e-001,\n",
       "         8.46153846e-001, 8.46153846e-001, 8.46153846e-001,\n",
       "         8.46153846e-001, 8.46153846e-001, 8.46153846e-001,\n",
       "         8.46153846e-001]]),\n",
       " array([0.19240364, 0.48923088, 0.62151716, 0.66912617, 0.69853484,\n",
       "        0.69961034, 0.70075612, 0.69207688, 0.69207688, 0.66679386]))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stdeval.metrics import CenterAveragePrecision\n",
    "Metric = CenterAveragePrecision(\n",
    "    dis_thrs=[1, 10],\n",
    "    conf_thrs=10,\n",
    "    match_alg='forloop',\n",
    "    # debug = True\n",
    "    )\n",
    "# For test multiple format of input.\n",
    "gt_img_list = []\n",
    "pred_img_list = [] \n",
    "\n",
    "gt_img_list_chw = []\n",
    "pred_img_list_chw = []\n",
    "\n",
    "gt_img_paths = []\n",
    "pred_img_paths = []\n",
    "for image_name in tbar:\n",
    "    tbar.set_description(f\"Reading image_name={image_name}\")\n",
    "    gt_image_path = os.path.join(gt_path, f\"{image_name}.png\")  # \n",
    "    pred_image_path = os.path.join(preds_path, f\"{image_name}.png\") \n",
    "\n",
    "    gt_img = cv2.imread(gt_image_path)\n",
    "    pred_img = cv2.imread(pred_image_path)\n",
    "\n",
    "    # for test [chw, chw, ...] format\n",
    "    gt_img_list_chw.append(gt_img.transpose(2,1,0))\n",
    "    pred_img_list_chw.append(pred_img.transpose(2,1,0))\n",
    "\n",
    "    # for test [hwc, hwc] format\n",
    "    gt_img_list.append(gt_img)\n",
    "    pred_img_list.append(pred_img)\n",
    "\n",
    "    # for test [path, path] format\n",
    "\n",
    "    gt_img_paths.append(gt_image_path)\n",
    "    pred_img_paths.append(pred_image_path)\n",
    "\n",
    "    # for test single img or img path\n",
    "    Metric.update(gt_img, pred_img)\n",
    "    Metric.update(gt_image_path, pred_image_path)\n",
    "\n",
    "print(\"Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\")\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of list [hwc, hwc, ...]\")\n",
    "Metric.update(gt_img_list, pred_img_list) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image_path of list, [img_path, img_path, ...]\")\n",
    "Metric.update(gt_img_paths, pred_img_paths)\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of np.array, bhwc\")\n",
    "Metric.update(np.stack(gt_img_list), np.stack(pred_img_list)) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of tensor, bchw\")\n",
    "Metric.update(torch.from_numpy(np.stack(gt_img_list_chw)), torch.from_numpy(np.stack(pred_img_list_chw))) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of list,  [chw, chw, ...]\")\n",
    "Metric.update(gt_img_list_chw, pred_img_list_chw) \n",
    "Metric.get()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Dis-Thr</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.00000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AP</th>\n",
       "      <td>0.192404</td>\n",
       "      <td>0.489231</td>\n",
       "      <td>0.621517</td>\n",
       "      <td>0.669126</td>\n",
       "      <td>0.698535</td>\n",
       "      <td>0.69961</td>\n",
       "      <td>0.700756</td>\n",
       "      <td>0.692077</td>\n",
       "      <td>0.692077</td>\n",
       "      <td>0.666794</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                0         1         2         3         4        5         6  \\\n",
       "Dis-Thr  1.000000  2.000000  3.000000  4.000000  5.000000  6.00000  7.000000   \n",
       "AP       0.192404  0.489231  0.621517  0.669126  0.698535  0.69961  0.700756   \n",
       "\n",
       "                7         8          9  \n",
       "Dis-Thr  8.000000  9.000000  10.000000  \n",
       "AP       0.692077  0.692077   0.666794  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Metric.table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Center PD_FA Metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\n",
      "CenterPdPixelFa.update() took 0.01s each time.\n",
      "+-----------+-----+-----+------+----------+-----------+-------------+\n",
      "| Threshold |  TD |  AT |  FD  |    NP    | target_Pd |   pixel_Fa  |\n",
      "+-----------+-----+-----+------+----------+-----------+-------------+\n",
      "|    1.0    | 148 | 318 | 2540 | 13107200 |  0.46541  | 1.93787e-04 |\n",
      "|    2.0    | 236 | 318 | 716  | 13107200 |  0.74214  | 5.46265e-05 |\n",
      "|    3.0    | 266 | 318 | 276  | 13107200 |  0.83648  | 2.10571e-05 |\n",
      "|    4.0    | 276 | 318 | 278  | 13107200 |  0.86792  | 2.12097e-05 |\n",
      "|    5.0    | 282 | 318 | 274  | 13107200 |  0.88679  | 2.09045e-05 |\n",
      "|    6.0    | 284 | 318 | 312  | 13107200 |  0.89308  | 2.38037e-05 |\n",
      "|    7.0    | 286 | 318 | 310  | 13107200 |  0.89937  | 2.36511e-05 |\n",
      "|    8.0    | 286 | 318 | 326  | 13107200 |  0.89937  | 2.48718e-05 |\n",
      "|    9.0    | 286 | 318 | 326  | 13107200 |  0.89937  | 2.48718e-05 |\n",
      "|    10.0   | 286 | 318 | 326  | 13107200 |  0.89937  | 2.48718e-05 |\n",
      "+-----------+-----+-----+------+----------+-----------+-------------+\n",
      "CenterPdPixelFa(conf_thr=[0.5], match_alg=forloop)\n",
      "Test image of list [hwc, hwc, ...]\n",
      "CenterPdPixelFa.update() took 0.01s each time.\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "| Threshold |  TD |  AT |  FD  |    NP   | target_Pd |   pixel_Fa  |\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "|    1.0    |  74 | 159 | 1270 | 6553600 |  0.46541  | 1.93787e-04 |\n",
      "|    2.0    | 118 | 159 | 358  | 6553600 |  0.74214  | 5.46265e-05 |\n",
      "|    3.0    | 133 | 159 | 138  | 6553600 |  0.83648  | 2.10571e-05 |\n",
      "|    4.0    | 138 | 159 | 139  | 6553600 |  0.86792  | 2.12097e-05 |\n",
      "|    5.0    | 141 | 159 | 137  | 6553600 |  0.88679  | 2.09045e-05 |\n",
      "|    6.0    | 142 | 159 | 156  | 6553600 |  0.89308  | 2.38037e-05 |\n",
      "|    7.0    | 143 | 159 | 155  | 6553600 |  0.89937  | 2.36511e-05 |\n",
      "|    8.0    | 143 | 159 | 163  | 6553600 |  0.89937  | 2.48718e-05 |\n",
      "|    9.0    | 143 | 159 | 163  | 6553600 |  0.89937  | 2.48718e-05 |\n",
      "|    10.0   | 143 | 159 | 163  | 6553600 |  0.89937  | 2.48718e-05 |\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "CenterPdPixelFa(conf_thr=[0.5], match_alg=forloop)\n",
      "Test image_path of list, [img_path, img_path, ...]\n",
      "CenterPdPixelFa.update() took 0.01s each time.\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "| Threshold |  TD |  AT |  FD  |    NP   | target_Pd |   pixel_Fa  |\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "|    1.0    |  74 | 159 | 1270 | 6553600 |  0.46541  | 1.93787e-04 |\n",
      "|    2.0    | 118 | 159 | 358  | 6553600 |  0.74214  | 5.46265e-05 |\n",
      "|    3.0    | 133 | 159 | 138  | 6553600 |  0.83648  | 2.10571e-05 |\n",
      "|    4.0    | 138 | 159 | 139  | 6553600 |  0.86792  | 2.12097e-05 |\n",
      "|    5.0    | 141 | 159 | 137  | 6553600 |  0.88679  | 2.09045e-05 |\n",
      "|    6.0    | 142 | 159 | 156  | 6553600 |  0.89308  | 2.38037e-05 |\n",
      "|    7.0    | 143 | 159 | 155  | 6553600 |  0.89937  | 2.36511e-05 |\n",
      "|    8.0    | 143 | 159 | 163  | 6553600 |  0.89937  | 2.48718e-05 |\n",
      "|    9.0    | 143 | 159 | 163  | 6553600 |  0.89937  | 2.48718e-05 |\n",
      "|    10.0   | 143 | 159 | 163  | 6553600 |  0.89937  | 2.48718e-05 |\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "CenterPdPixelFa(conf_thr=[0.5], match_alg=forloop)\n",
      "Test image of np.array, bhwc\n",
      "CenterPdPixelFa.update() took 0.01s each time.\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "| Threshold |  TD |  AT |  FD  |    NP   | target_Pd |   pixel_Fa  |\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "|    1.0    |  74 | 159 | 1270 | 6553600 |  0.46541  | 1.93787e-04 |\n",
      "|    2.0    | 118 | 159 | 358  | 6553600 |  0.74214  | 5.46265e-05 |\n",
      "|    3.0    | 133 | 159 | 138  | 6553600 |  0.83648  | 2.10571e-05 |\n",
      "|    4.0    | 138 | 159 | 139  | 6553600 |  0.86792  | 2.12097e-05 |\n",
      "|    5.0    | 141 | 159 | 137  | 6553600 |  0.88679  | 2.09045e-05 |\n",
      "|    6.0    | 142 | 159 | 156  | 6553600 |  0.89308  | 2.38037e-05 |\n",
      "|    7.0    | 143 | 159 | 155  | 6553600 |  0.89937  | 2.36511e-05 |\n",
      "|    8.0    | 143 | 159 | 163  | 6553600 |  0.89937  | 2.48718e-05 |\n",
      "|    9.0    | 143 | 159 | 163  | 6553600 |  0.89937  | 2.48718e-05 |\n",
      "|    10.0   | 143 | 159 | 163  | 6553600 |  0.89937  | 2.48718e-05 |\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "CenterPdPixelFa(conf_thr=[0.5], match_alg=forloop)\n",
      "Test image of tensor, bchw\n",
      "CenterPdPixelFa.update() took 0.01s each time.\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "| Threshold |  TD |  AT |  FD  |    NP   | target_Pd |   pixel_Fa  |\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "|    1.0    |  74 | 159 | 1270 | 6553600 |  0.46541  | 1.93787e-04 |\n",
      "|    2.0    | 118 | 159 | 358  | 6553600 |  0.74214  | 5.46265e-05 |\n",
      "|    3.0    | 133 | 159 | 139  | 6553600 |  0.83648  | 2.12097e-05 |\n",
      "|    4.0    | 138 | 159 | 174  | 6553600 |  0.86792  | 2.65503e-05 |\n",
      "|    5.0    | 141 | 159 | 221  | 6553600 |  0.88679  | 3.37219e-05 |\n",
      "|    6.0    | 142 | 159 | 278  | 6553600 |  0.89308  | 4.24194e-05 |\n",
      "|    7.0    | 143 | 159 | 278  | 6553600 |  0.89937  | 4.24194e-05 |\n",
      "|    8.0    | 143 | 159 | 286  | 6553600 |  0.89937  | 4.36401e-05 |\n",
      "|    9.0    | 143 | 159 | 286  | 6553600 |  0.89937  | 4.36401e-05 |\n",
      "|    10.0   | 143 | 159 | 286  | 6553600 |  0.89937  | 4.36401e-05 |\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "CenterPdPixelFa(conf_thr=[0.5], match_alg=forloop)\n",
      "Test image of list,  [chw, chw, ...]\n",
      "CenterPdPixelFa.update() took 0.02s each time.\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "| Threshold |  TD |  AT |  FD  |    NP   | target_Pd |   pixel_Fa  |\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "|    1.0    |  74 | 159 | 1270 | 6553600 |  0.46541  | 1.93787e-04 |\n",
      "|    2.0    | 118 | 159 | 358  | 6553600 |  0.74214  | 5.46265e-05 |\n",
      "|    3.0    | 133 | 159 | 139  | 6553600 |  0.83648  | 2.12097e-05 |\n",
      "|    4.0    | 138 | 159 | 174  | 6553600 |  0.86792  | 2.65503e-05 |\n",
      "|    5.0    | 141 | 159 | 221  | 6553600 |  0.88679  | 3.37219e-05 |\n",
      "|    6.0    | 142 | 159 | 278  | 6553600 |  0.89308  | 4.24194e-05 |\n",
      "|    7.0    | 143 | 159 | 278  | 6553600 |  0.89937  | 4.24194e-05 |\n",
      "|    8.0    | 143 | 159 | 286  | 6553600 |  0.89937  | 4.36401e-05 |\n",
      "|    9.0    | 143 | 159 | 286  | 6553600 |  0.89937  | 4.36401e-05 |\n",
      "|    10.0   | 143 | 159 | 286  | 6553600 |  0.89937  | 4.36401e-05 |\n",
      "+-----------+-----+-----+------+---------+-----------+-------------+\n",
      "CenterPdPixelFa(conf_thr=[0.5], match_alg=forloop)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([0.46540881, 0.74213836, 0.83647799, 0.86792453, 0.88679245,\n",
       "        0.89308176, 0.89937107, 0.89937107, 0.89937107, 0.89937107]),\n",
       " array([1.93786621e-04, 5.46264648e-05, 2.12097168e-05, 2.65502930e-05,\n",
       "        3.37219238e-05, 4.24194336e-05, 4.24194336e-05, 4.36401367e-05,\n",
       "        4.36401367e-05, 4.36401367e-05]))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For test multiple format of input.\n",
    "from stdeval.metrics import CenterPdPixelFa\n",
    "\n",
    "\n",
    "gt_img_list = []\n",
    "pred_img_list = [] \n",
    "\n",
    "gt_img_list_chw = []\n",
    "pred_img_list_chw = []\n",
    "\n",
    "gt_img_paths = []\n",
    "pred_img_paths = []\n",
    "\n",
    "Metric = CenterPdPixelFa(\n",
    "    dis_thrs=[1, 10],\n",
    "    conf_thr=0.5,\n",
    "    match_alg='forloop',\n",
    "    # debug = True\n",
    "    )\n",
    "for image_name in tbar:\n",
    "    tbar.set_description(f\"Reading image_name={image_name}\")\n",
    "    gt_image_path = os.path.join(gt_path, f\"{image_name}.png\") \n",
    "    pred_image_path = os.path.join(preds_path, f\"{image_name}.png\") \n",
    "\n",
    "    gt_img = cv2.imread(gt_image_path)\n",
    "    pred_img = cv2.imread(pred_image_path)\n",
    "\n",
    "    # for test [chw, chw, ...] format\n",
    "    gt_img_list_chw.append(gt_img.transpose(2,1,0))\n",
    "    pred_img_list_chw.append(pred_img.transpose(2,1,0))\n",
    "\n",
    "    # for test [hwc, hwc] format\n",
    "    gt_img_list.append(gt_img)\n",
    "    pred_img_list.append(pred_img)\n",
    "\n",
    "    # for test [path, path] format\n",
    "    gt_img_paths.append(gt_image_path)\n",
    "    pred_img_paths.append(pred_image_path)\n",
    "\n",
    "    # # for test single img or img path\n",
    "    Metric.update(gt_img, pred_img)\n",
    "    Metric.update(gt_image_path, pred_image_path)\n",
    "\n",
    "print(\"Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\")\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of list [hwc, hwc, ...]\")\n",
    "Metric.update(gt_img_list, pred_img_list) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image_path of list, [img_path, img_path, ...]\")\n",
    "Metric.update(gt_img_paths, pred_img_paths)\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of np.array, bhwc\")\n",
    "Metric.update(np.stack(gt_img_list), np.stack(pred_img_list)) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of tensor, bchw\")\n",
    "Metric.update(torch.from_numpy(np.stack(gt_img_list_chw)), torch.from_numpy(np.stack(pred_img_list_chw))) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of list,  [chw, chw, ...]\")\n",
    "Metric.update(gt_img_list_chw, pred_img_list_chw) \n",
    "Metric.get()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dis_thr</th>\n",
       "      <th>TD</th>\n",
       "      <th>AT</th>\n",
       "      <th>FD</th>\n",
       "      <th>NP</th>\n",
       "      <th>target_Pd</th>\n",
       "      <th>pixel_Fa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>1270.0</td>\n",
       "      <td>6553600.0</td>\n",
       "      <td>0.465409</td>\n",
       "      <td>0.000194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>358.0</td>\n",
       "      <td>6553600.0</td>\n",
       "      <td>0.742138</td>\n",
       "      <td>0.000055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.0</td>\n",
       "      <td>133.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>139.0</td>\n",
       "      <td>6553600.0</td>\n",
       "      <td>0.836478</td>\n",
       "      <td>0.000021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>138.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>174.0</td>\n",
       "      <td>6553600.0</td>\n",
       "      <td>0.867925</td>\n",
       "      <td>0.000027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>221.0</td>\n",
       "      <td>6553600.0</td>\n",
       "      <td>0.886792</td>\n",
       "      <td>0.000034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6.0</td>\n",
       "      <td>142.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>278.0</td>\n",
       "      <td>6553600.0</td>\n",
       "      <td>0.893082</td>\n",
       "      <td>0.000042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7.0</td>\n",
       "      <td>143.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>278.0</td>\n",
       "      <td>6553600.0</td>\n",
       "      <td>0.899371</td>\n",
       "      <td>0.000042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8.0</td>\n",
       "      <td>143.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>286.0</td>\n",
       "      <td>6553600.0</td>\n",
       "      <td>0.899371</td>\n",
       "      <td>0.000044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9.0</td>\n",
       "      <td>143.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>286.0</td>\n",
       "      <td>6553600.0</td>\n",
       "      <td>0.899371</td>\n",
       "      <td>0.000044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10.0</td>\n",
       "      <td>143.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>286.0</td>\n",
       "      <td>6553600.0</td>\n",
       "      <td>0.899371</td>\n",
       "      <td>0.000044</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Dis_thr     TD     AT      FD         NP  target_Pd  pixel_Fa\n",
       "0      1.0   74.0  159.0  1270.0  6553600.0   0.465409  0.000194\n",
       "1      2.0  118.0  159.0   358.0  6553600.0   0.742138  0.000055\n",
       "2      3.0  133.0  159.0   139.0  6553600.0   0.836478  0.000021\n",
       "3      4.0  138.0  159.0   174.0  6553600.0   0.867925  0.000027\n",
       "4      5.0  141.0  159.0   221.0  6553600.0   0.886792  0.000034\n",
       "5      6.0  142.0  159.0   278.0  6553600.0   0.893082  0.000042\n",
       "6      7.0  143.0  159.0   278.0  6553600.0   0.899371  0.000042\n",
       "7      8.0  143.0  159.0   286.0  6553600.0   0.899371  0.000044\n",
       "8      9.0  143.0  159.0   286.0  6553600.0   0.899371  0.000044\n",
       "9     10.0  143.0  159.0   286.0  6553600.0   0.899371  0.000044"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Metric.table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Center NormalizedIoU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\n",
      "Test image of list [hwc, hwc, ...]\n",
      "CenterNormalizedIoU.update() took 0.34s each time.\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr  |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| nIoU-0.5 | 0.1232 | 0.1902 | 0.2051 | 0.2037 | 0.2030 | 0.2021 | 0.2021 | 0.2017 | 0.2017 | 0.2017 |\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterNormalizedIoU(conf_thr=0.5, match_alg=forloop, second_match=none)\n",
      "Test image_path of list, [img_path, img_path, ...]\n",
      "CenterNormalizedIoU.update() took 0.37s each time.\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr  |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| nIoU-0.5 | 0.1232 | 0.1902 | 0.2051 | 0.2037 | 0.2030 | 0.2021 | 0.2021 | 0.2017 | 0.2017 | 0.2017 |\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterNormalizedIoU(conf_thr=0.5, match_alg=forloop, second_match=none)\n",
      "Test image of np.array, bhwc\n",
      "CenterNormalizedIoU.update() took 0.35s each time.\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr  |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| nIoU-0.5 | 0.1232 | 0.1902 | 0.2051 | 0.2037 | 0.2030 | 0.2021 | 0.2021 | 0.2017 | 0.2017 | 0.2017 |\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterNormalizedIoU(conf_thr=0.5, match_alg=forloop, second_match=none)\n",
      "Test image of tensor, bchw\n",
      "CenterNormalizedIoU.update() took 0.33s each time.\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr  |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| nIoU-0.5 | 0.1232 | 0.1900 | 0.2048 | 0.1990 | 0.1964 | 0.1917 | 0.1917 | 0.1912 | 0.1912 | 0.1912 |\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterNormalizedIoU(conf_thr=0.5, match_alg=forloop, second_match=none)\n",
      "Test image of list, [chw, chw, ...]\n",
      "CenterNormalizedIoU.update() took 0.34s each time.\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr  |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| nIoU-0.5 | 0.1232 | 0.1900 | 0.2048 | 0.1990 | 0.1964 | 0.1917 | 0.1917 | 0.1912 | 0.1912 | 0.1912 |\n",
      "+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterNormalizedIoU(conf_thr=0.5, match_alg=forloop, second_match=none)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.12321951, 0.18997083, 0.20483381, 0.19901622, 0.1963652 ,\n",
       "       0.19166286, 0.19165954, 0.19124712, 0.19124712, 0.19124712])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stdeval.metrics import CenterNormalizedIoU\n",
    "Metric = CenterNormalizedIoU(\n",
    "    conf_thr=0.5,\n",
    "    dis_thrs = [1,10],\n",
    "    match_alg='forloop',\n",
    "    # debug = True\n",
    "    )\n",
    "# For test multiple format of input.\n",
    "gt_img_list = []\n",
    "pred_img_list = [] \n",
    "\n",
    "gt_img_list_chw = []\n",
    "pred_img_list_chw = []\n",
    "\n",
    "gt_img_paths = []\n",
    "pred_img_paths = []\n",
    "for image_name in tbar:\n",
    "    tbar.set_description(f\"Reading image_name={image_name}\")\n",
    "    gt_image_path = os.path.join(gt_path, f\"{image_name}.png\")  # \n",
    "    pred_image_path = os.path.join(preds_path, f\"{image_name}.png\") \n",
    "\n",
    "    gt_img = cv2.imread(gt_image_path)\n",
    "    pred_img = cv2.imread(pred_image_path)\n",
    "\n",
    "    # for test [chw, chw, ...] format\n",
    "    gt_img_list_chw.append(gt_img.transpose(2,1,0))\n",
    "    pred_img_list_chw.append(pred_img.transpose(2,1,0))\n",
    "\n",
    "    # for test [hwc, hwc] format\n",
    "    gt_img_list.append(gt_img)\n",
    "    pred_img_list.append(pred_img)\n",
    "\n",
    "    # for test [path, path] format\n",
    "\n",
    "    gt_img_paths.append(gt_image_path)\n",
    "    pred_img_paths.append(pred_image_path)\n",
    "\n",
    "    # for test single img or img path\n",
    "    # Metric.update(gt_img, pred_img)\n",
    "    # Metric.update(gt_image_path, pred_image_path)\n",
    "\n",
    "print(\"Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\")\n",
    "# Metric.get()\n",
    "# Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of list [hwc, hwc, ...]\")\n",
    "Metric.update(gt_img_list, pred_img_list) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image_path of list, [img_path, img_path, ...]\")\n",
    "Metric.update(gt_img_paths, pred_img_paths)\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of np.array, bhwc\")\n",
    "Metric.update(np.stack(gt_img_list), np.stack(pred_img_list)) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of tensor, bchw\")\n",
    "Metric.update(torch.from_numpy(np.stack(gt_img_list_chw)), torch.from_numpy(np.stack(pred_img_list_chw))) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of list, [chw, chw, ...]\")\n",
    "Metric.update(gt_img_list_chw, pred_img_list_chw) \n",
    "Metric.get()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Center PdPixelFaROC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\n",
      "CenterPdPixelFaROC.update() took 0.04s each time.\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| AUC-ROC | 0.2328 | 0.3711 | 0.4182 | 0.4340 | 0.4434 | 0.4452 | 0.4469 | 0.4454 | 0.4454 | 0.4405 |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterPdPixelFaROC(match_alg=forloop)\n",
      "Test image of list [hwc, hwc, ...]\n",
      "CenterPdPixelFaROC.update() took 0.05s each time.\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| AUC-ROC | 0.2328 | 0.3711 | 0.4182 | 0.4340 | 0.4434 | 0.4452 | 0.4469 | 0.4454 | 0.4454 | 0.4405 |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterPdPixelFaROC(match_alg=forloop)\n",
      "Test image_path of list, [img_path, img_path, ...]\n",
      "CenterPdPixelFaROC.update() took 0.07s each time.\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| AUC-ROC | 0.2328 | 0.3711 | 0.4182 | 0.4340 | 0.4434 | 0.4452 | 0.4469 | 0.4454 | 0.4454 | 0.4405 |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterPdPixelFaROC(match_alg=forloop)\n",
      "Test image of np.array, bhwc\n",
      "CenterPdPixelFaROC.update() took 0.09s each time.\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| AUC-ROC | 0.2328 | 0.3711 | 0.4182 | 0.4340 | 0.4434 | 0.4452 | 0.4469 | 0.4454 | 0.4454 | 0.4405 |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterPdPixelFaROC(match_alg=forloop)\n",
      "Test image of tensor, bchw\n",
      "CenterPdPixelFaROC.update() took 0.10s each time.\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| AUC-ROC | 0.2328 | 0.3711 | 0.4182 | 0.4340 | 0.4434 | 0.4452 | 0.4469 | 0.4454 | 0.4454 | 0.4405 |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterPdPixelFaROC(match_alg=forloop)\n",
      "Test image of list,  [chw, chw, ...]\n",
      "CenterPdPixelFaROC.update() took 0.12s each time.\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| Dis—Thr |  1.0   |  2.0   |  3.0   |  4.0   |  5.0   |  6.0   |  7.0   |  8.0   |  9.0   |  10.0  |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "| AUC-ROC | 0.2328 | 0.3711 | 0.4182 | 0.4340 | 0.4434 | 0.4452 | 0.4469 | 0.4454 | 0.4454 | 0.4405 |\n",
      "+---------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n",
      "CenterPdPixelFaROC(match_alg=forloop)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.        , 0.46540881, 0.46540881, 0.46540881, 0.46540881,\n",
       "         0.46540881, 0.46540881, 0.46540881, 0.46540881, 0.46540881],\n",
       "        [0.        , 0.74213836, 0.74213836, 0.74213836, 0.74213836,\n",
       "         0.74213836, 0.74213836, 0.74213836, 0.74213836, 0.74213836],\n",
       "        [0.        , 0.83647799, 0.83647799, 0.83647799, 0.83647799,\n",
       "         0.83647799, 0.83647799, 0.83647799, 0.83647799, 0.83647799],\n",
       "        [0.        , 0.86792453, 0.86792453, 0.86792453, 0.86792453,\n",
       "         0.86792453, 0.86792453, 0.86792453, 0.86792453, 0.86792453],\n",
       "        [0.        , 0.88679245, 0.88679245, 0.88679245, 0.88679245,\n",
       "         0.88679245, 0.88679245, 0.88679245, 0.88679245, 0.88679245],\n",
       "        [0.00628931, 0.89308176, 0.89308176, 0.89308176, 0.89308176,\n",
       "         0.89308176, 0.89308176, 0.89308176, 0.89308176, 0.89308176],\n",
       "        [0.01257862, 0.89937107, 0.89937107, 0.89937107, 0.89937107,\n",
       "         0.89937107, 0.89937107, 0.89937107, 0.89937107, 0.89937107],\n",
       "        [0.01886792, 0.89937107, 0.89937107, 0.89937107, 0.89937107,\n",
       "         0.89937107, 0.89937107, 0.89937107, 0.89937107, 0.89937107],\n",
       "        [0.01886792, 0.89937107, 0.89937107, 0.89937107, 0.89937107,\n",
       "         0.89937107, 0.89937107, 0.89937107, 0.89937107, 0.89937107],\n",
       "        [0.03773585, 0.89937107, 0.89937107, 0.89937107, 0.89937107,\n",
       "         0.89937107, 0.89937107, 0.89937107, 0.89937107, 0.89937107]]),\n",
       " array([[1.00000000e+00, 1.93786621e-04, 1.93786621e-04, 1.93786621e-04,\n",
       "         1.93786621e-04, 1.93786621e-04, 1.93786621e-04, 1.93786621e-04,\n",
       "         1.93786621e-04, 1.93786621e-04],\n",
       "        [1.00000000e+00, 5.46264648e-05, 5.46264648e-05, 5.46264648e-05,\n",
       "         5.46264648e-05, 5.46264648e-05, 5.46264648e-05, 5.46264648e-05,\n",
       "         5.46264648e-05, 5.46264648e-05],\n",
       "        [1.00000000e+00, 2.12097168e-05, 2.12097168e-05, 2.12097168e-05,\n",
       "         2.12097168e-05, 2.12097168e-05, 2.12097168e-05, 2.12097168e-05,\n",
       "         2.12097168e-05, 2.12097168e-05],\n",
       "        [1.00000000e+00, 2.65502930e-05, 2.65502930e-05, 2.65502930e-05,\n",
       "         2.65502930e-05, 2.65502930e-05, 2.65502930e-05, 2.65502930e-05,\n",
       "         2.65502930e-05, 2.65502930e-05],\n",
       "        [1.00000000e+00, 3.37219238e-05, 3.37219238e-05, 3.37219238e-05,\n",
       "         3.37219238e-05, 3.37219238e-05, 3.37219238e-05, 3.37219238e-05,\n",
       "         3.37219238e-05, 3.37219238e-05],\n",
       "        [9.90000000e-01, 4.24194336e-05, 4.24194336e-05, 4.24194336e-05,\n",
       "         4.24194336e-05, 4.24194336e-05, 4.24194336e-05, 4.24194336e-05,\n",
       "         4.24194336e-05, 4.24194336e-05],\n",
       "        [9.80000000e-01, 4.24194336e-05, 4.24194336e-05, 4.24194336e-05,\n",
       "         4.24194336e-05, 4.24194336e-05, 4.24194336e-05, 4.24194336e-05,\n",
       "         4.24194336e-05, 4.24194336e-05],\n",
       "        [9.70000000e-01, 4.36401367e-05, 4.36401367e-05, 4.36401367e-05,\n",
       "         4.36401367e-05, 4.36401367e-05, 4.36401367e-05, 4.36401367e-05,\n",
       "         4.36401367e-05, 4.36401367e-05],\n",
       "        [9.70000000e-01, 4.36401367e-05, 4.36401367e-05, 4.36401367e-05,\n",
       "         4.36401367e-05, 4.36401367e-05, 4.36401367e-05, 4.36401367e-05,\n",
       "         4.36401367e-05, 4.36401367e-05],\n",
       "        [9.40000000e-01, 4.36401367e-05, 4.36401367e-05, 4.36401367e-05,\n",
       "         4.36401367e-05, 4.36401367e-05, 4.36401367e-05, 4.36401367e-05,\n",
       "         4.36401367e-05, 4.36401367e-05]]),\n",
       " [0.23280129582627015,\n",
       "  0.3710964956223589,\n",
       "  0.4182495985690903,\n",
       "  0.4339755392974278,\n",
       "  0.4434130873770084,\n",
       "  0.44520975556763465,\n",
       "  0.4468762888398561,\n",
       "  0.4453673203186419,\n",
       "  0.4453673203186419,\n",
       "  0.4404612482418804])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stdeval.metrics import CenterPdPixelFaROC\n",
    "Metric = CenterPdPixelFaROC(\n",
    "    dis_thrs=[1, 10],\n",
    "    conf_thrs=10,\n",
    "    match_alg='forloop',\n",
    "    )\n",
    "\n",
    "gt_img_list = []\n",
    "pred_img_list = [] \n",
    "\n",
    "gt_img_list_chw = []\n",
    "pred_img_list_chw = []\n",
    "\n",
    "gt_img_paths = []\n",
    "pred_img_paths = []\n",
    "for image_name in tbar:\n",
    "    tbar.set_description(f\"Reading image_name={image_name}\")\n",
    "    gt_image_path = os.path.join(gt_path, f\"{image_name}.png\")  # \n",
    "    pred_image_path = os.path.join(preds_path, f\"{image_name}.png\") \n",
    "\n",
    "    gt_img = cv2.imread(gt_image_path)\n",
    "    pred_img = cv2.imread(pred_image_path)\n",
    "\n",
    "    # for test [chw, chw, ...] format\n",
    "    gt_img_list_chw.append(gt_img.transpose(2,1,0))\n",
    "    pred_img_list_chw.append(pred_img.transpose(2,1,0))\n",
    "\n",
    "    # for test [hwc, hwc] format\n",
    "    gt_img_list.append(gt_img)\n",
    "    pred_img_list.append(pred_img)\n",
    "\n",
    "    # for test [path, path] format\n",
    "\n",
    "    gt_img_paths.append(gt_image_path)\n",
    "    pred_img_paths.append(pred_image_path)\n",
    "\n",
    "    # for test single img or img path\n",
    "    Metric.update(gt_img, pred_img)\n",
    "    Metric.update(gt_image_path, pred_image_path)\n",
    "\n",
    "print(\"Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\")\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of list [hwc, hwc, ...]\")\n",
    "Metric.update(gt_img_list, pred_img_list) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image_path of list, [img_path, img_path, ...]\")\n",
    "Metric.update(gt_img_paths, pred_img_paths)\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of np.array, bhwc\")\n",
    "Metric.update(np.stack(gt_img_list), np.stack(pred_img_list)) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of tensor, bchw\")\n",
    "Metric.update(torch.from_numpy(np.stack(gt_img_list_chw)), torch.from_numpy(np.stack(pred_img_list_chw))) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of list,  [chw, chw, ...]\")\n",
    "Metric.update(gt_img_list_chw, pred_img_list_chw) \n",
    "Metric.get()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Box Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BoxAveragePrecision.update() took 1.15s each time.\n",
      "+----------+--------------+--------+--------+---------+---------+--------+---------+---------+--------+----------------+-----------------+------------------+\n",
      "| category | mAP@0.5:0.95 | mAP@50 | mAP@75 |  mAP_s  |  mAP_m  | mAP_l  |  mAR_s  |  mAR_m  | mAR_l  | mAR_max_dets@1 | mAR_max_dets@10 | mAR_max_dets@100 |\n",
      "+----------+--------------+--------+--------+---------+---------+--------+---------+---------+--------+----------------+-----------------+------------------+\n",
      "|   All    |    0.6515    | 0.7525 | 0.7525 | -1.0000 | -1.0000 | 0.6515 | -1.0000 | -1.0000 | 0.6500 |     0.6500     |      0.6500     |      0.6500      |\n",
      "|   car    |    0.3030    | 0.5050 | 0.5050 | -1.0000 | -1.0000 | 0.3030 | -1.0000 | -1.0000 | 0.3000 |     0.3000     |      0.3000     |      0.3000      |\n",
      "|   tea    |    1.0000    | 1.0000 | 1.0000 | -1.0000 | -1.0000 | 1.0000 | -1.0000 | -1.0000 | 1.0000 |     1.0000     |      1.0000     |      1.0000      |\n",
      "+----------+--------------+--------+--------+---------+---------+--------+---------+---------+--------+----------------+-----------------+------------------+\n",
      "BoxAveragePrecision(iou_type=('bbox',), box_format=xyxy, iou_threshold=0.5:0.95, rec_threshold=0.0:1.0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\project\\binarysosmetrics\\stdeval\\metrics\\target_level\\box_level\\box_mean_ap_ar.py:258: UserWarning: Category 'person', ID = 0 not in preds or targets, but in classwise.This information can be ignored if you are sure there is not a problem.\n",
      "  warnings.warn(\n",
      "d:\\project\\binarysosmetrics\\stdeval\\metrics\\target_level\\box_level\\box_mean_ap_ar.py:258: UserWarning: Category 'cycle', ID = 3 not in preds or targets, but in classwise.This information can be ignored if you are sure there is not a problem.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from stdeval.metrics import BoxAveragePrecision\n",
    "preds = [\n",
    "   dict(\n",
    "     boxes=torch.tensor([[258.0, 41.0, 606.0, 285.0],\n",
    "                   [158.0, 41.0, 462.0, 285.0]]),\n",
    "     scores=torch.tensor([0.536, 0.71]),\n",
    "     labels=torch.tensor([1, 2]),\n",
    "   ),\n",
    "    dict(\n",
    "     boxes=torch.tensor([[254.0, 413.0, 656.0, 245.0]]),\n",
    "     scores=torch.tensor([0.526]),\n",
    "     labels=torch.tensor([1]),\n",
    "   )\n",
    " ]\n",
    "target = [\n",
    "   dict(\n",
    "     boxes=torch.tensor([[214.0, 41.0, 562.0, 285.0],\n",
    "                   [158.0, 41.0, 462.0, 285.0]]),\n",
    "     labels=torch.tensor([1,2]),\n",
    "   ),\n",
    "       dict(\n",
    "     boxes=torch.tensor([[258.0, 41.0, 606.0, 285.0]]),\n",
    "     labels=torch.tensor([1]),\n",
    "   )\n",
    " ]\n",
    "\n",
    "metric =BoxAveragePrecision(iou_type=\"bbox\", class_metrics=True, extended_summary=False, max_detection_thresholds=[1,10,100], classwise={0:'person', 1:'car', 2:'tea', 3:'cycle'})\n",
    "metric.update(target, preds)\n",
    "results=metric.get()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>mAP@0.5:0.95</th>\n",
       "      <th>mAP@50</th>\n",
       "      <th>mAP@75</th>\n",
       "      <th>mAP_s</th>\n",
       "      <th>mAP_m</th>\n",
       "      <th>mAP_l</th>\n",
       "      <th>mAR_s</th>\n",
       "      <th>mAR_m</th>\n",
       "      <th>mAR_l</th>\n",
       "      <th>mAR_max_dets@1</th>\n",
       "      <th>mAR_max_dets@10</th>\n",
       "      <th>mAR_max_dets@100</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>All</td>\n",
       "      <td>0.6515</td>\n",
       "      <td>0.7525</td>\n",
       "      <td>0.7525</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>0.6515</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.6500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>car</td>\n",
       "      <td>0.3030</td>\n",
       "      <td>0.5050</td>\n",
       "      <td>0.5050</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>0.3030</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3000</td>\n",
       "      <td>0.3000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tea</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  category mAP@0.5:0.95  mAP@50  mAP@75    mAP_s    mAP_m   mAP_l    mAR_s  \\\n",
       "0      All       0.6515  0.7525  0.7525  -1.0000  -1.0000  0.6515  -1.0000   \n",
       "1      car       0.3030  0.5050  0.5050  -1.0000  -1.0000  0.3030  -1.0000   \n",
       "2      tea       1.0000  1.0000  1.0000  -1.0000  -1.0000  1.0000  -1.0000   \n",
       "\n",
       "     mAR_m   mAR_l mAR_max_dets@1 mAR_max_dets@10 mAR_max_dets@100  \n",
       "0  -1.0000  0.6500         0.6500          0.6500           0.6500  \n",
       "1  -1.0000  0.3000         0.3000          0.3000           0.3000  \n",
       "2  -1.0000  1.0000         1.0000          1.0000           1.0000  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric.table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'classes': tensor([1, 2], dtype=torch.int32),\n",
       " 'ALL': {'map': tensor([0.6515]),\n",
       "  'map_50': tensor([0.7525]),\n",
       "  'map_75': tensor([0.7525]),\n",
       "  'map_small': tensor([-1.]),\n",
       "  'map_medium': tensor([-1.]),\n",
       "  'map_large': tensor([0.6515]),\n",
       "  'mar_1': tensor([0.6500]),\n",
       "  'mar_10': tensor([0.6500]),\n",
       "  'mar_100': tensor([0.6500]),\n",
       "  'mar_small': tensor([-1.]),\n",
       "  'mar_medium': tensor([-1.]),\n",
       "  'mar_large': tensor([0.6500]),\n",
       "  'map_per_class': tensor([0.3030, 1.0000]),\n",
       "  'mar_100_per_class': tensor([0.3000, 1.0000])},\n",
       " 1: {'name': 'car',\n",
       "  'map': tensor(0.3030, dtype=torch.float64),\n",
       "  'map_50': tensor(0.5050, dtype=torch.float64),\n",
       "  'map_75': tensor(0.5050, dtype=torch.float64),\n",
       "  'map_small': tensor([-1]),\n",
       "  'map_medium': tensor([-1]),\n",
       "  'map_large': tensor(0.3030, dtype=torch.float64),\n",
       "  'mar': tensor(0.3000, dtype=torch.float64),\n",
       "  'mar_small': tensor([-1]),\n",
       "  'mar_medium': tensor([-1]),\n",
       "  'mar_large': tensor(0.3000, dtype=torch.float64),\n",
       "  'mar_1': tensor(0.3000, dtype=torch.float64),\n",
       "  'mar_10': tensor(0.3000, dtype=torch.float64),\n",
       "  'mar_100': tensor(0.3000, dtype=torch.float64)},\n",
       " 2: {'name': 'tea',\n",
       "  'map': tensor(1.0000, dtype=torch.float64),\n",
       "  'map_50': tensor(1.0000, dtype=torch.float64),\n",
       "  'map_75': tensor(1.0000, dtype=torch.float64),\n",
       "  'map_small': tensor([-1]),\n",
       "  'map_medium': tensor([-1]),\n",
       "  'map_large': tensor(1.0000, dtype=torch.float64),\n",
       "  'mar': tensor(1., dtype=torch.float64),\n",
       "  'mar_small': tensor([-1]),\n",
       "  'mar_medium': tensor([-1]),\n",
       "  'mar_large': tensor(1., dtype=torch.float64),\n",
       "  'mar_1': tensor(1., dtype=torch.float64),\n",
       "  'mar_10': tensor(1., dtype=torch.float64),\n",
       "  'mar_100': tensor(1., dtype=torch.float64)}}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BoxAveragePrecision.update() took 0.00s each time.\n",
      "+----------+--------------+--------+--------+--------+---------+---------+--------+---------+---------+----------------+-----------------+------------------+\n",
      "| category | mAP@0.5:0.95 | mAP@50 | mAP@75 | mAP_s  |  mAP_m  |  mAP_l  | mAR_s  |  mAR_m  |  mAR_l  | mAR_max_dets@1 | mAR_max_dets@10 | mAR_max_dets@100 |\n",
      "+----------+--------------+--------+--------+--------+---------+---------+--------+---------+---------+----------------+-----------------+------------------+\n",
      "|   All    |    0.2000    | 1.0000 | 0.0000 | 0.2000 | -1.0000 | -1.0000 | 0.2000 | -1.0000 | -1.0000 |     0.2000     |      0.2000     |      0.2000      |\n",
      "|    0     |    0.2000    | 1.0000 | 0.0000 | 0.2000 | -1.0000 | -1.0000 | 0.2000 | -1.0000 | -1.0000 |     0.2000     |      0.2000     |      0.2000      |\n",
      "+----------+--------------+--------+--------+--------+---------+---------+--------+---------+---------+----------------+-----------------+------------------+\n",
      "BoxAveragePrecision(iou_type=('segm',), box_format=xyxy, iou_threshold=0.5:0.95, rec_threshold=0.0:1.0)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'classes': tensor([0], dtype=torch.int32),\n",
       " 'ALL': {'map': tensor([0.2000]),\n",
       "  'map_50': tensor([1.]),\n",
       "  'map_75': tensor([0.]),\n",
       "  'map_small': tensor([0.2000]),\n",
       "  'map_medium': tensor([-1.]),\n",
       "  'map_large': tensor([-1.]),\n",
       "  'mar_1': tensor([0.2000]),\n",
       "  'mar_10': tensor([0.2000]),\n",
       "  'mar_100': tensor([0.2000]),\n",
       "  'mar_small': tensor([0.2000]),\n",
       "  'mar_medium': tensor([-1.]),\n",
       "  'mar_large': tensor([-1.]),\n",
       "  'map_per_class': tensor([-1.]),\n",
       "  'mar_100_per_class': tensor([-1.])},\n",
       " 0: {'name': '0',\n",
       "  'map': tensor(0.2000, dtype=torch.float64),\n",
       "  'map_50': tensor(1.0000, dtype=torch.float64),\n",
       "  'map_75': tensor(0., dtype=torch.float64),\n",
       "  'map_small': tensor(0.2000, dtype=torch.float64),\n",
       "  'map_medium': tensor([-1]),\n",
       "  'map_large': tensor([-1]),\n",
       "  'mar': tensor(0.2000, dtype=torch.float64),\n",
       "  'mar_small': tensor(0.2000, dtype=torch.float64),\n",
       "  'mar_medium': tensor([-1]),\n",
       "  'mar_large': tensor([-1]),\n",
       "  'mar_1': tensor(0.2000, dtype=torch.float64),\n",
       "  'mar_10': tensor(0.2000, dtype=torch.float64),\n",
       "  'mar_100': tensor(0.2000, dtype=torch.float64)}}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask_pred = [\n",
    "   [0, 0, 0, 0, 0],\n",
    "   [0, 0, 1, 1, 0],\n",
    "   [0, 0, 1, 1, 0],\n",
    "   [0, 0, 0, 0, 0],\n",
    "   [0, 0, 0, 0, 0],\n",
    " ]\n",
    "mask_tgt = [\n",
    "   [0, 0, 0, 0, 0],\n",
    "   [0, 0, 1, 0, 0],\n",
    "   [0, 0, 1, 1, 0],\n",
    "   [0, 0, 1, 0, 0],\n",
    "   [0, 0, 0, 0, 0],\n",
    " ]\n",
    "preds = [\n",
    "   dict(\n",
    "     masks=torch.tensor([mask_pred], dtype=torch.bool),\n",
    "     scores=torch.tensor([0.536]),\n",
    "     labels=torch.tensor([0]),\n",
    "   )\n",
    " ]\n",
    "target = [\n",
    "   dict(\n",
    "     masks=torch.tensor([mask_tgt], dtype=torch.bool),\n",
    "     labels=torch.tensor([0]),\n",
    "   )\n",
    " ]\n",
    "metric = BoxAveragePrecision(iou_type=\"segm\")\n",
    "metric.update(target, preds)\n",
    "metric.get()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>mAP@0.5:0.95</th>\n",
       "      <th>mAP@50</th>\n",
       "      <th>mAP@75</th>\n",
       "      <th>mAP_s</th>\n",
       "      <th>mAP_m</th>\n",
       "      <th>mAP_l</th>\n",
       "      <th>mAR_s</th>\n",
       "      <th>mAR_m</th>\n",
       "      <th>mAR_l</th>\n",
       "      <th>mAR_max_dets@1</th>\n",
       "      <th>mAR_max_dets@10</th>\n",
       "      <th>mAR_max_dets@100</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>All</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>0.2000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  category mAP@0.5:0.95  mAP@50  mAP@75   mAP_s    mAP_m    mAP_l   mAR_s  \\\n",
       "0      All       0.2000  1.0000  0.0000  0.2000  -1.0000  -1.0000  0.2000   \n",
       "1        0       0.2000  1.0000  0.0000  0.2000  -1.0000  -1.0000  0.2000   \n",
       "\n",
       "     mAR_m    mAR_l mAR_max_dets@1 mAR_max_dets@10 mAR_max_dets@100  \n",
       "0  -1.0000  -1.0000         0.2000          0.2000           0.2000  \n",
       "1  -1.0000  -1.0000         0.2000          0.2000           0.2000  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric.table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pixel Level"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PixelNormalizeIoU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\n",
      "PixelNormalizedIoU.update() took 0.00s each time.\n",
      "+----------+\n",
      "| nIoU-0.5 |\n",
      "+----------+\n",
      "|  0.2307  |\n",
      "+----------+\n",
      "PixelNormalizedIoU(conf_thr=0.5)\n",
      "Test image of list [hwc, hwc, ...]\n",
      "PixelNormalizedIoU.update() took 0.00s each time.\n",
      "+----------+\n",
      "| nIoU-0.5 |\n",
      "+----------+\n",
      "|  0.2307  |\n",
      "+----------+\n",
      "PixelNormalizedIoU(conf_thr=0.5)\n",
      "Test image_path of list, [img_path, img_path, ...]\n",
      "PixelNormalizedIoU.update() took 0.00s each time.\n",
      "+----------+\n",
      "| nIoU-0.5 |\n",
      "+----------+\n",
      "|  0.2307  |\n",
      "+----------+\n",
      "PixelNormalizedIoU(conf_thr=0.5)\n",
      "Test image of np.array, bhwc\n",
      "PixelNormalizedIoU.update() took 0.00s each time.\n",
      "+----------+\n",
      "| nIoU-0.5 |\n",
      "+----------+\n",
      "|  0.2307  |\n",
      "+----------+\n",
      "PixelNormalizedIoU(conf_thr=0.5)\n",
      "Test image of tensor, bchw\n",
      "PixelNormalizedIoU.update() took 0.01s each time.\n",
      "+----------+\n",
      "| nIoU-0.5 |\n",
      "+----------+\n",
      "|  0.2307  |\n",
      "+----------+\n",
      "PixelNormalizedIoU(conf_thr=0.5)\n",
      "Test image of list,  [chw, chw, ...]\n",
      "PixelNormalizedIoU.update() took 0.01s each time.\n",
      "+----------+\n",
      "| nIoU-0.5 |\n",
      "+----------+\n",
      "|  0.2307  |\n",
      "+----------+\n",
      "PixelNormalizedIoU(conf_thr=0.5)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.23073638166447524"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stdeval.metrics import PixelNormalizedIoU\n",
    "Metric = PixelNormalizedIoU(\n",
    "    conf_thr=0.5,\n",
    "    # debug = True\n",
    "    )\n",
    "# For test multiple format of input.\n",
    "gt_img_list = []\n",
    "pred_img_list = [] \n",
    "\n",
    "gt_img_list_chw = []\n",
    "pred_img_list_chw = []\n",
    "\n",
    "gt_img_paths = []\n",
    "pred_img_paths = []\n",
    "for image_name in tbar:\n",
    "    tbar.set_description(f\"Reading image_name={image_name}\")\n",
    "    gt_image_path = os.path.join(gt_path, f\"{image_name}.png\")  # \n",
    "    pred_image_path = os.path.join(preds_path, f\"{image_name}.png\") \n",
    "\n",
    "    gt_img = cv2.imread(gt_image_path)\n",
    "    pred_img = cv2.imread(pred_image_path)\n",
    "\n",
    "    # for test [chw, chw, ...] format\n",
    "    gt_img_list_chw.append(gt_img.transpose(2,1,0))\n",
    "    pred_img_list_chw.append(pred_img.transpose(2,1,0))\n",
    "\n",
    "    # for test [hwc, hwc] format\n",
    "    gt_img_list.append(gt_img)\n",
    "    pred_img_list.append(pred_img)\n",
    "\n",
    "    # for test [path, path] format\n",
    "\n",
    "    gt_img_paths.append(gt_image_path)\n",
    "    pred_img_paths.append(pred_image_path)\n",
    "\n",
    "    # for test single img or img path\n",
    "    Metric.update(gt_img, pred_img)\n",
    "    Metric.update(gt_image_path, pred_image_path)\n",
    "\n",
    "print(\"Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\")\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of list [hwc, hwc, ...]\")\n",
    "Metric.update(gt_img_list, pred_img_list) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image_path of list, [img_path, img_path, ...]\")\n",
    "Metric.update(gt_img_paths, pred_img_paths)\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of np.array, bhwc\")\n",
    "Metric.update(np.stack(gt_img_list), np.stack(pred_img_list)) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of tensor, bchw\")\n",
    "Metric.update(torch.from_numpy(np.stack(gt_img_list_chw)), torch.from_numpy(np.stack(pred_img_list_chw))) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of list,  [chw, chw, ...]\")\n",
    "Metric.update(gt_img_list_chw, pred_img_list_chw) \n",
    "Metric.get()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nIoU</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.230736</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       nIoU\n",
       "0  0.230736"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Metric.table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pixel ROC Precision Recall Metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\n",
      "PixelROCPrecisionRecall.update() took 0.03s each time.\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "|      AUC_ROC       | AUC_PR(AUC function) | AP(BinaryAveragePrecision function) |\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "| 0.6083542704582214 |  0.5896512269973755  |              0.20967612             |\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "PixelROCPrecisionRecall\n",
      "Test image of list [hwc, hwc, ...]\n",
      "PixelROCPrecisionRecall.update() took 0.04s each time.\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "|      AUC_ROC       | AUC_PR(AUC function) | AP(BinaryAveragePrecision function) |\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "| 0.6083542704582214 |  0.5896512269973755  |              0.20967612             |\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "PixelROCPrecisionRecall\n",
      "Test image_path of list, [img_path, img_path, ...]\n",
      "PixelROCPrecisionRecall.update() took 0.05s each time.\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "|      AUC_ROC       | AUC_PR(AUC function) | AP(BinaryAveragePrecision function) |\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "| 0.6083542704582214 |  0.5896512269973755  |              0.20967612             |\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "PixelROCPrecisionRecall\n",
      "Test image of np.array, bhwc\n",
      "PixelROCPrecisionRecall.update() took 0.06s each time.\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "|      AUC_ROC       | AUC_PR(AUC function) | AP(BinaryAveragePrecision function) |\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "| 0.6083542704582214 |  0.5896512269973755  |              0.20967612             |\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "PixelROCPrecisionRecall\n",
      "Test image of tensor, bchw\n",
      "PixelROCPrecisionRecall.update() took 0.07s each time.\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "|      AUC_ROC       | AUC_PR(AUC function) | AP(BinaryAveragePrecision function) |\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "| 0.6083542704582214 |  0.5896512269973755  |              0.20967612             |\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "PixelROCPrecisionRecall\n",
      "Test image of list,  [chw, chw, ...]\n",
      "PixelROCPrecisionRecall.update() took 0.09s each time.\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "|      AUC_ROC       | AUC_PR(AUC function) | AP(BinaryAveragePrecision function) |\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "| 0.6083542704582214 |  0.5896512269973755  |              0.20967612             |\n",
      "+--------------------+----------------------+-------------------------------------+\n",
      "PixelROCPrecisionRecall\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.6083542704582214,\n",
       " 0.5896512269973755,\n",
       " array([1.5132346e-05, 1.5132346e-05, 1.5132346e-05, 1.5132346e-05,\n",
       "        1.5132346e-05, 1.5132346e-05, 1.5132346e-05, 1.5132346e-05,\n",
       "        1.5132346e-05, 1.0000000e+00], dtype=float32),\n",
       " array([0.21672702, 0.21672702, 0.21672702, 0.21672702, 0.21672702,\n",
       "        0.21672702, 0.21672702, 0.21672702, 0.21672702, 1.        ],\n",
       "       dtype=float32),\n",
       " array([0.00172775, 0.9612221 , 0.9612221 , 0.9612221 , 0.9612221 ,\n",
       "        0.9612221 , 0.9612221 , 0.9612221 , 0.9612221 , 0.9612221 ,\n",
       "        1.        ], dtype=float32),\n",
       " array([1.        , 0.21672702, 0.21672702, 0.21672702, 0.21672702,\n",
       "        0.21672702, 0.21672702, 0.21672702, 0.21672702, 0.21672702,\n",
       "        0.        ], dtype=float32),\n",
       " array(0.20967612, dtype=float32))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stdeval.metrics import PixelROCPrecisionRecall\n",
    "Metric = PixelROCPrecisionRecall(\n",
    "    conf_thrs=10,\n",
    "    # debug = True\n",
    "    )\n",
    "# For test multiple format of input.\n",
    "gt_img_list = []\n",
    "pred_img_list = [] \n",
    "\n",
    "gt_img_list_chw = []\n",
    "pred_img_list_chw = []\n",
    "\n",
    "gt_img_paths = []\n",
    "pred_img_paths = []\n",
    "for image_name in tbar:\n",
    "    tbar.set_description(f\"Reading image_name={image_name}\")\n",
    "    gt_image_path = os.path.join(gt_path, f\"{image_name}.png\")  # \n",
    "    pred_image_path = os.path.join(preds_path, f\"{image_name}.png\") \n",
    "\n",
    "    gt_img = cv2.imread(gt_image_path)\n",
    "    pred_img = cv2.imread(pred_image_path)\n",
    "\n",
    "    # for test [chw, chw, ...] format\n",
    "    gt_img_list_chw.append(gt_img.transpose(2,1,0))\n",
    "    pred_img_list_chw.append(pred_img.transpose(2,1,0))\n",
    "\n",
    "    # for test [hwc, hwc] format\n",
    "    gt_img_list.append(gt_img)\n",
    "    pred_img_list.append(pred_img)\n",
    "\n",
    "    # for test [path, path] format\n",
    "\n",
    "    gt_img_paths.append(gt_image_path)\n",
    "    pred_img_paths.append(pred_image_path)\n",
    "\n",
    "    # for test single img or img path\n",
    "    Metric.update(gt_img, pred_img)\n",
    "    Metric.update(gt_image_path, pred_image_path)\n",
    "\n",
    "print(\"Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\")\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of list [hwc, hwc, ...]\")\n",
    "Metric.update(gt_img_list, pred_img_list) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image_path of list, [img_path, img_path, ...]\")\n",
    "Metric.update(gt_img_paths, pred_img_paths)\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of np.array, bhwc\")\n",
    "Metric.update(np.stack(gt_img_list), np.stack(pred_img_list)) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of tensor, bchw\")\n",
    "Metric.update(torch.from_numpy(np.stack(gt_img_list_chw)), torch.from_numpy(np.stack(pred_img_list_chw))) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of list,  [chw, chw, ...]\")\n",
    "Metric.update(gt_img_list_chw, pred_img_list_chw) \n",
    "Metric.get()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AUC_ROC</th>\n",
       "      <th>AUC_PR</th>\n",
       "      <th>AP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.608354</td>\n",
       "      <td>0.589651</td>\n",
       "      <td>0.209676</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    AUC_ROC    AUC_PR        AP\n",
       "0  0.608354  0.589651  0.209676"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Metric.table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Piexl Precision Recall F1 IoU "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\n",
      "PixelPrecisionRecallF1IoU.update() took 0.02s each time.\n",
      "+---------------+------------+--------+---------+\n",
      "| Precision-0.5 | Recall-0.5 | F1-0.5 | IOU-0.5 |\n",
      "+---------------+------------+--------+---------+\n",
      "|     0.9612    |   0.2167   | 0.3537 | 0.21485 |\n",
      "+---------------+------------+--------+---------+\n",
      "PixelPrecisionRecallF1IoU\n",
      "Test image of list [hwc, hwc, ...]\n",
      "PixelPrecisionRecallF1IoU.update() took 0.03s each time.\n",
      "+---------------+------------+--------+---------+\n",
      "| Precision-0.5 | Recall-0.5 | F1-0.5 | IOU-0.5 |\n",
      "+---------------+------------+--------+---------+\n",
      "|     0.9612    |   0.2167   | 0.3537 | 0.21485 |\n",
      "+---------------+------------+--------+---------+\n",
      "PixelPrecisionRecallF1IoU\n",
      "Test image_path of list, [img_path, img_path, ...]\n",
      "PixelPrecisionRecallF1IoU.update() took 0.05s each time.\n",
      "+---------------+------------+--------+---------+\n",
      "| Precision-0.5 | Recall-0.5 | F1-0.5 | IOU-0.5 |\n",
      "+---------------+------------+--------+---------+\n",
      "|     0.9612    |   0.2167   | 0.3537 | 0.21485 |\n",
      "+---------------+------------+--------+---------+\n",
      "PixelPrecisionRecallF1IoU\n",
      "Test image of np.array, bhwc\n",
      "PixelPrecisionRecallF1IoU.update() took 0.06s each time.\n",
      "+---------------+------------+--------+---------+\n",
      "| Precision-0.5 | Recall-0.5 | F1-0.5 | IOU-0.5 |\n",
      "+---------------+------------+--------+---------+\n",
      "|     0.9612    |   0.2167   | 0.3537 | 0.21485 |\n",
      "+---------------+------------+--------+---------+\n",
      "PixelPrecisionRecallF1IoU\n",
      "Test image of tensor, bchw\n",
      "PixelPrecisionRecallF1IoU.update() took 0.07s each time.\n",
      "+---------------+------------+--------+---------+\n",
      "| Precision-0.5 | Recall-0.5 | F1-0.5 | IOU-0.5 |\n",
      "+---------------+------------+--------+---------+\n",
      "|     0.9612    |   0.2167   | 0.3537 | 0.21485 |\n",
      "+---------------+------------+--------+---------+\n",
      "PixelPrecisionRecallF1IoU\n",
      "Test image of list,  [chw, chw, ...]\n",
      "PixelPrecisionRecallF1IoU.update() took 0.09s each time.\n",
      "+---------------+------------+--------+---------+\n",
      "| Precision-0.5 | Recall-0.5 | F1-0.5 | IOU-0.5 |\n",
      "+---------------+------------+--------+---------+\n",
      "|     0.9612    |   0.2167   | 0.3537 | 0.21485 |\n",
      "+---------------+------------+--------+---------+\n",
      "PixelPrecisionRecallF1IoU\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([0.96122209]),\n",
       " array([0.21672702]),\n",
       " array([0.35370424]),\n",
       " array([0.21484854]))"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stdeval.metrics import PixelPrecisionRecallF1IoU\n",
    "Metric = PixelPrecisionRecallF1IoU(\n",
    "    conf_thr=0.5,\n",
    "    # debug = True\n",
    "    )\n",
    "# For test multiple format of input.\n",
    "gt_img_list = []\n",
    "pred_img_list = [] \n",
    "\n",
    "gt_img_list_chw = []\n",
    "pred_img_list_chw = []\n",
    "\n",
    "gt_img_paths = []\n",
    "pred_img_paths = []\n",
    "for image_name in tbar:\n",
    "    tbar.set_description(f\"Reading image_name={image_name}\")\n",
    "    gt_image_path = os.path.join(gt_path, f\"{image_name}.png\")  # \n",
    "    pred_image_path = os.path.join(preds_path, f\"{image_name}.png\") \n",
    "\n",
    "    gt_img = cv2.imread(gt_image_path)\n",
    "    pred_img = cv2.imread(pred_image_path)\n",
    "\n",
    "    # for test [chw, chw, ...] format\n",
    "    gt_img_list_chw.append(gt_img.transpose(2,1,0))\n",
    "    pred_img_list_chw.append(pred_img.transpose(2,1,0))\n",
    "\n",
    "    # for test [hwc, hwc] format\n",
    "    gt_img_list.append(gt_img)\n",
    "    pred_img_list.append(pred_img)\n",
    "\n",
    "    # for test [path, path] format\n",
    "\n",
    "    gt_img_paths.append(gt_image_path)\n",
    "    pred_img_paths.append(pred_image_path)\n",
    "\n",
    "    # for test single img or img path\n",
    "    Metric.update(gt_img, pred_img)\n",
    "    Metric.update(gt_image_path, pred_image_path)\n",
    "\n",
    "print(\"Simultaneous test single image path and single image, (TP, FN, FP) should be two times as big as the following test.\")\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of list [hwc, hwc, ...]\")\n",
    "Metric.update(gt_img_list, pred_img_list) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image_path of list, [img_path, img_path, ...]\")\n",
    "Metric.update(gt_img_paths, pred_img_paths)\n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of np.array, bhwc\")\n",
    "Metric.update(np.stack(gt_img_list), np.stack(pred_img_list)) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "\n",
    "print(\"Test image of tensor, bchw\")\n",
    "Metric.update(torch.from_numpy(np.stack(gt_img_list_chw)), torch.from_numpy(np.stack(pred_img_list_chw))) \n",
    "Metric.get()\n",
    "Metric.reset()\n",
    "\n",
    "print(\"Test image of list,  [chw, chw, ...]\")\n",
    "Metric.update(gt_img_list_chw, pred_img_list_chw) \n",
    "Metric.get()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Precision-0.5</th>\n",
       "      <th>Recall-0.5</th>\n",
       "      <th>F1-0.5</th>\n",
       "      <th>IOU-0.5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.961222</td>\n",
       "      <td>0.216727</td>\n",
       "      <td>0.353704</td>\n",
       "      <td>0.214849</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Precision-0.5  Recall-0.5    F1-0.5   IOU-0.5\n",
       "0       0.961222    0.216727  0.353704  0.214849"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Metric.table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dr",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}