EDA-v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Image Pre-Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import libraries\n",
    "\n",
    "import os\n",
    "import math\n",
    "import cv2\n",
    "import struct\n",
    "import glob\n",
    "from tqdm import tqdm\n",
    "from PIL import Image\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "import seaborn as sns\n",
    "import xml.etree.ElementTree as ET"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First converting Images to GrayScale --> Then increasing channels to 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_data = glob.glob('marmot_dataset_v1.0/data/English/Positive/Raw' + '/*.bmp')\n",
    "neg_data = glob.glob('marmot_dataset_v1.0/data/English/Negative/Raw' + '/*.bmp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PROCESSED_DATA = 'marmot_processed'\n",
    "IMAGE_PATH = os.path.join(PROCESSED_DATA, 'image_v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_h, new_w = 1024, 1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, data in enumerate([neg_data, pos_data]):\n",
    "    \n",
    "    for j, img_path in enumerate(data):\n",
    "        \n",
    "        image_name = os.path.basename(img_path)\n",
    "        image = Image.open(img_path).convert('LA')\n",
    "        w, h = image.size\n",
    "        \n",
    "        #convert to RGB image\n",
    "        image = image.resize((new_h, new_w)).convert(\"RGB\")\n",
    "\n",
    "        #save images and masks\n",
    "        save_image_path = os.path.join(IMAGE_PATH, image_name.replace('bmp', 'jpg'))\n",
    "        \n",
    "        image.save(save_image_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_data = pd.read_csv('processed_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_data['img_path'] = processed_data['img_path'].apply(lambda x: x.replace('image', 'image_v2'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_data.to_csv('processed_data_v2.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>img_path</th>\n",
       "      <th>table_mask</th>\n",
       "      <th>col_mask</th>\n",
       "      <th>original_height</th>\n",
       "      <th>original_width</th>\n",
       "      <th>hasTable</th>\n",
       "      <th>table_count</th>\n",
       "      <th>col_count</th>\n",
       "      <th>table_bboxes</th>\n",
       "      <th>col_bboxes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>marmot_processed\\image_v2\\10.1.1.1.2000_4.jpg</td>\n",
       "      <td>marmot_processed\\table_mask\\10.1.1.1.2000_4_ta...</td>\n",
       "      <td>marmot_processed\\col_mask\\10.1.1.1.2000_4_col_...</td>\n",
       "      <td>1008</td>\n",
       "      <td>768</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>marmot_processed\\image_v2\\10.1.1.1.2004_4.jpg</td>\n",
       "      <td>marmot_processed\\table_mask\\10.1.1.1.2004_4_ta...</td>\n",
       "      <td>marmot_processed\\col_mask\\10.1.1.1.2004_4_col_...</td>\n",
       "      <td>1123</td>\n",
       "      <td>793</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>marmot_processed\\image_v2\\10.1.1.1.2004_5.jpg</td>\n",
       "      <td>marmot_processed\\table_mask\\10.1.1.1.2004_5_ta...</td>\n",
       "      <td>marmot_processed\\col_mask\\10.1.1.1.2004_5_col_...</td>\n",
       "      <td>1123</td>\n",
       "      <td>793</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>marmot_processed\\image_v2\\10.1.1.1.2005_12.jpg</td>\n",
       "      <td>marmot_processed\\table_mask\\10.1.1.1.2005_12_t...</td>\n",
       "      <td>marmot_processed\\col_mask\\10.1.1.1.2005_12_col...</td>\n",
       "      <td>1056</td>\n",
       "      <td>816</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>marmot_processed\\image_v2\\10.1.1.1.2005_13.jpg</td>\n",
       "      <td>marmot_processed\\table_mask\\10.1.1.1.2005_13_t...</td>\n",
       "      <td>marmot_processed\\col_mask\\10.1.1.1.2005_13_col...</td>\n",
       "      <td>1056</td>\n",
       "      <td>816</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>988</th>\n",
       "      <td>marmot_processed\\image_v2\\10.1.1.8.2182_6.jpg</td>\n",
       "      <td>marmot_processed\\table_mask\\10.1.1.8.2182_6_ta...</td>\n",
       "      <td>marmot_processed\\col_mask\\10.1.1.8.2182_6_col_...</td>\n",
       "      <td>1123</td>\n",
       "      <td>793</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>[[583, 58, 919, 127]]</td>\n",
       "      <td>[[586, 61, 721, 121], [739, 61, 916, 124]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>989</th>\n",
       "      <td>marmot_processed\\image_v2\\10.1.1.8.2185_13.jpg</td>\n",
       "      <td>marmot_processed\\table_mask\\10.1.1.8.2185_13_t...</td>\n",
       "      <td>marmot_processed\\col_mask\\10.1.1.8.2185_13_col...</td>\n",
       "      <td>1056</td>\n",
       "      <td>816</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>[[241, 541, 787, 682]]</td>\n",
       "      <td>[[241, 558, 262, 680], [272, 558, 330, 677], [...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>990</th>\n",
       "      <td>marmot_processed\\image_v2\\10.1.1.8.2185_14.jpg</td>\n",
       "      <td>marmot_processed\\table_mask\\10.1.1.8.2185_14_t...</td>\n",
       "      <td>marmot_processed\\col_mask\\10.1.1.8.2185_14_col...</td>\n",
       "      <td>1056</td>\n",
       "      <td>816</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>[[277, 282, 736, 360]]</td>\n",
       "      <td>[[281, 304, 329, 358], [340, 304, 387, 357], [...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>991</th>\n",
       "      <td>marmot_processed\\image_v2\\10.1.1.8.2198_11.jpg</td>\n",
       "      <td>marmot_processed\\table_mask\\10.1.1.8.2198_11_t...</td>\n",
       "      <td>marmot_processed\\col_mask\\10.1.1.8.2198_11_col...</td>\n",
       "      <td>1056</td>\n",
       "      <td>816</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>[[126, 609, 897, 794]]</td>\n",
       "      <td>[[127, 610, 265, 786], [294, 611, 461, 794], [...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>992</th>\n",
       "      <td>marmot_processed\\image_v2\\10.1.1.8.2198_13.jpg</td>\n",
       "      <td>marmot_processed\\table_mask\\10.1.1.8.2198_13_t...</td>\n",
       "      <td>marmot_processed\\col_mask\\10.1.1.8.2198_13_col...</td>\n",
       "      <td>1056</td>\n",
       "      <td>816</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>[[157, 119, 866, 373]]</td>\n",
       "      <td>[[166, 118, 333, 372], [371, 121, 860, 372]]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>993 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           img_path  \\\n",
       "0     marmot_processed\\image_v2\\10.1.1.1.2000_4.jpg   \n",
       "1     marmot_processed\\image_v2\\10.1.1.1.2004_4.jpg   \n",
       "2     marmot_processed\\image_v2\\10.1.1.1.2004_5.jpg   \n",
       "3    marmot_processed\\image_v2\\10.1.1.1.2005_12.jpg   \n",
       "4    marmot_processed\\image_v2\\10.1.1.1.2005_13.jpg   \n",
       "..                                              ...   \n",
       "988   marmot_processed\\image_v2\\10.1.1.8.2182_6.jpg   \n",
       "989  marmot_processed\\image_v2\\10.1.1.8.2185_13.jpg   \n",
       "990  marmot_processed\\image_v2\\10.1.1.8.2185_14.jpg   \n",
       "991  marmot_processed\\image_v2\\10.1.1.8.2198_11.jpg   \n",
       "992  marmot_processed\\image_v2\\10.1.1.8.2198_13.jpg   \n",
       "\n",
       "                                            table_mask  \\\n",
       "0    marmot_processed\\table_mask\\10.1.1.1.2000_4_ta...   \n",
       "1    marmot_processed\\table_mask\\10.1.1.1.2004_4_ta...   \n",
       "2    marmot_processed\\table_mask\\10.1.1.1.2004_5_ta...   \n",
       "3    marmot_processed\\table_mask\\10.1.1.1.2005_12_t...   \n",
       "4    marmot_processed\\table_mask\\10.1.1.1.2005_13_t...   \n",
       "..                                                 ...   \n",
       "988  marmot_processed\\table_mask\\10.1.1.8.2182_6_ta...   \n",
       "989  marmot_processed\\table_mask\\10.1.1.8.2185_13_t...   \n",
       "990  marmot_processed\\table_mask\\10.1.1.8.2185_14_t...   \n",
       "991  marmot_processed\\table_mask\\10.1.1.8.2198_11_t...   \n",
       "992  marmot_processed\\table_mask\\10.1.1.8.2198_13_t...   \n",
       "\n",
       "                                              col_mask  original_height  \\\n",
       "0    marmot_processed\\col_mask\\10.1.1.1.2000_4_col_...             1008   \n",
       "1    marmot_processed\\col_mask\\10.1.1.1.2004_4_col_...             1123   \n",
       "2    marmot_processed\\col_mask\\10.1.1.1.2004_5_col_...             1123   \n",
       "3    marmot_processed\\col_mask\\10.1.1.1.2005_12_col...             1056   \n",
       "4    marmot_processed\\col_mask\\10.1.1.1.2005_13_col...             1056   \n",
       "..                                                 ...              ...   \n",
       "988  marmot_processed\\col_mask\\10.1.1.8.2182_6_col_...             1123   \n",
       "989  marmot_processed\\col_mask\\10.1.1.8.2185_13_col...             1056   \n",
       "990  marmot_processed\\col_mask\\10.1.1.8.2185_14_col...             1056   \n",
       "991  marmot_processed\\col_mask\\10.1.1.8.2198_11_col...             1056   \n",
       "992  marmot_processed\\col_mask\\10.1.1.8.2198_13_col...             1056   \n",
       "\n",
       "     original_width  hasTable  table_count  col_count            table_bboxes  \\\n",
       "0               768         0            0          0                      []   \n",
       "1               793         0            0          0                      []   \n",
       "2               793         0            0          0                      []   \n",
       "3               816         0            0          0                      []   \n",
       "4               816         0            0          0                      []   \n",
       "..              ...       ...          ...        ...                     ...   \n",
       "988             793         1            1          2   [[583, 58, 919, 127]]   \n",
       "989             816         1            1          3  [[241, 541, 787, 682]]   \n",
       "990             816         1            1          9  [[277, 282, 736, 360]]   \n",
       "991             816         1            1          5  [[126, 609, 897, 794]]   \n",
       "992             816         1            1          2  [[157, 119, 866, 373]]   \n",
       "\n",
       "                                            col_bboxes  \n",
       "0                                                   []  \n",
       "1                                                   []  \n",
       "2                                                   []  \n",
       "3                                                   []  \n",
       "4                                                   []  \n",
       "..                                                 ...  \n",
       "988         [[586, 61, 721, 121], [739, 61, 916, 124]]  \n",
       "989  [[241, 558, 262, 680], [272, 558, 330, 677], [...  \n",
       "990  [[281, 304, 329, 358], [340, 304, 387, 357], [...  \n",
       "991  [[127, 610, 265, 786], [294, 611, 461, 794], [...  \n",
       "992       [[166, 118, 333, 372], [371, 121, 860, 372]]  \n",
       "\n",
       "[993 rows x 10 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "cce9c5b3b81fcb98a8d09dfcaff65dd7cd23b6605cc49ab6f14f654f1c8962b0"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}