siim_isic_tabular_only.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.vision import *\n",
    "from fastai.tabular import *\n",
    "from image_tabular.core import *\n",
    "from image_tabular.dataset import *\n",
    "from image_tabular.model import *\n",
    "from image_tabular.metric import *\n",
    "\n",
    "# use gpu by default if available\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=UserWarning, module=\"torch.nn.functional\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = Path(\"./data/siim-isic-melanoma-classification/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33126 10982\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv(data_path/\"train.csv\")\n",
    "test_df = pd.read_csv(data_path/\"test.csv\")\n",
    "\n",
    "print(len(train_df), len(test_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>sex</th>\n",
       "      <th>age_approx</th>\n",
       "      <th>anatom_site_general_challenge</th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>benign_malignant</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_2637011</td>\n",
       "      <td>IP_7279968</td>\n",
       "      <td>male</td>\n",
       "      <td>45.0</td>\n",
       "      <td>head/neck</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0015719</td>\n",
       "      <td>IP_3075186</td>\n",
       "      <td>female</td>\n",
       "      <td>45.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0052212</td>\n",
       "      <td>IP_2842074</td>\n",
       "      <td>female</td>\n",
       "      <td>50.0</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>nevus</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0068279</td>\n",
       "      <td>IP_6890425</td>\n",
       "      <td>female</td>\n",
       "      <td>45.0</td>\n",
       "      <td>head/neck</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0074268</td>\n",
       "      <td>IP_8723313</td>\n",
       "      <td>female</td>\n",
       "      <td>55.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     image_name  patient_id     sex  age_approx anatom_site_general_challenge  \\\n",
       "0  ISIC_2637011  IP_7279968    male        45.0                     head/neck   \n",
       "1  ISIC_0015719  IP_3075186  female        45.0               upper extremity   \n",
       "2  ISIC_0052212  IP_2842074  female        50.0               lower extremity   \n",
       "3  ISIC_0068279  IP_6890425  female        45.0                     head/neck   \n",
       "4  ISIC_0074268  IP_8723313  female        55.0               upper extremity   \n",
       "\n",
       "  diagnosis benign_malignant  target  \n",
       "0   unknown           benign       0  \n",
       "1   unknown           benign       0  \n",
       "2     nevus           benign       0  \n",
       "3   unknown           benign       0  \n",
       "4   unknown           benign       0  "
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.98237\n",
       "1    0.01763\n",
       "Name: target, dtype: float64"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extremely unbalanced dataset, most of the images are benign\n",
    "train_df[\"target\"].value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tabular data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6625"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# idx for validation\n",
    "val_idx = get_valid_index(train_df)\n",
    "len(val_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dep_var = 'target'\n",
    "cat_names = ['sex', 'anatom_site_general_challenge']\n",
    "cont_names = ['age_approx']\n",
    "procs = [FillMissing, Categorify, Normalize]\n",
    "bs=64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = TabularList.from_df(test_df, path=data_path,\n",
    "                                cat_names=cat_names, cont_names=cont_names)\n",
    "\n",
    "tab_data = (TabularList.from_df(train_df, path=data_path, cat_names=cat_names,\n",
    "                                cont_names=cont_names, procs=procs)\n",
    "            .split_by_idx(val_idx)\n",
    "            .label_from_df(cols=dep_var)\n",
    "            .add_test(test_data)\n",
    "            .databunch(bs=bs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>sex</th>\n",
       "      <th>anatom_site_general_challenge</th>\n",
       "      <th>age_approx_na</th>\n",
       "      <th>age_approx</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>female</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>False</td>\n",
       "      <td>-1.6641</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>male</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>False</td>\n",
       "      <td>-0.6195</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>#na#</td>\n",
       "      <td>torso</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0768</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>female</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>False</td>\n",
       "      <td>-0.9677</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>female</td>\n",
       "      <td>torso</td>\n",
       "      <td>False</td>\n",
       "      <td>0.7732</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>female</td>\n",
       "      <td>torso</td>\n",
       "      <td>False</td>\n",
       "      <td>0.4250</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>male</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>False</td>\n",
       "      <td>0.7732</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>female</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>False</td>\n",
       "      <td>-0.9677</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>male</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>False</td>\n",
       "      <td>-0.9677</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>female</td>\n",
       "      <td>torso</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0768</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# examples\n",
    "tab_data.show_batch(rows=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# adjust loss function weight because the dataset is extremely unbalanced\n",
    "weights = [1/(1-train_df[\"target\"].mean()), 1/train_df[\"target\"].mean()]\n",
    "loss_func = CrossEntropyFlat(weight=torch.FloatTensor(weights).to(device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# package everything in a fastai learner, add auc roc score as a metric\n",
    "learn = tabular_learner(tab_data, layers=[8,8], metrics=[accuracy, ROCAUC()],\n",
    "                        loss_func=loss_func)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>rocauc</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='98' class='' max='414', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      23.67% [98/414 00:02<00:06 1.7407]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3zV9fX48dfJIiEhAxJGEiAkYYW9lww3ooJbcVSrdY8Oa6vVOutP2zpaW2f7ldaBo05UUBEVUfbeM6wEEhLIDhk39/37434SbsLNgjvDeT4e98G978/nc+9JuLnnvrcYY1BKKaUaCvJ1AEoppfyTJgillFIuaYJQSinlkiYIpZRSLmmCUEop5VKIrwNwl/j4eJOSkuLrMJRSKqCsWrUq3xiT4OpYm0kQKSkprFy50tdhKKVUQBGRvY0d0yYmpZRSLmmCUEop5ZImCKWUUi5pglBKKeWSRxOEiEwVkW0islNE7ndxvKeILBCR9SLyvYgkOx2rEZG11m2OJ+NUSil1PI+NYhKRYOBF4GwgC1ghInOMMZudTnsGeMMY818ROQN4CrjOOnbUGDPUU/EppZRqmidrEKOBncaYTGNMFfAuMKPBORnAt9b971wcV0op5SOeTBBJwH6nx1lWmbN1wCXW/YuBDiLSyXocLiIrRWSpiFzkwTiVUsqv7Mor5cuNOb4Ow+ed1L8FJovIGmAykA3UWMd6GmNGAlcDfxORtIYXi8gtVhJZmZeX57WglVLKk178die3vbWKBVtyfRqHJxNENtDd6XGyVVbHGHPAGHOJMWYY8KBVVmj9m239mwl8Dwxr+ALGmNeMMSONMSMTElzOFFdKqYCzK68UgHv/t47swqM+i8OTCWIF0FtEeolIGHAVUG80kojEi0htDA8Ar1vlcSLSrvYcYALg3LmtlFJtkjGGzLwypvRNoNpm5+7Zq6musfskFo8lCGOMDbgL+ArYArxvjNkkIo+LyHTrtCnANhHZDnQBnrTK+wMrRWQdjs7rpxuMflJKqTYpr7SSkkobp/ftzFOXDmb1vkKe+WqbT2Lx6GJ9xpi5wNwGZQ873f8A+MDFdYuBQZ6MTSml/FFmXhkAveIjmdQngWWZh3n1h0zGp8czuY93m9J93UmtlFLKSW2CSE2IBOCPF2SQFBvBvxdlej0WTRBKKeVHMvNKCQ8NIjEmAoDw0GAuGpbI4l2HyS+t9GosmiCUUsqPZOaXkdIpkqAgqSu7cEgiNXbDvA0HvRqLJgillPIjmXmlpCVE1Svr1zWaPl2i+GydJgillDolVdns7C84Wtf/4OzCwYks33OEg0XemxehCUIppfzEviNl1NiN6wQxJBGAz71Yi9AEoZRSfmJX7Qim+KjjjqXERzI4OYbP1h/wWjyaIJRSyk/UzYFwUYMAmD4kkfVZRezOL/NKPJoglFLKT+zOLyU+qh3R4aEuj58/uBsi8Pk679QiNEEopZSfyMwrc9n/UKtbTASjUjp6rZlJE4RSSvmJzPwy0ppIEODorN6eW8qO3BKPx6MJQiml/EBheRVHyqpcdlA7O7t/FwAWbD3k8Zg0QSillB/Y1WANpsZ0jQlnQGI0327RBKGUUqeETGuToNSEpmsQAGf068yqfQUUlld5NCZNEEop5Qcy88sIDRa6x0U0e+7p/TpTYzcs3O7ZrZY1QSillB/IzCulR8f2hAQ3/7E8JDmWTpFhfOfhfghNEEop5QccQ1ybb14CCA4SJvdN4PvtedTYjcdi0gShlFI+VmM37D1cTmp80x3Uzs7s14XC8mrW7CvwWFyaIJRSyseyCsqpqrE3O4LJ2cQ+8YQEiUeHu2qCUEopHzu2zWjLmpgAosNDGZkS59F+CE0QSinlY5n5tau4trwGAY5mpq05JWQXemaPCE0QSinlY5l5pcREhNIxMqxV153erzMA33qoFqEJQimlfCwzr4xe8ZGISPMnO0lLiKRHx/Z8uyXXI3GFeORZlVJKtdju/DLGp3dq9XUiws/G9aTSZvdAVJoglFLKp8oqbeQUV5DWig5qZ7+YmOrmiI7RJiallPKh2t3herWyg9obNEEopZQP1Y1gasUcCG/RBKGUUj6UmVeKCKR00gShlFLKSWZeGYkxEYSHBvs6lONoglBKKR/and/0PtS+pAlCKaV8xBhDZl7pCY9g8jRNEEop5SOHSiopq6rRGoRSSqn6ahfp88chrqAJQimlfCYzv+X7UPuCJgillPKRzLwywkOD6BYd7utQXNIEoZRSPrI7v4yUTpEEBbVukT5v8WiCEJGpIrJNRHaKyP0ujvcUkQUisl5EvheRZKdj14vIDut2vSfjVEopX/DnEUzgwQQhIsHAi8B5QAYwU0QyGpz2DPCGMWYw8DjwlHVtR+ARYAwwGnhEROI8FatSSnlblc3O/oKjfjuCCTxbgxgN7DTGZBpjqoB3gRkNzskAvrXuf+d0/FxgvjHmiDGmAJgPTPVgrEop5VX7jpRRYzd+O4IJPJsgkoD9To+zrDJn64BLrPsXAx1EpFMLr0VEbhGRlSKyMi8vz22BK6WUp53IPtTe5utO6t8Ck0VkDTAZyAZqWnqxMeY1Y8xIY8zIhIQET8WolFJul+nHy3zX8uSGQdlAd6fHyVZZHWPMAawahIhEAZcaYwpFJBuY0uDa7z0Yq1JKeVVmXinxUWHERIT6OpRGebIGsQLoLSK9RCQMuAqY43yCiMSLSG0MDwCvW/e/As4RkTirc/ocq0wppdqE3fllpMb7b/MSeDBBGGNswF04Pti3AO8bYzaJyOMiMt06bQqwTUS2A12AJ61rjwBP4EgyK4DHrTKllGoTDhRWkBQX4eswmuTRPamNMXOBuQ3KHna6/wHwQSPXvs6xGoVSSrUpheVVxLUP83UYTfJ1J7VSSp1yKm01lFXV0DHSf/sfQBOEUkp5XWF5NQCxWoNQSinlrKC8CkCbmJRSStV3pMxKENrEpJRSylltE5PWIJRSStWjTUxKKaVcKrCamGLbaxOTUkopJwXl1bQPCyY8NNjXoTRJE4RSSnlZQQBMkgNNEEop5XUFZVV+P4IJNEEopZTXFZRXaw1CKaXU8QJhHSbQBKGUUl53pKyKOD8fwQSaIJRSyqtsNXaKK2x+vw4TaIJQSimvKjrqmEXdMVIThFJKKSe1s6j9fZIcaIJQSimvKgiQdZhAE4RSSnlV7TIb2sSklFKqHm1iUkop5ZI2MSmllHKpoLyKsJAg2of590J9oAlCKaW8qsCaJCcivg6lWZoglFLKiwJlHSbQBKGUUl4VKOswgSYIpZTyqiMBstQ3aIJQSimvKiyvDoh1mEAThFJKeY3dbig8Wk1HTRBKKaWclVTYqLGbgJgkB5oglFLKa2pnUQfCMhugCUIppbzmiJUgdBSTUkqpegoDaB0m0AShlFJeU1AWOJsFgSYIpZTymmMruWqCUEop5aSgvIrgICE6PMTXobSIJgillPISxzpMgbFQH2iCUEoprykoqwqY5iXwcIIQkakisk1EdorI/S6O9xCR70RkjYisF5FpVnmKiBwVkbXW7RVPxqmUUt5QUO5Y6jtQeKwhTESCgReBs4EsYIWIzDHGbHY67SHgfWPMyyKSAcwFUqxju4wxQz0Vn1JKeVtheTU9Orb3dRgt5skaxGhgpzEm0xhTBbwLzGhwjgGirfsxwAEPxqOUUj51pCxwlvoGzyaIJGC/0+Msq8zZo8C1IpKFo/Zwt9OxXlbT00IRmejqBUTkFhFZKSIr8/Ly3Bi6Ukq5lzHGsZJrgCz1Db7vpJ4J/McYkwxMA94UkSDgINDDGDMM+A0wW0SiG15sjHnNGDPSGDMyISHBq4ErpVRrlFfVUFVjD5iVXKGFCUJEIq0PbkSkj4hMF5Hm0mA20N3pcbJV5uwm4H0AY8wSIByIN8ZUGmMOW+WrgF1An5bEqpRS/uhIWWCtwwQtr0H8AISLSBLwNXAd8J9mrlkB9BaRXiISBlwFzGlwzj7gTAAR6Y8jQeSJSILVyY2IpAK9gcwWxqqUUn6nsNyxzEagrMMELU8QYowpBy4BXjLGXA4MaOoCY4wNuAv4CtiCY7TSJhF5XESmW6fdC9wsIuuAd4AbjDEGmASsF5G1wAfAbcaYI6394ZRSyl8E2lLf0PJhriIi44BrcDQLAQQ3d5ExZi6Ozmfnsoed7m8GJri47kPgwxbGppRSfi/Q1mGCltcgfgU8AHxs1QJSge88F5ZSSrUtBWVttAZhjFkILASwOqvzjTH3eDIwpZRqS4orbAB0CJCF+qDlo5hmi0i0iEQCG4HNInKfZ0NTSqm2o6zSRruQIEKDfT27oOVaGmmGMaYYuAiYB/TCMZJJKaVUC5RW2ohqFzi1B2h5ggi15j1cBMwxxlTjWCZDKaVUC5RV2ohsowniVWAPEAn8ICI9gWJPBaWUUm1NaWVNwCWIlnZSvwC84FS0V0RO90xISinV9pRV2ohq1+zsAL/S0k7qGBF5rnZhPBF5FkdtQimlVAuUVbXdJqbXgRLgCutWDMzyVFBKKdXWlAZgH0RLo00zxlzq9PgxaxkMpZRSLVBWaSMqLLASREtrEEdF5LTaByIyATjqmZCUUqrtKa0IvBpESxPEbcCLIrJHRPYA/wRu9VhUPvTp2mzGP7WASluNr0NRSrURdruhrKqmbXZSG2PWGWOGAIOBwdZGPmd4NDIf+WF7PgeKKtiTX+7rUJRSbUR5teMLZ1utQQBgjCm2ZlSDY6e3NmfLQcePtyuv1MeRKKXairJKxzpMbTpBNCBui8JPVNns7DhUAkCmJgillJuUWgmirS614UqbW2pjV14p1TWOHyszr8zH0Sil2opArUE0Ga2IlOA6EQgQ4ZGIfGjzAUfzUreYcG1iUkq5TWldggisTuomE4QxpoO3AvEHmw8W0y4kiDP6dWbO2gMYYxBpcy1pSikvK6t0dFKfSk1Mbc6Wg8X069qBPl06UFJpI6+k0tchKaXagLJTsA+iTTHGsPlgMRmJ0aQmOJaZ2qX9EEopNzgVO6nblJziCgrLq+nfLZq0hCgAMvO1H0IpdfICtZNaE4SltoM6o1s0XaPDiQgNZtchrUEopU5eWaUNEWgfFlid1JogLLUJol+3aIKChNSESK1BKKXcorSyhsiwkIAb9KIJwrIlp5iendrXtRGmJkTpUFellFs4thsNrNoDaIKos/lAMRndousep8ZHklVwlIpqXbRPKXVySgNwsyDQBAE4RhjsOVxOf6cEkdY5CmNgz2Hth1BKnRzHdqOaIALStpxjHdS1UuMdQ111yQ2l1Mkqq7QRGWCbBYEmCOBYB3X/RKcEkVCbILQfQil1ckora7SJKVBtPlhCTEQoiTHhdWXtw0JIjAnXyXJKqZPmaGLSTuqAtPmgo4O64RC01IQorUEopU6aYxST1iACTo3dsC2nuF4Hda20hEh25ZVhTJtb2Vwp5UUl2kkdmA6VVBAZFkJG4vEJIjUhilJdtE8pdRKqa+xU2ewBWYMIvIjdrFtMBCsfOgu7i0pC7ZpMu/LK6BwdfvwJSinVjEBdhwm0BgGAiBAcdPwU+GOrumo/hFLqxBxbyVU7qduU2kX7dC6EUupE1W4WpDWINiYoSEjvHMW23GJfh6KUClCl2sTkmohMFZFtIrJTRO53cbyHiHwnImtEZL2ITHM69oB13TYROdeTcTZlYFIMG7KKdCSTUuqEBOpucuDBBCEiwcCLwHlABjBTRDIanPYQ8L4xZhhwFfCSdW2G9XgAMBV4yXo+rxucHENxhY19R8p98fJKqQBX10mtS23UMxrYaYzJNMZUAe8CMxqcY4Da8aUxwAHr/gzgXWNMpTFmN7DTej6vG5QUA8D6rCJfvLxSKsAF6naj4NkEkQTsd3qcZZU5exS4VkSygLnA3a24FhG5RURWisjKvLw8d8VdT58uHQgLDmJjtiYIpVTrHRvmqqOYWmsm8B9jTDIwDXhTRFockzHmNWPMSGPMyISEBI8EGBYSRP9uHVzWILIKylmWedgjr6uUahvKqnQUkyvZQHenx8lWmbObgPcBjDFLgHAgvoXXes2g5Bg2ZhdhbzCb7vHPNnP9rOW6qZBSqlGllTZCgoR2Ib7+Pt56nox4BdBbRHqJSBiOTuc5Dc7ZB5wJICL9cSSIPOu8q0SknYj0AnoDyz0Ya5MGJ8VSUmljr1NHdUV1DYt25FNRbWfV3gJfhaaU8nO1C/UF2n7U4MEEYYyxAXcBXwFbcIxW2iQij4vIdOu0e4GbRWQd8A5wg3HYhKNmsRn4ErjTGOOzr+kD6zqqC+vKFu/K56hVc/hhh2f6P9oaY0xde6xSp4rSAF2oDzzcB2GMmWuM6WOMSTPGPGmVPWyMmWPd32yMmWCMGWKMGWqM+drp2iet6/oaY+Z5Ms7m9O4SRbuQIDY49UPM33yIqHYhDO8Ry6Lt+T6MLnC8+kMmo578pt7vUXlelc3OjtwSncvjI4G63Sj4vpM6IIQGB5GRGM16ayST3W5YsCWXyX0SOLN/FzYfLCa/VFd8bc5Hq7Mor6rh5jdWcqi4wtfhtHmbDhTx2GebGPvUAs5+/gem//Mnvt92SBOFl5VV1gTkCCbQBNFig5Ji2GR1VK/PLuJQSSVnZXRmYu94AH7aqbWIpuzOL2N7bilXjepO0dFqbnlzlXbue0h5lY2Zry3l/Bd+5O2l+xiX2okHp/WnoLyKG2at4MpXl7Jm36nXb7Z4Z75PmjhLA3SzINAE0WKDkmIoq6ohM7+MbzbnEhwknN63MwMSY4hrH8oP2szUpK825QBw95m9ef7KIazdX8gDH20I6G+zBWVVvLdiH9e/vpy+D83ju62HfB0SdrvhN++tY9nuwzw4rT/LHzyTF68Zzs2TUvn23ik8MWMAew6XceWrS/na+j9xtjG7iBcW7GhzyftwaSXX/N8yHv50k9dfW5uYTgGDk2MB2JBdyDdbchmVEkds+zCCg4Tx6fH8uDMvoD/sPO2rTTkMSoohKTaCqQO78Zuz+/Dxmmwe+mQjheVVx51/tKqG4opqH0TaMk/P28rIJ7/h9x9uIDO/lKh2Ibz6wy5fh8UzX2/jy005/GFaf26elEps+7C6Y2EhQVw3LoX5v55M/8Ro7nh7NV+sPwg4BhC8uWQPl7y0mOfmb+fOt1dTZbP76KdwvwOFFRgDH67O8nofWKBuNwqaIFosLSGSiNBg5m7IYWtOCWf171J3bFLveHKLK9lxSPeNcCW3uII1+wo5d8Cx39ndZ6Rzw/gUZi/fx6S/fMcrC3dRUV3D8t1H+N0H6xj15Dec8cz37Dvsf2tgbc0p5pWFuzgnowuf330aP9x3Or+YmMrSzCNszy3xWVwfrsripe93MXN0d246rVej58W0D+Wtm0YzrEcsd7+zmreX7eXud9bwx083MT69Ew+c148FWw9x9zurqa5pG0niYNFRAEKChD99sdmrX+Z0FNMpIMTqqJ6/OReAszOOfdid1tsxi3vRDm1mcqW2KePcAV3rykSER6cPYO49ExneM46n521l8KNfc8WrS/h8/UHOGdAFm91ww6zlFJQdX8Pwpefnb6dDuxCeumQQA5NiEBGuHNWdsJAg3liyxycxrdxzhAc+2sD4tE48PmNgs2PuO4SH8t8bRzMurRMPfryReRtz+N3Uvrx+/ShunZzGIxdm8NWmXH793lpsbSBJ5FiDIu48PZ1lu4/wtfV37GnGGKsPQjup27zahft6d46iZ6fIuvKk2AhSEyJZpPMhXPpqUy6p8ZGkd4467lj/btH85+ejmX3zGC4elsSzlw9hxYNn8dwVQ/nXz0aSVXiUm99Y6Tdt4huzi/hqUy43ntarXvNNx8gwLhycyMersynxctPYoZIK7nh7NYmx4bx0zXBCg1v2Z90+LIT/u34Ud52ezjs3j+WOKekEWTsr/nxCLx44rx+frz/IDbNWsG5/YTPP5t8OFlUQGizceXo6vTtH8dTcLV5pQquotmM3gbnMBmiCaJXByY4EcZZT7aHWpN4JLM08TKXNPz7I/EVReTVLMw9zzoCuTX6rHZ8Wz58vG8ylI5Lr/phGpXTk+SuGsmpfAb95f+1xS534wvPztxMTEcpNE49vwvnZuJ6UVdXw0erGV4VZva+Ac55fyFo3feDaauzc884aiiuqeeW6EfWSVkuEhwbz23P7MrpXx+OO3To5jSdmDGBDdhEzXvyJG2YtZ3WAjn7KKaqgS3Q4YSFBPHh+f/YcLvdKbS+QV3IFTRCtMiE9nn5dO3Dp8OMWluW09HhddsOFBVtzsdlNvf6H1jh/cDcenNafuRtyuOPt1T5tblqzr4AFWw9xy6RUosNDjzs+pHssQ5JjeHPpXpdt3LYaO3/4aAPbc0u5a/Zqio42XdPYf6ScfzQzoujZ+dtZmnmEJy8aRL+u0Y2ed6KuG5fCj78/nd9N7cu6/YVc8tJiPl6T5fbX8bSDRUfpFhMOwJS+nZnUJ4EXFuzgUIln5+ME8l4QoAmiVbpEh/PlryaR3rnDccfGpnUiNFhYsMX3Qx39yVebcugS3Y4h1iiwE3HTab34w7R+LNiay7l/+4GF233TlPfc/O10jAzjhvEpjZ5z3bgUdh4qZcmu41f5fWvpXrbmlHD7lDRyiiq4/8P1jXaW2mrs3DV7Nc/O386tb65yWTOdvzmXl7/fxczRPbh0RPIJ/1zN6RAeyh1T0vnx92cwsmccj3y6idwAm+iYU1RB15iIuscPX5BBhc3OIx4e9hrI242CJgi3iWoXwjkZXflwdZbftJf72tGqGhZuz+OcjK51bdsnQkS4ZVIaH98xgZiIUK5/fTmPfbbJq01OK/YcYdGOfG6bnNrkH/sFg7sR1z6U/yzeU+/DP6+kkmfnb2di73h+d25ffje1L/M25vDW0r0un+eVhbtYl1XEJcOTWLg9r96wU1uNnfdW7OM3761lYFI0j1zYcKNGz4hsF8JfLx9Cpc1REwqUYd3GGA4WVdTVIADSO0fxq7N6M29jDnM3HPTYawfydqOgCcKtrh3bk8Lyaj5f77k3XCD5alMOFdV2pg3q5pbnG5gUw2d3n8Y1Y3ow66c9/OjF2esvLNhBfFQY141NafK88NBgrh3bk68353Ln7NUcsZrE/vzlViqqa3h0+gBEhF+clsrpfRN44vMtbDpQf1z+5gPF/H3BDi4Y3I3nrhjKny4ayDdbHMNO52/O5by/L+L3H24gvUsUr1w7gvBQ742Q6RUfyX3n9mXB1kN8stZnK/C3SmF5NZU2O12jw+uV3zIxlUFJMTz86UaPNV2WVQXuZkGgCcKtxqZ2JL1zFG828q3wVPPBqiyS4yIY46ID9ESFhwbz8IUZxLUP5Z3l+9z2vE1ZtbeARTvyuWVSKhFhzf+h/+qsPvx+aj/mb87lnOd/4MXvdvLBqixuOi2VtATHSK6gIOHZK4YSFxnKtf9exqsLd1FeZaPKZufe/60jJiKMJ2YMBBxfPGqHnd78xkpsdsPL1wzno9vHkxzX3qM/uys/n9CL4T1ieXTOZo+34bvDwSJHjM41CHAMXf/LZYMpLK/m8c83e+S1SysdrQmBWoMIzKj9lIhw7ZgePPrZZtZnFdbNvj4VZRce5add+fzyzN4n1bzkSruQYC4bkcysn/aQV1JJQod2bn3+hv7x7Q46RoZxzZieLTo/OEi4fUoaU/om8Ov31vLXr7bRLSacu89Ir3dex8gw3rxpDH/6YgtPzdvKvxZlMigphi0Hi/nXz0YSF3lsRNLPJ/Qitn0oR6vsXD4yucVDWT0hOEj46+VDOO/vi/jZ/y0nOS6C0kob5VU1nNW/C3dMSSPEh/E1lFPsmCTXtUGCAMcw6ztPT+fvC3YwbVC3evOb3KFM+yCUs0tGJBMRGtxo2/Kp4uPVWRgDlw73TOfplaN6YLMbPljl2RE16/YX8v22PH4xsVer/8j7d4vm07sm8ND5/fnn1cNdXt+nSwfeuHE0H94+jn5do/luWx6XDE9y+UF18bBkrh7Tw6fJoVZaQhRPXjSQKpudA4UV2O2OVY+fm7+dK19byv4j/jMD/lgNIsLl8TtPT3ckitmr+WSNe5vNAj1BBGbUfiw6PJSLhiXx0eosHpyWQUz744dDtiWZeaX849udPHR+fzpFOb7JG+P44B6b2pHuHT3TBJLeOYrRvTry3op93Dop1e21lFr/+HYnMRGh/Gxcygld3y4kmF9MTG32vBE9O/LWL8aw81ApPTt5v9noRFw+sjuXj+xer+zTtdk89PFGpv19Eb87rx/VNjsbDxSxMbuI9M5RvHj1cK/vrJZTVEFwkDRa0wwLCWL2L8Zw+9ur+NV7a9lxqIR7z+7rlvdU3SimFjRN+iPffxVpg64d24NKm53/rdrv61A87l+LMvl4TTa/fHctNdaoolV7C9hzuJzLRnRv5uqTM3N0d/YcLmdp5vFDSt1hY3YR32zJ5abTenmtDTm9c5Rf1BBO1IyhScz95UR6d4nij59s5PHPN/Pjjnwi24Uwd0MOn6494PWYDhZV0LlDO4Kb+MCPiwzjjRvHMHN0d178bhe3v73KLTPiyypthIcG+VWTW2sEZtR+bkBiDCN6xvH2sn1+MfvXUyptNczdkENyXAQ/7szn799sBxyd0+3DgjlvYNdmnuHknDewGzERobyzwjOJ+J/f7qRDeAjXNzHvQR2ve8f2vH/rOD68fRzL/nAmyx88iw9uG8+Q5BienLvF66v0OuZAHN//0FBYSBD/7+JBPHxBBvM35zL1b4tYvOvkRsqVVtYEbAc1aILwmJ+N68nu/DK+3nz8mvttxcJteRQdreaJiwZy2YhkXvh2J/M2HOTz9QeZNqibx9tdw0ODuXhYEl9tzKkbTuouOw+V8OWmHG4Yn0JMRNtuJvSEkOAgRvTsSBdraGlwkPDERQPJL63k+fnbvRqL8yzq5ogIN57Wi//dNp6wkCCu/tcyHvts0wnPbQrkpb5BE4THnD+oG2kJkTzz9fa6ppe25tN1B+gUGcbE9HiemDGQfl07cOfs1ZRW2rjMgzN7nc0c3YOqGjsfrXZvZ/VrP2QSHhrU5Kxp1TqDk2O5enQP/rt4D5sPFHvlNWsnyXWNdt1B3ZgRPeP44p7TuH5cT2b9tPpk48sAABXGSURBVIdpLyw6oV34yiptAbvMBmiC8JiQ4CDuPacvOw+V8rGbR0b4g5KKar7ZnMsFg7sREhxERFgwr1w7gsiwELp3jGB0ivvmPjSlb9cOjEqJ45WFmS4nO721dC9XvrqkVVtN5hRV8PGabK4Y2b2u4125x33n9iUmIpSHP3VsFLVizxHeXra30fWrTlaJNfy2pTUIZ+3DQnhsxkDe/sUYKqpquPTlxTzz1bZWrQIbyHtBgCYIjzpvYFcGJcXw/PztbW6V16835VJpszN96LGFC1PiI/n4zgnMumG0x0YVufLY9IEUHa3ij59urFe+am8Bj87ZxLLdR/j7gh0tfr5ZP+2mxm64uQWjj1TrxLYP4/7z+rFybwFDH5/P5a8s4cGPN/LHTzayfPcRt79ejjXEtSV9EI2ZkB7Pl7+exCXDk/nndzu56MWf2JNf1qJry6oCdy8I0AThUSLCfef2JbvwKO8ub1sjmj5Zm033jhEM71F/MmB65yiX+z54UkZiNL88szefrz/IZ+sco2SKyqu55501dIsNZ8bQRP7vx90tatYorqjm7WX7OH9woseG6J7qLh/Rnd9N7csD5/Vj1g2jWHDvZKLDQzyyAkFjs6hbKzo8lGcuH8Jr140gq6D8uC8jtWYv28dzX2+rqw2VVdZoH4Rq3MTe8YxN7cg/vt1JeVXLmzn8WV5JJT/tzGfGkCSvj2lvzG2T0xjSPZY/frqRQ8UV3PfBOg6VVPDPmcN5bPoAYiNC+cPHG5rtD5q9bB+llTZunaS1B08JChLumJLOrZPTOL1fZ9ISorh8ZHe+3Jjj9qU7cooan0V9Is4Z0JVbJ6exaEf+cV84DhVX8Nhnm3jh25387RtHjVWbmFSTHLWIfuSXVvL6j7t9HU6dLzfmcOnLi09okbIv1h/AbmDG0EQPRHZiQoKDePbyIRytquHilxbz9eZcfj+1H0O6xxLbPoyHLujP2v2FzG5i/aZKWw2v/7ibib3jGWjtHqi845oxjpnx77m5pn2wqAIR6NzBPQkC4NoxPWkfFsxrP+yqV/7ywl3Y7Iaz+nfh7wt28P7K/TqKSTVvRM84zh3Qhb99s4Mf/WTf6neW72PV3gJ+fQI7tX2y9gAZ3aLp3eX4fTF8Kb1zFL+f2o/swqOc2a8zN512bNe3i4YmMSG9E3+Zt5WsgvJ6HaKVthrW7S/k6XlbOVRSya2T0nwR/iktNSGKib3jmb18X5N7YG/MLuLpeVt5e9leFu/M50Dh0SY7t3OKKoiPakdYiPs+6mLah3LVqB58tv4g2YWOGsqh4gpmL9vHJcOSePna4UzsHc8fPtpAeVVgz4MI3MgDzF8uG8IVryzhtrdW8f6t48hIdP/uXy1VVmljya7D9IqP5Ptteby8cBd3np7e/IU4djlbu7+Q+8/r5+EoT8wN41NIjA1nQnp8veYvEeGJGQOZ+vdFnPbn74gIDaZzdDsiQoPJzCujyvpQmtQngQnpnXwV/inturE9ueXNVSzYeohzB7ieZPmnLzazNLN+Z/bPxvXkcWvl24Ya7gPhLjeelsJ/l+zh9R9388cLMnjp+13U2A13n9Gb0OAgXrpmOFe8upQtB4s1QajmxUSE8p8bR3Hxi4v5+X+W8/EdE0iMbd3YbHf5cWc+VTV2nrx4IO8u38+zX29jWI9YxqfFN3tt7eYq57tpjwd3CwoSpg50HVtqQhT/u3UcSzMPk1dSSV5pJcVHq5ncN4GhybEM7h5LYky43/SrnGrO6NeZxJhw3lq612WC2Hu4jKWZR/jN2X24dEQye/LL+HRtNm8s2cuInnHMGHr8VsA5RRUeWdsqOa49Fw7uxrvL93HVqO7MXr6PS4cn08N6rQ7hocy6YRS/eX8tw3vGuf31vUUThBd1i4ngPzeO4vKXl3DDrOX877bxPpmlu2BLLh3CQxiV0pEhybFsOlDEPe+sZe49p9E5uulvW3M3HGRwckzAjvAZ0j2WId1P3WXY/VlIcBBXj+nBM19vJzOvlNSE+qPh3l+5nyCBK0Z2p2tMOEmxjr1GMvPK+MNHGxiUFHPcNQeLjjI21TNzcm6ZlMYnaw9wzb+XYbcb7mqwnHvXmHBm3zzWI6/tLdoH4WX9ukbz6nUj2JVXxj+/bfnYfHex2w3fbs1jcp8EQoODiGwXwsvXjqCs0sZds9dQ3UT77/4j5azLKuK8Rr6hK3WyrhzVg9Bg4b+L99Qrt9XY+WBVFpP7JNQbkRQSHMQLM4cRGhLEnbPX1FsSo6zSRnGFrd5e1O6UkRjNxN7xHCqp5LIRyQH7pakpmiB8YHx6PBcM7sbsZfsoKvfuwmUbsovIL63kzP6d68r6dOnA05cOYvmeIzw1d2uj187b6N/NSyrwJXRox2UjuvPWsn1szD62FesPO/LILa7kylHHrxCcGBvBc1cMYcvBYp5w2hkup9g9cyCa8quzepOWENniPrxAownCR26bnEZZVQ1vLt3j1dddsCWXIIEpfTrXK58xNImfT0jh9Z9282kjew1/sSGHgUnRde2sSnnC/ef1o1NkGPd9sL6uRvveiv10igzjjH6ud3w7o18XbpmUytvL9vHmkj2Ae2ZRN2dEz44suHdKm6w9gCYIn+nfLZopfROY9dOeE14p8kQs2HqIET3j6m1nWesP0/ozKiWO+z/cwNac+pOAsgrKWbe/kGlae1AeFhMRyp8uGsiWg8W8unAXeSWVLNhyiEuGJzU5XPV35/blrP5deHjOJuasO+C2WdSnMk0QPnT75DQOl1Xxv5XeWYYjp6iCTQeKObO/629hocFBvHj1cKLCQ7jtzVXkl1bWHZu3wbFsuTYvKW84Z0BXzh/cjRcW7OTZr7dhsxuXzUvOQoKD+OfVwxiV0pHfvLe2bvvQLs0MvFCN0wThQ6N7dWR4j1he/SGzyclB7rJgay4AZ/br3Og5naPDefma4RwsqmD6P36sawf+YsNBBiRG07NTpMfjVArg0QsH0L5dMO+u2M/wHrGkd25+YmZ4aDD/vn4kfbp04Med+XSMDCM8NHAXy/M1jyYIEZkqIttEZKeI3O/i+PMista6bReRQqdjNU7H5ngyTl8REW6bnEZWwVG+sOYXeNK3Ww7RvWNEs4vpjUzpyAe3jQfgslcW8+9FmazV5iXlZQkd2vHwBRkAXDW6R4uviw4P5b83jialU3t6xesXmpMhnliDHUBEgoHtwNlAFrACmGmM2dzI+XcDw4wxN1qPS40xLV4WdOTIkWblypUnH7iX2e2Gc/72AyFBwtx7Jrp9mezSShur9hawYvcR/rUok5mje/Do9AEtuja/tJLb31rFij2OjVK+/+0UUvQPTnnZxuwiMrpFt/pvo7TSRpXNTkcX/W3qGBFZZYwZ6eqYJyfKjQZ2GmMyrSDeBWYALhMEMBN4xIPx+KWgIOGu09P51Xtr+WB1FleMbLqdtTUe+2wT/128B7txbPk4KCmmVTukxUe14+1fjOXpeVspPFqlyUH5xIkunBjVLgR0v6eT4skEkQQ4975mAWNcnSgiPYFewLdOxeEishKwAU8bYz5xcd0twC0APXq0vArqb6YPSeTNpXv587ytnDugq1tmV3+/7RCzftrDjKGJXDYimWE94k5oTZiwkCAevjDjpONRSgUef+mkvgr4wBjjPN6zp1XtuRr4m4gct8SmMeY1Y8xIY8zIhIQEb8XqdkFBwmPTB3CkvMotG7qXV9l46JONpCVE8pfLBjOxd0JALximlPINTyaIbMC5vSTZKnPlKuAd5wJjTLb1bybwPTDM/SH6j4FJMVwzpgdvLNnDloMnt6H7377ZQVbBUZ66ZDDtQnQEh1LqxHgyQawAeotILxEJw5EEjhuNJCL9gDhgiVNZnIi0s+7HAxNovO+izfjtOY4N3R+Zs+mEN3DfmF3E//24m5mjuzO6l2cWKVNKnRo8liCMMTbgLuArYAvwvjFmk4g8LiLTnU69CnjX1P9E7A+sFJF1wHc4+iDafIKIbR/Gfef2Y/nuI8yx9lZujRq74YGPNhDXPoz7p/b3QIRKqVOJx4a5elugDnNtqMZuOP+FRQDM++XEVu1N8Nm6A9z9zhr+MXMYFw7xn+1AlVL+q6lhrv7SSa0swUHCdeN6sjWnhPVZRc1f4GTRjjxiIkJ1OQyllFtogvBD04ckEhHqWGKgNZZkHmZMr45un2ynlDo1aYLwQx3CQzl/cDfmrM2mrNLWomuyCsrZf+QoY1N1P2WllHtogvBTV43qTllVTYvXaKrdyH1cmiYIpZR7aILwUyN6xpGWEMl7Ts1MdrvhkU83cv3ry48bBrtk12Hi2ofSt0vzK14qpVRLaILwUyLCVaN6sGpvATtyS7DbDQ9+soH/LtnLwu15LN99pO5cYwxLMw8zplcn7X9QSrmNJgg/dvHwJEKDhXdX7OfRzzbxzvL93DyxFx3CQ+p1YGcVHCW78Kg2Lyml3EoX6PFj8VHtODujC7N+2o3dwC2TUnngvH5UVNt5b+V+Hrkwg9j2YSzZdRjQ/gellHtpDcLPXTOmJ3YDP5+QwgPn9UNEmDm6B1U2Ox9bWyouyTxMp8gwejezEZBSSrWG1iD83IT0eBbffwbdYsLrZlVnJEYzJDmGd5bv44bxKSzNPMzY1E6tmnWtlFLN0RpEAEiMjTjuw3/m6B5szy3lk7XZHCyqYKw2Lyml3EwTRIC6cEgikWHBPPaZYw3DcTpBTinlZpogAlRkuxCmD02isLyahA7tSEvQ7UCVUu6lCSKAzRzt2I9J+x+UUp6gndQBbFBSDL89pw+T+gTudqtKKf+lCSKAiQh3ndHb12EopdoobWJSSinlkiYIpZRSLmmCUEop5ZImCKWUUi5pglBKKeWSJgillFIuaYJQSinlkiYIpZRSLknDvY0DlYjkAYVAkYvDMQ3Km3pce99VWTyQ38rQGr5WS4+fSMzO908m5qbiaup4c2X+GLOrcn1/NO9UeX8EYsyuypt63NsYE+Py2Y0xbeYGvNaS8qYe195vpGylu2LyRMyu4j+RmE807ubK/DFmfX/o+6OtxXwy74+Gt7bWxPRZC8ubevxZE2XujKm54ycSs/P9k4m5Jde7Ot5cmT/G7Kpc3x/NO1XeH4EYs6vylr4/6mkzTUzeICIrjTEjfR1Ha2jM3hOIcWvM3hGIMYN2UrfWa74O4ARozN4TiHFrzN4RiDFrDUIppZRrWoNQSinlkiYIpZRSLp2SCUJEXheRQyKy8QSuHSEiG0Rkp4i8IE57fYrI3SKyVUQ2ichf3Bu1Z+IWkUdFJFtE1lq3af4es9Pxe0XEiEi8+yL22O/5CRFZb/2OvxaRRHfG7MG4/2q9p9eLyMciEhsAMV9u/Q3aRcRtHcMnE2sjz3e9iOywbtc7lTf5vveqExmbG+g3YBIwHNh4AtcuB8YCAswDzrPKTwe+AdpZjzsHSNyPAr8NpN+1daw78BWwF4j395iBaKdz7gFeCYTfNXAOEGLd/zPw5wCIuT/QF/geGOnrWK04UhqUdQQyrX/jrPtxTf1cvridkjUIY8wPwBHnMhFJE5EvRWSViCwSkX4NrxORbjj+0Jcax//kG8BF1uHbgaeNMZXWaxwKkLg9yoMxPw/8DnD7KAtPxGyMKXY6NTKA4v7aGGOzTl0KJAdAzFuMMdvcGefJxNqIc4H5xpgjxpgCYD4w1Zd/q66ckgmiEa8BdxtjRgC/BV5ycU4SkOX0OMsqA+gDTBSRZSKyUERGeTTaY042boC7rCaE10UkznOh1jmpmEVkBpBtjFnn6UCdnPTvWUSeFJH9wDXAwx6M1Zk73h+1bsTxjdbT3Bmzp7UkVleSgP1Oj2vj95efC4AQX72wPxGRKGA88D+n5r52rXyaEBzVxbHAKOB9EUm1vgV4hJvifhl4Asc32ieAZ3F8EHjEycYsIu2BP+Bo+vAKN/2eMcY8CDwoIg8AdwGPuC1IF9wVt/VcDwI24G33RNfo67gtZk9rKlYR+TnwS6ssHZgrIlXAbmPMxd6O9URpgnAIAgqNMUOdC0UkGFhlPZyD48PUuYqdDGRb97OAj6yEsFxE7DgW6Mrz57iNMblO1/0L+NyD8cLJx5wG9ALWWX+UycBqERltjMnx05gbehuYi4cTBG6KW0RuAC4AzvTkFx6Lu3/XnuQyVgBjzCxgFoCIfA/cYIzZ43RKNjDF6XEyjr6KbHz/cx3jq84PX9+AFJw6m4DFwOXWfQGGNHJdww6kaVb5bcDj1v0+OKqPEgBxd3M659fAu/4ec4Nz9uDmTmoP/Z57O51zN/BBgLyvpwKbgQRPxOvJ9wdu7qQ+0VhpvJN6N44O6jjrfseWvu+9dfPJi/r6BrwDHASqcXzzvwnHt9IvgXXWH8TDjVw7EtgI7AL+ybHZ6GHAW9ax1cAZARL3m8AGYD2Ob2bd/D3mBufswf2jmDzxe/7QKl+PY3G0pAB5f+zE8WVnrXVz6+grD8V8sfVclUAu8JUvY8VFgrDKb7R+vzuBn7fmfe+tmy61oZRSyiUdxaSUUsolTRBKKaVc0gShlFLKJU0QSimlXNIEoZRSyiVNEKpNE5FSL7/eYjc9zxQRKRLH6q9bReSZFlxzkYhkuOP1lQJNEEq1iog0ufqAMWa8G19ukXHM0h0GXCAiE5o5/yJAE4RyG00Q6pTT2AqcInKhtdjiGhH5RkS6WOWPisibIvIT8Kb1+HUR+V5EMkXkHqfnLrX+nWId/8CqAbxdu66/iEyzylZZ6/03ubyJMeYojklqtYsV3iwiK0RknYh8KCLtRWQ8MB34q1XrSDuJlUaVAjRBqFNTYytw/giMNcYMA97FsZx4rQzgLGPMTOtxPxxLNo8GHhGRUBevMwz4lXVtKjBBRMKBV3Gs8T8CSGguWGuF3d7AD1bRR8aYUcaYIcAW4CZjzGIcM+HvM8YMNcbsauLnVKpFdLE+dUppZrXQZOA9a03+MBzr49SaY32Tr/WFcez9USkih4Au1F+mGWC5MSbLet21ONbxKQUyjTG1z/0OcEsj4U4UkXU4ksPfzLHFCAeKyJ+AWCAKx8ZJrfk5lWoRTRDqVNPoCpzAP4DnjDFzRGQKjt32apU1OLfS6X4Nrv+WWnJOUxYZYy4QkV7AUhF53xizFvgPcJExZp210uoUF9c29XMq1SLaxKROKcaxs9tuEbkcQByGWIdjOLa08vWurneDbUCqiKRYj69s7gKrtvE08HurqANw0GrWusbp1BLrWHM/p1ItoglCtXXtRSTL6fYbHB+qN1nNN5uAGda5j+JoklkF5HsiGKuZ6g7gS+t1SoCiFlz6CjDJSix/BJYBPwFbnc55F7jP6mRPo/GfU6kW0dVclfIyEYkyxpRao5peBHYYY573dVxKNaQ1CKW872ar03oTjmatV30cj1IuaQ1CKaWUS1qDUEop5ZImCKWUUi5pglBKKeWSJgillFIuaYJQSinl0v8H2LDXeLCNpPcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# find learning rate\n",
    "learn.lr_find()\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>rocauc</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.761437</td>\n",
       "      <td>0.739343</td>\n",
       "      <td>0.293283</td>\n",
       "      <td>0.656587</td>\n",
       "      <td>00:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.673929</td>\n",
       "      <td>0.661448</td>\n",
       "      <td>0.672906</td>\n",
       "      <td>0.636872</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.631476</td>\n",
       "      <td>0.614779</td>\n",
       "      <td>0.865962</td>\n",
       "      <td>0.689458</td>\n",
       "      <td>00:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.654370</td>\n",
       "      <td>0.633675</td>\n",
       "      <td>0.963321</td>\n",
       "      <td>0.679780</td>\n",
       "      <td>00:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.622434</td>\n",
       "      <td>0.634781</td>\n",
       "      <td>0.817208</td>\n",
       "      <td>0.692977</td>\n",
       "      <td>00:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.621000</td>\n",
       "      <td>0.624831</td>\n",
       "      <td>0.847849</td>\n",
       "      <td>0.708333</td>\n",
       "      <td>00:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.626121</td>\n",
       "      <td>0.625286</td>\n",
       "      <td>0.870943</td>\n",
       "      <td>0.710524</td>\n",
       "      <td>00:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.594596</td>\n",
       "      <td>0.617269</td>\n",
       "      <td>0.848906</td>\n",
       "      <td>0.706307</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.623723</td>\n",
       "      <td>0.610537</td>\n",
       "      <td>0.854641</td>\n",
       "      <td>0.707314</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.608752</td>\n",
       "      <td>0.614446</td>\n",
       "      <td>0.848906</td>\n",
       "      <td>0.709546</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# train\n",
    "learn.fit_one_cycle(10, 1e-3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# make predictions for the test set\n",
    "preds, y = learn.get_preds(DatasetType.Test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# submit predictions to kaggle\n",
    "submit = pd.read_csv(data_path/\"sample_submission.csv\")\n",
    "submit[\"target\"] = preds[:, 1]\n",
    "submit.to_csv(data_path/\"tab.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}