notebooks/final/bagging-final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bootstrap Aggregating without online sampling\n",
    "* Not seperating by WSI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/media/rene/Data/camelyon/src\n",
      "True\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import glob\n",
    "import random\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import torch.utils.data\n",
    "import torchvision.models as models\n",
    "from torchvision import datasets, models, transforms\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchvision import transforms, utils\n",
    "\n",
    "from fastai.imports import *\n",
    "from fastai.transforms import *\n",
    "from fastai.conv_learner import *\n",
    "from fastai.model import *\n",
    "from fastai.dataset import *\n",
    "from fastai.sgdr import *\n",
    "from fastai.plots import *\n",
    "\n",
    "SEED = 101\n",
    "np.random.seed(SEED)\n",
    "from torchvision import datasets, models, transforms\n",
    "\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "# Add the src directory for functions\n",
    "src_dir = os.path.join(os.path.dirname(os.path.dirname(os.getcwd())), 'src')\n",
    "print(src_dir)\n",
    "sys.path.append(src_dir)\n",
    "\n",
    "# import my functions:\n",
    "from WSI_pytorch_utils import*\n",
    "from other_functions import*\n",
    "\n",
    "fast_ai_dir = '/media/rene/Data/fastai/'\n",
    "sys.path.append(fast_ai_dir)\n",
    "\n",
    "torch.cuda.set_device(0)\n",
    "print(torch.cuda.is_available())\n",
    "print(torch.cuda.current_device())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make dataset \n",
    "* csv with locations pointing to the right images to sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # reproduce\n",
    "# cp -r tiles_224_100t/train/tumor/. tiles_224_100t_all\n",
    "# cp -r tiles_224_100t/train/normal/. tiles_224_100t_all\n",
    "# cp -r tiles_224_100t/valid/tumor/. tiles_224_100t_all\n",
    "# cp -r tiles_224_100t/valid/normal/. tiles_224_100t_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get data\n",
    "data_loc = '/media/rene/Data/camelyon_out/tiles_224_100t_all'\n",
    "all_imgs = glob.glob(data_loc+'/*')\n",
    "img_names = [loc.rsplit('/', 1)[-1] for loc in all_imgs]\n",
    "\n",
    "# load train valid split\n",
    "ttv_split = np.load('/media/rene/Data/camelyon/other/ttv_split.p')\n",
    "normal_valid = ttv_split['normal_vaild_idx']\n",
    "tumor_valid = ttv_split['tumor_vaild_idx']\n",
    "normal_train = list(range(1, 161))\n",
    "normal_train = [num for num in normal_train if num not in normal_valid]\n",
    "tumor_train = list(range(1, 111))\n",
    "tumor_train = [num for num in tumor_train if num not in tumor_valid]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get classes corresponding to each file\n",
    "def get_label(name):\n",
    "    if name.rsplit('_', 1)[-2][-5:]=='tumor':\n",
    "        label = 'tumor'\n",
    "    else: \n",
    "        label = 'normal'\n",
    "    return label\n",
    "\n",
    "classes = [get_label(name) for name in img_names]\n",
    "\n",
    "# make csv\n",
    "labels_df = pd.DataFrame(\n",
    "    {'file_name': img_names,\n",
    "     'label': classes\n",
    "    })\n",
    "\n",
    "labels_df.to_csv('/media/rene/Data/camelyon_out/tiles_224_100t_all_other/tiles_224_100t_all_labels.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the validation indices from the big list\n",
    "def is_validation(name, normal_valid, tumor_valid):\n",
    "    normal_valid = [int(x) for x in normal_valid]\n",
    "    tumor_valid = [int(x) for x in tumor_valid]\n",
    "    num = int(name.split('_', 1)[1].split('_', 1)[0])\n",
    "    \n",
    "    if classes[idx] == 'normal':\n",
    "        return num in normal_valid\n",
    "    \n",
    "    elif classes[idx] == 'tumor':\n",
    "        return num in tumor_valid\n",
    "    else:\n",
    "        raise ValueError(\"tile isn't tumor or non tumor\")\n",
    "\n",
    "valid_idxs = []\n",
    "for idx, name in enumerate(img_names):\n",
    "    if is_validation(name, normal_valid, tumor_valid):\n",
    "        valid_idxs.append(idx)\n",
    "        \n",
    "# with open('/media/rene/Data/camelyon_out/tiles_224_100t_all_other/tiles_224_100t_all_val_idxs.p', 'wb') as fp:\n",
    "#     pickle.dump(valid_idxs, fp)\n",
    "\n",
    "def make_validation_mask(data_df, normal_valid, tumor_valid):\n",
    "    img_names = data_df['file_name'].tolist()\n",
    "    valid_idxs = [is_validation(name, normal_valid, tumor_valid) for name in img_names]\n",
    "    return valid_idxs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Standard Bagging with WSI\n",
    "* Sample WSIs with replacement, same number as full dataset\n",
    "* How does the number of models affect performance?\n",
    "* Test a few models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the indices of slides to use\n",
    "def make_sample_replacement(labels_df, normal_train, tumor_train):\n",
    "    normal_train_subset = np.random.choice(normal_train, len(normal_train))\n",
    "    normal_train_subset = [int(x) for x in normal_train_subset]\n",
    "    \n",
    "    tumor_train_subset = np.random.choice(tumor_train, len(tumor_train))\n",
    "    tumor_train_subset = [int(x) for x in tumor_train_subset]\n",
    "    \n",
    "    drop_rows = []\n",
    "    for index, row in labels_df.iterrows():\n",
    "        label = row['label']\n",
    "        num = int(row['file_name'].split('_', 1)[1].split('_', 1)[0])\n",
    "        \n",
    "        if label == 'normal' and num not in normal_train_subset:\n",
    "            drop_rows.append(index)\n",
    "        elif label == 'tumor' and num not in tumor_train_subset:\n",
    "            drop_rows.append(index)\n",
    "    \n",
    "    subset_labels_df = labels_df.drop(labels_df.index[drop_rows])\n",
    "    return subset_labels_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6f8828d190241bfa41b7a8177bd1e13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/370 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.303586   0.26227    0.891908  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3826395217bf4c889caf1b10a4ae6b6c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.246768   0.22702    0.905894  \n",
      "    1      0.219619   0.196038   0.922677                    \n",
      "    2      0.204463   0.192088   0.924076                    \n",
      "    3      0.191473   0.181394   0.928871                    \n",
      "    4      0.189459   0.176      0.927672                    \n",
      "    5      0.178979   0.175404   0.929071                    \n",
      "                                                             \r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 1/5 [17:55<1:11:41, 1075.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    6      0.178621   0.173561   0.931069  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "897a07d6d8b945a187281e15f172f338",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.316013   0.272014   0.885086  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00c73bfebbda4d24943759a31561619e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.256364   0.223344   0.905495  \n",
      "    1      0.22047    0.191064   0.918681                    \n",
      "    2      0.208122   0.187783   0.922763                    \n",
      "    3      0.19095    0.178684   0.925432                    \n",
      "    4      0.18874    0.168514   0.93281                     \n",
      "    5      0.1822     0.166081   0.932339                    \n",
      "                                                             \r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 2/5 [35:39<53:29, 1069.99s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    6      0.185036   0.168756   0.931083  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc9ed0be59cf4dcb9b24161e55d18d95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.310994   0.262538   0.894505  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6dac28f808804c3d9300b4146731f7ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.254063   0.209205   0.911774  \n",
      "    1      0.213229   0.173254   0.928728                    \n",
      "    2      0.198619   0.170055   0.930298                    \n",
      "    3      0.182448   0.158144   0.939403                    \n",
      "    4      0.176268   0.152095   0.941601                    \n",
      "    5      0.173545   0.148307   0.943485                    \n",
      "                                                             \r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|██████    | 3/5 [54:17<36:11, 1085.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    6      0.170409   0.149654   0.941287  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "085b782cf89f40bdaf819a50d852e5b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.332892   0.289354   0.878681  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd559a842c214d1da73d7856f0d536e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.267732   0.230144   0.902637  \n",
      "    1      0.237092   0.206564   0.916484                    \n",
      "    2      0.223482   0.200381   0.916923                    \n",
      "    3      0.202618   0.184569   0.922198                    \n",
      "    4      0.203909   0.178505   0.925055                    \n",
      "    5      0.194919   0.177799   0.924396                    \n",
      "                                                             \r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|████████  | 4/5 [1:11:23<17:50, 1070.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    6      0.19858    0.178692   0.924176  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "795cc87e6a194fc6b47d017159b65de5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.311349   0.268104   0.885495  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "962de73c7a814b13905ffd57cb005b82",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 94/348 [00:33<01:31,  2.76it/s, loss=0.31] "
     ]
    }
   ],
   "source": [
    "models_arch = resnet18\n",
    "models_name = 'resnet18'\n",
    "\n",
    "lr=np.array([1e-3,5e-3,1e-2])\n",
    "\n",
    "sz = 224\n",
    "PATH = '/media/rene/Data/camelyon_out/tiles_224_100t_all_other'\n",
    "train_folder = '/media/rene/Data/camelyon_out/tiles_224_100t_all'\n",
    "\n",
    "for i in tqdm(range(10, 15)):\n",
    "    # create the new csv and save it\n",
    "    subset_labels_df = make_sample_replacement(labels_df, normal_train, tumor_train)\n",
    "    subset_labels_df.to_csv('/media/rene/Data/camelyon_out/tiles_224_100t_all_other/tiles_224_100t_all_subset_tmp2.csv',  \n",
    "                            index = False)\n",
    "    csv_fname = '/media/rene/Data/camelyon_out/tiles_224_100t_all_other/tiles_224_100t_all_subset_tmp2.csv'\n",
    "    \n",
    "    # get the validation indices based on this\n",
    "    val_idxs = make_validation_mask(subset_labels_df, normal_valid, tumor_valid)\n",
    "\n",
    "    tfms = tfms_from_model(models_arch, sz, aug_tfms=transforms_top_down, max_zoom=1)\n",
    "    data = ImageClassifierData.from_csv(PATH, train_folder, csv_fname, tfms=tfms, \n",
    "                                        val_idxs=val_idxs, bs=128)\n",
    "    learn = ConvLearner.pretrained(models_arch, data, precompute=False)\n",
    "    \n",
    "    lr =.001\n",
    "    learn.fit(lr, 1, cycle_len=1, cycle_mult=1) # train last few layers\n",
    "    lrs = np.array([lr/4,lr/2,lr])\n",
    "    learn.unfreeze()\n",
    "    learn.fit(lrs, 3, cycle_len=1, cycle_mult=2, best_save_name=models_name+'_bagging_proper_wsi_'+str(i)) # train whole model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "models_arch = resnet50\n",
    "models_name = 'resnet50'\n",
    "\n",
    "lr=np.array([1e-3,5e-3,1e-2])\n",
    "\n",
    "sz = 224\n",
    "PATH = '/media/rene/Data/camelyon_out/tiles_224_100t_all_other'\n",
    "train_folder = '/media/rene/Data/camelyon_out/tiles_224_100t_all'\n",
    "\n",
    "for i in tqdm(range(15)):\n",
    "    # create the new csv and save it\n",
    "    subset_labels_df = make_sample_replacement(labels_df, normal_train, tumor_train)\n",
    "    subset_labels_df.to_csv('/media/rene/Data/camelyon_out/tiles_224_100t_all_other/tiles_224_100t_all_subset_tmp2.csv',  \n",
    "                            index = False)\n",
    "    csv_fname = '/media/rene/Data/camelyon_out/tiles_224_100t_all_other/tiles_224_100t_all_subset_tmp2.csv'\n",
    "    \n",
    "    # get the validation indices based on this\n",
    "    val_idxs = make_validation_mask(subset_labels_df, normal_valid, tumor_valid)\n",
    "\n",
    "    tfms = tfms_from_model(models_arch, sz, aug_tfms=transforms_top_down, max_zoom=1)\n",
    "    data = ImageClassifierData.from_csv(PATH, train_folder, csv_fname, tfms=tfms, \n",
    "                                        val_idxs=val_idxs, bs=64)\n",
    "    learn = ConvLearner.pretrained(models_arch, data, precompute=False)\n",
    "    \n",
    "    lr =.001\n",
    "    learn.fit(lr, 1, cycle_len=1, cycle_mult=1) # train last few layers\n",
    "    lrs = np.array([lr/4,lr/2,lr])\n",
    "    learn.unfreeze()\n",
    "    learn.fit(lrs, 3, cycle_len=1, cycle_mult=2, best_save_name=models_name+'_bagging_proper_wsi_'+str(i)) # train whole model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/2 [00:00<?, ?it/s]"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be469d67e6404ab498a3ea8e688bcf58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/986 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.249017   0.216515   0.907493  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b68d2c5b7b8647e788e8c6240ee809d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.19583    0.178313   0.930869  \n",
      "    1      0.181065   0.15699    0.935864                    \n",
      "    2      0.164168   0.152913   0.938262                    \n",
      "    3      0.142469   0.142039   0.942458                    \n",
      "    4      0.155367   0.137674   0.946254                    \n",
      "    5      0.137972   0.132742   0.948851                    \n",
      "                                                             \r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 1/2 [49:40<49:40, 2980.44s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    6      0.135903   0.132269   0.946853  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "567770246fa641a0a1ee51559a63b189",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.249105   0.21259    0.909576  \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d56fc57ba0c4e3f8ed648642089fb4c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.217661   0.178476   0.926688  \n",
      "    1      0.181426   0.153965   0.938305                    \n",
      "    2      0.162695   0.148798   0.943014                    \n",
      "    3      0.16952    0.140335   0.944584                    \n",
      "    4      0.154633   0.130883   0.948509                    \n",
      "    5      0.139956   0.126873   0.949294                    \n",
      " 62%|██████▏   | 604/976 [03:51<02:22,  2.61it/s, loss=0.137]"
     ]
    }
   ],
   "source": [
    "models_arch = dn169\n",
    "models_name = 'dn169'\n",
    "\n",
    "lr=np.array([1e-3,5e-3,1e-2])\n",
    "\n",
    "sz = 224\n",
    "PATH = '/media/rene/Data/camelyon_out/tiles_224_100t_all_other'\n",
    "train_folder = '/media/rene/Data/camelyon_out/tiles_224_100t_all'\n",
    "\n",
    "for i in tqdm(range(13, 15)):\n",
    "    # create the new csv and save it\n",
    "    subset_labels_df = make_sample_replacement(labels_df, normal_train, tumor_train)\n",
    "    subset_labels_df.to_csv('/media/rene/Data/camelyon_out/tiles_224_100t_all_other/tiles_224_100t_all_subset_tmp2.csv',  \n",
    "                            index = False)\n",
    "    csv_fname = '/media/rene/Data/camelyon_out/tiles_224_100t_all_other/tiles_224_100t_all_subset_tmp2.csv'\n",
    "    \n",
    "    # get the validation indices based on this\n",
    "    val_idxs = make_validation_mask(subset_labels_df, normal_valid, tumor_valid)\n",
    "\n",
    "    tfms = tfms_from_model(models_arch, sz, aug_tfms=transforms_top_down, max_zoom=1)\n",
    "    data = ImageClassifierData.from_csv(PATH, train_folder, csv_fname, tfms=tfms, \n",
    "                                        val_idxs=val_idxs, bs=48)\n",
    "    learn = ConvLearner.pretrained(models_arch, data, precompute=False)\n",
    "    \n",
    "    lr =.001\n",
    "    learn.fit(lr, 1, cycle_len=1, cycle_mult=1) # train last few layers\n",
    "    lrs = np.array([lr/4,lr/2,lr])\n",
    "    learn.unfreeze()\n",
    "    learn.fit(lrs, 3, cycle_len=1, cycle_mult=2, best_save_name=models_name+'_bagging_proper_wsi_'+str(i)) # train whole model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resnet18_bagging_proper_wsi_0 :    0.9279640718562874\n",
      "resnet18_bagging_proper_wsi_1 :    0.9259880239520958\n",
      "resnet18_bagging_proper_wsi_2 :    0.9306586826347305\n",
      "resnet18_bagging_proper_wsi_3 :    0.9332934131736527\n",
      "resnet18_bagging_proper_wsi_4 :    0.9268862275449102\n",
      "resnet18_bagging_proper_wsi_5 :    0.9264670658682634\n",
      "resnet18_bagging_proper_wsi_6 :    0.9226347305389222\n",
      "resnet18_bagging_proper_wsi_7 :    0.9261676646706587\n",
      "resnet18_bagging_proper_wsi_8 :    0.9286826347305389\n",
      "resnet18_bagging_proper_wsi_9 :    0.931497005988024\n",
      "resnet18_bagging_proper_wsi_10 :    0.9281437125748503\n",
      "resnet18_bagging_proper_wsi_11 :    0.92\n",
      "resnet18_bagging_proper_wsi_12 :    0.9273652694610779\n",
      "resnet18_bagging_proper_wsi_13 :    0.9259281437125748\n",
      "resnet18_bagging_proper_wsi_14 :    0.9255089820359281\n",
      "0.9319161676646707\n",
      "resnet50_bagging_proper_wsi_0 :    0.9338323353293413\n",
      "resnet50_bagging_proper_wsi_1 :    0.9302395209580838\n",
      "resnet50_bagging_proper_wsi_2 :    0.9411976047904191\n",
      "resnet50_bagging_proper_wsi_3 :    0.938562874251497\n",
      "resnet50_bagging_proper_wsi_4 :    0.9332335329341317\n",
      "resnet50_bagging_proper_wsi_5 :    0.9275449101796407\n",
      "resnet50_bagging_proper_wsi_6 :    0.9376047904191617\n",
      "resnet50_bagging_proper_wsi_7 :    0.9381437125748503\n",
      "resnet50_bagging_proper_wsi_8 :    0.9364071856287425\n",
      "resnet50_bagging_proper_wsi_9 :    0.9252694610778444\n",
      "resnet50_bagging_proper_wsi_10 :    0.9308982035928144\n",
      "resnet50_bagging_proper_wsi_11 :    0.9265269461077844\n",
      "resnet50_bagging_proper_wsi_12 :    0.932934131736527\n",
      "resnet50_bagging_proper_wsi_13 :    0.9383233532934132\n",
      "resnet50_bagging_proper_wsi_14 :    0.9376047904191617\n",
      "0.942874251497006\n",
      "dn169_bagging_proper_wsi_0 :    0.9379041916167664\n",
      "dn169_bagging_proper_wsi_1 :    0.9311976047904191\n",
      "dn169_bagging_proper_wsi_2 :    0.9412574850299401\n",
      "dn169_bagging_proper_wsi_3 :    0.9425149700598803\n",
      "dn169_bagging_proper_wsi_4 :    0.9386826347305389\n",
      "dn169_bagging_proper_wsi_5 :    0.9376047904191617\n",
      "dn169_bagging_proper_wsi_6 :    0.9390419161676646\n",
      "dn169_bagging_proper_wsi_7 :    0.9376047904191617\n",
      "dn169_bagging_proper_wsi_8 :    0.9199401197604791\n",
      "dn169_bagging_proper_wsi_9 :    0.944251497005988\n",
      "dn169_bagging_proper_wsi_10 :    0.9416167664670658\n",
      "dn169_bagging_proper_wsi_11 :    0.9336526946107785\n",
      "dn169_bagging_proper_wsi_12 :    0.9416167664670658\n",
      "dn169_bagging_proper_wsi_13 :    0.9330538922155689\n",
      "dn169_bagging_proper_wsi_14 :    0.9294610778443114\n",
      "0.9454491017964072\n"
     ]
    }
   ],
   "source": [
    "PATH = '/media/rene/Data/camelyon_out/tiles_224_100t/'\n",
    "models_path = '/media/rene/Data/camelyon_out/tiles_224_100t_all_other'\n",
    "models_arch = [resnet18]*15\n",
    "models_name = ['resnet18_bagging_proper_wsi_'+str(i) for i in range (15)]\n",
    "\n",
    "acc_r18, acc_list_r18, int_acc_list_r18 = ensemble_acc_dir(models_arch, \n",
    "                                                                   models_name, models_path, PATH, weights=None)\n",
    "\n",
    "models_arch = [resnet50]*15\n",
    "models_name = ['resnet50_bagging_proper_wsi_'+str(i) for i in range (15)]\n",
    "\n",
    "acc_r50, acc_list_r50, int_acc_list_r50 = ensemble_acc_dir(models_arch, \n",
    "                                                                   models_name, models_path, PATH, weights=None)\n",
    "\n",
    "models_arch = [dn169]*15\n",
    "models_name = ['dn169_bagging_proper_wsi_'+str(i) for i in range (15)]\n",
    "\n",
    "acc_dn169, acc_list_dn169, int_acc_list_dn169 = ensemble_acc_dir(models_arch, \n",
    "                                                                   models_name, models_path, PATH, weights=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAggAAAGECAYAAABTZthdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd4VFX6wPHvm0ZLCBBAAhFCh9ASigQRQVwpCvYKKFjWsmD9WXBtuKLIimtf24qAoKi4KhYQRVBWAanSAgSpgWDoISSknt8f5yZMhiRM6kyS9/M898nce88999yZycw755x7jhhjUEoppZRy5eftAiillFLK92iAoJRSSqnTaICglFJKqdNogKCUUkqp02iAoJRSSqnTaICglFJKqdNogKCqDBGZJiJfe7sc3uKL1y8il4lIvIhkicg0L5ajp4gYEYksxjE+93wqVZE0QFAl5nyAGpfloIh8LSIdvFSke4FR3jixy3PxuNv2Ac72ht4olw/4D/AZ0AL7+pxGRBYX9Nw5+z5x9r1ezuWs1ERkrohki8hF3i6Lqjo0QFCl9QMQ7iyDgFrA594oiDHmmDHmqDfO7TgJPCwijbxYhjInIoElPK4e0BD4zhiz1xhzrIjke4CbRURcjg8DLnX2qUKISDhwIfAScJuXi4OI+ImIv7fLoUpPAwRVWunGmP3Oshr7IdVBRGrlJhCR50Vki4ikichOEfmniNR0zUREHhWRP0UkRURmiMhTIrLTZX+AiLwkIkec5SUReVNEFrukyVcl7Pwy/beIPOfUbiSJyBQR8XNJc5bz6ytNRHaJyM0iskFEJpTguVgE7ASeKCxBQTUKIhLpbOvplmaoiKxyyrZERCJEpL+I/O48T187X6Lu53jc5bl83+21EBF5WET+cPJdLyKjCijLDSLyo4ikAXcUci31RWS683qkicgPItIp9xqAI07SH508BxTx3M0DggHXNKOA5cB2t/PWEJGXnWs8KSLLROQ8tzRDRGSzs38J0K6A8p8rIj+JSKqI7HXeT3ULK6CInO+cK0VEjonIchHpXEjaSSKyqoDtv4rIK87jLiKyUESSReS487peUOgzVLgxwHzgVeDSQt4To53XOt153qa57KvrXHui83zFich1zr4xIpLille+93BuGhG5WEQ2ABlARxHpJSILnP+9ZBH5n4j0ccurwHOLSB3nmKvd0l8kIpkiclYJnidVTBogqDIjIiHAdcB6Y0yay64TwC1AR+BvwPXAYy7HXQ885WzrDsQBD7hl/yD2g/A2IBb73h3hQbFGAlnAucA44D6njLmmY6u/BwKXYb+UWrhd1zRxCVaKkAOMB+4UkdYepD+Tp53y9gbqAx8DTwK3Y79IOwET3I7pD3TD/qK8ClurM9ll/0TgVmAsEAVMAt4WkUvc8pkE/NtJ80Uh5ZvmlO0y4BwgFZjvBCS/OuXDKUe4s60wmcAM7Psk1y3AewWk/Sf2NbwFiAHWO+cNBxCRs50yfw9EA685x+QRkS7AAmAu9vm60kk7taDCiUgA8CXwPyd9b+AVILuQ6/kA6C4uzW0i0hLoA8x0Nn0IJGKfuxjsa3mykPwKJCKCfR5mGmN2YwOqG93S3AG8DbwPdAUuBja6HD8P+765Gft6P4D9ki+OmsDj2GAyCtgFhGCfh37ONa4FvnUJLAo9tzHmBPAR+d8POOtfG2P+LGb5VEkYY3TRpUQL9gsiC0hxFgPsBjqf4bg7gW0u60uBt9zSLAB2uqwnAuNd1gXYDCx2K8/XLuuLgaVu+X4P/Md53N4pc6zL/rOxH/oTXLZNAhZ68Fx87TxeBMx2Hg9wztGwoHVnW6SzradbmsEuacY527q7bJsAbHArw1Eg2GXbKCAdqOMsaUA/t7K/DHzrVpb/O8P1tnXSne+yLRQ4BtzmrDd00gw4Q16LgdexAeQJoC7Q08mrdu5+J20d7JfXTS7H+wN/ABOd9eeArYC4pHncKUuksz4DeM+tHNFOmsYFvKYNnH39i/H/sQZ4xq0MW1zWk4HRpfwfvAA4BAQ567dgA3TXNAnA84UcfxE2sO1YyP4xQIrbtnzvYSeNAXqcoayC/T8e5eG5e2I/X5o56/Wd9++w0jxnuni+aA2CKq2fsR+s0dhfVT8CC5xfcQCIyNVO9eJ+p7ryJaC5Sx4dgN/c8l3ucnwo0MQ1jbGfGCs8KN86t/V9QGOX8+YAK13y3eOkwWXbo8aYCz04V66HgWvEaTIoBdey5/5iWu+2rTH5rTPGuFYJLwWCgNbYX2g1sb+2U3IX4C5nv6uVFK0j9rlbmrvB2D4G653zFJsxJg74HbgBW8sx2xiT6pasNRAI/OJyXLZTjtzzdgSWOe+RXEvJrwcwyu15yM3ztNofY8xhbMDwnYh8IyIPuL7HCzGT/LVcIzlVewDwL+A/TlPOY1Kyzr23Ap8YY3J/8c8BWotIbwARaQw0AxYWcnwMkOg896WRha0hyCMijUXkbRHZKiLHgOPY92vu/36R5zbGrMS+n0Y7m0Zgm63mlbKsykMaIKjSSjXGbHOW37AfWHWx1eCISCwwG/gOGI79UHgc+yHvypNpRUsy9WhmAXnkvu+FcmCMWYHtuT+5gN05BZy7sE6ArmU3Tt7u24rzP5ybdjingrpobFPAILe0J86QV1HPXWmmiJ2Kraa+gYKr+3PPW9A5jFuaovhh77BwfR66YWtG1hZ0gDHmZmwQ/DO28+RWERlcxDk+BCJFpI+IdMcGpLNc8pvAqSacc4F1IuJepV4osZ1ArwJuF3sbaRZwGNtROLez4pmeizPtzykgTUHv13QnUHM1HegF3I+9vmhsbUaQh+cG+xrd7Dy+BZhWwHlUOdEAQZU1g/1Qqe2s9wX2GmOeMcasMMbE49bGj20qOMdtW96688t0v+s2p/2yVynLGof9H+jhkm8E0LSU+QL8Hdv2OsRt+wHnb7jLtugyOF+uLiJSx2U9Flsl/wewCdvc0MIlqMtddhXzPJuwz11epzOng18XZ19JfYztUJhgjFlewP5t2OvJ65Qotsd8H5fzbgJ6O++RXLFu+awGOhXwPGwz+fvP5GOM+d0YM9kYMwDb9DG6iLSJ2Bq1kc7yqzFmu1uaeGPMq8aYS7D9LYpzF8JI7PupG/kDnduB60SkjrFt9XuxfVIKshoIF5GOhew/ANR267zp6fv1POA1Y8w3xpiN2BoE1/f9mc4NtsalmYiMw/ZPet/Dc6syoAGCKq0aItLEWTpiO4QFA185+7di/8FHikgrEbkL++vQ1SvAGBG5RUTaisjD2F9qxi3NwyJyhYi0B17EftiU+NeqMWYLtmbjLRGJFZFo7AdQqmu+YnukF1ZFW1je24B3OP3e/23Y2/YmiEg7ERmErVEpKwHAVBHpJPae+OeBd40xJ4wxx4EpwBTnuW4jItEicqeI3F6ckziB3pfYDo79nE5/M7Ht6h+WtPBOGZtx+hd67v4TwJvA806v+Y7O+lnYTpUAb2H7UrwsIu2dnvB3umU1GThHRN4SkRjnuRgmIm8XdF4RaSn2bpxzRaSF2LsNunLmYGgmtkPl9bg0L4hILRF5Q+wdAZFOk8B5rvmJvQtjXBF53wrMMcZscF2wv9xzONUZ91ngPhG533nPRYvI/zn7FmKb8z4TkcHOdV4kIpc7+5dja5MmOc/RVdiOxp7Yim3GiRKRXtiaRNfOj2c6d+6Pg0+x/+8/O+87VUE0QFCl9Rdsx6NE7D97L+AaY8xiAGPMV8AL2I5w67Adk550zcAYMxt4BvtltgbojP2Qd+3RPQXbI/p9YJmz7XOK2eu7AGOw1Z6LsT3aZwFJbvmGU0C7tAf+gW2bzeM0EVwPtMK2tz+NrW0oKz9he6gvwj4/P2L7ROR6Atu58UEn3ffYauodJTjXzdh+IXOdv7WBIUX9AveEseNZpBSR5BHgE+x7YS32i3qI84sdY3vzX4mtvfkdW8U93u0c64DzsYHET066SZzq6+EuFVuz8Sn2i2869r1SUDOSq8+wz0sjp8y5srGd7qYDW7Cv1VLy373THtvR8zROk0UMts9BPk5/hLk4tRHGmDexd638FdiAvSWyk7MvBxiK7X8xE1ur9gpOM4DT92Ik9v92PbZ2otDbeN3cgv2xsAobHEzF3gacW84iz+3iPWdbQXe0qHIk+fvxKOUbRORzIMAYM7yINKuBX4wxd5fheRtiOyneYIz5rKzyVUqVjNgxGd4GmhbQaVWVowBvF0ApEamN7Uk/H/uL+yrsvfVXuaRpAQzG/toLwP6S6eb8Lc25B2Lv116P7WH9LHDQKYtSykucz4VIbA3buxocVDxtYlC+wGCrGn/GNjFcB9xojHEdsjkHuAlblb0M20Y91LkVqjQCsYMHrcf2m0jD3tt/pl78Sqny9TC26ecwtglSVTBtYlBKKaXUabQGQSmllFKn0QBBKaWUUqep1p0UGzZsaCIjI71dDKWUUqrCrFq16qAx5ozT0lfrACEyMpKVK0vbx00ppZSqPETEo5FTtYlBKaWUUqfRAEEppZRSp9EAQSmllFKnqdZ9EAqSmZlJQkICJ0+Wdoh/VdZq1qxJREQEgYGFzY6slFKqrGiA4CYhIYGQkBAiIyPJP1us8iZjDIcOHSIhIYGWLVt6uzhKKVXlaRODm5MnTxIWFqbBgY8REcLCwrRmRymlKogGCAXQ4MA36euilFIVRwMEH+Tv7090dDSdO3dm+PDhHD16tET5DBgwgJ49e+atr1y5kgEDBhR5zM6dO/nwww/z1g8dOsQFF1xAcHAw48aNy5f2o48+okuXLnTt2pUhQ4Zw8ODBEpVTKaWU79EAwQfVqlWLtWvXsmHDBho0aMAbb7xR4rySkpKYN2+ex+ndA4SaNWvyzDPPMGXKlHzpsrKyuPfee1m0aBHr1q2ja9euvP766yUup1JKKd+iAYKP69OnD3v37s1bf+GFF+jVqxddu3blqaeeAuDEiRNccskldOvWjc6dO/Pxxx/npX/ooYeYOHHiaflmZ2fz0EMP5eX19ttvAzB+/HiWLFlCdHQ0L730EnXq1OG8886jZs2a+Y43xmCM4cSJExhjSE5OpmnTpuXxFCillPICvYuhCE9/tZFN+5LLNM+opnV5angnj9JmZ2ezcOFCbr31VgAWLFhAfHw8v/32G8YYLr30Un7++WcOHDhA06ZN+eabbwA4duxYXh59+vTh888/Z9GiRYSEhORtf++99wgNDWXFihWkp6fTt29fBg0axPPPP8+UKVP4+uuviyxbYGAgb775Jl26dKFOnTq0bdu2VDUdSimlfIvWIPigtLQ0oqOjCQsL4/Dhw1x00UWADRAWLFhATEwM3bt3Z/PmzcTHx9OlSxd++OEHHnnkEZYsWUJoaGi+/B5//PHTahEWLFjAjBkziI6Opnfv3hw6dIj4+HiPy5iZmcmbb77JmjVr2LdvH127dmXSpEmlv3ilVIntP7GfdQfWkWNyvF0UVQVoDUIRPP2lX9Zy+yAcO3aMYcOG8cYbb3DPPfdgjOHRRx/ljjvuOO2YVatW8e233/Loo48yaNAgnnzyybx9AwcO5IknnmDZsmV524wxvPbaawwePDhfPosXL/aojGvXrgWgdevWAFx77bU8//zzxb1UpVQJ5Zgcth3dxtqktaxOWs2aP9ew78Q+AJqHNGdExxFc3uZy6gTW8XJJVWWlAYIPCw0N5dVXX+Wyyy7jrrvuYvDgwTzxxBOMHDmS4OBg9u7dS2BgIFlZWTRo0IBRo0YRHBzMtGnTTsvrscce484776RVq1YADB48mDfffJOBAwcSGBjI1q1badasGSEhIRw/fvyMZWvWrBmbNm3iwIEDNGrUiO+//56OHTuW9VOglHKczDrJhoMbWJO0hjVJa1h7YC3HM+z/asNaDYlpHMOoqFGEBIXw6ZZPef6353ltzWtc0eYKRnQYwdl1z/byFajKRgMEHxcTE0O3bt2YPXs2N954I3FxcfTp0weA4OBgZs6cybZt23jooYfw8/PL6xvg7uKLL6ZRo1PTf992223s3LmT7t27Y4yhUaNGfPHFF3Tt2pWAgAC6devGmDFjuP/++4mMjCQ5OZmMjAy++OILFixYQFRUFE899RTnn38+gYGBtGjRosDARClVMkdOHskLBtYkrWHjoY1k5WQB0Cq0FYNaDKL7Wd2JaRRDREhEvnFCLm9zOesOrGNW3Cxmb57NrLhZ9I/oz8iokfRu0lvHFFEeEWOMt8vgNT179jQrV67Mty0uLk5/CfswfX1UVWSMYc/xPbapIGkNq/9czc7knQAE+gXSKawTMWfF0L1xd6IbRVOvZj2P805KTeLjLR8zZ+scDp88TJt6bRjZcSSXtLqEWgG1yumKlC8TkVXGmJ5nTKcBggYIlYm+PqoqyMzJZPOhzXm1A6uTVnP45GEA6gbVJaZxDNGNo+neuDudGnaihn+NUp8zPTudb7d/y6y4WWw5soXQGqFc1fYqbuhwA03qNCl1/hUlPTudNUlr2H50O03qNCEiJIKI4AhqB9b2dtEqDU8DBG1iUEqpcpaSkcLvB37PCwjWH1xPWlYaAM2Cm9G3aV9izoohplEMreq1wk/K/gazGv41uKLtFVze5nJW/bmKWXGzmLZxGtM3TufC5hcyKmoU0Y2ifa75IcfksPnwZpYlLmPZvmWsTlpNenb6aeka1GxAREgEZ4ecTURwRF7gEBESQePajcvlOa3qNEBQSqlysOPYDj7a/BFrktaw9chWckwOfuJH+/rtubLtlcQ0jiGmcQyNazeu0HKJCD2b9KRnk57sTdnL7M2z+Sz+MxbsWkBUWBSjOo5icORggvyDKrRcrhKOJ9iAIHEZyxOXczTdDjffpl4brml3DX2a9qF9/fYcSDtAwvEEElIS7N/jCaxNWsu8HfPy3eoZ6BdIs+BmpwcQWvtQJG1i0CaGSkVfH+XrUjNTeXvd28zYNIMACSC6cXReMNC1UVefvO0wNTOVr/74ilmbZ7Hj2A7CaoZxXfvruKb9NTSs1bDcz3/05FF+2/8bSxOXsmzfMhJSEgBoXKsxsU1jiQ23S6Pajc6Qk5WZk0liSmL+4MH5u+f4HlIyU/Klz619iAh2AogqXvugfRA8oAFC5aOvj/JVxhgW7FrACyte4M/UP7ms9WXc1+O+CvmCLSs5Joel+5YyK24WS/YuIdAvkCGRQxgZNZJOYWU3LkxuP4Kl+5ayLHEZcYfiMBjqBNahV5NexIbH0qdpH1rWbVnmTR7GGJIzkvOCBdfah4SUBBJPJBZa+9Cibgs6NOhAVFgUrUJbEeBXOSvhtQ+CUkpVkO3HtjNp+SSWJS6jff32vND/BWIax3i7WMXmJ370bdaXvs36svPYTj7c/CFfbPuCr7Z/RffG3RnRcQQXNr+w2F+Muf0IcgOCNUlrSM9OJ0AC6NqoK3dF30Wf8D50bti53L90RYTQGqGE1gilU8PTg57MnEz2p+xnT8qefIFDwvEEVv25Kq/vSE3/mrRv0J6osKi8pTIHDQXRGgQfrEHw9/enS5cuZGVl0bJlSz744APq1fP8tqZcAwYMICUlhdxrXLlyJQ8++GCRoyXu3LmTX3/9lREjRuStd+zYkfbt2wMQGxvLW2+9BdjRG8eMGUNaWhoXX3wxr7zySrl3cPKF10epXK7NCbX8azEuZhzXtr+2Sn1JHM84zufxn/Ph5g/Zm7KXJnWacH3767m63dWE1ggt9LiE4wl5TQa/7f8tXz+C3BqCnmf1rFTt/9k52exK3sXGQxvZdGgTmw5tYvPhzaRmpQI2aGjXoB2dwjr5dNCgNQiVWO5QywCjR4/mjTfe4LHHHitRXrnTPQ8dOtSj9LnTPecGCGCHU84tj6u77rqLd955h9jYWC6++GLmz5/v8XmUqsyqQnOCp0KCQrip002M7DiSnxJ+YlbcLF5e/TJv/f4Ww1oPY2SHkbSp34ajJ4+yfP/yvLsN8voR1G5M/4j+xDaNpXeT3h73I/BF/n7+tKrXilb1WjG89XDACRqO72LjwVNBw5fbvuSjzR8Bp4KGqAZRdGrYyWeDhoL4fgmruT59+rBu3bq89RdeeIFPPvmE9PR0rrjiCp5++mlOnDjBtddeS0JCAtnZ2TzxxBNcd911wKnpnt2/uLOzsxk/fjyLFy8mPT2dsWPHcscddzB+/Hji4uKIjo5m9OjRXHHFFQWWKzExkeTk5LxRHW+66Sa++OILDRBUlefanNChQQem9J9CdONobxer3Pn7+TOw+UAGNh/IlsNb+GjzR3z1x1fM2TqH5iHN2XN8DwZDcGAwvZr04saoG4ltGlsu/Qh8ib+fP61CW9EqtOigYe4fc5m9ZTZQeYIG3yqNr5k3HvavL9s8m3SBoZ5NauQL0z3v3LmTHTt2EBMTQ926dZk4cSL9+vVj7969RERE5OUXERHB3r17S/30KOWr3JsTHj3n0SrXnOCp9g3aM+HcCdzb/V4+i/+M1X+uZnjr4cSGx1ZIPwJfV1TQsOnQprzAobCgISrMBg7eDhqq96voo3Kne965cyc9evQocLpngJSUFOLj4+nXrx8PPvggjzzyCMOGDaNfv3758sud7nny5Ml52xYsWMC6deuYM2cOYIOK+Ph4goLy3/scHh7O7t27CQsLY9WqVVx++eVs3LiRgvquVOVfCVWRMYb5O+cze/NsgvyDqBtUl7o16tq/hTwOrRFKcGBwlbvtqygFNSfc3+N+wmqFebtoXle/Zn1u63IbdPF2SXyfa9AwrNUwwHbe3Jm884xBw6sXvOqV95sGCEXx8Jd+WfOl6Z5r1KhBjRp2mNcePXrQunVrtm7dSkREBAkJCXnpEhISaNq0aVlcvqoAB9MOMnHZRBbuXkjL0JaEBIWw/8R+kjOSSc5IzpsUqCCCEBIU4lFAkRtU5G6rbMFFdW1OUBXDT/yKDBo2HdpE/JF46tUofif1sqABgg/zhemeDxw4QIMGDfD392f79u3Ex8fTqlUrGjRoQEhICMuWLaN3797MmDGDu+++u6KeGlVCxhjm7ZjHc789R1pmGvf3uJ+bom7KV41pjCEtKy0vWEhOT+ZYxjGS05Pzbct7nJHscXDhJ34EBwYTViuM7o2706dpH3o36V2syYcqgjYnKG8pKGjwFn23+zhvT/fcvHlznnzySQICAvD39+ett96iQYMGALz55pt5tzkOHTpUOyj6ONdagy4NuzCx70Ra1Wt1WjoRoXZgbWoH1i72JD4FBRfJGckcSz+Wb1viiUS+2/kdn8V/hiB0aNAhb8S87o27UzOgZllddrHLr80JSlk6DoIPjoOgCqevT/G51xqMjRl7Wq2BN2TlZLHx0Ma8wXN+P/A7WTlZBPkFEXNWjL1XPrwPHRp0wN/Pv9zLs/3Ydp5b/hzLE5fToUEHHuv9mDYnqCrJJ4ZaFpEhwCuAP/AfY8zzbvtbAFOBRsBhYJQxJsFlf10gDvjcGDPO2bYYCAfSnGSDjDFJIlIDmAH0AA4B1xljdhZVPg0QKh99fYrH01oDX5CamcqqP1fZwXUSlxF/JB6A0BqhnNPknLzBdc4OObvMz/vWurf4YNMH1PKvxd3d7+badtdWSFCilDd4faAkEfEH3gAuAhKAFSIy1xizySXZFGCGMWa6iAwEJgE3uux/BvipgOxHGmNWum27FThijGkjItcDk4HryuhylKpUPOlr4GtqB9amX0Q/+kXYu3AOph1keaIdeGfpvqV8v+t7wE6PHBsemzfwTv2a9Ut0vtzmhH+u+CdJqUlc3uZy7ut+nzYnKOUoz0+Lc4BtxpjtACIyG7gMcA0QooD7nceLgC9yd4hID+AsYD5wxkjHyXuC83gO8LqIiKnObSiqxIwxZOZkenXK25KqTLUGRWlYqyGXtLqES1pdgjGGnck780bpK23/BffmhBf7v6jNCUq5Kc8AoRmwx2U9AejtluZ34CpsM8QVQIiIhAFHgBextQkXFpD3+yKSDXwGTHSCgLzzGWOyROQYEAYcdD1QRG4Hbgdo3rx5aa5PVQHGGA6dPMS2o9v44+gfp5Zjf5Ccnsx5zc7jmnbX0C+in0//+obKWWvgKRGhZWhLWoa25IYON+T1X1i2bxnLEpfxwaYPeH/D+2fsv5CvOSGgFn/v/XdtTlCqEOX5yVHQqDnuv+YfxP7SHwP8DOwFsoC/Ad8aY/YUMPjOSGPMXhEJwQYIN2L7HnhyPowx7wDvgO2D4PHVqEqtqEDgWPqpkSfrBtWlTb02DGoxiNoBtfl2x7fcs+geGtduzJVtr+TKNlcSHhzuxSsp2MG0gzyz9Bl+3PMjXRt25Zm+z1TKWgNPBfgF0K1RN7o16sYd3e7I67+wLNEGDK+sfoVXeIW6QXXpHd6b2PBYagbU5JXVr2hzglIeKs8AIQFw7U0UAexzTWCM2QdcCSAiwcBVxphjItIH6CcifwOCgSARSTHGjDfG7HWOPS4iH2KbMma4nC9BRAKAUGzHR1WNlCQQaF2vNa3rtaZNvTaE1QzLNyLkfT3u4+eEn/l066e8/fvbvP372z5Vq+Bea/BAjwe4KeqmaveL2NP+C9qcoJTnyvPTbQXQVkRaYmsGrgdGuCYQkYbAYWNMDvAo9o4GjDEjXdKMAXoaY8Y7X/z1jDEHRSQQGAb84CSdC4wGlgJXAz9W1v4HudM9Z2ZmEhAQwOjRo7nvvvvw8yv/EeimTZvGLbfcwtq1a+natSsAnTt35uuvvyYyMrLQ415++WVuv/12ate2U7c+9thjzJgxgyNHjpCSkpIv7SeffMKECRMQEbp168aHH34IwCOPPJI3n4TrhFMFKSwQ2HZ0G8kZyXnpPA0EChPgF5A3Qc2+lH18Fv8Zn8d/bmsVajXmirZXcFXbq7xSq1Ddag2Kw73/wq7kXexL2Ufv8N7VLnhSqqTKLUBw+gGMA77D3uY41RizUUT+Aaw0xswFBgCTRMRgmxjGniHbGsB3TnDgjw0O3nX2vQd8ICLbsDUH15f1NVUU1+mek5KSGDFiBMeOHePpp5+ukPNHRETw7LPP8vHHH3t8zMsvv8yoUaPyAoThw4czbtw42rZtmy9dfHw8kyZN4pdffqF+/fokJSUB8M0337B69WrWrl1Leno6/fv3Z+jDqftXAAAgAElEQVTQodStWxdjDKlZqZzMOsnR9KOMnje60EBgcOTgEgUCnmga3JS7Y+7mrm538XPCz8zZOod31r3DO+ve4bxm53F1u6s5P+L8cq9V0FqD4hERIkMjiQyN9HZRlKpUyvWTzBjzLfCt27YnXR7Pwd5xUFQe04BpzuMT2HEOCkp3ErimVAX2QY0bN+add96hV69eTJgwgZycnAKnaV68eDETJkygYcOGbNiwgR49ejBz5kxEhPHjxzN37lwCAgIYNGgQU6ZM4cCBA9x5553s3r0bsF/wffv2BWDYsGH8/PPPbNmyhfbt2+crz4IFC3jqqadIT0+ndevWvP/++0ydOpV9+/ZxwQUX0LBhQxYtWkRsbGyB1/Puu+8yduxY6tevn3d9AJs2baJ///4EBATkjeQ4f/58hl8xnMQTiaRl2WEvTmadBCjXQOBM3GsV/hv/Xz6P/5x7F92bV6twZdsraRpc9nNTaK2BUqqiVP7uzeVo8m+T2Xx4c5nm2aFBBx4555FiHdOqVStycnJISkriyy+/LHCaZoA1a9awceNGmjZtSt++ffnll1+Iiori888/Z/PmzYgIR48eBeDee+/l/vvv57zzzmP37t0MHjyYuLg4APz8/Hj44Yd57rnnmD59el45Dh48yMSJE/nhhx+oU6cOkydP5l//+hdPPvkk//rXv1i0aBENGzYs8lq2bt0KQN++fcnOzmbChAkMGTKEbt268fTTT/PAAw+QmprKokWLaNa6GZ2OdSLAL4CmwU0JDgrG74Af03tOL/IcFalpcFPGxYzjzm53lmutgtYaKKUqmgYIlURud4qipmk+55xziIiIAMibLjo2NpaaNWty2223cckllzBsmJ3844cffmDTplNDUiQnJ+ebpGnEiBE8++yz7NixI2/bsmXL2LRpU15NQ0ZGRt68EJ7KysoiPj6exYsXk5CQQL9+/diwYQODBg1ixYoVnHvuudQLq0enHp3IMBmE1QqjUa1GPv9FWJ61ClproJTyBg0QilDcX/rlZfv27fj7+9O4ceMip2nOnZYZbEfHrKwsAgIC+O2331i4cCGzZ8/m9ddf58cffyQnJ4elS5dSq1atAs8ZEBDA//3f/zF58uS8bcYYLrroIj766KMSX0tERASxsbEEBgbSsmVL2rdvT3x8PL169eKBhx/g+rHXczLrJI/e9SixXWOLPVmQLyiqVqFvs75c0+4aj2oVjDF8u+NbJv02SWsNlFIVrvJMzF5N5fYVGDduHCKSN01zZmYmYKvsT5w4UejxKSkpHDt2jIsvvpiXX345r/PjoEGDeP311/PS5W53NWbMGH744QcOHDgAQGxsLL/88gvbtm0DIDU1Na/JwH2a6MJcfvnlLFq0CLBNFlu3bqV5ZHP2HNvD6h2rycrJ4sjOI2yP286wod6d6rS0cmsV/v2XfzP/qvnc3vV2th7eyr2L7mXwnMG8tuY19qXsK/DYg2kHuW/RfYxfMp4WIS34dPin3Nz5Zg0OlFIVRmsQfFBaWhrR0dF5tzneeOONPPDAA0Dh0zQX5vjx41x22WWcPHkSYwwvvfQSAK+++ipjx46la9euZGVlcf755/PWW2/lOzYoKIh77rmHe++9F4BGjRoxbdo0brjhBtLT0wGYOHEi7dq14/bbb2fo0KGEh4ezaNEiHn74YT788ENSU1OJiIjgtttuY8KECQwePJgFCxYQFRWFv78/Tz37FEf8j5B6PJWbL72ZAL8A6taty8yZMwkIqDpvz4JqFd5d9y7vrnuXvs36cnW7q+kf0R9/8ddaA6WUT9DpnnU2R69IzUwl8UQiJ7NOUjuwNuF1wj0aQ78qvT77Uvbx+bbP+e/W/5KUlkSjWo2IDI1kxf4VdG3k9DUI1b4GSqmy5fXZHJUqSFZOFkmpSRw5eYQAvwCahTQjNCi0wm5R9CVNg5syNnosd3S9gyUJS5gTP4f1B9ZrrYFSyidogKAqhDGGI+lHSEpNIjsnu9LcnVARAvwCuKD5BVzQ/AJvF0UppfJogKDK3WnNCXU9a05QSinlPRogFMAYUy2rvMtaWTcnVOf+MkopVdE0QHBTs2ZNDh06RFhYxQ3dW9WUR3OCMYZDhw5Rs6bWPCilVEXQAMFNREQECQkJeff+q+LJyM7gWMYxMrMzCfIPIrRGKEf8jnCEI6XOu2bNmnkjRSqllCpfGiC4yR3hTxXPkZNHeGX1K/w3/r80rNWQB3s+yMCWA7UWRimlKikNEFSpZOdk81n8Z7yy+hVOZJ7gpqibuLPbnQQHBXu7aEoppUpBAwRVYusOrOPZ5c+y6dAmejXpxd/P+Ttt6rfxdrGUUkqVAQ0QVLHlNid8Fv8ZjWo1YnK/yQxtOVSbE5RSqgrRAEEVy/aj27lp/k2kZKQwOmq0NicopVQVpQGCKpZ31r9DRnYGnw7/lLb123q7OEoppcqJTvesPLb/xH7m75jPVW2v0uBAKaWqOA0QlMdmbpoJwI1RN3q5JEoppcqbBgjKI8czjjMnfg6DIgfRNLipt4ujlFKqnGmAoDwyZ+scTmSeYEynMd4uilJKqQqgAYI6o8zsTGbGzaR3k95EhUV5uzhKFZ8xkHHC26VQqlLRuxjUGc3bOY+k1CQm9Jng7aIo5bmUJNi++NSSvBfCo6H9UGg3BMK7gY7doVShNEBQRTLGMG3jNNrUa8N5zc7zdnGUKlx6Cuz69VRAkLTRbq9VH1r2h4YjYMfPsPh5WDwJQsKh3WBoNxRa9YfAWt4svVI+RwMEVaRf9/1K/JF4nun7jI6UqHxLdhbsW30qINjzG+Rkgn8NaNEHuk6AVgOgSVdwnWr8xEGIXwBb5sH6ObBqGgTUsmnbDba1C3XDK/56lPIxGiCoIk3bOI3GtRpzSctLvF0UVd0ZAwfjTwUEO5dAejIgtrmgz1j7Jd88tujagDoNIXqEXbLSYdcvsGU+bJ1nF3BpihhsH2twrKohDRBUoeIOxbEscRn3db+PQP9AbxdHVUfH/4QdP+XvRwBQPxI6X2kDgpb9oXaDkuUfUANaD7TL0MmQFAdb59tFmyKUt2Wm2fd9+6FeOb0GCKpQ0zdNp3ZAba5pf423i6Kqi3z9CBZB0ia7PbcfQauHbFDQoGXZn1sEzoqyS78HimiK6G+bIbQpQpWn3cvhy7/B4e1w9+ryec+fgQYIqkC5wyqP6DiCukF1vV0cVVW59iP4YxEk/AY5WS79CK6FVhc4/Qgq+K7sIpsi5ts04dE2UGg/RJsiVNnITIMfJ8LSNyD0bLjxc68EB6ABgirEB5s+AGBUx1FeLokqV0d2Op37siv2vCeP2jsKdv4vfz+Cc++2NQRn9/atqnz3pogDm23Nwtb58NNk+Ol5l6aIIba2I6i2t0utKpvcWoND26DnLXDRP6BGiNeKU64BgogMAV4B/IH/GGOed9vfApgKNAIOA6OMMQku++sCccDnxphxIlIb+BRoDWQDXxljxjtpxwAvAE4jJa8bY/5TjpdXZSVnJDNn6xwGRw7WYZWrmpxsSFhx6svtwGbvlaV+JHS+yulHcH7J+xFUNBFo3NEueU0R39uahfWf5W+KaH0h1GsOIU1sAFGnYf47KpSC02sNbvrS/l94WbkFCCLiD7wBXAQkACtEZK4xZpNLsinADGPMdBEZCEwCXGcCegb4yS3rKcaYRSISBCwUkaHGGKfrMR8bY8aVywVVI3O2ziE1K5XRnUZ7uyhlJycbsk5C5kn7N3fJW0+z1ciZzt+sNHtceAyEd4XK3EnzZDL8sdBWj8cvgLTD4BcALc6F7qOdjncV/Gs3sJb90qwK6jSE6BvskpUBu/53elNELvGD4LPsEhIOIc5f9/U6jTSQqC58rNbAVXnWIJwDbDPGbAcQkdnAZYBrgBAF3O88XgR8kbtDRHoAZwHzgZ4AxphUJx3GmAwRWQ1ElOM1VDuZ2ZnM2jTLt4ZVPr4f1n1iq6Ldv+TzvtDdv/Dd1nMyS37+gFoQ0dPePnd2LJzdC2qGlt31lYfDO071xt/5i73+WvWh7SBbDd76QqhVz9ulrHoCgvI3RSTvs+/f44l2SfnTefwnHEuwtTmpB0/PR/ygTmOn5sFZgpvkX9dAonLLSIVFz/pcrYGr8gwQmgF7XNYTgN5uaX4HrsI2Q1wBhIhIGHAEeBFbm3BhQZmLSD1guHNsrqtE5HxgK3C/MWZPQceqws3bOY+ktCSe7vu0t4tiv/x/fR3+9xJkOuPoB9SCwJoQ4LLkrgcFQ+2Ghe8PqGl/uQbUsPkE1Ch6PTsD9q60Ef7upbDkRTA5gMBZnW3AkLuEejlOzcm2fQlyg4LcpoOG7SH2LnubVMQ54K/djiqMCIQ2s0tRsjLgRJINGo4nQsp+l6DiTzi2F/aughMHCjhHbiDhUhNRPxKiLoOw1uVyWaoM7F4GX/wNDv8BPW+Fi572mVoDV+X5aVFQd17jtv4g8LrTf+BnbP+BLOBvwLfGmD0Fjd4nIgHAR8CruTUUwFfAR8aYdBG5E5gODCzg2NuB2wGaN29egsuqulyHVe7btK83C2JvK/thAiQnQIdhttqtQauK7yVe72zodIV9nH4cElbCHidgWPshrHjX7gs923asax4LzfvY9uny/mV38hj88WPhTQftBuuXRGUQEGQDzDMFmdmZdn6J3OAhL5BwlmSXQGLh0zYg7Ha9ff9Wlv4dVd1ptQZzbROfjxJj3L+zyyhjkT7ABGPMYGf9UQBjzKRC0gcDm40xESIyC+gH5ADBQBDwb5cOiVOBFGPMPYXk5Q8cNsYUWQ/cs2dPs3LlyhJdX1X0y95fuPOHO5nYdyKXtbnMO4XY8xvMf9T+cm/SFQY/By37eacsZ5KdBX9usL8Gdi+1f1P22301Qm1TRG7A0LR72fRqz2062DLP3naXk+XSdDAE2lzo+80fqnwl77NNcr/PhgNx4B9k3xvdboA2f7EBiap4PlRrICKrjDE9z5iuHAOEAGxV/4XYmoEVwAhjzEaXNA2xX+Q5IvIskG2MedItnzFAz9zOhyIyEegIXGOMyXFJF26MSXQeXwE8YoyJLaqMGiDkd9uC29hxdAfzr5pf8SMnHt1taww2fGarSS980n6gVab2VWPg6C6XgGG5/YAG+8s+PPpUk8TZsRDc6Mx55jUdzLM1BQe32O0N29t779sNhYhe2nSgTmcM7F9nA4X1n9qahVoNoMvVtmahaffKN26DMXDoDzuI1tFdNuBpcZ5vv//daw0ue93rtQZeDxCcQlwMvIy9zXGqMeZZEfkHsNIYM1dErsbeuWCwTQxjjTHpbnmMwQkQRCQC269hM5Cb7nVjzH9EZBJwKbaJ4jBwlzGmyHu4NEA4Je5QHNd+fS3397ifWzrfUnEnTj9u+xgsfcOun3s39L0PagRXXBnKU+ph+wW/Z5kNHPausn0bAMLa2EAht5YhrLX9wD55DLYttDUF8Qsg7YjTdND31PwADVp597pU5ZKdaZujfv8INn8L2enQsJ0NFLpca5vSfFXKAWe47UXwx2Lb7Agg/mCyoXYYdBxum1J8LVjwoVoDVz4RIPg6DRBOGb9kPIt2L+L7a76vmJETc7Jh7Sx772/Kn9DlGrjwKd/+oCoLWemwb+2pJok9y2wAALaDZYOWsG+NNh2o8pN2FDZ9aWsWdv8KCESeZ2vsoi71/hdYxgnYtdQGBNsX22Y8sO//lv1tT/9WA2ynzG0/wMbPYet3tiOzrwQLGan2s23Zv32m1sCVBgge0ADBSkxJZOh/hzKi4wge7vVw+Z9wx8/w3d9h/3pbPT7keXsbYXWUkwOH4k8FDIf+sEMMtxsKZ59TuZpYVOVzeIfTX+EjOLLD3r3TcbitWWg1oGLef9lZkLjWDrW9fbHtBJyTaftONI89FRCERxdenozUwoOFqMshsl/FBQs+WmvgSgMED2iAYL2w4gVmxc1i3pXzCA8ux8lnDv0BC56ALd/YqPovE+xIepWtHVSpqsYY2xS2brbtB3TymB13oes1tmbhrE5le67cfgTbF8OOJZB+zO5r0vVUQNC8T8k69uYGC5u+sP12KipY8PFaA1caIHhAAwQ7rPJFn17EgLMHMPn8yeVzkrQj8NML8Ns7doyB8+6HPmN9a6x9pZSVeRLiv7NNEPELbHNXky42UOh8tR1zobhSkmC767TdTj+C0ObQesCpabvrNCyzywDsWCrx35d/sFAJag1caYDgAQ0QYOqGqby06iU+GfYJHcM6lm3m2Zmw8n1YPMkGCTGjYOATJfuAUUpVvBMHbY3C7x/ZvjHib0eJ7HY9dLik8CA/44TLtN2LXfoR1LPzbrQaAK0vgPotK64GsTyCBddag3pnw6W+W2vgSgMED1T3ACEzO5Mhnw2hZb2W/GdQGc5rZYz9R1zwGBzcav/xBj9n5zRQSlVOB7bYWoV1H9tBmWrUtSM2drvB9pfZt/ZUQFBgP4IL7IydvtCvpiyChV1L4cuxttag123wl6crzd1XGiB4oLoHCF9u+5LHf3mcN//yJuc1O69sMv1zkw0M/vjR3oo3aCK0v1j7GShVVeTkwM4lNliImwsZKfY23Jwsuz+3H0HrC+xtvL4+7XVxg4WMVPjxGVj2ZqWqNXClAYIHqnOAYIzhyrlXAvDfS/9LQUNaF8uJg3YwkFXTbNtb/0eg11911DalqrKME7D5G9v8ENHLNh+UdT+CipSZdupuCNdgocMwe+ukfyDMvadS1hq48jRA8KERJVRF+mXfL2w7uo2JfSeWLjjISoflb8HPU+yHRa/boP94qBNWdoVVSvmmoDrQ9Vq7VAWBzm2eHYfnDxbWz4HV022aes19fg6FsqIBQjU1beM0GtdqzMUtLy5ZBsbY6sXvn4QjO+2APoMmQqP2ZVpOpZTyioKChaN7oPtNlbLWoCQ0QKiG4g7FsTxxOff3uL9kcy7sWwPfPWYnC2rUEUb91470p5RSVVFusFDNaIBQDU3bOI06gXW4pt01xT949zKYOsROH3vJv+y0wr409rlSSqkyoZ/s1UxiSiLf7fyOkR1HEhJUgoE8fnsXataFu1fZuQKUUkpVSX7eLoCqWB/EfQDAqI6jin9w2hGI+8rO/qbBgVJKVWkaIFQjyRnJfLb1M4a0HFKyORfWz7HTxHa/sewLp5RSyqdogFCNzNk6h9SsVMZ0GlOyDFbPsGOyh3cr03IppZTyPRogVBOZ2ZnM2jSL2PBYOjToUPwMEn+H/esg5qayL5xSSimfowFCNfHtjm9JSksqee3BmpngXwO6XF2m5VJKKeWbNECoBowxTNs4jbb123Ju03OLn0HmSVj3CXQcZm9vVEopVeVpgFAN5A6rPKbTmJINq7z5azh51E7XrJRSqlrQAKEamLZxGo1rN2Zo5NCSZbBmJoQ2h5YDyrRcSimlfJcGCFXcpkObWJ64nFEdR5VsWOWju+387jEjwU/fLkopVV3oJ34VN33jdOoE1uHqdiXsXLj2Q/s3ekTZFUoppZTP0wChCtuXso/vdn7H1W2vLtmwyjk5sGYWtBpgpzhVSilVbWiAUIXNjJuJIIyKKmHnwh0/wbHd2jlRKaWqIQ0QqqjcYZUHtxxMkzpNSpbJmg+gZj3oMKxsC6eUUsrnaYBQRX265dPSDaucehjivoau10JgzTItm1JKKd+nAUIVlJmdyay4UgyrDKcmZorRiZmUUqo60gChCvpmxzccSDvAzZ1uLnkmaz6AJl0hvGvZFUwppVSloQFCFWOMYfrG6bSt35Y+TfuULJPciZm668RMSilVXWmAUMWUelhl0ImZlFJKaYBQ1UzbUMphlfMmZhoOteqXbeGUUkpVGuUaIIjIEBHZIiLbRGR8AftbiMhCEVknIotFJMJtf10R2Ssir7ts6yEi6508XxXnZ7KINBCR70Uk3vlb7b7dNh3axPL9pRhWGXRiJqWUUkA5Bggi4g+8AQwFooAbRCTKLdkUYIYxpivwD2CS2/5ngJ/ctr0J3A60dZYhzvbxwEJjTFtgobNerUzbOK10wyqD7ZwY2hxa9i+7gimllKp0yrMG4RxgmzFmuzEmA5gNXOaWJgr7ZQ6wyHW/iPQAzgIWuGwLB+oaY5YaYwwwA7jc2X0ZMN15PN1le7WwL2UfC3YuKPmwygBHdsH2n3RiJqWUUuUaIDQD9risJzjbXP0OXOU8vgIIEZEwEfEDXgQeKiDPhELyPMsYkwjg/G1c6iuoREo9rDLoxExKKaXylGeAUFAXeuO2/iDQX0TWAP2BvUAW8DfgW2PMHrf0nuRZdKFEbheRlSKy8sCBA8U51GflDqs8pOWQkg+rnJMDa3ViJqWUUlZAOeadAJztsh4B7HNNYIzZB1wJICLBwFXGmGMi0gfoJyJ/A4KBIBFJAV5x8ikozz9FJNwYk+g0RSQVVChjzDvAOwA9e/YsVnDhq0o9rDLAjsVwbA9c9HRZFUsppVQlVp41CCuAtiLSUkSCgOuBua4JRKSh05wA8CgwFcAYM9IY09wYE4mtZZhhjBnvNB0cF5FY5+6Fm4AvnePnAqOdx6NdtldpGdkZzIqbRZ/wPrRv0L7kGa2ZaSdman9J2RVOKaVUpVVuAYIxJgsYB3wHxAGfGGM2isg/RORSJ9kAYIuIbMV2SHzWg6zvAv4DbAP+AOY5258HLhKReOAiZ73K+3bHtxxIO1C62oO8iZmu04mZlFJKASD2ZoDqqWfPnmblypXeLkaJGWO4cu6V+Ikfc4bPKfnIicvfgXkPwR1LdO4FpZSq4kRklTGm55nS6b1sldh7G94r/bDKAGtmQHg3DQ6UUkrl0QChEjLG8ObaN3ll9SsMbTmUi1teXPLMEn+H/et1WmellFL5lOddDKocGGN4efXLTN0wlcvbXM6EPhPw9/MveYarP9CJmZRSSp1GA4RKxBjD5BWTmRU3i+vaX8ffe/8dPylFJVDmSVivEzMppZQ6nQYIlUSOyWHisol8uvVTboy6kYd6PlS6fgfgTMx0DLpr84JSSqn8NECoBLJzsnny1yeZ+8dc/trlr9wdc3fpgwOwEzPVaw6R55c+L6WUUlWKdlL0cZk5mTy65FHm/jGXcdHjuKf7PWUTHBzZBdsXQ/QonZhJKaXUabQGwYdlZGfw8M8Ps3D3Qh7o8QA3d7657DJf+yEgOjGTUkqpAmmA4KPSs9O5f9H9LNm7hPHnjGdkx5Fll3lOtp2YqfUFUO/sM6dXSilV7Wjdsg9KzUxl7MKx/G/v/3iqz1NlGxwA7PjJTswUU4qpoZVSSlVpWoPgY1IyUhi7cCxrD6zl2fOeZXjr4WV/ktUf2NsaOwwr+7yVUkpVCRog+JDkjGTu+v4uNh3axOTzJzMkckjZnyT1sL29scfNEFCj7PNXSilVJWiA4COOnDzCHd/fwbaj23hxwIsMbD6wfE60/lPIztCxD5RSShVJAwQfcDDtIH9d8Ff2HN/DqwNf5bxm55XPiYyxzQvh3aBJl/I5h1JKqSpBOyl62Z8n/uTm+TezN2Uvb1z4RvkFB2AnZvpTJ2ZSSil1ZlqD4EX7UvZx63e3ciT9CG9f9DYxjWPK94RrdGImpZRSntEAwUt2J+/m1gW3ciLzBO9e9C5dGpVzlX9mmu1/EHWpTsyklFLqjDRA8ILtR7dz24LbyMrJYurgqXRo0KH8TxrnTMykYx8opZTygPZBqGBbDm/h5u9uxmAqLjgAnZhJKaVUsWiAUIE2HtrIrQtuJdAvkPcHv0+b+m0q5sRHdtrRE3ViJqWUUh4647eFiIwTEW20LqW1SWu57bvbCA4MZtqQaUSGRlbgyXViJqWUUsXjyc/JJsAKEflERIZImcw1XL2s2L+C27+/nbBaYUwbMo2IkIiKO3lONqzRiZmUUkoVzxkDBGPM40Bb4D1gDBAvIs+JSOtyLluV8OveX/nbD3+jaZ2mvD/4fZrUaVKxBdi+GJITdOwDpZRSxeJRg7QxxgD7nSULqA/MEZF/lmPZKr2f9vzEuB/H0aJuC6YOmUqj2o0qvhBrcidmuqTiz62UUqrS8qQPwj0isgr4J/AL0MUYcxfQA7iqnMtXaX2/63vuW3Qf7eq3473B79GgZoOKL0TqYdj8DXS9TidmUkopVSyejIPQELjSGLPLdaMxJkdEdL7gAnyz/Rse+99jdGnYhX//5d+EBIV4pyDrPrETM+nYB0oppYrJkyaGb4HDuSsiEiIivQGMMXHlVbDK6vP4z3l0yaN0P6s7b1/0tveCA2Ns80J4tE7MpJRSqtg8CRDeBFJc1k8425Sb2Ztn8+SvT9KnaR/euPANagfW9l5hEtfCnxu09kAppVSJeBIgiNNJEbBNC+gQzadZtHsRzy5/lgERA3ht4GvUCqjl3QKtmQkBNaHLNd4th1JKqUrJkwBhu9NRMdBZ7gW2l3fBKpu+zfryQI8H+NeAfxHkH+TdwmSmwbpPoeNwqFXPu2VRSilVKXkSINwJnAvsBRKA3sDt5VmoyijIP4ibO99MoH+gt4tiJ2ZKP6ZjHyillCqxMzYVGGOSgOsroCyqrKyZAfVaQGQ/b5dEKaVUJeXJOAg1RWSsiPxbRKbmLp5k7gzNvEVEtonI+AL2txCRhSKyTkQWi0iEy/ZVIrJWRDaKyJ3O9hBnW+5yUERedvaNEZEDLvtuK95TUUUc2Qk7fradE3ViJqWUUiXkyTfIB9j5GAYDPwERwPEzHSQi/sAbwFAgCrhBRKLckk0BZhhjugL/ACY52xOBc40x0dgmjfEi0tQYc9wYE527ALuA/7rk97HL/v94cG1Vz5pZgEC3G7xdEqWUUpWYJwFCG2PME8AJY8x04BLAkxvrzwG2GWO2G2MygNnAZW5pooCFzuNFufuNMRnGmHRne42CyikibYHGwBIPylI95GTbmRtbD9SJmZRSSpWKJwFCpvP3qIh0BkKBSA+OawbscVlPcLa5+p1TwzVfAYSISBiAiJwtIuucPCYbY/a5HcjnPksAACAASURBVHsDtsbAuGy7ymmumCMiBX5DisjtIrJSRFYeOHDAg8uoRLYvciZm0rEPlFJKlY4nAcI7IlIfeByYC2wCJntwXEHTQhu39QeB/iKyBuiPvVMiC8AYs8dpemgDjBaRs9yOvR74yGX9KyDSOeYHYHpBhTLGvGOM6WmM6dmokRcmTypPa2bqxExKKaXKRJF3MYiIH5BsjDkC/Ay0KkbeCYDrr/gIIF8tgFMrcKVzrmDgKmPMMfc0IrIR6AfMcdJ2AwKMMatc0h1yOexdPAtiqo7ciZl63qITMymllCq1ImsQnFETx5Uw7xVAWxFpKSJB2F/8c10TiEhDJwgBeBSY6myPEJFazuP6QF9gi8uhN5C/9gARCXdZvRSoXvNE6MRMSimlypAnQyZ/LyIPAh9j52EAwBhzuPBDwBiTJSLjgO8Af2CqMWajiPwDWGmMmQsMACaJiMHWUIx1Du8IvOhsF2CKMWa9S/bXAhe7nfIeEbkU20RxGBjjwbVVDToxk1JKqTIm+fv4FZBAZEcBm40xpjjNDT6pZ8+eZuXKld4uRuntWwPvDIBLXoRe1XP4B6WUUp4RkVXGmJ5nSufJSIoty6ZIqtys/sBOzNT5am+XRCmlVBVxxgBBRG4qaLsxZkbZF0cVW2YarJ8DHS/ViZmUUkqVGU/6IPRyeVwTuBBYDWiA4AvivnImZtLOiUoppcqOJ00Md7uui0godvhl5QvWfKATMymllCpzJZnNJxVoW9YFUSVw6A+dmEkppVS58KQPwlecGgHRDzt/wiflWSjloe+fhMDaEHOjt0uilFKqivGkD8IUl8dZwC5jTEI5lUd5Kv4H2Pw1XPgU1A0/c3qllFKqGDwJEHYDicaYkwAiUktEIo0xO8u1ZKpwWekw7yEIawN9SjrQpVJKKVU4TxquPwVyXNaznW3KW359FQ5vh6H/hIAgb5dGKaVUFeRJgBBgjMnIXXEe67eStxzdDT+/CFGXQZsLvV0apZRSVZQnAcIBZ44DAETkMuBg+RVJFWn+oyACg5/zdkmUUkpVYZ70QbiT/2/vzuOkKq/8j39OLzT7vssqi4Bxb1HcQJAEo9GoWSQGjDhjjFvGuMeZzPycKMkETeIyLkHcxiXuYRRFbVFAHaFZFdlBoAGh2ZeGXs/vj7qF1Rt0Q92u7qrv+/WqV9V96t5b5ypQp57nueeB583s4WA7D6iyuqKEbPn7305MbNUt0dGIiEgSq0mhpJXA6WbWnMjiTrvDD0sqKSmEd26Hdv00MVFEREJ3yCEGM7vPzFq7+x53321mbczs93URnMSITkz8viYmiohI+GoyB+F8d98R3XD37cD3wwtJKtm+5tuJiX2GJzoaERFJATVJENLNLCu6YWZNgKyD7C/xNvW3mpgoIiJ1qiaTFP8HyDGzp4Ltq4BnwgtJytHERBERSYCaTFL8LzNbCJwHGPAu0DPswAQo3g9TbtPERBERqXM16UEA+IZINcWfAKuB10KLSL716UOwfTWMeUMTE0VEpE5VmyCYWX/gcmA0sBX4O5HbHM+to9hS2/Y1MEMTE0VEJDEO1oOwBJgB/MDdVwCY2c11EpVoYqKIiCTUwe5iuIzI0MI0M/ubmY0gMgdBwhadmDj0dk1MFBGRhKg2QXD3N9z9p8AA4CPgZqCTmT1qZt+to/hST+zExNOvT3Q0IiKSog5ZB8Hd97r78+5+IdANmA/cGXpkqSo6MVEVE0VEJIFqUijpAHff5u6Pu7tmzYVh+xqYMQEG/VATE0VEJKFqlSBIyKb+FiwNvndvoiMREZEUpwShvlj2niYmiohIvaEEoT4o3v/tUs6amCgiIvVATSspSpgOVEx8UxMTRUSkXlAPQqKVm5ioIpUiIlI/KEFItHfvAktXxUQREalXQk0QzGyUmS01sxVmVql2gpn1NLMcM1toZh+ZWbeY9jlmNt/MFpnZtTHHfBScc37w6Bi0Z5nZ34PP+tzMeoV5bXGx7D1Y+nYwMfGoREcjIiJyQGgJgpmlA48A5wODgNFmNqjCbhOAZ939eOAeYHzQvhE4w91PBE4D7jSzrjHHXeHuJwaPzUHb1cB2d+8L/Bn4YygXFi/RiYnt+8Pp1yU6GhERkXLC7EEYDKxw91XuXgS8BFxcYZ9BQE7welr0fXcvcvfCoD2rhnFeDDwTvH4VGGFm9XftiE8fjExMPF8VE0VEpP4JM0E4ClgXs50XtMVaQGRRKIBLgBZm1g7AzLqb2cLgHH909w0xxz0VDC/8W0wScODz3L0E2Am0qxiUmV1jZrlmlpufn39kV3i4oks5H3uJJiaKiEi9FGaCUNWvd6+wfSsw1MzmAUOB9UAJgLuvC4Ye+gJXmlmn4Jgr3P044OzgMaYWn4e7P+Hu2e6e3aFDh9peU3xEJyZ+VxUTRUSkfgozQcgDusdsdwNiewFw9w3ufqm7nwTcHbTtrLgPsIhIMoC7rw+edwMvEBnKKPd5ZpYBtAK2xfeS4kATE0VEpAEIM0GYDfQzs95m1gi4HJgcu4OZtTezaAx3AZOC9m5m1iR43QY4E1hqZhlm1j5ozwQuBL4Mjp8MXBm8/hHwobtX6kFIKE1MFBGRBiK0SoruXmJmNwBTgXRgkrsvMrN7gFx3nwwMA8abmQPTgWid4YHA/UG7ARPc/QszawZMDZKDdOAD4G/BMU8Cz5nZCiI9B5eHdW2HLToxcew/NDFRRETqNatvP7LrUnZ2tufm5tbNh23/Gh45DY45H378dN18poiISAVmNsfdsw+1nyop1pV3f6uJiYfB3Zk0czWn3vsBD3+4nMKS0kSHJCKSEpQg1IVlUzUx8TDsLy7llpcXcM9bX9E8K4MJ7y1j1F9m8NHSzYc+WEREjogShLBpYuJhWb9jHz967FPemL+e34zsT85vhvLsuMEY8IunZvPL53LJ216Q6DBFRJKWEoSwffpgZP7B9/+kiYk19NnKrfzgoZms2VLAxLHZ3DSiH2lpxjn9O/DOv5zNbd87hunLtnDeAx9r2EFEJCRKEMK0/etvKyYePSzBwdR/0fkGP3/yc9o2a8Q/bjiTEQM7ldsnKyOd68/tywe3DOXcYzoy4b1lfO/P0zXsICISZ0oQwqSKiTUWO99gxICOvHHdGRzdoXm1+x/VugmP/vwUnh03mDQzfvHUbK55Npd12zTsICISD0oQwrJsKiydAsPu0MTEQ8jbXsBlj0bmG9wysj+P/fwUWjTOrNGx0WGH20cdw4zlWxj55495KGc5+4s17CAiciRUByGMOgjF++G/T4P0RnDtJ5p7cBCfrtzCDS/Mo7ikjL+OPpHhAzod+qBqrN+xj3vf/oopX3xDr3ZN+feLjuXcYzrGMVoRkYZPdRAS6ZO/amLiIbg7T85czZgnZx2Yb3AkyQFEhh3++4pvhx2u0rCDiMhhU4IQb9u/hpkPwLGXamJiNfYXl/Kblxfwn299xXkDO/Lm9WcedL5BbVUcdjjvAQ07iIjUlhKEeDswMfH3iY6kXorON3gzmG/w6BWn0Dwr/kuCZGWkc92wyN0OIwZ25P73lzHqL9OZprsdRERqRAlCPGli4kF9unILFz38CWu3FvDkldncGNQ3CFN02OG5qzXsICJSG5qkGK9JigcmJmbBtTM19yBGdL7B+HeWcHT7Zjw+5pS4DinUVGFJKU/OXM1DOSsoc+eGc/vyz+ccTePM9DqPRUQkUTRJsa6tmgbb12hiYgX7ikq5+e/z+f3bizlvYEfeiPN8g9qIDjvkxAw7fE/DDiIiVVIPQjxvc9y6Etr1id/5Gri87QX88rk5fLVxF7eM7M91w/qGPqRQGzOW5/PvkxexKn8vIwd14ncXDqJ726aJDktEJFQ17UFQghBGHQTh0xVbuP6FuZSUOX+9/MjqG4SpqKSMJ2eu5sGc5ZS5c/25fblGww4iksQ0xCAJ4e5MnLGKMZNm0b55FpNvOKveJgcAjTLS+NWwPuTcMpTzBnbigeiwwxINO4hIalOCIHFT1XyD3u2bJTqsGunaugmPXHEyz109mPQ046qnZ/NPz8xm9tfbSOVeNhFJXRpi0BBDXKzbVsC1/1N/5xvURnTY4b+nrWB3YQn9OjZn9OAeXHryUbRuqgmoItKwaQ5CDShBiI/Y+QYPXn4S5w5IjvUPCopKeGvBRp6ftZYF63bQKCONC47rws9O60F2zzaYNcwESERSmxKEGlCCcGSi9Q3um7KYPh2a88TY7AYzpFBbizbs5KVZ63hz3nr1KohIg6YEoQaUIBy+fUWl3PX6Qt6cv4FRx3Zmwk9OCKVkcn0T7VV4YdZa5qtXQUQaICUINaAE4fCs2xapb7D4m13c+t1juG5Yn5T8Yvxqwy5enLVWvQoi0qAoQagBJQi1tzJ/Dz969NOkm29wJAqKSnhr4UZe+Fy9CiJS/9U0QUj+PmGJq3vfXkxJmTP5hrOSdr5BbTVtlMFPsrvzk+zu5XoV3pi3Xr0KItJgqQdBPQg19smKLVwx8XPuOn8AvxyqktIHU12vwujBPTi1l3oV6sq6bQXMXLGFmSu2sGj9TjLS08jKSKNxZvpBn7MO8X5Vz40z08jKiGw31Ft8JTVoiKEGlCDUXGmZc+FDM9m1r5icW4aqFHEtVJyr0DfoVbhMvQpxt31vEZ+t2hpJCpZvYW2wrHfnlo05uWdrAAqLy9hfUlr1c3EphSVlFJaUHVEcjdLTyMpMo1mjDLq1aUKPdk3p0bb8o0OLLCWKkhBKEGpACULNvTonj1tfWcCDo0/iohO6JjqcBinaq/DirLXMW6tehXjYX1zKnDXbmbF8C5+s2MKXG3biDs2zMjj96Hac3a89Z/ZtT58OzWr137eszCkqLaOwuIzCklL2xzxXTCYO9rx7fwl52wtYt62Ajbv2E/vPbePMNLq3CRKGCglE97ZNlYRLaJQg1IAShJrZV1TKuRM+olOrxrx53Rn6IouDrzbs4qXZa3ljrnoVaqOszPlq464DCcHsr7dRWFJGRppxco82nBUkBCd0a0VGev2qJF9YUsr67ftYsy2SMKzdWsDabZHHum0F7C0qLbd/xxZZ5RKGnu3U+yDxoQShBpQg1MxDOcu5//1lvPzLIQzu3TbR4SSVir0KaRZZFyL2iyH2l2Xrppkp98WwdmtkHsEnK7bw6cotbC8oBuCYTi04q197zurbnsG929KsAdfhcHe27S0qlzCs2frt64P1PpT7M9KuKd3aNKFJZnrK/TmRmlOCUANKEA5t8+79DPvTR5zdrz2Pjznknyc5Aos37uLdL79hzda9wRfFPrbsKSy3T4usjHLd0bFfDl1bN6FRRv361Xw4tu8t4tOVWw8kBbHzCKIJwRl929GxReMER1p3or0P0YRh7YHHPtZu3Vup96FRehotm2TSqkkGrZpk0qpJJq2bNqJVk8ygvfpH48w0JRdJrl7c5mhmo4C/AunARHf/Q4X3ewKTgA7ANuDn7p4XtL8eHJcJPOTuj5lZU+AVoA9QCvyvu98ZnOsXwJ+A9cHpH3b3iWFeXyr4ywfLKSop487zByY6lKQ3sEtLBnZpWa6toKiEddv2ffuFECQPyzbtJmfJZopiJtOlGXRpFel96Nmu4fQ+7C8uJffr7cHdBvks2rAL90gydHqfdlx9Vu/DmkeQTLIy0jm6Q3OO7tC80nsVex827NjPjn1F7NpXzM7gkb+nkBX5e9hZUMzuwhIO9ruwquQimmBUl1y0bJJB44x0soI7OdJ1F0dSCK0HwczSgWXASCAPmA2MdvevYvZ5BXjL3Z8xs+HAVe4+xswaBbEVmllz4EvgDGAHcJq7Twv2yQHuc/d3ggQh291vqGmM6kE4uGWbdjPqL9MZO6QX/3HRsYkORyooK3M27y6M+TUZ7ZreW23vQ/e2lSfFdW3dmPS0uu152LmvmE9XRucRbKeogcwjSAalZc6e/SXs2Fd0IIGo+NhVYXtHQeR59/6SGn1GZrqVSxiyMtMObDfO+PaW0HLPmek0jrnFNHa70nNmGm2aNqJTy9TpRYqn+tCDMBhY4e6rgoBeAi4GvorZZxBwc/B6GvAmgLsXxeyTBaQF7QXBfrh7kZnNBbqFeA0pbfyUxTTLyuCmEf0SHYpUIS3N6NyqMZ1bNa5ybkjF3odo1/Tyzbv5cGn53odEGdC5BWNO75kU8wgaivQ0o1XTTFo1zaz1saVlzu79lROK3ftLDnlXR2HM9pY9JZXuDok+10a/js0ZMbAT5w3syEk92qjnIs7C/Nt4FLAuZjsPOK3CPguAy4gMQ1wCtDCzdu6+1cy6A28DfYHb3H1D7IFm1hr4QXBs1GVmdg6Rnoub3T3286UWZi7fwrSl+fz2+wNo20yz6huipo0yOKZzC47p3KLSe7G9Dxt37jtol3MYsjLSOKVXm5SaR5AM0tOM1k0bhXanjXvk9tJowlDVbabR5w079vHhks1MnLGKxz5eSZummZx7TEdGDOzEOf3b06Jx7RMgKS/MBKGqVK7iP0O3Ag8HwwPTicwfKAEIvtyPN7OuwJtm9qq7bwIwswzgReDBaA8F8L/Ai8GwxLXAM8DwSkGZXQNcA9CjR48ju8IkVVrm3DtlMd3aNGHskF6JDkdCENv7IFJfmFlQjTI6/ezg/unso9m5r5jpy/L5cMlmPly6mdfnrScz3Rjcuy0jBnTivIGd6NGuafjBJ6Ew5yAMAf7D3b8XbN8F4O7jq9m/ObDE3SsNGZjZU8Db7v5qsD0J2OPuN1VzrnRgm7u3OliMmoNQtVdy13HbqwtVFElEGpSS0jLmrt1BzpJN5CzezIrNe4BvhyJGDOzIyRqKSPxtjsGv/GXACCI9A7OBn7n7oph92hP5Ii8zs3uBUnf/nZl1A7a6+z4zawN8Dlzm7l+Y2e+BgcCP3b0s5lxd3H1j8PoS4A53P/1gMSpBqGxfUSnDJkyjc6smKookIg3amq17+WDxZnIWb2LW6m2UlHm5oYiz+7enZQoORSR8kqK7l5jZDcBUIrcrTnL3RWZ2D5Dr7pOBYcB4M3MiQwzXB4cPBO4P2g2YECQH3YC7gSXA3ODLK3o7401mdhGRIYptwC/CurZkNnHGKjbtKuThn52s5EBEGrSe7Zpx9Vm9ufqs3uzaHxmKyFn87VBERppx2tEaiqiOCiWpB+GAaFGkc/p14LExpyQ6HBGRUJSUljFv3Q4+WFx+KKJvx+aMGNiR8wZ2SuqhiIQPMTQEShDKu+v1L3gldx3v/2Yovds3S3Q4IiJ1Ys3WveQs3kzOkk18vqr8UMTwgR05p3+HpBqKSPgQgzQsyzbt5u+z1zJ2SC8lByKSUnq2a8a4s3oz7hBDEWf2bU/75llVVpNs2ij51r9QgiDAt0WRfq2iSCKSwlo2zuTC47ty4fFdKS1z5q7dfmAo4r/eXVrtcZnpRsvGmQfWu2jdtKqS1FWXqq6vyYUSBClXFKmNiiKJiACRwlCn9mrLqb3actf5A9m1v5idBVWXp65Yqnrb3iJWb9nLjoJidu0vPmgxskMlF78a1oemjer+61oJQoorLXN+//ZXKookInIILRtn0rJxJt1reVxZmbO7sKTSGheHSi6i278a1ieU6zkUJQgp7vW5eSz5ZjcPjT6JxpnpiQ5HRCTppKXZgd6Aw0kuEjX6oAQhhRUUlTDhvaWc2L01Fx7fJdHhiIhIBWkJvNVSa6mmsIkzVrNpVyF3XzCwXk6QERGRxFGCkKI2797PYx+vZNSxnTm1V+WlgkVEJLUpQUhRf35/OUUlZdx5/oBEhyIiIvWQEoQUtPSbSFGkMUN60ktFkUREpApKEFLQ+HcW0zwrg5uGqyiSiIhUTQlCipmxPJ+PluZz4/B+KookIiLVUoKQQkrLnHvfXhwpinRGz0SHIyIi9ZgShBTyWlAU6Y5RA8jKUFEkERGpnhKEFFFQVML9KookIiI1pAQhRUSLIv2riiKJiEgNKEFIAZt3RYoinf+dzmSrKJKIiNSAEoQU8OcPllFcWsYdo1QUSUREakYJQpKLFEVax5jTe6kokoiI1JgShCR335RIUaQbh/dNdCgiItKAKEFIYtOX5fPxMhVFEhGR2lOCkKRKy5z7piyme1sVRRIRkdpTgpCkVBRJRESOhBKEJFRQVMKEqZGiSBccp6JIIiJSe0oQktDfpq9m8+5C/u1CFUUSEZHDowQhyWzetZ/Hp6/k+8d15pSeKookIiKHRwlCknng/UhRpNu/p6JIIiJy+JQgJJEl3+zi5VwVRRIRkSOnBCGJjJ+yhOZZGdw0QkWRRETkyChBSBLRokg3jehH66YqiiQiIkdGCUISiC2KNGaIiiKJiMiRCzVBMLNRZrbUzFaY2Z1VvN/TzHLMbKGZfWRm3WLa55jZfDNbZGbXxhxzipl9EZzzQQvu4zOztmb2vpktD57bhHlt9clrc1QUSURE4iu0BMHM0oFHgPOBQcBoMxtUYbcJwLPufjxwDzA+aN8InOHuJwKnAXeaWdfgvUeBa4B+wWNU0H4nkOPu/YCcYDvpFRSVMOG9pZzUQ0WRREQkfsLsQRgMrHD3Ve5eBLwEXFxhn0FEvswBpkXfd/cidy8M2rOicZpZF6Clu3/m7g48C/ww2O9i4Jng9TMx7Untiemr2Ly7kH+9QEWRREQkfsJMEI4C1sVs5wVtsRYAlwWvLwFamFk7ADPrbmYLg3P80d03BMfnVXPOTu6+ESB47lhVUGZ2jZnlmllufn7+YV9cfbB+xz4e/3iViiKJiEjchZkgVPVz1its3woMNbN5wFBgPVAC4O7rgqGHvsCVZtaphuc8KHd/wt2z3T27Q4cOtTm0XtlRUMSVk2aRkWbcMUpFkUREJL4yQjx3HtA9ZrsbsCF2h6BX4FIAM2sOXObuOyvuY2aLgLOBT4LzVHXOTWbWxd03BkMRm+N5MfXJvqJSxj09m7XbCnjmqsH0bKeiSCIiEl9h9iDMBvqZWW8zawRcDkyO3cHM2ptZNIa7gElBezczaxK8bgOcCSwNhg52m9npwd0LY4F/BMdPBq4MXl8Z055UikvLuO75Ocxft4MHLz+RIX3aJTokERFJQqElCO5eAtwATAUWAy+7+yIzu8fMLgp2GwYsNbNlQCfg3qB9IPC5mS0APgYmuPsXwXu/AiYCK4CVwDtB+x+AkWa2HBgZbCeVsjLnjtcWMm1pPr//4XGM+o7uWhARkXBY5GaA1JSdne25ubmJDqPG7puymCemr+I3I/tz04h+iQ5HREQaIDOb4+7Zh9pPlRQbiCemr+SJ6asYO6QnNw7XWgsiIhIuJQgNwGtz8rhvyhIuOK4L//6DY1XvQEREQqcEoZ6btmQzt7+2kDP7tuOBn55AepqSAxERCZ8ShHpszprt/Or5OQzs0oLHx2RrnQUREakzShDqqeWbdjPu6dl0btmYp68aTPOsMEtWiIiIlKcEoR7asGMfYyfNolFGGs+OO432zbMSHZKIiKQYJQj1zPa9RYx58nP27C/hmasG06Nd00SHJCIiKUj91vVIQVEJVz09m3Xb9/HsuMEM6toy0SGJiEiKUg9CPREpoTyXhXmREsqnH60SyiIikjjqQagHysqcO15dyEdL87nvEpVQFhGRxFMPQj0w/p3FvD5vPb8Z2Z+fndYj0eGIiIgoQUi0xz9eyd9mrFYJZRERqVeUICTQq3PyGP/OEi44XiWURUSkflGCkCAfLtnEHdESyj9RCWUREalflCAkwJw127ju+bkqoSwiIvWWEoQ6tmzTbsY9nasSyiIiUq8pQahD63fsY+yTkRLKz12tEsoiIlJ/KUGoI9v2FjH2yc/ZWxgpody9rUooi4hI/aX+7TpQUFTCOJVQFhGRBkQ9CCErX0L5JJVQFhGRBkE9CCEqK3NuD0ooj7/0OEZ9p3OiQxIREakR9SCExN25b8pi3pi3nltG9mf0YJVQFhGRhkMJQkgen76KiTNXc+WQntygEsoiItLAKEEIwSu56/hDUEL5dyqhLCIiDZAShDjLWbyJO1//QiWURUSkQVOCEEdz1mzj+hfmMqhLS5VQFhGRBk0JQpxESyh3adWEp646VSWURUSkQVOCECcG9GrfjGfHDVYJZRERafD0MzdO+nVqwZvXnaEJiSIikhTUgxBHSg5ERCRZKEEQERGRSkJNEMxslJktNbMVZnZnFe/3NLMcM1toZh+ZWbeg/UQz+8zMFgXv/TTmmBlmNj94bDCzN4P2YWa2M+a934V5bSIiIskstDkIZpYOPAKMBPKA2WY22d2/itltAvCsuz9jZsOB8cAYoAAY6+7LzawrMMfMprr7Dnc/O+YzXgP+EXO+Ge5+YVjXJCIikirC7EEYDKxw91XuXgS8BFxcYZ9BQE7welr0fXdf5u7Lg9cbgM1Ah9gDzawFMBx4M7QrEBERSVFhJghHAetitvOCtlgLgMuC15cALcys3HrIZjYYaASsrHDsJUCOu++KaRtiZgvM7B0zO7aqoMzsGjPLNbPc/Pz82l2RiIhIiggzQahqSr9X2L4VGGpm84ChwHqg5MAJzLoAzwFXuXtZhWNHAy/GbM8Ferr7CcBDVNOz4O5PuHu2u2d36NChql1ERERSXpgJQh7QPWa7G7Ahdgd33+Dul7r7ScDdQdtOADNrCbwN/Ku7/1/scUEvw+Dg/ei5drn7nuD1FCDTzNrH/apERERSQJgJwmygn5n1NrNGwOXA5NgdzKy9mUVjuAuYFLQ3At4gMoHxlSrO/WPgLXffH3OuzhYUIgiGJdKArXG+JhERkZQQWoLg7iXADcBUYDHwsrsvMrN7zOyiYLdhwFIzWwZ0Au4N2n8CnAP8Iua2xRNjTn855YcXAH4EfGlmC4AHgcvdveKQhoiIiNSApfJ3aHZ2tufm5iY6DBERkTpjZnPcPftQ+6mSooiIiFSiBEFEREQqSekhBjPLB9bE8ZTtgS1xPF99petMLrrO5KLrTC5hXGdPdz/kff4pnSDElVEJAgAAB8FJREFUm5nl1mRcp6HTdSYXXWdy0XUml0Rep4YYREREpBIlCCIiIlKJEoT4eiLRAdQRXWdy0XUmF11ncknYdWoOgoiIiFSiHgQRERGpRAlCnJjZKDNbamYrzOzORMcTBjPrbmbTzGyxmS0ys18nOqYwmVm6mc0zs7cSHUtYzKy1mb1qZkuC/69DEh1TGMzs5uDP7Jdm9qKZNU50TPFgZpPMbLOZfRnT1tbM3jez5cFzm0TGGA/VXOefgj+3C83sDTNrncgY46Gq64x571Yz87pchFAJQhyYWTrwCHA+MAgYbWaDEhtVKEqAW9x9IHA6cH2SXmfUr4msI5LM/gq86+4DgBNIwus1s6OAm4Bsd/8OkE5kPZdk8DQwqkLbnUCOu/cDcoLthu5pKl/n+8B33P14YBmRBf8auqepfJ2YWXdgJLC2LoNRghAfg4EV7r7K3YuAl4CLExxT3Ln7RnefG7zeTeTL5KjERhUOM+sGXABMTHQsYQmWVD8HeBLA3YvcfUdiowpNBtDEzDKAplRYer6hcvfpwLYKzRcDzwSvnwF+WKdBhaCq63T394JFAQH+D+hW54HFWTX/PwH+DNwO1OmkQSUI8XEUsC5mO48k/eKMMrNewEnA54mNJDR/IfIXsizRgYToaCAfeCoYSploZs0SHVS8uft6YAKRX18bgZ3u/l5iowpVJ3ffCJGkHuiY4HjqwjjgnUQHEYZg9eP17r6grj9bCUJ8WBVtSXt7iJk1B14D/sXddyU6nngzswuBze4+J9GxhCwDOBl41N1PAvaSHN3R5QRj8BcDvYGuQDMz+3lio5J4MbO7iQx/Pp/oWOLNzJoCdwO/S8TnK0GIjzyge8x2N5KkC7MiM8skkhw87+6vJzqekJwJXGRmXxMZLhpuZv+T2JBCkQfkuXu0F+hVIglDsjkPWO3u+e5eDLwOnJHgmMK0ycy6AATPmxMcT2jM7ErgQuAKT8579vsQSWwXBP8edQPmmlnnuvhwJQjxMRvoZ2a9zawRkQlQkxMcU9yZmREZr17s7g8kOp6wuPtd7t7N3XsR+X/5obsn3S9Od/8GWGdmxwRNI4CvEhhSWNYCp5tZ0+DP8AiScDJmjMnAlcHrK4F/JDCW0JjZKOAO4CJ3L0h0PGFw9y/cvaO79wr+PcoDTg7+7oZOCUIcBBNlbgCmEvmH52V3X5TYqEJxJjCGyC/q+cHj+4kOSo7IjcDzZrYQOBG4L8HxxF3QQ/IqMBf4gsi/e0lRhc/MXgQ+A44xszwzuxr4AzDSzJYTmfn+h0TGGA/VXOfDQAvg/eDfoscSGmQcVHOdiYsnOXtlRERE5EioB0FEREQqUYIgIiIilShBEBERkUqUIIiIiEglShBERESkEiUIIkkgWOXt/pjtW83sP+J07qfN7EfxONchPufHwYqS0yq09wqu7z9j2tqbWbGZPVzLz9gTj31EUoESBJHkUAhcWpdLwdZEsNJpTV0NXOfu51bx3ioiFfOifgwkY60RkXpDCYJIcighUvzn5opvVOwBiP5CNrNhZvaxmb1sZsvM7A9mdoWZzTKzL8ysT8xpzjOzGcF+FwbHp5vZn8xstpktNLNfxpx3mpm9QKQwUcV4Rgfn/9LM/hi0/Q44C3jMzP5UxfXtAxabWXaw/VPg5Zhz9jSznCCOHDPrEbT3NrPPghj/M/aEZnZbTOz/r4o4u5jZ9KAIz5dmdnYVcYkkLSUIIsnjEeAKM2tVi2NOAH4NHEekSmZ/dx9MZJnrG2P26wUMJbIE9mNm1pjIL/6d7n4qcCrwz2bWO9h/MHC3uw+K/TAz6wr8ERhOpHLjqWb2Q3e/B8glUlP/tmpifQm4PFiKu5Ty6508DDzr7scTWbTnwaD9r0QWozoVOFCe1sy+C/QL4jwROMXMzqnweT8Dprr7icF/p/nVxCWSlJQgiCSJYGXNZ4GbanHYbHff6O6FwEogugzyF0SSgqiX3b3M3ZcT6e4fAHwXGGtm84ks+92OyJcuwCx3X13F550KfBQsnBRdga/iF3N13iVSOng08PcK7w0BXgheP0ekNwIi5cFfjGmP+m7wmEekBPOAmNijZgNXBXM5jnP33TWMUyQpKEEQSS5/IfLLvllMWwnB3/VgsaJGMe8Vxrwui9kuI7IcdFTFmuxOZJnzG939xODR292jCcbeauKramn0GnH3ImAOcAuRFUUPuns1r2PjGB8Te193f7LC500nkrysB54zs7GHG7tIQ6QEQSSJuPs2ImPzsYu8fA2cEry+GMg8jFP/2MzSgnkJRwNLiSxO9qtgCXDMrL+ZNTvYSYj0NAwN7kJIJ9Ib8HEt4rgfuMPdt1Zo/5TIypsAVwAzg9efVGiPmgqMM7PmQexHmVnH2BOaWU9gs7v/jcgqpsm4FLZItTIOvYuINDD3E1ldNOpvwD/MbBaQQ/W/7g9mKZEv8k7Ate6+38wmEhmGmBv0TOQDPzzYSdx9o5ndBUwj8it+irvXeDniYJXUqu5euAmYZGa3BXFcFbT/GnjBzH5NTK+Du79nZgOBzyKhswf4ObA55pzDgNvMrDh4Xz0IklK0mqOIiIhUoiEGERERqUQJgoiIiFSiBEFEREQqUYIgIiIilShBEBERkUqUIIiIiEglShBERESkEiUIIiIiUsn/B5kncmEz5hLcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(8, 6))\n",
    "\n",
    "x = list(range(15))\n",
    "plt.plot(x, int_acc_list_r18)\n",
    "plt.plot(x, int_acc_list_r50)\n",
    "plt.plot(x, int_acc_list_dn169)\n",
    "plt.title('Bagging: Number of Models vs. Accuracy', fontsize=14)\n",
    "plt.xlabel('Number of Models')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend(['ResNet18', 'ResNet50', 'DenseNet169'], loc='upper left')\n",
    "plt.show()\n",
    "\n",
    "fig.savefig('/media/rene/Data/camelyon_out/figures/bagging_num_models.jpeg', bbox_inches='tight', pad_inches=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'acc_r18' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-16e0b92c2a84>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0macc_r18\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0macc_r50\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0macc_dn169\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'acc_r18' is not defined"
     ]
    }
   ],
   "source": [
    "print(acc_r18, acc_r50, acc_dn169)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Add in above graph, mention how don't see much improvement from bagging after a few models, and how ResNet18 was really affected by model variance.\n",
    "* Need a bar graph comparing a bunch of model's perfromances. Show how bagging loses to just averaging models.\n",
    "* Need a graph comparing the different resnet sizes and their perfromance on average and after bagging. Show somehting about how model size is related to how much bagging helps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.01"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}