demo_resnet_epoch532_lr0.001_dropout0.25_auc0.8998.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Run ADBench \n",
    "- Here we provide a demo for testing AD algorithms on the datasets proposed in ADBench.\n",
    "- Feel free to evaluate any customized algorithm in ADBench.\n",
    "- For reproducing the complete experiment results in ADBench, please run the code in the run.py file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\autosoftware\\anconda\\lib\\site-packages\\pandas\\compat\\_optional.py:138: UserWarning: Pandas requires version '2.7.0' or newer of 'numexpr' (version '2.6.9' currently installed).\n",
      "  warnings.warn(msg, UserWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from glob import glob\n",
    "import pickle\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 加载训练集的pkl文件，划分训练集与验证集\n",
    "训练集的label存放在pkl里面，可以通过它并区分正常片段和异常片段\n",
    "注意需要输入训练集对应的路径"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 28389/28389 [01:37<00:00, 292.52it/s]\n"
     ]
    }
   ],
   "source": [
    "ind_pkl_files = []#存放标签为0的文件\n",
    "ood_pkl_files = []#存放标签为1的文件\n",
    "data_path='Train'#存放数据的路径\n",
    "pkl_files = glob(data_path+'/*.pkl')\n",
    "for each_path in tqdm(pkl_files):\n",
    "    pic = open(each_path,'rb')\n",
    "    this_pkl_file= pickle.load(pic)#下载pkl文件\n",
    "    if this_pkl_file[1]['label'] == '00':\n",
    "        ind_pkl_files.append(each_path)\n",
    "    else:\n",
    "        ood_pkl_files.append(each_path)\n",
    "\n",
    "random.seed(0)\n",
    "#排序并打乱存放车辆序号的集合\n",
    "random.shuffle(ind_pkl_files)\n",
    "random.shuffle(ood_pkl_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23735\n",
      "4654\n"
     ]
    }
   ],
   "source": [
    "print(len(ind_pkl_files))\n",
    "print(len(ood_pkl_files))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22456\n",
      "7096\n"
     ]
    }
   ],
   "source": [
    "train_pkl_files=[]\n",
    "for j in range(len(ind_pkl_files)//4,len(ind_pkl_files)):\n",
    "#for j in range(len(ind_pkl_files)):\n",
    "    train_pkl_files.append(ind_pkl_files[j])\n",
    "#for i in range(len(ind_pkl_files)//4):\n",
    "#    train_pkl_files.append(ind_pkl_files[i])\n",
    "for j in range(len(ood_pkl_files)):\n",
    "    train_pkl_files.append(ood_pkl_files[j])\n",
    "print(len(train_pkl_files))\n",
    "test_pkl_files=[]\n",
    "#for j in range(len(ind_pkl_files)//4,len(ind_pkl_files)):\n",
    " #   test_pkl_files.append(ind_pkl_files[j])\n",
    "for i in range(len(ind_pkl_files)//4):\n",
    "    test_pkl_files.append(ind_pkl_files[i])\n",
    "#for item in ood_pkl_files:\n",
    "#    test_pkl_files.append(item)\n",
    "\n",
    "for j in range(len(ood_pkl_files)//4):\n",
    "    test_pkl_files.append(ood_pkl_files[j])\n",
    "print(len(test_pkl_files))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 定义加载函数，并对数据进行正则化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def  load_data(pkl_list,label=True):\n",
    "    '''\n",
    "    输入pkl的列表，进行文件加载\n",
    "    label=True用来加载训练集\n",
    "    label=False用来加载真正的测试集，真正的测试集无标签\n",
    "    '''\n",
    "    X = []\n",
    "    y = []\n",
    "    \n",
    "\n",
    "    for  each_pkl in pkl_list:\n",
    "        pic = open(each_pkl,'rb')\n",
    "        item= pickle.load(pic)#下载pkl文件\n",
    "        # 此处选取的是每个滑窗的最后一条数据，仅供参考，可以选择其他的方法，比如均值或者其他处理时序数据的网络\n",
    "        # 此处选取了前7个特征，可以需求选取特征数量\n",
    "        feature = item[0][:,0:7][-1]\n",
    "        #feature = item[0][:,0:7][-1]\n",
    "        #feature = item[0][:,0:7].mean(axis=0)\n",
    "        #feature = np.append(item[0][:,0:7][-1],(item[0][:,3][-1] - item[0][:,4][-1])) #加max_single_volt - min_single_volt 一列为特征\n",
    "        feature=np.append(feature,item[1][\"mileage\"])\n",
    "        X.append(feature)\n",
    "        if label:\n",
    "            y.append(int(item[1]['label'][0]))\n",
    "    X = np.vstack(X)\n",
    "    if label:\n",
    "        y = np.vstack(y)\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,y_train=load_data(train_pkl_files)\n",
    "X_test,y_test=load_data(test_pkl_files)\n",
    "_mean = np.mean(X_train, axis=0)\n",
    "_std = np.std(X_train, axis=0)\n",
    "X_train = (X_train - _mean) / (_std + 1e-4)\n",
    "X_test = (X_test - _mean) / (_std + 1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.utils import shuffle\n",
    "# 进行随机打乱，这里random_state指定为固定值，则打乱结果相同\n",
    "X_train,y_train = shuffle(X_train,y_train,random_state=40)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path3='Test_A'\n",
    "test1_files = glob(data_path3+'/*.pkl')\n",
    "X_val,_=load_data(test1_files,label=False)\n",
    "_mean = np.mean(X_val, axis=0)\n",
    "_std = np.std(X_val, axis=0)\n",
    "X_val = (X_val - _mean) / (_std + 1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.metrics\n",
    "from sklearn.model_selection import train_test_split\n",
    "import sklearn.preprocessing\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "# import zero\n",
    "import delu # the new version of zero package\n",
    "import rtdl\n",
    "import scipy\n",
    "import platform\n",
    "from myutils import Utils\n",
    "utils = Utils() # utils function\n",
    "\n",
    "\n",
    "class FTTransformer():\n",
    "    '''\n",
    "    The original code: https://yura52.github.io/rtdl/stable/index.html\n",
    "    The original paper: \"Revisiting Deep Learning Models for Tabular Data\", NIPS 2019\n",
    "    '''\n",
    "    def __init__(self, seed:int, model_name:str, n_epochs=100, batch_size=64):\n",
    "\n",
    "        self.seed = seed\n",
    "        self.model_name = model_name\n",
    "        self.utils = Utils()\n",
    "\n",
    "        # device\n",
    "        if model_name == 'FTTransformer':\n",
    "            self.device = self.utils.get_device(gpu_specific=True)\n",
    "        else:\n",
    "            self.device = self.utils.get_device(gpu_specific=False)\n",
    "\n",
    "        # Docs: https://yura52.github.io/zero/0.0.4/reference/api/zero.improve_reproducibility.html\n",
    "        # zero.improve_reproducibility(seed=self.seed)\n",
    "        delu.improve_reproducibility(base_seed=int(self.seed))\n",
    "\n",
    "        # hyper-parameter\n",
    "        self.n_epochs = n_epochs # default is 1000\n",
    "        self.batch_size = batch_size # default is 256\n",
    "\n",
    "    def apply_model(self, x_num, x_cat=None):\n",
    "        if isinstance(self.model, rtdl.FTTransformer):\n",
    "            return self.model(x_num, x_cat)\n",
    "        elif isinstance(self.model, (rtdl.MLP, rtdl.ResNet)):\n",
    "            assert x_cat is None\n",
    "            return self.model(x_num)\n",
    "        else:\n",
    "            raise NotImplementedError(\n",
    "                f'Looks like you are using a custom model: {type(self.model)}.'\n",
    "                ' Then you have to implement this branch first.'\n",
    "            )\n",
    "\n",
    "    @torch.no_grad()\n",
    "    def evaluate(self, X, y=None):\n",
    "        self.model.eval()\n",
    "        score = []\n",
    "        # for batch in delu.iter_batches(X[part], 1024):\n",
    "        for batch in delu.iter_batches(X, self.batch_size):\n",
    "            score.append(self.apply_model(batch))\n",
    "        score = torch.cat(score).squeeze(1).cpu().numpy()\n",
    "        score = scipy.special.expit(score)\n",
    "\n",
    "        # calculate the metric\n",
    "        if y is not None:\n",
    "            target = y.cpu().numpy()\n",
    "            metric = self.utils.metric(y_true=target, y_score=score)\n",
    "        else:\n",
    "            metric = {'aucroc': None, 'aucpr': None}\n",
    "\n",
    "        return score, metric['aucpr']\n",
    "\n",
    "    def fit(self, X_train, y_train, ratio=None,X_test=X_test,y_test=y_test):\n",
    "        # set seed\n",
    "        self.utils.set_seed(self.seed)\n",
    "       \n",
    "        #X_train, X_test_val, y_train, y_test_val = train_test_split(X_train, y_train, test_size=0.33, random_state=42)\n",
    "        # training set is used as the validation set in the anomaly detection task\n",
    "        X = {'train': torch.from_numpy(X_train).float().to(self.device),\n",
    "             'val': torch.from_numpy(X_train).float().to(self.device)}\n",
    "\n",
    "        y = {'train': torch.from_numpy(y_train).float().to(self.device),\n",
    "             'val': torch.from_numpy(y_train).float().to(self.device)}\n",
    "        \n",
    "       #training set is used as the validation set in the anomaly detection task\n",
    "#         X = {'train': torch.from_numpy(X_train).float().to(self.device),\n",
    "#             'val': torch.from_numpy(X_test_val).float().to(self.device)}\n",
    "\n",
    "#         y = {'train': torch.from_numpy(y_train).float().to(self.device),\n",
    "#             'val': torch.from_numpy(y_test_val).float().to(self.device)}\n",
    "\n",
    "        task_type = 'binclass'\n",
    "        n_classes = None\n",
    "        d_out = n_classes or 1\n",
    "\n",
    "\n",
    "        if self.model_name == 'ResNet':\n",
    "            self.model = rtdl.ResNet.make_baseline(\n",
    "                d_in=X_train.shape[1],\n",
    "                d_main=128,\n",
    "                d_hidden=256,\n",
    "                dropout_first=0.25,\n",
    "                dropout_second=0,\n",
    "                n_blocks=2,\n",
    "                d_out=d_out,\n",
    "            )\n",
    "            lr = 0.001\n",
    "            weight_decay = 0.0\n",
    "        \n",
    "        elif self.model_name == 'MLP':\n",
    "            self.model = rtdl.MLP.make_baseline(\n",
    "            d_in=X_train.shape[1],\n",
    "            d_layers= [128, 256, 128],\n",
    "            dropout=0.25,\n",
    "            d_out=d_out,\n",
    "            )\n",
    "            lr = 0.001\n",
    "            weight_decay = 0.0\n",
    "\n",
    "        elif self.model_name == 'FTTransformer':\n",
    "            self.model = rtdl.FTTransformer.make_default(\n",
    "                n_num_features=X_train.shape[1],\n",
    "                cat_cardinalities=None,\n",
    "                last_layer_query_idx=[-1],  # it makes the model faster and does NOT affect its output\n",
    "                d_out=d_out,\n",
    "            )\n",
    "            \n",
    "        elif self.model_name == 'FTTransformer_baseline':\n",
    "            self.model = rtdl.FTTransformer.make_baseline(\n",
    "                n_num_features=X_train.shape[1],\n",
    "                cat_cardinalities=None,\n",
    "                d_token=X_train.shape[1],\n",
    "                n_blocks=2,\n",
    "                attention_dropout=0.2,\n",
    "                ffn_d_hidden=6,\n",
    "                ffn_dropout=0.2,\n",
    "                residual_dropout=0.0,\n",
    "                d_out=d_out,\n",
    "            ) \n",
    "        else:\n",
    "            raise NotImplementedError\n",
    "\n",
    "        self.model.to(self.device)\n",
    "        optimizer = (\n",
    "            self.model.make_default_optimizer()\n",
    "            if isinstance(self.model, rtdl.FTTransformer)\n",
    "            else torch.optim.AdamW(self.model.parameters(), lr=lr, weight_decay=weight_decay)\n",
    "        )\n",
    "        loss_fn = (\n",
    "            F.binary_cross_entropy_with_logits\n",
    "            if task_type == 'binclass'\n",
    "            else F.cross_entropy\n",
    "            if task_type == 'multiclass'\n",
    "            else F.mse_loss\n",
    "        )\n",
    "\n",
    "        # Create a dataloader for batches of indices\n",
    "        # Docs: https://yura52.github.io/zero/reference/api/zero.data.IndexLoader.html\n",
    "        train_loader = delu.data.IndexLoader(len(X['train']), self.batch_size, device=self.device)\n",
    "\n",
    "        # Create a progress tracker for early stopping\n",
    "        # Docs: https://yura52.github.io/zero/reference/api/zero.ProgressTracker.html\n",
    "        progress = delu.ProgressTracker(patience=100)\n",
    "\n",
    "        # training\n",
    "        # report_frequency = len(X['train']) // self.batch_size // 5\n",
    "\n",
    "        for epoch in range(1, self.n_epochs + 1):\n",
    "            loss_tmp = []\n",
    "            for iteration, batch_idx in enumerate(train_loader):\n",
    "                self.model.train()\n",
    "                optimizer.zero_grad()\n",
    "                x_batch = X['train'][batch_idx]\n",
    "                y_batch = y['train'][batch_idx]\n",
    "                loss = loss_fn(self.apply_model(x_batch).squeeze(1), y_batch)\n",
    "                loss_tmp.append(loss.item())\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "                # if iteration % report_frequency == 0:\n",
    "                #     print(f'(epoch) {epoch} (batch) {iteration} (loss) {loss.item():.4f}')\n",
    "\n",
    "            loss_.append(sum(loss_tmp)/len(loss_tmp))\n",
    "            _, val_metric = self.evaluate(X=X['val'], y=y['val'])\n",
    "            print(f'Epoch {epoch:03d} | Validation metric: {val_metric:.4f}', end='')\n",
    "            progress.update((-1 if task_type == 'regression' else 1) * val_metric)\n",
    "            if progress.success:\n",
    "                print(' <<< BEST VALIDATION EPOCH', end='')\n",
    "            print()\n",
    "            # 验证\n",
    "            # output predicted anomaly score on testing set\n",
    "            score = self.predict_score(X_test)\n",
    "            # evaluation\n",
    "            result = utils.metric(y_true=y_test, y_score=score)\n",
    "            aucroc.append(result['aucroc'])\n",
    "            aucpr.append(result['aucpr'])\n",
    "            if progress.fail:\n",
    "                break\n",
    "\n",
    "        return self\n",
    "\n",
    "    def predict_score(self, X):\n",
    "        X = torch.from_numpy(X).float().to(self.device)\n",
    "        score, _ = self.evaluate(X=X, y=None)\n",
    "        return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 001 | Validation metric: 0.8804 <<< BEST VALIDATION EPOCH\n",
      "Epoch 002 | Validation metric: 0.9405 <<< BEST VALIDATION EPOCH\n",
      "Epoch 003 | Validation metric: 0.9575 <<< BEST VALIDATION EPOCH\n",
      "Epoch 004 | Validation metric: 0.9711 <<< BEST VALIDATION EPOCH\n",
      "Epoch 005 | Validation metric: 0.9709\n",
      "Epoch 006 | Validation metric: 0.9792 <<< BEST VALIDATION EPOCH\n",
      "Epoch 007 | Validation metric: 0.9836 <<< BEST VALIDATION EPOCH\n",
      "Epoch 008 | Validation metric: 0.9865 <<< BEST VALIDATION EPOCH\n",
      "Epoch 009 | Validation metric: 0.9855\n",
      "Epoch 010 | Validation metric: 0.9902 <<< BEST VALIDATION EPOCH\n",
      "Epoch 011 | Validation metric: 0.9921 <<< BEST VALIDATION EPOCH\n",
      "Epoch 012 | Validation metric: 0.9911\n",
      "Epoch 013 | Validation metric: 0.9902\n",
      "Epoch 014 | Validation metric: 0.9931 <<< BEST VALIDATION EPOCH\n",
      "Epoch 015 | Validation metric: 0.9923\n",
      "Epoch 016 | Validation metric: 0.9932 <<< BEST VALIDATION EPOCH\n",
      "Epoch 017 | Validation metric: 0.9950 <<< BEST VALIDATION EPOCH\n",
      "Epoch 018 | Validation metric: 0.9939\n",
      "Epoch 019 | Validation metric: 0.9951 <<< BEST VALIDATION EPOCH\n",
      "Epoch 020 | Validation metric: 0.9955 <<< BEST VALIDATION EPOCH\n",
      "Epoch 021 | Validation metric: 0.9953\n",
      "Epoch 022 | Validation metric: 0.9966 <<< BEST VALIDATION EPOCH\n",
      "Epoch 023 | Validation metric: 0.9959\n",
      "Epoch 024 | Validation metric: 0.9968 <<< BEST VALIDATION EPOCH\n",
      "Epoch 025 | Validation metric: 0.9959\n",
      "Epoch 026 | Validation metric: 0.9968 <<< BEST VALIDATION EPOCH\n",
      "Epoch 027 | Validation metric: 0.9973 <<< BEST VALIDATION EPOCH\n",
      "Epoch 028 | Validation metric: 0.9965\n",
      "Epoch 029 | Validation metric: 0.9979 <<< BEST VALIDATION EPOCH\n",
      "Epoch 030 | Validation metric: 0.9971\n",
      "Epoch 031 | Validation metric: 0.9961\n",
      "Epoch 032 | Validation metric: 0.9976\n",
      "Epoch 033 | Validation metric: 0.9972\n",
      "Epoch 034 | Validation metric: 0.9977\n",
      "Epoch 035 | Validation metric: 0.9974\n",
      "Epoch 036 | Validation metric: 0.9972\n",
      "Epoch 037 | Validation metric: 0.9978\n",
      "Epoch 038 | Validation metric: 0.9973\n",
      "Epoch 039 | Validation metric: 0.9984 <<< BEST VALIDATION EPOCH\n",
      "Epoch 040 | Validation metric: 0.9979\n",
      "Epoch 041 | Validation metric: 0.9984\n",
      "Epoch 042 | Validation metric: 0.9980\n",
      "Epoch 043 | Validation metric: 0.9969\n",
      "Epoch 044 | Validation metric: 0.9984\n",
      "Epoch 045 | Validation metric: 0.9981\n",
      "Epoch 046 | Validation metric: 0.9980\n",
      "Epoch 047 | Validation metric: 0.9982\n",
      "Epoch 048 | Validation metric: 0.9980\n",
      "Epoch 049 | Validation metric: 0.9983\n",
      "Epoch 050 | Validation metric: 0.9984 <<< BEST VALIDATION EPOCH\n",
      "Epoch 051 | Validation metric: 0.9981\n",
      "Epoch 052 | Validation metric: 0.9984\n",
      "Epoch 053 | Validation metric: 0.9979\n",
      "Epoch 054 | Validation metric: 0.9988 <<< BEST VALIDATION EPOCH\n",
      "Epoch 055 | Validation metric: 0.9983\n",
      "Epoch 056 | Validation metric: 0.9980\n",
      "Epoch 057 | Validation metric: 0.9981\n",
      "Epoch 058 | Validation metric: 0.9981\n",
      "Epoch 059 | Validation metric: 0.9987\n",
      "Epoch 060 | Validation metric: 0.9972\n",
      "Epoch 061 | Validation metric: 0.9980\n",
      "Epoch 062 | Validation metric: 0.9982\n",
      "Epoch 063 | Validation metric: 0.9988 <<< BEST VALIDATION EPOCH\n",
      "Epoch 064 | Validation metric: 0.9969\n",
      "Epoch 065 | Validation metric: 0.9988 <<< BEST VALIDATION EPOCH\n",
      "Epoch 066 | Validation metric: 0.9982\n",
      "Epoch 067 | Validation metric: 0.9989 <<< BEST VALIDATION EPOCH\n",
      "Epoch 068 | Validation metric: 0.9988\n",
      "Epoch 069 | Validation metric: 0.9990 <<< BEST VALIDATION EPOCH\n",
      "Epoch 070 | Validation metric: 0.9988\n",
      "Epoch 071 | Validation metric: 0.9989\n",
      "Epoch 072 | Validation metric: 0.9986\n",
      "Epoch 073 | Validation metric: 0.9988\n",
      "Epoch 074 | Validation metric: 0.9988\n",
      "Epoch 075 | Validation metric: 0.9989\n",
      "Epoch 076 | Validation metric: 0.9991 <<< BEST VALIDATION EPOCH\n",
      "Epoch 077 | Validation metric: 0.9988\n",
      "Epoch 078 | Validation metric: 0.9989\n",
      "Epoch 079 | Validation metric: 0.9985\n",
      "Epoch 080 | Validation metric: 0.9990\n",
      "Epoch 081 | Validation metric: 0.9986\n",
      "Epoch 082 | Validation metric: 0.9987\n",
      "Epoch 083 | Validation metric: 0.9992 <<< BEST VALIDATION EPOCH\n",
      "Epoch 084 | Validation metric: 0.9988\n",
      "Epoch 085 | Validation metric: 0.9986\n",
      "Epoch 086 | Validation metric: 0.9991\n",
      "Epoch 087 | Validation metric: 0.9989\n",
      "Epoch 088 | Validation metric: 0.9991\n",
      "Epoch 089 | Validation metric: 0.9990\n",
      "Epoch 090 | Validation metric: 0.9983\n",
      "Epoch 091 | Validation metric: 0.9991\n",
      "Epoch 092 | Validation metric: 0.9990\n",
      "Epoch 093 | Validation metric: 0.9990\n",
      "Epoch 094 | Validation metric: 0.9990\n",
      "Epoch 095 | Validation metric: 0.9991\n",
      "Epoch 096 | Validation metric: 0.9991\n",
      "Epoch 097 | Validation metric: 0.9987\n",
      "Epoch 098 | Validation metric: 0.9991\n",
      "Epoch 099 | Validation metric: 0.9993 <<< BEST VALIDATION EPOCH\n",
      "Epoch 100 | Validation metric: 0.9986\n",
      "Epoch 101 | Validation metric: 0.9989\n",
      "Epoch 102 | Validation metric: 0.9987\n",
      "Epoch 103 | Validation metric: 0.9987\n",
      "Epoch 104 | Validation metric: 0.9991\n",
      "Epoch 105 | Validation metric: 0.9994 <<< BEST VALIDATION EPOCH\n",
      "Epoch 106 | Validation metric: 0.9990\n",
      "Epoch 107 | Validation metric: 0.9991\n",
      "Epoch 108 | Validation metric: 0.9994\n",
      "Epoch 109 | Validation metric: 0.9990\n",
      "Epoch 110 | Validation metric: 0.9990\n",
      "Epoch 111 | Validation metric: 0.9990\n",
      "Epoch 112 | Validation metric: 0.9991\n",
      "Epoch 113 | Validation metric: 0.9979\n",
      "Epoch 114 | Validation metric: 0.9989\n",
      "Epoch 115 | Validation metric: 0.9987\n",
      "Epoch 116 | Validation metric: 0.9989\n",
      "Epoch 117 | Validation metric: 0.9992\n",
      "Epoch 118 | Validation metric: 0.9990\n",
      "Epoch 119 | Validation metric: 0.9989\n",
      "Epoch 120 | Validation metric: 0.9994 <<< BEST VALIDATION EPOCH\n",
      "Epoch 121 | Validation metric: 0.9989\n",
      "Epoch 122 | Validation metric: 0.9990\n",
      "Epoch 123 | Validation metric: 0.9992\n",
      "Epoch 124 | Validation metric: 0.9990\n",
      "Epoch 125 | Validation metric: 0.9993\n",
      "Epoch 126 | Validation metric: 0.9995 <<< BEST VALIDATION EPOCH\n",
      "Epoch 127 | Validation metric: 0.9992\n",
      "Epoch 128 | Validation metric: 0.9989\n",
      "Epoch 129 | Validation metric: 0.9994\n",
      "Epoch 130 | Validation metric: 0.9991\n",
      "Epoch 131 | Validation metric: 0.9992\n",
      "Epoch 132 | Validation metric: 0.9995 <<< BEST VALIDATION EPOCH\n",
      "Epoch 133 | Validation metric: 0.9994\n",
      "Epoch 134 | Validation metric: 0.9993\n",
      "Epoch 135 | Validation metric: 0.9994\n",
      "Epoch 136 | Validation metric: 0.9993\n",
      "Epoch 137 | Validation metric: 0.9993\n",
      "Epoch 138 | Validation metric: 0.9991\n",
      "Epoch 139 | Validation metric: 0.9991\n",
      "Epoch 140 | Validation metric: 0.9994\n",
      "Epoch 141 | Validation metric: 0.9992\n",
      "Epoch 142 | Validation metric: 0.9992\n",
      "Epoch 143 | Validation metric: 0.9992\n",
      "Epoch 144 | Validation metric: 0.9991\n",
      "Epoch 145 | Validation metric: 0.9991\n",
      "Epoch 146 | Validation metric: 0.9992\n",
      "Epoch 147 | Validation metric: 0.9991\n",
      "Epoch 148 | Validation metric: 0.9989\n",
      "Epoch 149 | Validation metric: 0.9991\n",
      "Epoch 150 | Validation metric: 0.9992\n",
      "Epoch 151 | Validation metric: 0.9988\n",
      "Epoch 152 | Validation metric: 0.9991\n",
      "Epoch 153 | Validation metric: 0.9992\n",
      "Epoch 154 | Validation metric: 0.9989\n",
      "Epoch 155 | Validation metric: 0.9995\n",
      "Epoch 156 | Validation metric: 0.9994\n",
      "Epoch 157 | Validation metric: 0.9990\n",
      "Epoch 158 | Validation metric: 0.9991\n",
      "Epoch 159 | Validation metric: 0.9993\n",
      "Epoch 160 | Validation metric: 0.9993\n",
      "Epoch 161 | Validation metric: 0.9996 <<< BEST VALIDATION EPOCH\n",
      "Epoch 162 | Validation metric: 0.9993\n",
      "Epoch 163 | Validation metric: 0.9990\n",
      "Epoch 164 | Validation metric: 0.9995\n",
      "Epoch 165 | Validation metric: 0.9993\n",
      "Epoch 166 | Validation metric: 0.9991\n",
      "Epoch 167 | Validation metric: 0.9993\n",
      "Epoch 168 | Validation metric: 0.9996 <<< BEST VALIDATION EPOCH\n",
      "Epoch 169 | Validation metric: 0.9992\n",
      "Epoch 170 | Validation metric: 0.9989\n",
      "Epoch 171 | Validation metric: 0.9985\n",
      "Epoch 172 | Validation metric: 0.9993\n",
      "Epoch 173 | Validation metric: 0.9987\n",
      "Epoch 174 | Validation metric: 0.9993\n",
      "Epoch 175 | Validation metric: 0.9992\n",
      "Epoch 176 | Validation metric: 0.9995\n",
      "Epoch 177 | Validation metric: 0.9993\n",
      "Epoch 178 | Validation metric: 0.9993\n",
      "Epoch 179 | Validation metric: 0.9988\n",
      "Epoch 180 | Validation metric: 0.9994\n",
      "Epoch 181 | Validation metric: 0.9992\n",
      "Epoch 182 | Validation metric: 0.9993\n",
      "Epoch 183 | Validation metric: 0.9992\n",
      "Epoch 184 | Validation metric: 0.9993\n",
      "Epoch 185 | Validation metric: 0.9992\n",
      "Epoch 186 | Validation metric: 0.9994\n",
      "Epoch 187 | Validation metric: 0.9993\n",
      "Epoch 188 | Validation metric: 0.9993\n",
      "Epoch 189 | Validation metric: 0.9992\n",
      "Epoch 190 | Validation metric: 0.9996\n",
      "Epoch 191 | Validation metric: 0.9991\n",
      "Epoch 192 | Validation metric: 0.9992\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 193 | Validation metric: 0.9995\n",
      "Epoch 194 | Validation metric: 0.9996 <<< BEST VALIDATION EPOCH\n",
      "Epoch 195 | Validation metric: 0.9992\n",
      "Epoch 196 | Validation metric: 0.9993\n",
      "Epoch 197 | Validation metric: 0.9993\n",
      "Epoch 198 | Validation metric: 0.9989\n",
      "Epoch 199 | Validation metric: 0.9996\n",
      "Epoch 200 | Validation metric: 0.9993\n",
      "Epoch 201 | Validation metric: 0.9995\n",
      "Epoch 202 | Validation metric: 0.9994\n",
      "Epoch 203 | Validation metric: 0.9995\n",
      "Epoch 204 | Validation metric: 0.9995\n",
      "Epoch 205 | Validation metric: 0.9993\n",
      "Epoch 206 | Validation metric: 0.9992\n",
      "Epoch 207 | Validation metric: 0.9992\n",
      "Epoch 208 | Validation metric: 0.9995\n",
      "Epoch 209 | Validation metric: 0.9992\n",
      "Epoch 210 | Validation metric: 0.9995\n",
      "Epoch 211 | Validation metric: 0.9993\n",
      "Epoch 212 | Validation metric: 0.9991\n",
      "Epoch 213 | Validation metric: 0.9993\n",
      "Epoch 214 | Validation metric: 0.9995\n",
      "Epoch 215 | Validation metric: 0.9993\n",
      "Epoch 216 | Validation metric: 0.9995\n",
      "Epoch 217 | Validation metric: 0.9993\n",
      "Epoch 218 | Validation metric: 0.9990\n",
      "Epoch 219 | Validation metric: 0.9993\n",
      "Epoch 220 | Validation metric: 0.9994\n",
      "Epoch 221 | Validation metric: 0.9995\n",
      "Epoch 222 | Validation metric: 0.9995\n",
      "Epoch 223 | Validation metric: 0.9996 <<< BEST VALIDATION EPOCH\n",
      "Epoch 224 | Validation metric: 0.9995\n",
      "Epoch 225 | Validation metric: 0.9993\n",
      "Epoch 226 | Validation metric: 0.9992\n",
      "Epoch 227 | Validation metric: 0.9992\n",
      "Epoch 228 | Validation metric: 0.9983\n",
      "Epoch 229 | Validation metric: 0.9996\n",
      "Epoch 230 | Validation metric: 0.9996\n",
      "Epoch 231 | Validation metric: 0.9992\n",
      "Epoch 232 | Validation metric: 0.9987\n",
      "Epoch 233 | Validation metric: 0.9996\n",
      "Epoch 234 | Validation metric: 0.9994\n",
      "Epoch 235 | Validation metric: 0.9996\n",
      "Epoch 236 | Validation metric: 0.9990\n",
      "Epoch 237 | Validation metric: 0.9995\n",
      "Epoch 238 | Validation metric: 0.9996\n",
      "Epoch 239 | Validation metric: 0.9995\n",
      "Epoch 240 | Validation metric: 0.9991\n",
      "Epoch 241 | Validation metric: 0.9987\n",
      "Epoch 242 | Validation metric: 0.9994\n",
      "Epoch 243 | Validation metric: 0.9996\n",
      "Epoch 244 | Validation metric: 0.9996\n",
      "Epoch 245 | Validation metric: 0.9994\n",
      "Epoch 246 | Validation metric: 0.9997 <<< BEST VALIDATION EPOCH\n",
      "Epoch 247 | Validation metric: 0.9995\n",
      "Epoch 248 | Validation metric: 0.9996\n",
      "Epoch 249 | Validation metric: 0.9994\n",
      "Epoch 250 | Validation metric: 0.9996\n",
      "Epoch 251 | Validation metric: 0.9996\n",
      "Epoch 252 | Validation metric: 0.9995\n",
      "Epoch 253 | Validation metric: 0.9995\n",
      "Epoch 254 | Validation metric: 0.9995\n",
      "Epoch 255 | Validation metric: 0.9995\n",
      "Epoch 256 | Validation metric: 0.9996\n",
      "Epoch 257 | Validation metric: 0.9993\n",
      "Epoch 258 | Validation metric: 0.9995\n",
      "Epoch 259 | Validation metric: 0.9997\n",
      "Epoch 260 | Validation metric: 0.9998 <<< BEST VALIDATION EPOCH\n",
      "Epoch 261 | Validation metric: 0.9996\n",
      "Epoch 262 | Validation metric: 0.9995\n",
      "Epoch 263 | Validation metric: 0.9990\n",
      "Epoch 264 | Validation metric: 0.9993\n",
      "Epoch 265 | Validation metric: 0.9995\n",
      "Epoch 266 | Validation metric: 0.9994\n",
      "Epoch 267 | Validation metric: 0.9995\n",
      "Epoch 268 | Validation metric: 0.9997\n",
      "Epoch 269 | Validation metric: 0.9990\n",
      "Epoch 270 | Validation metric: 0.9996\n",
      "Epoch 271 | Validation metric: 0.9988\n",
      "Epoch 272 | Validation metric: 0.9996\n",
      "Epoch 273 | Validation metric: 0.9996\n",
      "Epoch 274 | Validation metric: 0.9994\n",
      "Epoch 275 | Validation metric: 0.9996\n",
      "Epoch 276 | Validation metric: 0.9992\n",
      "Epoch 277 | Validation metric: 0.9995\n",
      "Epoch 278 | Validation metric: 0.9997\n",
      "Epoch 279 | Validation metric: 0.9996\n",
      "Epoch 280 | Validation metric: 0.9992\n",
      "Epoch 281 | Validation metric: 0.9995\n",
      "Epoch 282 | Validation metric: 0.9996\n",
      "Epoch 283 | Validation metric: 0.9996\n",
      "Epoch 284 | Validation metric: 0.9994\n",
      "Epoch 285 | Validation metric: 0.9995\n",
      "Epoch 286 | Validation metric: 0.9997\n",
      "Epoch 287 | Validation metric: 0.9994\n",
      "Epoch 288 | Validation metric: 0.9997\n",
      "Epoch 289 | Validation metric: 0.9995\n",
      "Epoch 290 | Validation metric: 0.9995\n",
      "Epoch 291 | Validation metric: 0.9995\n",
      "Epoch 292 | Validation metric: 0.9997\n",
      "Epoch 293 | Validation metric: 0.9997\n",
      "Epoch 294 | Validation metric: 0.9989\n",
      "Epoch 295 | Validation metric: 0.9997\n",
      "Epoch 296 | Validation metric: 0.9995\n",
      "Epoch 297 | Validation metric: 0.9993\n",
      "Epoch 298 | Validation metric: 0.9996\n",
      "Epoch 299 | Validation metric: 0.9997\n",
      "Epoch 300 | Validation metric: 0.9998\n",
      "Epoch 301 | Validation metric: 0.9994\n",
      "Epoch 302 | Validation metric: 0.9993\n",
      "Epoch 303 | Validation metric: 0.9997\n",
      "Epoch 304 | Validation metric: 0.9997\n",
      "Epoch 305 | Validation metric: 0.9997\n",
      "Epoch 306 | Validation metric: 0.9995\n",
      "Epoch 307 | Validation metric: 0.9997\n",
      "Epoch 308 | Validation metric: 0.9995\n",
      "Epoch 309 | Validation metric: 0.9996\n",
      "Epoch 310 | Validation metric: 0.9990\n",
      "Epoch 311 | Validation metric: 0.9997\n",
      "Epoch 312 | Validation metric: 0.9996\n",
      "Epoch 313 | Validation metric: 0.9995\n",
      "Epoch 314 | Validation metric: 0.9996\n",
      "Epoch 315 | Validation metric: 0.9991\n",
      "Epoch 316 | Validation metric: 0.9996\n",
      "Epoch 317 | Validation metric: 0.9996\n",
      "Epoch 318 | Validation metric: 0.9997\n",
      "Epoch 319 | Validation metric: 0.9995\n",
      "Epoch 320 | Validation metric: 0.9997\n",
      "Epoch 321 | Validation metric: 0.9995\n",
      "Epoch 322 | Validation metric: 0.9993\n",
      "Epoch 323 | Validation metric: 0.9996\n",
      "Epoch 324 | Validation metric: 0.9995\n",
      "Epoch 325 | Validation metric: 0.9996\n",
      "Epoch 326 | Validation metric: 0.9992\n",
      "Epoch 327 | Validation metric: 0.9997\n",
      "Epoch 328 | Validation metric: 0.9994\n",
      "Epoch 329 | Validation metric: 0.9995\n",
      "Epoch 330 | Validation metric: 0.9996\n",
      "Epoch 331 | Validation metric: 0.9996\n",
      "Epoch 332 | Validation metric: 0.9997\n",
      "Epoch 333 | Validation metric: 0.9996\n",
      "Epoch 334 | Validation metric: 0.9997\n",
      "Epoch 335 | Validation metric: 0.9996\n",
      "Epoch 336 | Validation metric: 0.9997\n",
      "Epoch 337 | Validation metric: 0.9997\n",
      "Epoch 338 | Validation metric: 0.9995\n",
      "Epoch 339 | Validation metric: 0.9997\n",
      "Epoch 340 | Validation metric: 0.9996\n",
      "Epoch 341 | Validation metric: 0.9997\n",
      "Epoch 342 | Validation metric: 0.9998 <<< BEST VALIDATION EPOCH\n",
      "Epoch 343 | Validation metric: 0.9995\n",
      "Epoch 344 | Validation metric: 0.9991\n",
      "Epoch 345 | Validation metric: 0.9997\n",
      "Epoch 346 | Validation metric: 0.9997\n",
      "Epoch 347 | Validation metric: 0.9989\n",
      "Epoch 348 | Validation metric: 0.9997\n",
      "Epoch 349 | Validation metric: 0.9997\n",
      "Epoch 350 | Validation metric: 0.9997\n",
      "Epoch 351 | Validation metric: 0.9996\n",
      "Epoch 352 | Validation metric: 0.9995\n",
      "Epoch 353 | Validation metric: 0.9997\n",
      "Epoch 354 | Validation metric: 0.9997\n",
      "Epoch 355 | Validation metric: 0.9998\n",
      "Epoch 356 | Validation metric: 0.9992\n",
      "Epoch 357 | Validation metric: 0.9998\n",
      "Epoch 358 | Validation metric: 0.9996\n",
      "Epoch 359 | Validation metric: 0.9997\n",
      "Epoch 360 | Validation metric: 0.9998\n",
      "Epoch 361 | Validation metric: 0.9996\n",
      "Epoch 362 | Validation metric: 0.9998 <<< BEST VALIDATION EPOCH\n",
      "Epoch 363 | Validation metric: 0.9997\n",
      "Epoch 364 | Validation metric: 0.9995\n",
      "Epoch 365 | Validation metric: 0.9996\n",
      "Epoch 366 | Validation metric: 0.9991\n",
      "Epoch 367 | Validation metric: 0.9997\n",
      "Epoch 368 | Validation metric: 0.9998\n",
      "Epoch 369 | Validation metric: 0.9997\n",
      "Epoch 370 | Validation metric: 0.9998\n",
      "Epoch 371 | Validation metric: 0.9998\n",
      "Epoch 372 | Validation metric: 0.9998\n",
      "Epoch 373 | Validation metric: 0.9996\n",
      "Epoch 374 | Validation metric: 0.9998\n",
      "Epoch 375 | Validation metric: 0.9998\n",
      "Epoch 376 | Validation metric: 0.9998\n",
      "Epoch 377 | Validation metric: 0.9997\n",
      "Epoch 378 | Validation metric: 0.9989\n",
      "Epoch 379 | Validation metric: 0.9998\n",
      "Epoch 380 | Validation metric: 0.9997\n",
      "Epoch 381 | Validation metric: 0.9996\n",
      "Epoch 382 | Validation metric: 0.9997\n",
      "Epoch 383 | Validation metric: 0.9997\n",
      "Epoch 384 | Validation metric: 0.9997\n",
      "Epoch 385 | Validation metric: 0.9997\n",
      "Epoch 386 | Validation metric: 0.9997\n",
      "Epoch 387 | Validation metric: 0.9998\n",
      "Epoch 388 | Validation metric: 0.9997\n",
      "Epoch 389 | Validation metric: 0.9984\n",
      "Epoch 390 | Validation metric: 0.9996\n",
      "Epoch 391 | Validation metric: 0.9998\n",
      "Epoch 392 | Validation metric: 0.9996\n",
      "Epoch 393 | Validation metric: 0.9997\n",
      "Epoch 394 | Validation metric: 0.9995\n",
      "Epoch 395 | Validation metric: 0.9996\n",
      "Epoch 396 | Validation metric: 0.9986\n",
      "Epoch 397 | Validation metric: 0.9998\n",
      "Epoch 398 | Validation metric: 0.9997\n",
      "Epoch 399 | Validation metric: 0.9994\n",
      "Epoch 400 | Validation metric: 0.9998\n",
      "Epoch 401 | Validation metric: 0.9997\n",
      "Epoch 402 | Validation metric: 0.9997\n",
      "Epoch 403 | Validation metric: 0.9998\n",
      "Epoch 404 | Validation metric: 0.9994\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 405 | Validation metric: 0.9997\n",
      "Epoch 406 | Validation metric: 0.9995\n",
      "Epoch 407 | Validation metric: 0.9997\n",
      "Epoch 408 | Validation metric: 0.9997\n",
      "Epoch 409 | Validation metric: 0.9998 <<< BEST VALIDATION EPOCH\n",
      "Epoch 410 | Validation metric: 0.9998\n",
      "Epoch 411 | Validation metric: 0.9997\n",
      "Epoch 412 | Validation metric: 0.9998\n",
      "Epoch 413 | Validation metric: 0.9996\n",
      "Epoch 414 | Validation metric: 0.9997\n",
      "Epoch 415 | Validation metric: 0.9998\n",
      "Epoch 416 | Validation metric: 0.9998\n",
      "Epoch 417 | Validation metric: 0.9998\n",
      "Epoch 418 | Validation metric: 0.9997\n",
      "Epoch 419 | Validation metric: 0.9997\n",
      "Epoch 420 | Validation metric: 0.9998\n",
      "Epoch 421 | Validation metric: 0.9997\n",
      "Epoch 422 | Validation metric: 0.9996\n",
      "Epoch 423 | Validation metric: 0.9998\n",
      "Epoch 424 | Validation metric: 0.9993\n",
      "Epoch 425 | Validation metric: 0.9998\n",
      "Epoch 426 | Validation metric: 0.9997\n",
      "Epoch 427 | Validation metric: 0.9996\n",
      "Epoch 428 | Validation metric: 0.9996\n",
      "Epoch 429 | Validation metric: 0.9992\n",
      "Epoch 430 | Validation metric: 0.9998\n",
      "Epoch 431 | Validation metric: 0.9998 <<< BEST VALIDATION EPOCH\n",
      "Epoch 432 | Validation metric: 0.9998\n",
      "Epoch 433 | Validation metric: 0.9998\n",
      "Epoch 434 | Validation metric: 0.9996\n",
      "Epoch 435 | Validation metric: 0.9997\n",
      "Epoch 436 | Validation metric: 0.9998\n",
      "Epoch 437 | Validation metric: 0.9997\n",
      "Epoch 438 | Validation metric: 0.9998\n",
      "Epoch 439 | Validation metric: 0.9997\n",
      "Epoch 440 | Validation metric: 0.9998\n",
      "Epoch 441 | Validation metric: 0.9996\n",
      "Epoch 442 | Validation metric: 0.9995\n",
      "Epoch 443 | Validation metric: 0.9998\n",
      "Epoch 444 | Validation metric: 0.9995\n",
      "Epoch 445 | Validation metric: 0.9997\n",
      "Epoch 446 | Validation metric: 0.9997\n",
      "Epoch 447 | Validation metric: 0.9998\n",
      "Epoch 448 | Validation metric: 0.9997\n",
      "Epoch 449 | Validation metric: 0.9996\n",
      "Epoch 450 | Validation metric: 0.9998\n",
      "Epoch 451 | Validation metric: 0.9996\n",
      "Epoch 452 | Validation metric: 0.9997\n",
      "Epoch 453 | Validation metric: 0.9997\n",
      "Epoch 454 | Validation metric: 0.9997\n",
      "Epoch 455 | Validation metric: 0.9998\n",
      "Epoch 456 | Validation metric: 0.9998\n",
      "Epoch 457 | Validation metric: 0.9998\n",
      "Epoch 458 | Validation metric: 0.9994\n",
      "Epoch 459 | Validation metric: 0.9997\n",
      "Epoch 460 | Validation metric: 0.9994\n",
      "Epoch 461 | Validation metric: 0.9998\n",
      "Epoch 462 | Validation metric: 0.9992\n",
      "Epoch 463 | Validation metric: 0.9998\n",
      "Epoch 464 | Validation metric: 0.9998\n",
      "Epoch 465 | Validation metric: 0.9998\n",
      "Epoch 466 | Validation metric: 0.9997\n",
      "Epoch 467 | Validation metric: 0.9998\n",
      "Epoch 468 | Validation metric: 0.9995\n",
      "Epoch 469 | Validation metric: 0.9998\n",
      "Epoch 470 | Validation metric: 0.9998\n",
      "Epoch 471 | Validation metric: 0.9997\n",
      "Epoch 472 | Validation metric: 0.9998 <<< BEST VALIDATION EPOCH\n",
      "Epoch 473 | Validation metric: 0.9997\n",
      "Epoch 474 | Validation metric: 0.9998\n",
      "Epoch 475 | Validation metric: 0.9996\n",
      "Epoch 476 | Validation metric: 0.9996\n",
      "Epoch 477 | Validation metric: 0.9997\n",
      "Epoch 478 | Validation metric: 0.9998\n",
      "Epoch 479 | Validation metric: 0.9996\n",
      "Epoch 480 | Validation metric: 0.9998\n",
      "Epoch 481 | Validation metric: 0.9994\n",
      "Epoch 482 | Validation metric: 0.9999 <<< BEST VALIDATION EPOCH\n",
      "Epoch 483 | Validation metric: 0.9995\n",
      "Epoch 484 | Validation metric: 0.9998\n",
      "Epoch 485 | Validation metric: 0.9997\n",
      "Epoch 486 | Validation metric: 0.9994\n",
      "Epoch 487 | Validation metric: 0.9994\n",
      "Epoch 488 | Validation metric: 0.9998\n",
      "Epoch 489 | Validation metric: 0.9998\n",
      "Epoch 490 | Validation metric: 0.9998\n",
      "Epoch 491 | Validation metric: 0.9998\n",
      "Epoch 492 | Validation metric: 0.9998\n",
      "Epoch 493 | Validation metric: 0.9997\n",
      "Epoch 494 | Validation metric: 0.9997\n",
      "Epoch 495 | Validation metric: 0.9998\n",
      "Epoch 496 | Validation metric: 0.9998\n",
      "Epoch 497 | Validation metric: 0.9999 <<< BEST VALIDATION EPOCH\n",
      "Epoch 498 | Validation metric: 0.9999\n",
      "Epoch 499 | Validation metric: 0.9997\n",
      "Epoch 500 | Validation metric: 0.9998\n",
      "Epoch 501 | Validation metric: 0.9995\n",
      "Epoch 502 | Validation metric: 0.9997\n",
      "Epoch 503 | Validation metric: 0.9998\n",
      "Epoch 504 | Validation metric: 0.9997\n",
      "Epoch 505 | Validation metric: 0.9998\n",
      "Epoch 506 | Validation metric: 0.9997\n",
      "Epoch 507 | Validation metric: 0.9998\n",
      "Epoch 508 | Validation metric: 0.9998\n",
      "Epoch 509 | Validation metric: 0.9997\n",
      "Epoch 510 | Validation metric: 0.9997\n",
      "Epoch 511 | Validation metric: 0.9998\n",
      "Epoch 512 | Validation metric: 0.9998\n",
      "Epoch 513 | Validation metric: 0.9998\n",
      "Epoch 514 | Validation metric: 0.9999\n",
      "Epoch 515 | Validation metric: 0.9999\n",
      "Epoch 516 | Validation metric: 0.9997\n",
      "Epoch 517 | Validation metric: 0.9998\n",
      "Epoch 518 | Validation metric: 0.9998\n",
      "Epoch 519 | Validation metric: 0.9989\n",
      "Epoch 520 | Validation metric: 0.9998\n",
      "Epoch 521 | Validation metric: 0.9998\n",
      "Epoch 522 | Validation metric: 0.9998\n",
      "Epoch 523 | Validation metric: 0.9998\n",
      "Epoch 524 | Validation metric: 0.9997\n",
      "Epoch 525 | Validation metric: 0.9998\n",
      "Epoch 526 | Validation metric: 0.9998\n",
      "Epoch 527 | Validation metric: 0.9994\n",
      "Epoch 528 | Validation metric: 0.9994\n",
      "Epoch 529 | Validation metric: 0.9998\n",
      "Epoch 530 | Validation metric: 0.9996\n",
      "Epoch 531 | Validation metric: 0.9997\n",
      "Epoch 532 | Validation metric: 0.9998\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 6234/6234 [00:00<00:00, 520997.29it/s]\n"
     ]
    }
   ],
   "source": [
    "seed = 42\n",
    "clf=FTTransformer(seed,\"ResNet\",n_epochs=532,batch_size=256)\n",
    "aucroc = []\n",
    "aucpr = []\n",
    "loss_ = []\n",
    "clf = clf.fit(X_train=X_train, y_train=y_train.squeeze(1),X_test=X_test,y_test=y_test)\n",
    "import platform\n",
    "y_val_scores = clf.predict_score(X_val)   #返回未知数据上的异常值 (分值越大越异常) # outlier scores\n",
    "#记录文件名和对应的异常得分\n",
    "predict_result={}\n",
    "for i in tqdm(range(len(test1_files))):\n",
    "    file=test1_files[i]\n",
    "    #如果是window系统：\n",
    "    if platform.system().lower() == 'windows':\n",
    "        name=file.split('\\\\')[-1]\n",
    "    #如果是linux系统\n",
    "    elif platform.system().lower() == 'linux':\n",
    "        name=file.split('/')[-1]\n",
    "    predict_result[name]=y_val_scores[i]\n",
    "predict_score=pd.DataFrame(list(predict_result.items()),columns=['file_name','score'])#列名必须为这俩个\n",
    "predict_score.to_csv(f'submision.csv',index = False) #保存为比赛要求的csv文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlkAAADlCAYAAABgb+JgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAABFjElEQVR4nO3deXxU9dX48c+ZLXsChJ2wKYigLGrAXXDHpYpa61IVt/LY1trNx0drF+tTq1Xb/upj61Kliq2iYrVqaREXXAEJq7IJIkICQlYge2bm/P64N8MQAhlgQnKn5/16zSt37v3eO2ey3Jz5rqKqGGOMMcaY5PJ1dADGGGOMManIkixjjDHGmHZgSZYxxhhjTDuwJMsYY4wxph1YkmWMMcYY0w4syTLGGGOMaQeBjg6gpe7du+ugQYM6OgxjzEG0cOHCMlXt0dFxJIPdw4z5z7K3+1enS7IGDRpEUVFRR4dhjDmIROTLjo4hWeweZsx/lr3dv6y50BhjjDGmHViSZYwxxhjTDizJMsYYY4xpB232yRKRqcD5wFZVPbKV4wL8ATgXqAWuVdVF7rHJwE/dor9S1aeTFbgxxiRCRCbi3KP8wBOqel+L4zcB3wUiQDUwRVVXuMfuAG5wj92iqrMOZuzGeFFTUxPFxcXU19d3dChJlZ6eTkFBAcFgMOFzEun4/hTwMDBtD8fPAYa6j2OBR4BjRaQb8AugEFBgoYi8qqqVCUdnjDEHQET8wB+BM4FiYIF7H1oRV+xZVX3ULX8B8DtgooiMAC4HjgD6Am+KyGGqGjmob8IYjykuLiYnJ4dBgwbh1MN4n6pSXl5OcXExgwcPTvi8NpsLVfU9oGIvRS4EpqljHtBFRPoAZwOzVbXCTaxmAxMTjswYYw7cOGCtqq5T1UZgOs49K0ZVt8c9zcL5UIhbbrqqNqjqF8Ba93rGmL2or68nPz8/ZRIsABEhPz9/n2vnkjGFQz9gY9zzYnffnvZ3mMZwlKgqaQEfdU0RMoJ+1m6tJhTwEYkqjZEoIb+PbXVN1DdFyUrz0y0rRE5akJKqOipqGtm8rY6miBL0CznpAYor60gP+snLCJKfFWJbXRNVdU3kpgepbmgiIxQgNz1AVW0TPXLSKKtuoKy6Eb9At+w0stP8hPx+Nm+rIxxVVMHvg8raJvwipIf8RCJR/D4hPeinKaJkhvw0hCP0zEmnIRyhS2aIzdvqqGmI4BNhe30TAZ+Qmx5k87Z6jh7YhYamKKXVDTRFotQ2RsjLCOL3CQGfEI4q4YhSXt1Ar7x0ahrCVNeH8fmErpkhstL8+ESIRJWKmkZCAR9dMoOsK60hM+SnS2aIusYwDeEoNQ0RumQG3Rij1DSEiSrUNoYRIDMtgABRhYZwhKhC79x0An5hW20ToYCPytpG0oN+0gM+KmqbQJWg30dGyM/2uiay0gI0hKM0hqMUdM3gi/Ia8rNCVNeHSQv6CUeUqrpGumSEiKrz/crNCFLXGKGuKUJjOEpeRpCKmkYiqqQH/Ph9zh9RTUOYuqYI2+vC9O+WgU+ErTvqEYRQwEdDOEJWKEBUFRGhMRwl6BcCfp/z+gGf+3NUFIhElez0ACG/j6raJnLSA9Q3RWkIR0gPOj/HoN9Hl4wQtY1hAn4hHFEawlHA+fQU9Pvw+ZzXCviEYMBHyO9zYgC214fpkhmkvimK3wc+EVTB5xMqaxrJSgvgE6huCBMJN5KVkUF60E9dY5jq+iZyMkKkB/2gSnVDhKgqkaiiKL1y0mmKRFFgR32YUMBHVihAUyRKUyRKTWMYQcgI+Qn5faQFfAT8PsqrG7h8XH+G9MzpuD94R2v3oWNbFhKR7wI/AkLAaXHnzmtxbtLuYR+uLePFoo3cPelIctMTb34wxgtSKcFqtj/vqVPMkyUiU4ApAAMGDEjKNasbwtQ2hvnnss08v2AjZdWNlFU3ABD0C00RbeMKe+YjShQfoIySdXyig9FdKgUVkLivOx0qJfSXrazX3qzXPrsc8xMhgp+eVHKUbw1+ogSI8oX2ZqUOIIrPfV0Qoru8Zhd2EMFPIwHO8hVRot0pJ5fr/f/ilvAlVJJLd7YhKKV0iZ0XJEwEH3lUc4X/HaZGj6JU8zjat4YKzeFL7U0ZebHYhCi51FJPiGv9s0DqeCh8HvmynR5UUUMGW7ULpXTBR5RjfSuJqI+rAm9SrRm8EJlAjtQiKD2lip5Uck/kbGrI4CTfJ5RpHoKSQQN1pDHBt5RB8hUNEuSJ8LnkUEcDAXKoo5+U8bF24ZLA+7wXGcVQXzHVGmIbWdRqGo1ST5nm0kO2EcZPnYYopQvD5UtmRMYxOTibIBEWRobQSICeUkW1ZjDEV8JgalkQHUYEH3cFp9GN7azUgXTx1bEjmsZG7cExvs9YIwPZHO1GgDDV0TSaJMgAfxm51NIlWkWJdmez5rNBe3KYVHCS7xMG+7bQpH58RKmVdN7To6mKpJPuCxPSRvKkllN8SwF4U8fyop7O5fIGNZLJDs2gi26ni+7ALxHqNUR33w7mR4aRL4qPKFWaSW+pZGV0AKNkI319FQSIMMi3hV5UUKa5vBo5gXN9yznct5E6DfFGtBBFiPpCdJcqurGddBpYGhlEWANspQuj/NsYzWo+ioygu2871WSxPZpOgZSyRXowhC9ZHe1PpeZwhH8r5V1/xJCep+7339nBpKp/BP4oIlfi9COdvC/n78897IuyGl5ZsomfnDfckixjkiw7O5vq6uqODiMpSVYJ0D/ueYG7rwSY0GL/nNYuoKqPA48DFBYW7nf2o6pU1TYxb105P/vHp5RVN+5yPCPop1/XDIb1yqFXbjorN28nzRdm5IAerPxyM2eOOYSAQBqNdN/wLxrVx2c9z6FofSWF29+gflsZV6e9S96ONShCU1ZvQjWbY9dvTO+OL9KAP9JAQ3o+ocZKGrsMIRoJE/ZnsHHQpRxR9JNY+Zqex/BVNI/6IecwdNVjhKrWtvkew5k98TVswxdpoLjwJ2SULqHk0MsZNu8u0mo3t3rOxD61NKTn079kJhINAxDJ6Ia/zmkFjqR3xV/vdJW7jed3/Z6Kj9r+E8jYPJ9wdh9ClU6MjTkDCO3YAMAtgVfajLvZlYG3d9t3W/AFmnIHEty+9/kor/HP3uOxy/xzEo4B4GfBv8W2r/Pv4fXY9fVGyhfgCwJNjMCJtXtoPbAeGnfsbHzvOQLScmHjSg6l9Z9JUJxuPTnpaZzXOB8k3Gq5M2QBZ8gCCGRAMAPqq8AXQDO6IdVfxcqNCKzf/eQ9vK/usp3rA/+OPc+QRi70f4QGMpD0XEjv4pxcVswQf/Fu5x8a2LTzSfN79gUh2sQJPqerk/oCSK8ftB7AwbWn+9OeTMfpV7pP5+7PPcznfirW/f+8Z4zp5JKRZL0K3Cwi03Gq4bep6mYRmQX8WkS6uuXOAu5Iwuvt0e9nf8ZDb+9MVCYM68H3ThvKEX1zSYvWIitfg/Q8yO8Nn82C0eNg6nlOIwA4t8+MblC3swvaKSMXc2OwDkpfd3a4eZuguyRYAKH6sth2upvwpIeCsOlTAI4oW7xL+aytCzkUoGz3xINuh0DFOhh1GXzxHuxwrheo3RorUlD0awDyv5y51+9Lr9IPd9vnj3uPzQkW/Y+FjfN3Fhp+AVL+OVkNpdBUE0uwACfBOvWnkJYN/759r6/PhJ9AUy2sf9/5Z7zRbYH5xjOw4M/wxXttJlgMPQvWvNH6sQv/COVroctAePd+2OEmAaEcJ/n5xjT4/B1Y+Jed5/QZDWNvhIyu8MX7UFcJY66AZy7aWabwBuhxOMz7E1R+Aef9Fo66Bv5+I6z4B9z4NhQc4/yXLF8LDxfCuClw7gPO+a//CIqehFPvdH6WS5+DnkfAeQ/CmtngC8CpP4HGGti6ArJ7Qk5fZ/+aWbB0Ogw5HWor4OhrnFhVIdqE+AKw5VPYsQWyusNzV8AVz0HvURBpgLI18Ph4J46bPoSqDTDgOOfaG+bCujlw4g+cuHqPhC3LkV4jIC2uea9qg/O7l94FGqud+EI5zuv5/FBTBn92a6p+sgnC9fDMJECQK1+ArPy9/0wPjgXAUBEZjPMXfjlwZXwBERmqqmvcp+cBzduvAs+KyO9wOr4PBT5OVmDNLQ9Ry7KMaTeqym233ca//vUvRISf/vSnXHbZZWzevJnLLruM7du3Ew6HeeSRRzjhhBO44YYbKCoqQkS4/vrr+eEPf3hAr5/IFA7P4dRIdReRYpwRg0E3+EeBmTjTN6zFmcLhOvdYhYj8L85NDuBuVd1bB/oD8mnJtl0SrA9vP41+XTKcJ+FGePZy+PKDti9U1yLET17cvczPymD7Jti8FEZcAB89DF0HQUYXWPVPOP67MON6OO1nMOgkWPsWzPwxVK6HAcfD1a84/+xWvAIv3eBc80crYcmzULoKJt4HmfnQVAehzJ2vu/5DyO4FXQY4//RKFsKXHznbgTQ45jrnH1//cU7S0FQLgXR44FDn/Gv/CduKYc598J15sGkx5PRyErrGWue13r0f3rkHjr8Zzr5n1/cdjTjJRcMOyOwGw7/m7G+ohnd+tbPctz9y/jHv+AqqvoQjL971OpEm2F7ifM9GXOBcs+9RzvuadqGTANxRAsULoMcwqCl1kqIvP3JiHnGhk3B98R58/S87/1sBFF4Hq/8NJUUw4Q4nUeg22DnnnN8451R96SRYzZrfR7zCG5ykSgSOvtr5WWR2c459o8VAWxHoPtR53/lDdr9WMBNGTHKSrLN/BQNPcB7N0rKdn1m8Yec4j5ZEwJfmbPcZDc0tzreu3lnGH4C+Y+D2Dc7vQddB0Dtu9pXDznYe4Pz8AQbs1k3J+XkcddXu++OPBzKc9xIIOY9vtfKBoQOpalhEbgZm4dTtTVXV5SJyN1Ckqs0fEs8AmoBK3KZCt9wLwAogDHw3mSMLfdIcY7KuaIxp6e9//ztLlixh6dKllJWVMXbsWE455RSeffZZzj77bO68804ikQi1tbUsWbKEkpISPv3UqRipqqo64NcX7WR/4YWFhbqv635Fo8ohP3Fqc56cXMixh+STnebmjw074KGjoWbrXq7g+t4iyD/Uuet9/pZz7ro5sPCpnWWuegmGnLFP8QHOP2kEguk79zXWwIvXwhm/hF4j9v2aiVo3B7ofBrl92y5bvw3e+BmccdfOpKLN678L0y6Acx90msoGnbj/sYYbnJqb3D5tl20PNWXgD0F67oFf68u58JeJ8J350PNwJzFP5GfgJeEGEL+T2B0AEVmoqoVJiqpDJXoPe2HBRm57aRkf/M+pFHTNbLO8MV6xcuVKhg8fDsAvX1vOik3b2zhj34zom8svvnbEXss098n64Q9/yMiRI7n++usBuPrqq7n00kvp0qUL119/PVdddRWTJk1izJgxVFZWUlhYyLnnnst5553HWWedhc+36yQM8e+t2d7uXykx4/uS4ioAzhrRi9MO77kzwQKYOnH3BKvXkc4n7uzecNx3nYQiq6dTowNObcGQM+CIi2Dib+DoyTtrPvqM2b8ggxm7JlgAoSz45ovtm2ABHDIh8X/u6XlwwUOJJ1gAh4yHm4uc79GBJFjg1Mh1VIIFTlNYMhIsgIHHw13bnAQLUi/BAufndYAJ1n8qsZosYzrMKaecwnvvvUe/fv249tprmTZtGl27dmXp0qVMmDCBRx99lBtvvLHtC7UhJe6O731Wigj85pJRO4dYNtY4TUNbnGo/Bp7oNpOcA10HOv8c4ptYTtpDu2sw3Uk6VJ0ap7Ts9n0zXtV9aEdHYIyniHV8N/8B2qpxam8nn3wyjz32GJMnT6aiooL33nuPBx54gC+//JKCggK+9a1v0dDQwKJFizj33HMJhUJccsklDBs2jKuu2kt3iQSlRJK1avMODumeRdes0M6d9x8K4TqnturMu2HkpeA7gIo7EUuwjDFJE+uThWVZxrSXiy66iLlz5zJ69GhEhPvvv5/evXvz9NNP88ADDxAMBsnOzmbatGmUlJRw3XXXEY068xTee++9B/z6KZFkbaioZUC3uD4NTfVOggVwws0w+rKOCcwYY/Zg5+jCjo3DmFTUPEeWiPDAAw/wwAMP7HJ88uTJTJ68+3R4ixYtSmocnu+TtWbLDlZs3k7/+CSrdOXO7THfPPhBGWNMG3bOk2VZljGpyvNJ1mtLnTmRjh0cNyfPendeqO8v27cO3MYYc5BZTZYxqcvzzYUVtY10zQxy3qg+zhD5RdNgzr3OKLkuyVmixxhjks0Xm9/NsixjUpXnk6zKmqadHd5f+/7OWcGPvmbXSSqNMaYTsT5ZJpWpasotEr0/Tfueby6sqGkkvznJaohbDHLEpA6JxxhjEmFrF5pUlZ6eTnl5eUr1N1RVysvLSU9Pb7twHO/XZNU27hxZ6ItbETere8cEZIwxCWj+jG9rF5pUU1BQQHFxMaWlpR0dSlKlp6dTUFCwT+d4Pskqr2lkTP8uzpNtxTsPZPXokHiMMSYRNhmpSVXBYJDBgwd3dBidgqebC5siUcqqG+iZ61bf1VftPBjK6pCYjDEmETv7ZFmWZUyq8nRNVumOBlShd26683Gwfhsccx0cc21Hh2aMMXvlS7FOwcaY3Xm6Juur7fUA9M5Lg8Zq0KizyHPfMR0bmDHGtMH6ZBmT+jydZG11k6yeOelQ+pmzMz23AyMyxpjENC+lajmWManL00lWXVMEgKy0ADxxmrMzPa8DIzLGmMSIW5dlNVnGpK6EkiwRmSgiq0VkrYjc3srxgSLylogsE5E5IlIQd+x+EVkuIitF5CFJ4uxkjWFnpexQIO5tBDKSdXljjGk3zXdCS7GMSV1tJlki4gf+CJwDjACuEJERLYo9CExT1VHA3cC97rknACcCo4AjgbHA+GQF3xhxbk9Bf1zeZjVZxhgPsAWijUl9idRkjQPWquo6VW0EpgMXtigzAnjb3X4n7rgC6UAISAOCwJYDDbpZk1uTleYTED8cMgEGHp+syxtjTLuxZXWMSX2JJFn9gI1xz4vdffGWAhe72xcBOSKSr6pzcZKuze5jlqqubPkCIjJFRIpEpGhfZohtjDhJVjC8AzQCQ89K+FxjjOlItqyOMakvWR3fbwXGi8hinObAEiAiIkOA4UABTmJ2moic3PJkVX1cVQtVtbBHj8Rnam+uyQo2VDg7Mm0pHWOMN9gUDsakvkQmIy0B+sc9L3D3xajqJtyaLBHJBi5R1SoR+RYwT1Wr3WP/Ao4H3k9C7DRGoohAoHG7syOjSzIua4wx7c6W1TEm9SVSk7UAGCoig0UkBFwOvBpfQES6i0jzte4AprrbG3BquAIiEsSp5dqtuXB/NUaiBP0+pMFNstJsjixjzK4SGB39IxFZ4Y6OfktEBsYdi4jIEvfxastzDywu56t1fDcmdbWZZKlqGLgZmIWTIL2gqstF5G4RucAtNgFYLSKfAb2Ae9z9M4DPgU9w+m0tVdXXkhV8U1hJ8/ugYYezIy0nWZc2xqSABEdHLwYK3dHRM4D7447VqeoY93EBSRTrk5XMixpjOpWE1i5U1ZnAzBb7fh63PQPn5tTyvAjwXwcY4x41RiIEAz6od2uybLZ3Y8yuYqOjAUSkeXT0iuYCqvpOXPl5wFUHIzBbINqY1OfpGd+bwurMkWXNhcaY1iUyOjreDcC/4p6nuyOf54nIpD2dtD8jpH2x5sKEihtjPCihmqzOqikSdWZ7t+ZCY8wBEpGrgEJ2nTB5oKqWiMghwNsi8omqft7yXFV9HHgcoLCwMMG0yZbVMSbVebomq8Ht+E79dghlg8/f0SEZYzqXNkdHA4jIGcCdwAWq2tC8X1VL3K/rgDnAUckKzGfL6hiT8jydZDWFo4T8Pmjc4SRZxhizq0RGRx8FPIaTYG2N299VRNLc7e44S4StIEnEltUxJuV5urmwsbm5sKkegrYwtDFmV6oaFpHm0dF+YGrz6GigSFVfBR4AsoEX3cRngzuScDjwmIhEcT6Q3qeqSUuyrE+WManP00lWU3NzYbgOAukdHY4xphNKYHT0GXs47yNgZHvFJbE+We31CsaYjubx5kJ1mgub6iFoSZYxxjtsMlJjUp+na7L6dc0gPeiD7fUQsOZCY4x37Jwnq2PjMMa0H08nWb+/bIyz8US9dXw3xnhK84zvNr7QmNTl6ebCGOv4bozxGKvJMib1pUaSFa63ju/GGE+JrV1oSZYxKcuSLGOM6QDNjYU247sxqSs1kqymOhtdaIzxlObJSC3JMiZ1pUaSFW6w0YXGGE9pnozUGJO6UiTJsposY4y3WE2WMakvoSRLRCaKyGoRWSsit7dyfKCIvCUiy0RkjogUxB0bICJviMhKEVkhIoOSGD9EwhANW58sY4yn2LI6xqS+NpMsEfEDfwTOAUYAV4jIiBbFHgSmqeoo4G7g3rhj04AHVHU4MA7YSjJtL3G+ZuYn9bLGGNOebFkdY1JfIjVZ44C1qrpOVRuB6cCFLcqMAN52t99pPu4mYwFVnQ2gqtWqWpuUyJttWuR87XtUUi9rjDHtyZbVMSb1JZJk9QM2xj0vdvfFWwpc7G5fBOSISD5wGFAlIn8XkcUi8oBbM5Y8Feucrz0OT+pljTGmPYk1FxqT8pLV8f1WYLyILAbGAyVABGfZnpPd42OBQ4BrW54sIlNEpEhEikpLS/ftlSNNzlfrk2WM8ZDYZKS2rI4xKSuRJKsE6B/3vMDdF6Oqm1T1YlU9CrjT3VeFU+u1xG1qDAOvAEe3fAFVfVxVC1W1sEePHvv2DqJhQMCXGgMljTH/GWxZHWNSXyKZyQJgqIgMFpEQcDnwanwBEekuIs3XugOYGnduFxFpzpxOA1YceNhxIk3gDyb1ksYY095sWR1jUl+bSZZbA3UzMAtYCbygqstF5G4RucAtNgFYLSKfAb2Ae9xzIzhNhW+JyCc4K0n8OanvIBoGXyCplzTGmPZmy+oYk/oSyk5UdSYws8W+n8dtzwBm7OHc2cCoA4hx76Jh8FlNljHGWyTWJ8sYk6q835Ep0gR+q8kyxniLTeFgTOrzfpJlzYXGGA+yPlnGpL4USbKsudAY4y3WJ8uY1Of9JMuaC40xe5HA2qs/ctdVXeauwTow7thkEVnjPiYnMy6ryTIm9Xk/ybLmQmPMHiS49upioNBde3UGcL97bjfgF8CxOMuL/UJEuiYvOOeL1WQZk7pSIMlqsuZCY8yetLn2qqq+E7em6jycCZcBzgZmq2qFqlYCs4GJyQrMJ22XMcZ4m/eTrEjYmguNMXuSyNqr8W4A/rWf5+6T5ikcrCbLmNTl/ezEmguNMUkgIlcBhTjrr+7ruVOAKQADBgxI6ByfLRBtTMrzfk2WNRcaY/aszbVXAUTkDJx1Vy9Q1YZ9ORf2b/1VobkmK6HixhgPSoEkK2I1WcaYPUlk7dWjgMdwEqytcYdmAWeJSFe3w/tZ7r6kEOv4bkzK8352EmmCQKijozDGdEKqGhaR5rVX/cDU5rVXgSJVfRV4AMgGXnT7SW1Q1QtUtUJE/hcnUQO4W1UrkhVb8xQOxpjU5f0kK9oEvqyOjsIY00klsPbqGXs5dyowtT3iitVkWXuhMSkrBZoLreO7McZ7fLZAtDEpz/tJViQMfuv4bozxFltWx5jU5/0kK9pkNVnGGM8Rm8LBmJSXAkmWNRcaY7xHYmsXWpZlTKpKKMlKYIHVge7CqstEZI6IFLQ4nisixSLycLICj7HmQmOMR/nE+mQZk8raTLISXGD1QWCau8Dq3cC9LY7/L/DegYfbCqvJMsZ4lIhYnyxjUlgiNVltLrCKk3y97W6/E39cRI4BegFvHHi4rbA+WcYYj/KJzfhuTCpLJMlKZJHUpcDF7vZFQI6I5IuID/gtcOveXkBEpohIkYgUlZaWJhZ5s0iTNRcaYzzJ7xObJ8uYFJasju+3AuNFZDHO4qolQAT4DjBTVYv3dvL+rPsVY8vqGGM8Kujz0RiJdnQYxph2kkh20uYiqaq6CbcmS0SygUtUtUpEjgdOFpHv4CxbERKRalXdrfP8frPmQmOMRwUDPposyTImZSWSncQWWMVJri4HrowvICLdgQpVjQJ34C5DoarfjCtzLVCY1AQLrLnQGONZAZ8QjlhzoTGpqs3mQlUNA80LrK4EXmheYFVELnCLTQBWi8hnOJ3c72mneFsGB2rNhcYYbwr6rbnQmFSWUHaSwAKrM4AZbVzjKeCpfY5wb6Jh56vParKMMd4TCvisJsuYFObtGd+bkyy/1WQZY7wn4BPrk2VMCvN2khVpcr5ac6ExxoOCfh9NVpNlTMrydpJlzYXGGA8L+q0my5hUlhpJljUXGmM8KOj3EY5akmVMqvJ2kmXNhcYYDwv4haawNRcak6q8nWRFm5Msay40xnhP0O+jyWqyjElZHk+yIs5Xq8kyxniQ0/HdkixjUpW3k6zm5kLrk2WM8aCg32Z8NyaVeTvJstGFxpg2iMhEEVktImtFZLdlvUTkFBFZJCJhEfl6i2MREVniPl5NdmwBm/HdmJTm7SqgqHV8N8bsmYj4gT8CZwLFwAIReVVVV8QV2wBcC9zayiXqVHVMe8UX8tuM78akMm9nJ5HmKRysJssY06pxwFpVXQcgItOBC4FYkqWq691jB71KyWZ8Nya1pUhzobdzRWNMu+kHbIx7XuzuS1S6iBSJyDwRmZTUyIBgwDq+G5PKvJ2dWHOhMaZ9DVTVEhE5BHhbRD5R1c9bFhKRKcAUgAEDBiR88aBPbFkdY1KYt2uyYqMLrbnQGNOqEqB/3PMCd19CVLXE/boOmAMctYdyj6tqoaoW9ujRI+HgbAoHY1Kbt5Os2DxZlmQZY1q1ABgqIoNFJARcDiQ0SlBEuopImrvdHTiRuL5cyRAK+GgMW5JlTKpKKMlKYAj0QBF5S0SWicgcESlw948Rkbkistw9dllSo481F/qTelljTGpQ1TBwMzALWAm8oKrLReRuEbkAQETGikgxcCnwmIgsd08fDhSJyFLgHeC+FqMSD1hawE84qoStNsuYlNRmZ6YEh0A/CExT1adF5DTgXuBqoBa4RlXXiEhfYKGIzFLVqqREb82Fxpg2qOpMYGaLfT+P216A04zY8ryPgJHtGVt60Pmc2xiJEvB7u2HBGLO7RP6qY0OgVbURaB4CHW8E8La7/U7zcVX9TFXXuNubgK1A4h0W2qJuc6FYTZYxxnvSAs4tuKHJarKMSUWJJFmJDIFeClzsbl8E5IhIfnwBERkHhIDdRubsN3VH5Yh9AjTGeE9a0PmAWB+OdHAkxpj2kKzs5FZgvIgsBsbjjN6J3TVEpA/wDHCdqu72kU1Eprhz0RSVlpYm/qqxJEsOJHZjjOkQVpNlTGpLJMlqcwi0qm5S1YtV9SjgTndfFYCI5AL/BO5U1XmtvcD+Dn8Gq8kyxnhXWsCpyWqwEYbGpKREspM2h0CLSHeRWKZzBzDV3R8CXsbpFD8jeWG7dq8UM8YYz2ju+N5gzYXGpKQ2k6xEhkADE4DVIvIZ0Au4x93/DeAU4Nq4lezHJC16ay40xniY1WQZk9oSWo8mgSHQM4DdaqpU9a/AXw8wxr1F5n61JMsY4z1pbk1WfZPVZBmTirzdmam5udD6ZBljPMg6vhuT2rydnVhzoTHGw6y50JjU5u0ky5oLjTEeZh3fjUlt3k6ybDJSY4yHNddk1VmfLGNSkrezk1ifLKvJMsZ4T2aam2Q1WpJlTCrydpJlzYXGGA/LdJfVqbUky5iU5O0ky5oLjTEeFvD7CAV81DSGOzoUY0w78HZ2Ys2FxhiPywr5qW2wmixjUpG3k6wYS7KMMd6UGQpYTZYxKcrbSZY1FxpjPC4rzWqyjElV3s5OrLnQGONxmaEAtTaFgzEpydtJVmx0oTHGeJNTk2XNhcakIm8nWdZcaIzxuMxQgGpLsoxJSd7OTqy50Bjjcb1z0ympqkPVauaNSTXeTrJsMlJjTBtEZKKIrBaRtSJyeyvHTxGRRSISFpGvtzg2WUTWuI/J7RHfoT2y2FEfprS6oT0ub4zpQN5Osqy50BizFyLiB/4InAOMAK4QkREtim0ArgWebXFuN+AXwLHAOOAXItI12TEe0iMbgC9Ka5J9aWNMB0soO0ngk+BAEXlLRJaJyBwRKYg71n6fBK250Bizd+OAtaq6TlUbgenAhfEFVHW9qi4Doi3OPRuYraoVqloJzAYmJjvAblkhAKrqmpJ9aWNMB2szyUrwk+CDwDRVHQXcDdzrntvOnwStudAYs1f9gI1xz4vdfe19bsJy04MA7Ki3zu/GpJpEarLa/CSIk3y97W6/E3e8fT8JWnOhMaYTEJEpIlIkIkWlpaX7dG52egCAHfVWk2VMqkkkO0nk09xS4GJ3+yIgR0TyEzx3/29Q1lxojNm7EqB/3PMCd19Sz1XVx1W1UFULe/TosU8B5rhJVrXVZBmTcpJVBXQrMF5EFgPjcW5ECU9hvP83KGsuNMbs1QJgqIgMFpEQcDnwaoLnzgLOEpGubjeHs9x9SRX0+0gP+thhc2UZk3ISSbLa/DSnqptU9WJVPQq4091Xlci5BySWY1lzoTFmd6oaBm7GSY5WAi+o6nIRuVtELgAQkbEiUgxcCjwmIsvdcyuA/8VJ1BYAd7v7ki47LWjNhcakoEACZWKfBHESpMuBK+MLiEh3oEJVo8AdwFT30Czg13Gd3c9yjyeHNRcaY9qgqjOBmS32/TxuewHOB8DWzp3KzvtZu8lND1jHd2NSUJtVQIl8EgQmAKtF5DOgF3CPe247fxK05kJjjPflZ4f4alt9R4dhjEmyRGqyEvkkOAOYsYdz2++TYGx0oSVZxhjvGt4nl5cWFhONKj6f3c+MSRXe7sxkzYXGmBQwok8uNY0RNlbWdnQoxpgk8naShWJNhcYYr2teWmd9uSVZxqQSbydZqjay0BjjeYPyMwH4stzWLzQmlXg7Q9GoNRUaYzyvR04a2WkBlmys6uhQjDFJ5O0ky5oLjTEpQET42ui+vL5sM+FIy3WqjTFe5e0ky5oLjTEpYmS/PBrDUUqrGzo6FGNMkng7Q7HmQmNMiuidlwbAZpsvy5iU4e0ky5oLjTEpolduOgBbLMkyJmV4O8my5kJjTIro1yUDgHVlNsLQmFTh7QxF1ZoLjTEpoUtmiBHuzO81DbaOoTGpwNtJljUXGmNSyHdOPZR1ZTVMX7Cxo0MxxiSBt5Msay40xqSQ80f1ZWS/PJ77eANVtY1EotrRIRljDoC3MxSNWkWWMSalXHP8QNZurWbM3bN56K01HR2OMeYAeDvJsuZCY0yKGdE3N7b97melHRiJMeZAeTvJsuZCY0yK6d8tM7Z9SI+sDozEGHOgEspQRGSiiKwWkbUicnsrxweIyDsislhElonIue7+oIg8LSKfiMhKEbkjqdHbZKTGmBSTmx6MbVfX2yhDY7yszSRLRPzAH4FzgBHAFSIyokWxnwIvqOpRwOXAn9z9lwJpqjoSOAb4LxEZlKTYseZCY0wqevk7JzCgWyZvrNjCq0s3dXQ4xpj9lEhN1jhgraquU9VGYDpwYYsyCjR3JMgDNsXtzxKRAJABNALbDzjq2Ktac6ExJvUcNaArw/vkAHDLc4s7OBpjzP5KJEPpB8RP2lLs7ot3F3CViBQDM4HvuftnADXAZmAD8KCqVhxIwLuw5kJjTIrKSgvEttfbLPDGeFKyqoGuAJ5S1QLgXOAZEfHh1IJFgL7AYODHInJIy5NFZIqIFIlIUWnpvoymseZCY0xq+sHph5Gb7iRaEx6cw6qvktcIYIw5OBJJskqA/nHPC9x98W4AXgBQ1blAOtAduBL4t6o2qepW4EOgsOULqOrjqlqoqoU9evRIPHprLjTGtCGBgTtpIvK8e3x+c79RERkkInUissR9PHow4x6Qn8m9F4+KPZ/+sc0Cb4zXJJKhLACGishgEQnhdGx/tUWZDcDpACIyHCfJKnX3n+buzwKOA1YlJ3Rs7UJjzF4lOHDnBqBSVYcAvwd+E3fsc1Ud4z5uOihBxzl3ZG9e/95JDOmZzVMfrWf1VzsOdgjGmAPQZpKlqmHgZmAWsBJnFOFyEblbRC5wi/0Y+JaILAWeA65VVcW5uWWLyHKcZO0vqroseeFbc6ExZq8SGbhzIfC0uz0DOF2kc3x6ExGO7JfHId2d+bJ++dpytmyv7+CojDGJCrRdBFR1Jk6H9vh9P4/bXgGc2Mp51TjTOLQPay40nVxTUxPFxcXU19s/RoD09HQKCgoIBoNtF06O1gbuHLunMqoaFpFtQL57bLCILMYZFf1TVX2/neNt1b0Xj2RdWQ0ffV7Osb9+i5+cezhTTjm0I0IxxuyDhJKsTsvWLjSdXHFxMTk5OQwaNIhOUjnSYVSV8vJyiouLGTx4cEeHk4jNwABVLReRY4BXROQIVd2tB7qITAGmAAwYMCDpgeRnpzHzlpO5+sn5zP+igl/PXMXDb6/l7985kSE9s5P+esaY5PB4NZA1F5rOrb6+nvz8/P/4BAucpq/8/PyDXauXyMCdWBl3Tr88oFxVG1S1HEBVFwKfA4e19iL7PXhnH4QCPqZPOY45t06gW1aI7fVhzvjdu/xt/pft8nrGmAPn7STLmguNB1iCtVMHfC8SGbjzKjDZ3f468Laqqoj0cDvO4049MxRYd5DibpWIMKh7FkV3nhHbd+fLn/LNJ+bxZbnNpWVMZ+PtDMUmIzXG7EWCA3eeBPJFZC3wI6B5modTgGUisgSnQ/xNSZ1M+QD4fMLVxw2MPf9wbTnXTP24AyMyxrTG232yrLnQmINKVVFVfD7vfD5LYOBOPa0M0FHVl4CX2j3A/fS/k47kf845nCN/MQuAkso6/vXJZgbkZzK8dy4+n90bjelo3rlTtsaaC41p06RJkzjmmGM44ogjePzxxwHIzt7ZWXrGjBlce+21AGzZsoWLLrqI0aNHM3r0aD766CPWr1/PsGHDuOaaazjyyCPZuHEj//3f/82RRx7JyJEjef7552PX+s1vfsPIkSMZPXo0t9++27yfJsmy0wIs/flZ/OjMwwhHlW//bRHnPfQB3312UUeHZozB6zVZ1lxoPOSXry1nxabkLo0yom8uv/jaEXstM3XqVLp160ZdXR1jx47lkksu2WPZW265hfHjx/Pyyy8TiUSorq6msrKSNWvW8PTTT3Pcccfx0ksvsWTJEpYuXUpZWRljx47llFNOYcmSJfzjH/9g/vz5ZGZmUlHRKVrWUl5eZpCbxh/KhopaZiwsBuBfn37FOX94n1H98vjxWYcR8Pu4718rKVpfydu3TujYgI35D+LtJMuaC41p00MPPcTLL78MwMaNG1mzZs0ey7799ttMmzYNAL/fT15eHpWVlQwcOJDjjjsOgA8++IArrrgCv99Pr169GD9+PAsWLODdd9/luuuuIzMzE4Bu3bq18zszzUIBHw9eOprvnz6UF4s28tDba1m5eTsrN2/n+aJdl+N5Y/lXnHVE79jzX89cyePvrWP9fecd7LCNSXneTrKsudB4SFs1Tu1hzpw5vPnmm8ydO5fMzEwmTJhAfX39LqP8EplSISsrqz3DNEnSv1smPzzzMOauKyc/K41/L/9qtzJTnlnI+7edytrSak4d1pPH33MGTFY3hMlO8/a/BGM6G29nKNZcaMxebdu2ja5du5KZmcmqVauYN28eAL169WLlypVEo9FYLRfA6aefziOPPAJAJBJh27Ztu13z5JNP5vnnnycSiVBaWsp7773HuHHjOPPMM/nLX/5CbW0tgDUXdhAR4cWbTuDRq49h7T3ncNvEYbuVOfn+d7juLwt477PS2L4jfzGL9WU2DYQxyeTtJAuw5kJj9mzixImEw2GGDx/O7bffHmvyu++++zj//PM54YQT6NOnT6z8H/7wB9555x1GjhzJMcccw4oVK3a75kUXXcSoUaMYPXo0p512Gvfffz+9e/dm4sSJXHDBBRQWFjJmzBgefPDBg/Y+TesCfh83xS2/8+Mzd51LteW0DxMenENNQxiArTvqKa6sbf8gjUlh4qzj3HkUFhZqUVFRYoWfuxKqvoRvf9i+QRmzn1auXMnw4cM7OoxOpbXviYgsVNXCDgopqfbpHnaQzFhYzLBeOWSl+blm6sd889iBLN+0jdeXbd6t7OG9czh2cDeenuvMJP/fZw/ju6cO4d6ZKzljRC/GDtpzX7vm/yc2Aa/5T7K3+5e3G+CtudAYY9r09WMKYtsf/M9pgJMQXXnsAF5buolNVfW86zYdrvpqB6u+2hEr/8Cs1Xywpoy568p57L11vH/bqfTv5gxuKKmq4//eWsN3Tx3CxY98ROmOBs4+ohePXV1IXWMEv08IBVKgwcSY/eTx334bXWiMMftDRDjh0O7ce/Eonr5+HIf3ztnleN+89Nj23HXlse2fvPxJbPtHzy9h+oKN3PzcYkp3NAAwa/kWVJUj75rF9U8taOd3YUzn5vGaLLWaLGOMSYJ//+AUSqrqKFpfwWG9cji8dw5NEeXSx+aydGMVAJeP7c/0BRv5v7fWMKp/F+Z/4QxuaD7e7KmP1hOJKh+sLQMgHIkCTh8xgK3b68lKC5CVwGjGSFSpbQyTkx5M0js15uDxdpKFTeFgjDHJ0q9LBv3G9Is9DwWEl799Ah+vryArFMDng+kLNvLb2Z/tdq5PIOp28f3lazsHTLyyuIRf/XMF4wZ3496LRvFffy1i3jonOXtyciHDeuewobyWN1Zsob4pwn2XjNrlur99YzV/mvM5z954LCcM6d4O79qY9pNQkiUiE4E/AH7gCVW9r8XxAcDTQBe3zO3uemGIyCjgMSAXiAJj3bXCDpxGseZCY4xpPz6fcNwh+cDOju0t/e+FRzD+sJ7c+conFHTN4LmPNzKsVw6VtY384PklAMz85CtmfrLrvF03PL37AIGff20EbyzfwvGH5rNy83aK1lcCMOez0gNOslTVOuWbg6rNJEtE/MAfgTOBYmCBiLyqqvFju3+Ks7r9IyIyAmcx1kEiEgD+ClytqktFJB9oSlr01lxojDEHjYjw/m2nkhb00SM7jQ/WlnHCod3xu4tRP3PDsdQ1RuiRncZl4waQHQrwu9mriajy13kbdrnW6II8lhbvPg/biJ/PavW1F37pJFvNidKSjVX831tr+OGZh1FR00jP3DQKumby3PwNHN4nh2G9cvjvGcv47TdGk50WYMIDc+jfLYMXbzphj+/v4y8qGN0/j7SAf3+/RXtVUlVHvy4Z7XJts++2bK9n3rpyLoyrvU22RGqyxgFrVXUdgIhMBy4E4pMsxampAsgDNrnbZwHLVHUpgKqWk1TWXGiMMQdT88hCgJOH9tjteEbIz4/O2jkB6i8vPBKA3PQgf5rzOU9cU8jpw3sCMG9dBaGA8NHacg7tmc13/rbnha0/Kd7Gg7NW8/A7azlzRC/eXV1KYyTKW6u2tlr+5KHdeX9NGdM/3sBf523gq+31fLW9nvqmCHWNEbpmhWJlaxrCFFfW8Y3H5nLpMQX85pJR+HzC1U/OJy3g44nJY2NlI1HlwTdWMzg/i0ff/Zx/3HxiQv3F5q0r5/LH5/HwlUdx/qi+bZYH+HBtGaMK8qw/WjuZPPVjVn21gzOG90qof+D+SOSq/YD4xa+KgWNblLkLeENEvgdkAWe4+w8DVERmAT2A6ap6f8sXEJEpwBSAAQMGJB69NRca02lEIhH8/vapATDed9vEw7lt4uG77Dv+UKcZ8piBztxbn/3qHN5Y8RVpAT9zPy/n9WWbGNkvjwvG9OX705fw8DtrAZi9Ykubr/f+GqfT/YNv7Np/7PCf/Tu2/edrCtlYUcvdr++sM3hxYTEvLizm8N45saksHpy1mvNH9+Hw3rks2lDJI3M+j5Vfvml7rDm12ba6JvIydk2MlruLw89bVx5Lsh6Z8zlDe2Zzxoheu8VfVt3AN5+Yz2mH92TqtWN3Ow5QWdO4S7Jo9k1xZR3gLCnVkUlWIq4AnlLV34rI8cAzInKke/2TgLFALfCWO2nXW/Enq+rjwOPgTOSX8Ktac6ExbZo0aRIbN26kvr6e73//+0yZMoXs7Gyqq6sBmDFjBq+//jpPPfUUW7Zs4aabbmLdOmc9u0ceeYS+ffsyceJEjjnmGBYtWsQRRxzBtGnTyMzMZNCgQVx22WXMnj2b2267jcsvv7wj36rxuFDAF0tAzhzRi59/bQQA0ajy1bZ6MkN+xvTvykuLigG49oRBLC2uIhxRXlu2iZ+dP4Lc9CBj73kzodf71rQ9TxobP1fYw++s5eF31nLjSYMpHNR1l3I/e+VTvnnsAJYVb+OHZx7GK4tL+O3sz/jzNYWc6SZP2+qaeMetcXMHWhKJKr/59yoA1v36XBZuqKRwYNdYn7ESNwGYt27XBqDaxjDrSmvYXtfElU/M55kbxrVao9hctnRHAwPz92/t0TVbdvD2qq381/hD2y7sQW4rN9vrmuiVm773wvspkSSrBOgf97zA3RfvBmAigKrOFZF0oDtOrdd7qloGICIzgaOBt0gKay40HvKv2+GrT9outy96j4Rz7ttrkalTp9KtWzfq6uoYO3Ysl1xyyR7L3nLLLYwfP56XX36ZSCRCdXU1lZWVrF69mieffJITTzyR66+/nj/96U/ceuutAOTn57No0Z6beYw5UD6f7PKPfmRBXmx7UHcngbgkbsLVV28+kQ0VtYT8PqY8s5BJY/oytFcOpw7rycD8TLbuaODmZxfFapcKB3al6MtKhvbMZs3W6th1Hvj6KP57xrLY8yc++IJFGyp3iW3N1mruckdT/n3xzn+N35pWRE5agAuP6ktVbVNsOovnPt7Acx/v2j/tuqcW8O5npbzwX8dTUdNAj5w0vvfsYgBqGyOxcqrKhQ9/yJqt1Zw7sjcA89dVxJKsT4q30TM3jZcXl1BV20RTJMqTH3zB3248liP75jHvi3KOPzSf3ASbHy99bC5VtU1cPnYAeZnt12T5ack2umWF6Luf/dXeWrmFgN/H+MNaTzb3xOdmWdvrk9dVvKVEkqwFwFARGYyTXF0OXNmizAbgdOApERkOpAOlwCzgNhHJBBqB8cDvkxS7U5NlzYXG7NVDDz0UWwR648aNrFmzZo9l3377baZNmwaA3+8nLy+PyspK+vfvz4knngjAVVddxUMPPRRLsi677LJ2fgfG7JtRBV0YVdAFgDX3nEPAJ7uMKhycFuCft5wce15R08jVT87n2hMGxZKqop+eQffsNDZvq+etVVtjc4Et2lAVO+/HZx6223QWw/vkMrogj+kLNrKjIbxbh//WNM+2/43H5rZ6fPAd/+TRq47h7tdWUFLl1HA1j9Ssbggz/eMN/H1xCR9/UUFeRpBtdbsmDVc/OT82vcakMX3pmZvO4b1zuPjoAvamqta5zsbKWu585XNGFeQxsl8XPtuyg/XlNUwa04/R/bsAsOqr7QzunkVawE9TJMrvZ39Gj5w0NlbU8b3ThuyxWVNVOf//PiAvI8jSX5zV5veqNc2jVONXI2jNtrom/v3pZr5R2B8RwSfNSVZ4v143EW0mWaoaFpGbcRImPzBVVZeLyN1Akaq+CvwY+LOI/BCnE/y16oz1rRSR3+EkagrMVNV/Ji16ay40XtJGjVN7mDNnDm+++SZz584lMzOTCRMmUF9fv8s/nPr6tmdUaTnsPf55Vtb+NUUYczAE/W23dnTLCsWSrp656WSF/HTPTgPgltOHcvOpQ3jsvXWx5j2AowZ04XunD+Wckb1ZsXkHM5dt5t/Lv+LZG4+la1aIO84Zzui73wAgLeDjrguOYOv2Bn7/ppOU/WrSkYzp34Xz/++DPcbVPTtEWXUjqvBfzyxstcwrS0piyRCwW4IFO+cvA3h92WbC7o5XlmziN5eM5IrH53HL6UM55bAezPxkM6cO67nLLP/NMbZc6/IvH67n4SuP4oM1ZUxf4HTdvm3iMMYUdOFPcf3Wpn74BS99+3iG9c5l+scb+NU/V3Jkv1yOG5zP5eMG7BJ3OBLlwj9+yBF9c7n46IJYf7d/LClhWfE2bps4LDb6syEcYVvce39lcQnHH5rP0QO6xmqp4t316nJeXlzCYb1yOGpA112aC5tjKN3RQFaanz55yRkFmlCfLHfOq5kt9v08bnsFcOIezv0rzjQO7cCaC43Zm23bttG1a1cyMzNZtWoV8+bNA6BXr16sXLmSYcOG8fLLL5OT4yypcvrpp/PII4/wgx/8INZcCLBhwwbmzp3L8ccfz7PPPstJJ53UYe/JmPbUWpOTzyd8e8KhfPO4ARx992zuPG841504GIAhPXMY0jOHc4/szaaq+liNTV5mkPdvO5X6pghDezl/X+vLaliysZLJJwxiwjBnhOWjVx3N3M/Lmf9FBWkBH3+5bhwbKmr57RurefSqYzjiF86UFlkhPzWNESYe0ZustAAvLSrmWycP5s/vfwE4Sd9it5bN7xOK7jyDt1ZtpbKmkTdWfMUCd76x5gQrOy3Ae5+Vcvy9bwPwoxeWxr3j5Ql/v252mzWb3f/v1a2W++YT86lvisaef1qynU9LtrM4brWAUx+cwxdlNU4Em7bzQlExT103lg/XlsXeZ9DvY0NFDX3zMlj11Y5YMyzg1CrOhhtPGswPzzws1pm9rLqBq56YH+tnt2ZLNWP6d6GsuhFwarLqmyLc8NQCitypQqZdP45T9rH5sTWyp8nlOso+rWA/dSL4AnDt6+0blDH7aeXKlQwfPrzDXr+hoYFJkyaxfv16hg0bRlVVFXfddRdlZWX8z//8Dz169KCwsJDq6upYx/cpU6awbt06/H4/jzzyCH369GHixIkUFhaycOFCRowYwTPPPBPr+F5UVET37olPEtna92Rvq9h7zT7dw4znRKPaai3JgWq59FCz8uoGRISmSJTTHpzDX64bx/A+OWzd0cDg/Czufn0F2+ub+OUFR5CdFuC5jzcydlDXWGLX7P5/r2LJxio++ryc2885nKuPGxhL4MYO6hpLwlp6+MqjuPXFpTz7reNQVS55ZC73XHQkeRlBPinexvtryggGfLstrfS10X15bekm+uSls3nb3mvLmxPIZiP75fFJye5zqO3N6Yf33GU6j/ysEK/fchIvFhXzu1ZWKIg3sl8eJVV1VNQ0xvZdddwAfjVpZEKvvbf7l7eTrCfPhkAIJr/WvkEZs586OslKhvXr13P++efz6aefJuV6BzvJSmDFijRgGnAMUA5cpqrr3WN34AzsiQC3qGrrM2XGsSTLdFaqyrqyGg7tkQ3AnNVbaYooZ47oRXFlLTlpQfx+4W/zvqRwUDf6d8ugZ86uo+4iUY1NPtvy2o2RKBc+/CETj+zND844jB31TQT9Pn72yqe8uLCYi47qx1XHDWBTVT3Deudwx98/Ycv2ep6YXMjHX1QwZ3Up/+/yMeSmB9lR30RdY4Rrpn7M0QO7csHovlz+uFMT//yU4/jr/A28tnRT7PX/duOxfPOJ+Xt873eeO5yzjujFjU8X7TK4oaWzRvTijRVb+OEZh/H9M4Ym9H1N4STrLAhmwDX/aN+gjNlPlmTt7mAmWe6KFZ8Rt2IFcEX8ihUi8h1glKreJCKXAxep6mXu6hXP4UzI3Bd4EzhMVSMtXyeeJVnG7Gp9WQ3fn76Y/7viaAbk77ljeltWbNpOfnaIXrnphCNRSqrqYtNTqCqPvruO80f1oX+3TB5993OmfbSeIb1y+N5pQxg7yJmLrXRHAw/MWsULRcVcfFQ/fnTWYdz92gq+PeFQjhrQdW8vv0d7u395e4HoEZPAbzPhGtOeBg0alLQEqwMksmLFhTgTKgPMAB4Wp2f/hTgTKDcAX4jIWvd6rQ8BM8a0alD3LP5x84H34xzRNze2HfD7dpn/S8TpN9fspvGHclMr83v1yEnjV5NGMqRnNt88diBZaQEev6b9eip4O8k6/jsdHYExpnNLZMWKWBl3NPU2IN/dP6/Fue23yJkx5qAIBXxMOeXgTLBqQ/OMaWedrUm+I6Xq90JEpohIkYgUlZaWdnQ4xphOwpIsY9pReno65eXlKZtc7AtVpby8nPT09lm+Yg8SWbEiVkZEAjiL3JcneC7gLA2mqoWqWtijx4EP+zbGpAZvNxca08kVFBRQXFyM1W440tPTKSjY+yzTSZbIihWvApNx+lp9HXhbVVVEXgWedSdU7gsMBT4+aJEbYzzPkixj2lEwGGTw4MEdHcZ/rARXrHgSZ1H7tUAFTiKGW+4FnE7yYeC7bY0sNMaYeJZkGWNSWgIrVtQDl+7h3HuAe9o1QGNMyrI+WcYYY4wx7cCSLGOMMcaYdtDpZnwXkVLgy304pTtQ1mapzsFLsYK34vVSrOCteA9GrANVNSWG5e3jPcxLvwfgrXi9FCt4K16LdVd7vH91uiRrX4lIkVcWlvVSrOCteL0UK3grXi/F6jVe+956KV4vxQreitdiTZw1FxpjjDHGtANLsowxxhhj2kEqJFmPd3QA+8BLsYK34vVSrOCteL0Uq9d47XvrpXi9FCt4K16LNUGe75NljDHGGNMZpUJNljHGGGNMp+PZJEtEJorIahFZKyK3d3Q8ACIyVUS2isincfu6ichsEVnjfu3q7hcReciNf5mIHH2QY+0vIu+IyAoRWS4i3+/k8aaLyMcistSN95fu/sEiMt+N63kRCbn709zna93jgw5mvG4MfhFZLCKvd+ZYRWS9iHwiIktEpMjd1yl/D1JJZ7uHeen+5cbgmXuY3b/aPdZOew/zZJIlIn7gj8A5wAjgChEZ0bFRAfAUMLHFvtuBt1R1KPCW+xyc2Ie6jynAIwcpxmZh4MeqOgI4Dviu+z3srPE2AKep6mhgDDBRRI4DfgP8XlWHAJXADW75G4BKd//v3XIH2/eBlXHPO3Osp6rqmLihzp319yAldNJ72FN45/4F3rqH2f2r/XXOe5iqeu4BHA/Mint+B3BHR8flxjII+DTu+Wqgj7vdB1jtbj8GXNFauQ6K+x/AmV6IF8gEFgHH4kwyF2j5e4GzIPDx7nbALScHMcYCnD/s04DXAenEsa4HurfY1+l/D7z86Kz3MK/ev9wYPHEPs/tXu8Tbae9hnqzJAvoBG+OeF7v7OqNeqrrZ3f4K6OVud5r34FbvHgXMpxPH61ZfLwG2ArOBz4EqVQ23ElMsXvf4NiD/IIb7/4DbgKj7PJ/OG6sCb4jIQhGZ4u7rtL8HKcIr30dP/B544R5m96921WnvYYH2urDZnaqqiHSq4Zwikg28BPxAVbeLSOxYZ4tXVSPAGBHpArwMHN6xEbVORM4HtqrqQhGZ0MHhJOIkVS0RkZ7AbBFZFX+ws/0emI7RWX8PvHIPs/tXu+q09zCv1mSVAP3jnhe4+zqjLSLSB8D9utXd3+HvQUSCODenv6nq393dnTbeZqpaBbyDU2XdRUSaPyzExxSL1z2eB5QfpBBPBC4QkfXAdJwq9z900lhR1RL361acm/84PPB74HFe+T526t8DL97D7P6VfJ35HubVJGsBMNQd7RACLgde7eCY9uRVYLK7PRmn30Dz/mvckQ7HAdviqjbbnTgf954EVqrq7zwQbw/3EyAikoHT92Ilzs3q63uIt/l9fB14W90G+PamqneoaoGqDsL53XxbVb/ZGWMVkSwRyWneBs4CPqWT/h6kEK/cwzrt74GX7mF2/2o/nf4edjA7pyXzAZwLfIbTrn1nR8fjxvQcsBlowmnnvQGnbfotYA3wJtDNLSs4o4s+Bz4BCg9yrCfhtGMvA5a4j3M7cbyjgMVuvJ8CP3f3HwJ8DKwFXgTS3P3p7vO17vFDOuh3YgLwemeN1Y1pqftY3vy31Fl/D1Lp0dnuYV66f7kxeOYeZvevdo2xU9/DbMZ3Y4wxxph24NXmQmOMMcaYTs2SLGOMMcaYdmBJljHGGGNMO7AkyxhjjDGmHViSZYwxxhjTDizJMsYYY4xpB5ZkGWOMMca0A0uyjDHGGGPawf8HLjUVjD/AxOIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "471\n",
      "471\n"
     ]
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "plt.figure(figsize=((10,8)))\n",
    "plt.subplot(221)\n",
    "plt.plot(aucroc,label=\"aucroc\")\n",
    "plt.plot(aucpr,label=\"aucpr\")\n",
    "plt.legend()\n",
    "plt.subplot(222)\n",
    "plt.plot(loss_,label=\"loss\")\n",
    "plt.legend()\n",
    "plt.show()\n",
    "print(aucroc.index(max(aucroc)))\n",
    "print(aucpr.index(max(aucpr)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ResNet(\n",
       "  (first_layer): Linear(in_features=8, out_features=128, bias=True)\n",
       "  (blocks): Sequential(\n",
       "    (0): Block(\n",
       "      (normalization): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (linear_first): Linear(in_features=128, out_features=256, bias=True)\n",
       "      (activation): ReLU()\n",
       "      (dropout_first): Dropout(p=0.25, inplace=False)\n",
       "      (linear_second): Linear(in_features=256, out_features=128, bias=True)\n",
       "      (dropout_second): Dropout(p=0, inplace=False)\n",
       "    )\n",
       "    (1): Block(\n",
       "      (normalization): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (linear_first): Linear(in_features=128, out_features=256, bias=True)\n",
       "      (activation): ReLU()\n",
       "      (dropout_first): Dropout(p=0.25, inplace=False)\n",
       "      (linear_second): Linear(in_features=256, out_features=128, bias=True)\n",
       "      (dropout_second): Dropout(p=0, inplace=False)\n",
       "    )\n",
       "  )\n",
       "  (head): Head(\n",
       "    (normalization): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (activation): ReLU()\n",
       "    (linear): Linear(in_features=128, out_features=1, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm1d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "34.243M\n",
      "133.889K\n"
     ]
    }
   ],
   "source": [
    "from thop import clever_format\n",
    "from thop import profile\n",
    "input = torch.randn(256, 8)\n",
    "flops, params =profile(clf.model, inputs=(input,))\n",
    "flops, params = clever_format([flops, params], \"%.3f\")\n",
    "print(flops)\n",
    "print(params)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}