notebooks/5_model_tensorrt.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import pickle\n",
    "import itertools\n",
    "\n",
    "import h5py\n",
    "import numpy as np\n",
    "\n",
    "import onnx\n",
    "import pycuda.driver as cuda\n",
    "import pycuda.autoinit\n",
    "import tensorrt as trt\n",
    "\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_models = \"../models/\"\n",
    "path_converted_model = path_models + \"full_model.onnx\"\n",
    "path_dataset = \"../datasets/dataset.h5\"\n",
    "path_labels = \"../datasets/labels.pickle\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path_labels, 'rb') as handle:\n",
    "    labels = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "h5f = h5py.File(path_dataset, 'r')\n",
    "\n",
    "X_train = np.array(h5f['X_train'])\n",
    "X_test = np.array(h5f['X_test'])\n",
    "y_train = np.array(h5f['y_train'])\n",
    "y_test = np.array(h5f['y_test'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train.astype(np.float32)\n",
    "X_test = X_test.astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_batch(array: np.array, size=32):\n",
    "    for i in range(array.shape[0] // size):\n",
    "        \n",
    "        yield array[size*i : size*(i+1)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "OUT_SIZE = len(labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert to TensorRT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensor RT engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRT_LOGGER = trt.Logger(trt.Logger.VERBOSE)\n",
    "flags = 1 << (int)(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)\n",
    "\n",
    "def build_engine(model_path):\n",
    "    \"\"\"Build and return the TensorRT engine and context.\"\"\"\n",
    "    \n",
    "    with trt.Builder(TRT_LOGGER) as builder, builder.create_network(flags) as network, trt.OnnxParser(network, TRT_LOGGER) as parser: \n",
    "        \n",
    "        # Set builder 1GB Vram, batch size 1 & fp16 if possible\n",
    "        builder.max_workspace_size = 1 << 30\n",
    "        builder.max_batch_size = BATCH_SIZE\n",
    "        \n",
    "        if builder.platform_has_fast_fp16:\n",
    "            builder.fp16_mode = True\n",
    "        \n",
    "        # Load ONNX model\n",
    "        with open(model_path, \"rb\") as f:\n",
    "            parser.parse(f.read())\n",
    "        \n",
    "        # Generate an engine optimized for the target platform\n",
    "        engine = builder.build_cuda_engine(network)\n",
    "        context = engine.create_execution_context()\n",
    "\n",
    "        return engine, context"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Serialize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "engine, context = build_engine(path_converted_model)\n",
    "\n",
    "# Serialize and save the model\n",
    "serialized_engine = engine.serialize()\n",
    "\n",
    "with open(path_models + \"model.engine\", \"wb\") as file:\n",
    "    file.write(engine.serialize())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deserialize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This step is not needed, and is provided as an example. We already have a deserialized engine as we build it above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path_models + \"model.engine\", \"rb\") as file, trt.Runtime(TRT_LOGGER) as runtime:\n",
    "    engine = runtime.deserialize_cuda_engine(file.read())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inference pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This function allocate memory on the CPU and GPU. It has to be called before any inference. Once the memory has been allocated, the data can be passed and an inference can be done."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def allocate_buffer(engine):\n",
    "    \n",
    "    # Host CPU memory\n",
    "    h_in_size = trt.volume(engine.get_binding_shape(0))\n",
    "    h_out_size = trt.volume(engine.get_binding_shape(1))\n",
    "    h_in_dtype = trt.nptype(engine.get_binding_dtype(0))\n",
    "    h_out_dtype = trt.nptype(engine.get_binding_dtype(1))\n",
    "    \n",
    "    in_cpu = cuda.pagelocked_empty(h_in_size, h_in_dtype)\n",
    "    out_cpu = cuda.pagelocked_empty(h_out_size, h_out_dtype)\n",
    "    \n",
    "    # Allocate GPU memory\n",
    "    in_gpu = cuda.mem_alloc(in_cpu.nbytes)\n",
    "    out_gpu = cuda.mem_alloc(out_cpu.nbytes)\n",
    "    \n",
    "    stream = cuda.Stream()\n",
    "    return in_cpu, out_cpu, in_gpu, out_gpu, stream"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This function take allocated space, an input matrix, and run the inference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inference(engine, context, inputs, out_cpu, in_gpu, out_gpu, stream):\n",
    "    \n",
    "    cuda.memcpy_htod(in_gpu, inputs)\n",
    "    context.execute(1, [int(in_gpu), int(out_gpu)])\n",
    "    cuda.memcpy_dtoh(out_cpu, out_gpu)\n",
    "    \n",
    "    return out_cpu"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "engine, context = build_engine(path_converted_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []\n",
    "t1 = time.time()\n",
    "\n",
    "for batch in get_batch(X_test, BATCH_SIZE):\n",
    "    \n",
    "    in_cpu, out_cpu, in_gpu, out_gpu, stream = allocate_buffer(engine)\n",
    "    results.append(inference(engine, context, batch.reshape(-1), out_cpu, in_gpu, out_gpu, stream))\n",
    "\n",
    "cost_time = time.time() - t1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Output post-processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Batch inferences flatten all results. We have to get back result as array of 6 for each images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Flatten the list\n",
    "flatten = np.array(list(itertools.chain(*results)))\n",
    "\n",
    "# Get the amount of predicted values\n",
    "flatten_lenght = int(flatten.shape[0] / OUT_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "reshaped = flatten.reshape(flatten_lenght, OUT_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inference time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inference time: **0.3408** sec, for **755** images.\n",
    "\n",
    "Time per image: **0.004 sec**/image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "736"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_lenght = len(reshaped)\n",
    "result_lenght"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost time:  0.11856484413146973\n"
     ]
    }
   ],
   "source": [
    "print(\"cost time: \", cost_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time per image:  0.0001610935382221056\n"
     ]
    }
   ],
   "source": [
    "print(\"time per image: \", cost_time / result_lenght)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction = reshaped.argmax(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcoAAAHSCAYAAABsGYsqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAnk0lEQVR4nO3de3xV5ZX/8e86SSh3UGuBXBQU6qUyQI304tSi/BSqIjitqD9RdHw1/hQrjq13W1tblI5XrB1tqlas9YK2jjdGcQTH2lYElSoEiqAUkhDwBt5QclnzB6f5AcIJhGeffbL3593XfiVnn5Oz13KfZrGe/ewn5u4CAADblok7AAAAChmFEgCAHCiUAADkQKEEACAHCiUAADlQKAEAyKE46gPcVjEhNfefnLtmTtwhICK39Dk87hDyaljzx3GHkDeHvj037hDyqmljnUX13o1vvxH8933J5/eJLN4dRUcJAEAOkXeUAICUaGmOO4JI0FECAJADHSUAIAxviTuCSNBRAgCQAx0lACCMlmR2lBRKAEAQztArAADpQ0cJAAgjoUOvdJQAAORARwkACCOh1ygplACAMFiZBwCA9KGjBACEkdChVzpKAAByoKMEAISR0NtDKJQAgCBYmQcAgBSiowQAhJHQoVc6SgAAcqCjBACEwTVKAADSh44SABBGQpewo1ACAMJg6BUAgPShowQAhMHtIQAApA8dJQAgjIReo6RQAgDCYOgVAID0SUxHaRnTt5/4qT5qeE//dcb1OvyGKvX7yv7a+MEGSdKcC36ld2pWxhxleKOOGqEbbrhKRZmM7vzNffr3a38Zd0iRSUuuafksD517m5o/3CBvaZE3NWvRty5S1wP7q//Us1TUrbM+rV2r5ZNuUvOHG+IONbikfpbduY+yoA0+c7TeW1avTt27tO57Ycp9emPmvBijilYmk9HN06Zo9NEnq7Z2tV74y0w99vgsLV78etyhBZemXNP0WV58wo/U9O4HrY8HXHeOVl51lz54oUZ7nnSE+p09TrXX3hdjhOGl6bOcFIkYeu3Wd3ftdcRQLb7v2bhDyavhhwzT8uUr9OabK9XY2KgZMx7RcWNGxR1WJNKSa1o/y//QeZ9++uCFGknS+uf+qt2P+WrMEYWX6M+yt4TfCkCbhdLM9jezi83s5ux2sZkdkI/gdtTXfzxBL1x9n9TiW+wfftF4nTDran39ylOU6ZSY5rlVaVlfraqtb31cW7dapaV9Y4woOmnJNU2fZXfX/vddqYOevFZ7nnKkJGnD0lXabfRwSdLux35dnUo/H2eIkUj0Z7mlJfxWAHIWSjO7WNL9kkzSi9nNJN1nZpdEH17b9ho5VJ+8877efm3FFvvnTp2h+0dcqN8f+yN9rld3DTv72HgCBHZQ2j7LNeMu18JRP9CSU36mPqd/Sz2+cqDeuOCX6jNxtA568loVde+ilo1NcYcJtHmN8kxJX3L3xs13mtkNkhZJmrqtHzKzKklVkvR/ew/XN7oPChDqtvWt/KL2PvLL2uvwISr6XIlKenTREdPO1uzJt0qSWjY26W8zntOQs46OLIa41Nc1qKK8tPVxeVk/1dc3xBhRdNKQa9o+y40N70qSmt5Zr/eenKtuwwap4bZHtOTkqyRtGobtPfLgOEOMRKI/ywUyVBpaW0OvLZJKt7G/X/a5bXL3anevdPfKKIukJL348xm6Z/h5+t3X/03/PemXqv9TjWZPvlVdv9C79TX9Rx2sd/9WG2kccZg3f4EGDhyg/v0rVFJSovHjx+qxx2fFHVYk0pBrmj7LmS6fU6Zb59bve31ziDYsWaniPXpteoGZSiefoLW/fSrGKKORhs9y0rTVUZ4v6Rkze13Squy+vSQNlHRuhHHtspE3n63Oe/SUmfT2opV67tI74w4puObmZk0+/wrNfOJeFWUyumv6A6qpWRp3WJFIU65bS+JnuWTP3hp0x8WSJCvO6J2H/6j1z76iPmceoz6nf0uS9N5/vaC37p8dZ5iRSPRnOaF/ZsvcPfcLzDKShksqy+6qkzTPd/CGmdsqJuQ+QIKcu2ZO3CEgIrf0OTzuEPJqWPPHcYeQN4e+PTfuEPKqaWOdRfXen7z4YPDf952HnxBZvDuqzelz7t4i6YU8xAIAQMFJxjxzAED8CuR2jtASseAAAABRoaMEAISR0ttDAABINTpKAEAYCb1GSaEEAISR0ELJ0CsAADnQUQIAgkjqH26mowQAIAc6SgBAGAm9RkmhBACEwX2UAACkDx0lACCMhA690lECAJADHSUAIIyEXqOkUAIAwmDoFQCA9KGjBACEkdChVzpKAAByoKMEAITBNUoAANKHjhIAEEZCO0oKJQAgDCbzAACQPnSUAIAwEjr0SkcJAOjQzOxOM1trZgs327e7mT1tZq9nv+6W3W9mdrOZLTOzV83sy229P4USABCGt4TfdsxdkkZvte8SSc+4+yBJz2QfS9K3JA3KblWSbm3rzSmUAIAwWlrCbzvA3Z+T9O5Wu8dKmp79frqkcZvtv9s3eUFSbzPrl+v9KZQAgIJlZlVmNn+zrWoHf7SPu6/Oft8gqU/2+zJJqzZ7XW1233YxmQcAEEYEt4e4e7Wk6l18Dzczb+/P01ECAJJozT+GVLNf12b310mq2Ox15dl92xV5R3numjlRH6JgfPjir+IOIa+6Dz8r7hDyJk2fY6DdCuv2kEclTZQ0Nfv1kc32n2tm90v6iqT1mw3RbhNDrwCAMGIqlGZ2n6QRkj5vZrWSrtSmAjnDzM6U9HdJ47MvnynpaEnLJH0s6Yy23p9CCQDo0Nz95O08NXIbr3VJk3bm/SmUAIAwvN3zZQoak3kAAMiBjhIAEEZhTeYJho4SAIAc6CgBAGEktKOkUAIAwuAPNwMAkD50lACAMBI69EpHCQBADnSUAIAwErrgAIUSABAGQ68AAKQPHSUAIAw6SgAA0oeOEgAQRkIXHKBQAgCC8JZkznpl6BUAgBzoKAEAYTCZBwCA9KGjBACEkdDJPHSUAADkQEcJAAgjobNeKZQAgDCYzAMAQPrQUQIAwqCjBAAgfegoAQBh8IebAQDIgaFXAADSJ3GFctRRI7Ro4XNaUvO8LrpwUtzhBPGjW+/XiO9eqX/5/rWt+2b95a86/vv/rqEn/UCLlq/a4vV3PPyMjj3vah13/lT9acGSfIcbmSSe21zIN7kSm2uLh98KQKIKZSaT0c3TpujYMRM0eMjhOvHEcTrggEFxh7XLxn7zEN166Xe32Dewoq9u/P7pOviAfbbYv7y2QU/++RX94fqL9B+XfVdX3/kHNSdgOCSp53Z7yDe5+aYp16RIVKEcfsgwLV++Qm++uVKNjY2aMeMRHTdmVNxh7bKDD9xXPbt33WLfPuV91L/0C5957bPzFmn014epU0mxyr+whyr67KGFy1bmK9TIJPXcbg/5JjffROfqLeG3AtDuQmlmZ4QMJITSsr5aVVvf+ri2brVKS/vGGFH+rXlvvfp8vnfr4z579Nbad9fHF1AgaTu35JvcfBOdK0Ovn/GTYFEAAFCgct4eYmavbu8pSX1y/FyVpCpJsqJeymS6tTvAnVFf16CK8tLWx+Vl/VRf35CXYxeKPrv10pq317U+XvPOOn1h917xBRRI2s4t+SY33yTn6gmYD7EtbXWUfSSdJmnMNrZ3tvdD7l7t7pXuXpmvIilJ8+Yv0MCBA9S/f4VKSko0fvxYPfb4rLwdvxB8s/JLevLPr2hjY5Nq176jlQ1v66CBe8Ud1i5L27kl3+Tmm6Zck6KtBQcel9Td3Rds/YSZPRtFQLuiublZk8+/QjOfuFdFmYzumv6AamqWxh3WLrt42m81v2a51n3wkY48+yqdfcIo9ereVVN/87Dee/9Dnfvz27Xf3qW67fKzNLCir4762lAd//1/V1Emo8v+9V9UlOn4c7aSem63h3yTm2+icy2Qa4qhmUe85FBxp7Jk/pfbhg9f/FXcIeRV9+FnxR0CgJ3UtLHOonrvj6acFvz3fbfL744s3h3FEnYAgDAK5HaO0CiUAIAwEjr02vEvXgEAECE6SgBAGCm9PQQAgFSjowQAhJHQa5QUSgBAGAmd9crQKwAAOdBRAgDCSOjQKx0lAAA50FECAIJI6l8PoVACAMJg6BUAgPShowQAhEFHCQBA+tBRAgDCYMEBAADSh44SABBGQq9RUigBAEF4QgslQ68AAORARwkACIOOEgCA9KGjBACEwVqvAADkwNArAADpQ0cJAAiDjhIAgMJjZv9mZovMbKGZ3Wdmnc1sgJnNNbNlZvaAmXVq7/tTKAEAQbh78K0tZlYm6TxJle5+kKQiSSdJ+rmkG919oKT3JJ3Z3rwolACAMFo8/LZjiiV1MbNiSV0lrZZ0hKSHss9PlzSuvWlRKAEABcvMqsxs/mZb1ebPu3udpOskrdSmArle0kuS1rl7U/ZltZLK2hsDk3kAAGFEMJnH3aslVW/veTPbTdJYSQMkrZP0oKTRIWOgUAZUftgFcYeQVx8vfSTuEPKm6xfHxh0CgG37P5LedPe3JMnM/iDpUEm9zaw421WWS6pr7wEYegUABOEtHnzbASslfdXMupqZSRopqUbSHEnfyb5moqR2/8ueQgkA6LDcfa42Tdp5WdJr2lTXqiVdLOkCM1smaQ9Jd7T3GAy9AgDCiGnBAXe/UtKVW+1+Q9LwEO9PoQQAhJHMNdEZegUAIBc6SgBAEDs4+abDoaMEACAHOkoAQBgJ7SgplACAMJjMAwBA+tBRAgCCYDIPAAApREcJAAgjodcoKZQAgCAYegUAIIXoKAEAYSR06JWOEgCAHOgoAQBBeEI7SgolACCMhBZKhl4BAMiBjhIAEERSh17pKAEAyIGOEgAQBh0lAADpQ0cJAAgiqdcoKZQAgCCSWigZegUAIAc6SgBAEHSUAACkEB0lACAMt7gjiASFEgAQBEOvHcSoo0Zo0cLntKTmeV104aS4w4lcz149dOfdN+sv85/Un+f9lyqHD407pF32w+ur9c3xZ+v4qotb9z313FyN++5F+qfRE7Ro6Rut++sa3lLlmNP1nbMv1XfOvlRXTbsjjpAjkbbPcpryTVOuSZCojjKTyejmaVM0+uiTVVu7Wi/8ZaYee3yWFi9+Pe7QInP1z6/Q7P/+o/71tPNUUlKiLl07xx3SLht71Dd08nFH6vJrb2vdN6h/uW780fm66uY7P/P6in599NCt1+QzxMil7bOcpnyTnKu3JHPoNVEd5fBDhmn58hV6882Vamxs1IwZj+i4MaPiDisyPXp219e+Xql77n5QktTY2Kj3138Qc1S7rnLwAerVo/sW+/bZq0wDKkpjiij/0vZZTlO+aco1KdoslGa2v5mNNLPuW+0fHV1Y7VNa1lerautbH9fWrVZpad8YI4rW3ntX6J133tMvbp2q2X/8T930iynq2rVL3GHlXV3DWzrhnMt0+g9+qpdeWxJ3OEGk7bOcpnyTnKu3hN8KQc5CaWbnSXpE0vckLTSzsZs9fXWUgaFtxcVF+qchB+o3d9yrI74xTh99/LHOu6Aq7rDyas/de2vWPdP04H9crQvPmqCLp/5SH370cdxhAankbsG3QtBWR/ldSQe7+zhJIyT90MwmZ5/bbgZmVmVm881sfkvLR0EC3RH1dQ2qKP//w3PlZf1UX9+Qt+PnW31dg+rrGvTy/FclSY/951MaMuRLMUeVX506lah3zx6SpC8NGqCK0j76e13HP+dp/CynJd805ZoUbRXKjLt/KEnuvkKbiuW3zOwG5SiU7l7t7pXuXpnJdAsVa5vmzV+ggQMHqH//CpWUlGj8+LF67PFZeTt+vq1d+7bq6ho0cOAASdJhI76mvy1ZFnNU+fXuuvfV3LxpfGbV6rVaWdeg8r5fiDmqXZe2z3Ka8k1yrkkdem1r1usaMxvq7gskyd0/NLNjJd0paXDUwe2s5uZmTT7/Cs184l4VZTK6a/oDqqlZGndYkbr0wp/qttuvU0mnEv19Ra2+d84lcYe0yy665hbNe3Wx1q3/QCNPOVeTTv2OevXopqv/Y7reW/+Bzvnhtdp/3731q6sv0UuvLdEv735IxcVFymQy+uF5/6pePbu3fZACl7bPcpryTVOuSWHuvv0nzcolNbn7Z8YFzOxQd/9TWwco7lS2/QMkTO/O+eueC0H9q/fGHULedP3i2LZfBHQATRvrIrvwt+qQkcF/31fMeyb2C5U5O0p3r83xXJtFEgCAji5RCw4AAOKTY4CyQ6NQAgCCYGUeAABSiI4SABAEHSUAAClERwkACILJPAAA5MDQKwAAKURHCQAIolD+2kdodJQAAORARwkACKJQ/tpHaBRKAEAQLQy9AgCQPnSUAIAgmMwDAEAK0VECAIJgwQEAAFKIjhIAEARrvQIAkANDrwAApBAdJQAgCBYcAAAghegoAQBBJHXBAQolACCIpM56ZegVAIAc6CgBAEEwmQcAgAJkZr3N7CEzW2Jmi83sa2a2u5k9bWavZ7/u1t73p1ACAIJwt+DbDpom6Ul331/SEEmLJV0i6Rl3HyTpmezjdqFQAgCCcA+/tcXMekk6TNIdm2Lwje6+TtJYSdOzL5suaVx786JQAgA6sgGS3pL0GzN7xcxuN7Nukvq4++rsaxok9WnvASiUAIAgWtyCb2ZWZWbzN9uqtjpssaQvS7rV3YdJ+khbDbO6u0tq980rzHoNaN0nH8UdQl51/eLYuEPImw2rZscdQl51qTgi7hAASZK7V0uqzvGSWkm17j43+/ghbSqUa8ysn7uvNrN+kta2NwY6SgBAEHFM5nH3BkmrzGy/7K6RkmokPSppYnbfREmPtDcvOkoAQEf3PUm/M7NOkt6QdIY2NYIzzOxMSX+XNL69b06hBAAEEdeCA+6+QFLlNp4aGeL9KZQAgCASutQr1ygBAMiFjhIAEARrvQIAkEJ0lACAIPjDzQAA5NASdwARYegVAIAc6CgBAEG4kjn0SkcJAEAOdJQAgCBaErriAIUSABBEC0OvAACkDx0lACAIJvMAAJBCdJQAgCBYcAAAgBSiowQABJHUa5QUSgBAEAy9AgCQQnSUAIAg6CgBAEghOkoAQBBM5gEAIIeWZNZJhl4BAMiFjhIAEAR/PQQAgBSiowQABJHQv9tMoQQAhMF9lB3EqKNGaNHC57Sk5nlddOGkuMOJXJryTWKuV0ydpsOOO1XjJp7buu+pOc9r7GmTNPibY7Vwyeut+xubmnTZlBt1/MTvacyEc/Trex6MI+TIJPH8bk+ack2CRBXKTCajm6dN0bFjJmjwkMN14onjdMABg+IOKzJpyjepuY4bPVK3XfvjLfYNHLC3bvrZpTp4yJe22D9rzp+0sbFJD0//hWbcfqMefPQp1a1ek8doo5PU87stSc61xSz4VgjaLJRmNtzMDsl+f6CZXWBmR0cf2s4bfsgwLV++Qm++uVKNjY2aMeMRHTdmVNxhRSZN+SY118qhB6lXz+5b7Nu3f4UG7FX+mdeaSRs++URNTc369NNPVVJcrO7duuYr1Egl9fxuS5pyTYqchdLMrpR0s6RbzewaSbdI6ibpEjO7PA/x7ZTSsr5aVVvf+ri2brVKS/vGGFG00pRvmnLdniNHHKounTvr8OMn6sgTztTpJ41Tr5494g4riDSd3yTn6hFshaCtyTzfkTRU0uckNUgqd/f3zew6SXMlTYk2PAD/8NripSrKZDT74bv0/gcfauK5l+qrlUNVkZBfskChamvotcndm939Y0nL3f19SXL3DcoxwcnMqsxsvpnNb2n5KGC4udXXNaiivLT1cXlZP9XXN+Tt+PmWpnzTlOv2zHz6OR36lS+rpLhYe+zWW0MH769FS5bFHVYQaTq/Sc61JYKtELRVKDea2T8ughz8j51m1ks5cnD3anevdPfKTKZbgDB3zLz5CzRw4AD171+hkpISjR8/Vo89Pitvx8+3NOWbply3p1+fPfXiy69Kkj7e8IleXbRUA/YuizmqMNJ0fpOca4uF3wpBW0Ovh7n7p5Lk7psXxhJJEyOLqp2am5s1+fwrNPOJe1WUyeiu6Q+opmZp3GFFJk35JjXXC39yrea9slDr1r+vkd8+Q+eccbJ69eyha6ZV691163XOxVdp/4H7qPr6n+jk44/WFVOnaexpk+QujTt6pPbbd0DcKQSR1PO7LWnKNSnMPdrLpcWdygrleizQbhtWzY47hLzqUnFE3CEgIk0b6yLr035XOiH47/tT6u+Jva9M1H2UAACExhJ2AIAgkjp8SKEEAARRKJNvQmPoFQCAHOgoAQBBFMp9j6HRUQIAkAMdJQAgCCbzAACQA5N5AABIITpKAEAQTOYBACCF6CgBAEHQUQIAkEJ0lACAIDyhs14plACAIBh6BQAghegoAQBB0FECAJBCdJQAgCBY6xUAgBxY6xUAgBSiowQABMFkHgAAUoiOEgAQRFI7SgolACCIpM56ZegVAIAc6CgBAEFwewgAAAXKzIrM7BUzezz7eICZzTWzZWb2gJl1au97UygBAEG0RLDthMmSFm/2+OeSbnT3gZLek3Rmu5IShRIA0MGZWbmkYyTdnn1sko6Q9FD2JdMljWvv+3ONEgAQRIyzXm+SdJGkHtnHe0ha5+5N2ce1ksra++YUSmAHdKk4Iu4Q8mpD/R/jDiFvupR+I+4QEqMlglJpZlWSqjbbVe3u1Zs9f6ykte7+kpmNCB6AKJQAgAKWLYrVOV5yqKTjzOxoSZ0l9ZQ0TVJvMyvOdpXlkuraGwPXKAEAQcQxmcfdL3X3cnfvL+kkSbPd/RRJcyR9J/uyiZIeaW9eFEoAQBJdLOkCM1umTdcs72jvGzH0CgAIIu4l7Nz9WUnPZr9/Q9LwEO9LoQQABJHURdEZegUAIAc6SgBAEKz1CgBACtFRAgCCiGLBgUJAoQQABJHMMsnQKwAAOdFRAgCC4PYQAABSiI4SABAEk3kAAMghmWWSoVcAAHKiowQABMFkHgAAUoiOEgAQRFIn89BRAgCQAx0lACCIZPaTFEoAQCBM5gEAIIXoKAEAQXhCB1/pKAEAyIGOEgAQRFKvUVIoAQBBcB8lAAApREcJAAgimf0kHSUAADnRUQIAguAaZQcx6qgRWrTwOS2peV4XXTgp7nAil6Z805SrlMx8r7j6Bh12zEkaN+H/te57avYfNfaUszT4n4/WwsVLW/c3Njbqiik36PhTz9a/TDxHL778ahwhRyKJ51baNOs19FYIElUoM5mMbp42RceOmaDBQw7XiSeO0wEHDIo7rMikKd805SolN99xRx+p22742Rb7Bu6zt266+oc6eOhBW+x/6NEnJUkP//ZW/fqmq3XdLb9WS0uh/Opsv6Se2yRLVKEcfsgwLV++Qm++uVKNjY2aMeMRHTdmVNxhRSZN+aYpVym5+VYOHaxePXtssW/f/ntpwN7ln3nt8hUrNfzgIZKkPXbrrR7du2nRktfzEmeUknpupU0r84T+XyHY6UJpZndHEUgIpWV9taq2vvVxbd1qlZb2jTGiaKUp3zTlKqUv323Zb+AAPfv8C2pqalZtfYNq/rZMDWveijusXca57XhyTuYxs0e33iXpcDPrLUnuflxEcQFIueOPGaU3VqzSiWeep9K+X9DQgw5QpihRg2CJ0/EHxretrVmv5ZJqJN2uTbfImKRKSdfn+iEzq5JUJUlW1EuZTLddj3QH1Nc1qKK8tPVxeVk/1dc35OXYcUhTvmnKVUpfvttSXFykiyef1fr4lLMuUP+KshgjCoNz2/G09c+zSkkvSbpc0np3f1bSBnf/H3f/n+39kLtXu3ulu1fmq0hK0rz5CzRw4AD171+hkpISjR8/Vo89Pitvx8+3NOWbplyl9OW7LRs++UQfb/hEkvTnF19WcVGR9h2wd8xR7bokn9ukXqPM2VG6e4ukG83swezXNW39TJyam5s1+fwrNPOJe1WUyeiu6Q+opmZp2z/YQaUp3zTlKiU33wuvnKp5r7yqdeve18hxE3TOmaeqV8/uuubGW/XuuvU658Irtf+gfVR94xS9+956nfVvl8syGfXZcw9d86MfxB1+EEk9t1Jyh17NfccrtpkdI+lQd79sR3+muFNZYfyTAMAO21D/x7hDyJsupd+IO4S8atpYZ1G998T+3w7++376it9HFu+O2qnu0N2fkPRERLEAADqwlp1ovDoSppABAJBDwV5vBAB0LMnsJymUAIBAWBQdAIAUoqMEAARRKPc9hkZHCQBADnSUAIAgkrrgAIUSABAEk3kAAEghOkoAQBBM5gEAIIXoKAEAQSR1Mg8dJQAAOdBRAgCC2Jk/29iRUCgBAEFwewgAAClERwkACILJPAAApBAdJQAgiKQuOEChBAAEwWQeAABSiI4SABBEUu+jpKMEACAHOkoAQBBJvT2EQgkACCKps14ZegUAIAc6SgBAENweAgBAgTGzCjObY2Y1ZrbIzCZn9+9uZk+b2evZr7u19xgUSgBAEO4efNsBTZK+7+4HSvqqpElmdqCkSyQ94+6DJD2TfdwuFEoAQIfl7qvd/eXs9x9IWiypTNJYSdOzL5suaVx7j8E1SgBAEHFfozSz/pKGSZorqY+7r84+1SCpT3vfl0IZUMYs7hDyqiWhq3BA6lL6jbhDyJsNq2bHHUJiRHF7iJlVSarabFe1u1dv43XdJf1e0vnu/r5t9vvY3d3M2h0chRIAULCyRfEzhXFzZlaiTUXyd+7+h+zuNWbWz91Xm1k/SWvbGwPXKAEAQbS4B9/aYptaxzskLXb3GzZ76lFJE7PfT5T0SHvzoqMEAHRkh0o6VdJrZrYgu+8ySVMlzTCzMyX9XdL49h6AQgkACCKOWQvu/ryk7U0QGRniGBRKAEAQcc96jQrXKAEAyIGOEgAQBB0lAAApREcJAAhiB9dm7XAolACAIBh6BQAghegoAQBBRLHWayGgowQAIAc6SgBAEEmdzENHCQBADnSUAIAgkjrrlUIJAAiCoVcAAFKIjhIAEERSh17pKAEAyIGOEgAQRFIXHKBQAgCCaGEyDwAA6UNHCQAIIqlDr3SUAADkQEcJAAgiqdcoKZQAgCAYegUAIIXoKAEAQSR16JWOEgCAHOgoAQBBcI2ygxh11AgtWvicltQ8r4sunBR3OJGq/tV1ql21QK+8/N9xh5IXaTq3Evl2dFdMnabDjjtV4yae27rvqTnPa+xpkzT4m2O1cMnrrfsbm5p02ZQbdfzE72nMhHP063sejCNkbEeiCmUmk9HN06bo2DETNHjI4TrxxHE64IBBcYcVmbt/+6COHTMh7jDyIm3nlnw7fr7jRo/Ubdf+eIt9AwfsrZt+dqkOHvKlLfbPmvMnbWxs0sPTf6EZt9+oBx99SnWr1+Qx2jBa3INvhWCnCqWZ/bOZXWBmR0UV0K4YfsgwLV++Qm++uVKNjY2aMeMRHTdmVNxhReb55+fqvffWxR1GXqTt3JJvx8+3cuhB6tWz+xb79u1foQF7lX/mtWbShk8+UVNTsz799FOVFBere7eu+Qo1GI/gf4UgZ6E0sxc3+/67km6R1EPSlWZ2ScSx7bTSsr5aVVvf+ri2brVKS/vGGBFCSdu5Jd9k57u1I0ccqi6dO+vw4yfqyBPO1OknjVOvnj3iDgtZbU3mKdns+ypJR7r7W2Z2naQXJE2NLDIASInXFi9VUSaj2Q/fpfc/+FATz71UX60cqooO9o8F95a4Q4hEW0OvGTPbzcz2kGTu/pYkuftHkpq290NmVmVm881sfkvLRwHDza2+rkEV5aWtj8vL+qm+viFvx0d00nZuyTfZ+W5t5tPP6dCvfFklxcXaY7feGjp4fy1asizusJDVVqHsJeklSfMl7W5m/STJzLpLsu39kLtXu3ulu1dmMt2CBduWefMXaODAAerfv0IlJSUaP36sHnt8Vt6Oj+ik7dySb7Lz3Vq/PnvqxZdflSR9vOETvbpoqQbsXRZzVDuvRR58KwQ5h17dvf92nmqRdHzwaHZRc3OzJp9/hWY+ca+KMhndNf0B1dQsjTusyPz27lt02GFf0+c/v7veWD5PV/30et111/1xhxWJtJ1b8u34+V74k2s175WFWrf+fY389hk654yT1atnD10zrVrvrluvcy6+SvsP3EfV1/9EJx9/tK6YOk1jT5skd2nc0SO1374D4k5hp3mBzFINzaJOrLhTWTL/y21DxrbbZCdSoUzdBnbFhlWz4w4hr0r67BfZL6q9dh8c/JfCyndfi/0XKyvzAACCKJSh0tASteAAAACh0VECAIJI6jVKCiUAIIikzltg6BUAgBzoKAEAQRTK2qyh0VECAJADHSUAIIikTuahowQAIAc6SgBAEEldcIBCCQAIgqFXAABSiI4SABAECw4AAJBCdJQAgCCSeo2SQgkACCKps14ZegUAIAc6SgBAEEkdeqWjBAAgBzpKAEAQSb09hEIJAAiCP7MFAEAK0VECAIJI6tArHSUAADnQUQIAguD2EAAAUoiOEgAQRFJnvVIoAQBBMPQKAEABMrPRZvY3M1tmZpeEfn86SgBAEHF0lGZWJOmXko6UVCtpnpk96u41oY5BRwkA6MiGS1rm7m+4+0ZJ90saG/IAFEoAQBAewbYDyiSt2uxxbXZfMJEPvTZtrLOoj7EtZlbl7tVxHDvf0pSrRL5JlqZcpeTlG8XvezOrklS12a7qfP83S3JHWdX2SxIjTblK5JtkacpVSl++O83dq929crNt6yJZJ6lis8fl2X3BJLlQAgCSb56kQWY2wMw6STpJ0qMhD8CsVwBAh+XuTWZ2rqSnJBVJutPdF4U8RpILZWLG/XdAmnKVyDfJ0pSrlL58I+HuMyXNjOr9LakrKQAAEALXKAEAyCFxhTLqpYwKiZndaWZrzWxh3LHkg5lVmNkcM6sxs0VmNjnumKJiZp3N7EUz+2s215/EHVPUzKzIzF4xs8fjjiVqZrbCzF4zswVmNj/ueJBbooZes0sZLdVmSxlJOjnkUkaFxMwOk/ShpLvd/aC444mamfWT1M/dXzazHpJekjQuiefXzExSN3f/0MxKJD0vabK7vxBzaJExswskVUrq6e7Hxh1PlMxshaRKd3877ljQtqR1lJEvZVRI3P05Se/GHUe+uPtqd385+/0HkhYr8AochcI3+TD7sCS7JedftVsxs3JJx0i6Pe5YgK0lrVBGvpQRCoOZ9Zc0TNLcmEOJTHYocoGktZKedvfE5irpJkkXSWqJOY58cUmzzOyl7MozKGBJK5RIATPrLun3ks539/fjjicq7t7s7kO1aaWR4WaWyOF1MztW0lp3fynuWPLon939y5K+JWlS9jIKClTSCmXkSxkhXtnrdb+X9Dt3/0Pc8eSDu6+TNEfS6JhDicqhko7LXre7X9IRZnZPvCFFy93rsl/XSnpYmy4boUAlrVBGvpQR4pOd4HKHpMXufkPc8UTJzPY0s97Z77to0wS1JbEGFRF3v9Tdy929vzb9f3a2u0+IOazImFm37GQ0mVk3SUdJSsXM9Y4qUYXS3Zsk/WMpo8WSZoReyqiQmNl9kv4iaT8zqzWzM+OOKWKHSjpVmzqOBdnt6LiDikg/SXPM7FVt+gfg0+6e+NsmUqKPpOfN7K+SXpT0hLs/GXNMyCFRt4cAABBaojpKAABCo1ACAJADhRIAgBwolAAA5EChBAAgBwolAAA5UCgBAMiBQgkAQA7/C53fndpK2OvdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "conf_matrix = confusion_matrix(y_test[:result_lenght], prediction)\n",
    "\n",
    "plt.figure(figsize=(8, 8))\n",
    "sns.heatmap(conf_matrix, annot=True, fmt='g')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}