ULMFiT_MultiLabel_Classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ULMFiT MultiLabel Classification.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "16Yx-UbBDwA8"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/shishir-joshi/notebooks/blob/master/ULMFiT_MultiLabel_Classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "473TxqQAixA0",
        "colab_type": "text"
      },
      "source": [
        "References:\n",
        "\n",
        "1. https://medium.com/dair-ai/pytorch-1-2-quickstart-with-google-colab-6690a30c38d\n",
        "\n",
        "2. https://medium.com/dair-ai/building-rnns-is-fun-with-pytorch-and-google-colab-3903ea9a3a79\n",
        "\n",
        "3. https://medium.com/dair-ai/a-simple-neural-network-from-scratch-with-pytorch-and-google-colab-c7f3830618e0\n",
        "\n",
        "4. https://colab.research.google.com/drive/1gCgtlnMPVWY0l1ra8ssS0EzJiWaVR4Pk\n",
        "\n",
        "5. !!\n",
        "https://www.analyticsvidhya.com/blog/2018/11/tutorial-text-classification-ulmfit-fastai-library/\n",
        "\n",
        "6. !!\n",
        "https://docs.fast.ai/basic_data.html#DataBunch\n",
        "\n",
        "7. !!\n",
        "https://docs.fast.ai/text.html\n",
        "\n",
        "8. !!\n",
        "https://www.aclweb.org/anthology/C18-1330/\n",
        "\n",
        "9. !!\n",
        "https://forums.fast.ai/t/a-quick-and-easy-way-to-make-ulmfit-work-for-multi-label-problems/17222\n",
        "\n",
        "10. !!\n",
        "https://docs.fast.ai/text.learner.html#text.learner\n",
        "\n",
        "11. !!\n",
        "https://nbviewer.jupyter.org/github/fastai/course-v3/blob/master/nbs/dl1/lesson3-planet.ipynb\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kqCKXyC-ylpt",
        "colab_type": "text"
      },
      "source": [
        "# Trying out ULMFiT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bXsqiWN5z0tO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import fastai\n",
        "from fastai import *\n",
        "from fastai.text import * \n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from functools import partial\n",
        "import io\n",
        "import os"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uSjX56qIlNIJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !pip install tensorflow-gpu==2.0.0b1\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "# BeautifulSoup is useful for parsing HTML and XML tags\n",
        "from bs4 import BeautifulSoup\n",
        "# NLTK is the NLP library\n",
        "from nltk.corpus import stopwords\n",
        "# Regular Expressions useful for quickly searching corpus of text\n",
        "import re\n",
        "import string\n",
        "# from tensorflow import keras\n",
        "import tarfile\n",
        "import random\n",
        "from pathlib import Path\n",
        "from sklearn.model_selection import train_test_split\n",
        "from random import shuffle\n",
        "# from sklearn.preprocessing import MultiLabelBinarizer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5JA9Li-4xCUp",
        "colab_type": "code",
        "outputId": "6f6170cf-e2cc-4d08-b682-26d8ac642d62",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "torch.backends.cudnn.enabled"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ISUgWmJ5xClB",
        "colab_type": "code",
        "outputId": "c1470da7-1fcb-456c-9bdb-a481a037a81f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "torch.cuda.is_available()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RpAxN_G54Vll",
        "colab_type": "text"
      },
      "source": [
        "### Download/ Prepare/ Preprocess Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qsEJe8yz4K_2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PHrHLIzjlhl2",
        "colab_type": "code",
        "outputId": "05f79e85-392e-44ad-d292-92ed87e552e8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 100
        }
      },
      "source": [
        "!git clone https://github.com/Jaspereclipse/kaggle-transfer-learning-on-stack-exchange-tags"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'kaggle-transfer-learning-on-stack-exchange-tags'...\n",
            "remote: Enumerating objects: 179, done.\u001b[K\n",
            "remote: Total 179 (delta 0), reused 0 (delta 0), pack-reused 179\u001b[K\n",
            "Receiving objects: 100% (179/179), 52.91 MiB | 17.47 MiB/s, done.\n",
            "Resolving deltas: 100% (86/86), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1amEiS4MarDs",
        "colab_type": "code",
        "outputId": "1a13064b-7a9f-4959-a638-73179856f63f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "train_dir = !ls */*/train/*\n",
        "test_dir = !ls */*/test/*\n",
        "print(test_dir)\n",
        "print(train_dir)\n",
        "train_df = []\n",
        "for c in train_dir:\n",
        "  train_df.append(pd.read_csv(c))\n",
        "for tdf in train_df:\n",
        "  tdf[\"text\"] = tdf[\"title\"]+tdf[\"content\"]\n",
        "  tdf.pop(\"id\")\n",
        "  tdf.pop(\"title\")\n",
        "  tdf.pop(\"content\")\n",
        "test_df = []\n",
        "for c in test_dir:\n",
        "  test_df.append(pd.read_csv(c))\n",
        "for tdf in test_df:\n",
        "  tdf[\"text\"] = tdf[\"title\"]+tdf[\"content\"]\n",
        "  tdf.pop(\"id\")\n",
        "  tdf.pop(\"title\")\n",
        "  tdf.pop(\"content\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['kaggle-transfer-learning-on-stack-exchange-tags/data/test/test.csv']\n",
            "['kaggle-transfer-learning-on-stack-exchange-tags/data/train/biology.csv', 'kaggle-transfer-learning-on-stack-exchange-tags/data/train/cooking.csv', 'kaggle-transfer-learning-on-stack-exchange-tags/data/train/crypto.csv', 'kaggle-transfer-learning-on-stack-exchange-tags/data/train/diy.csv', 'kaggle-transfer-learning-on-stack-exchange-tags/data/train/robotics.csv', 'kaggle-transfer-learning-on-stack-exchange-tags/data/train/travel.csv']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ShmkbIoGkW0E",
        "colab_type": "code",
        "outputId": "0231d030-51f1-4a03-d7f0-3e662a7a37a2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        }
      },
      "source": [
        "train_df[1].head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tags</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>baking cookies texture</td>\n",
              "      <td>How can I get chewy chocolate chip cookies?&lt;p&gt;...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>oven cooking-time bacon</td>\n",
              "      <td>How should I cook bacon in an oven?&lt;p&gt;I've hea...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>eggs</td>\n",
              "      <td>What is the difference between white and brown...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>substitutions please-remove-this-tag baking-so...</td>\n",
              "      <td>What is the difference between baking soda and...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>sauce pasta tomatoes italian-cuisine</td>\n",
              "      <td>In a tomato sauce recipe, how can I cut the ac...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                tags                                               text\n",
              "0                             baking cookies texture  How can I get chewy chocolate chip cookies?<p>...\n",
              "1                            oven cooking-time bacon  How should I cook bacon in an oven?<p>I've hea...\n",
              "2                                               eggs  What is the difference between white and brown...\n",
              "3  substitutions please-remove-this-tag baking-so...  What is the difference between baking soda and...\n",
              "4               sauce pasta tomatoes italian-cuisine  In a tomato sauce recipe, how can I cut the ac..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ca2mAnkzLDYO",
        "colab_type": "text"
      },
      "source": [
        "#### Cleaning the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uH5DzUgtlIX0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "uri_re = r'(?i)\\b((?:https?://|www\\d{0,3}[.]|[a-z0-9.\\-]+[.][a-z]{2,4}/)(?:[^\\s()<>]+|\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\))+(?:\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\)|[^\\s`!()\\[\\]{};:\\'\".,<>?«»“”‘’]))'\n",
        "\n",
        "# Vectorization parameters\n",
        "# Limit on the number of features. We use the top 20K features.\n",
        "TOP_K = 100000000\n",
        "\n",
        "# Limit on the length of text sequences. Sequences longer than this\n",
        "# will be truncated.\n",
        "MAX_SEQUENCE_LENGTH = 400\n",
        "\n",
        "# Mapping function to strip the CSVs of html tags and all\n",
        "def stripTagsAndUris(x):\n",
        "  if x:\n",
        "      # BeautifulSoup on content\n",
        "      soup = BeautifulSoup(x, \"html.parser\")\n",
        "      # Stripping all <code> tags with their content if any\n",
        "      if soup.code:\n",
        "          soup.code.decompose()\n",
        "      # Get all the text out of the html\n",
        "      text =  soup.get_text()\n",
        "      # Returning text stripping out all uris\n",
        "      return re.sub(uri_re, \"\", text)\n",
        "  else:\n",
        "      return \"\"\n",
        "      \n",
        "def removePunctuation(x):\n",
        "    # Lowercasing all words\n",
        "  x = x.lower()\n",
        "  # Removing non ASCII chars\n",
        "  x = re.sub(r'[^\\x00-\\x7f]',r' ',x)\n",
        "  # Removing (replacing with empty spaces actually) all the punctuations\n",
        "  return re.sub(\"[\"+string.punctuation+\"]\", \" \", x)\n",
        "\n",
        "\n",
        "  \n",
        "for tdf in train_df:\n",
        "  tdf[\"text\"] = tdf[\"text\"].apply(stripTagsAndUris)\n",
        "  tdf[\"text\"] = tdf[\"text\"].apply(removePunctuation)\n",
        "\n",
        "  \n",
        "for tdf in test_df:\n",
        "  tdf[\"text\"] = tdf[\"text\"].apply(stripTagsAndUris)\n",
        "  tdf[\"text\"] = tdf[\"text\"].apply(removePunctuation)\n",
        "# Unrolling \n",
        "train_df = pd.concat(train_df)\n",
        "test_df = pd.concat(test_df)\n",
        "\n",
        "# Sampling the dataset for experimentation\n",
        "train_df = train_df.sample(900)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gDPA39-zaa3N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "_, val_df =train_test_split(train_df, test_size=0.2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZxCECjHbaazy",
        "colab_type": "code",
        "outputId": "eca487cb-fcc3-49a8-f42d-95d60a9d4077",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 573
        }
      },
      "source": [
        "train_df"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tags</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>11974</th>\n",
              "      <td>repair lawn-mower</td>\n",
              "      <td>how can i remove this stuck nut on my riding m...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4185</th>\n",
              "      <td>evolution sociality</td>\n",
              "      <td>evolution of bee hiveshow do the instinct to c...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14158</th>\n",
              "      <td>meat turkey fats</td>\n",
              "      <td>does 85  lean ground turkey have the same amou...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6021</th>\n",
              "      <td>wine</td>\n",
              "      <td>wine selection when dining on horse meat i kno...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12518</th>\n",
              "      <td>winterizing</td>\n",
              "      <td>why is the pressure vacuum breaker leaking whi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>918</th>\n",
              "      <td>homomorphic-encryption</td>\n",
              "      <td>is there difference between algebraic homomorp...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1533</th>\n",
              "      <td>japan hotels mail</td>\n",
              "      <td>can you receive mail at hotels weeks before yo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5238</th>\n",
              "      <td>doors installation</td>\n",
              "      <td>how do i hang a prehung door i want to put up ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1899</th>\n",
              "      <td>baking bread</td>\n",
              "      <td>baking bread     without crust my fiancee does...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>118</th>\n",
              "      <td>equipment cookware sauteing stainless-steel te...</td>\n",
              "      <td>which is better to saut  with  stainless steel...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>900 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    tags                                               text\n",
              "11974                                  repair lawn-mower  how can i remove this stuck nut on my riding m...\n",
              "4185                                 evolution sociality  evolution of bee hiveshow do the instinct to c...\n",
              "14158                                   meat turkey fats  does 85  lean ground turkey have the same amou...\n",
              "6021                                                wine  wine selection when dining on horse meat i kno...\n",
              "12518                                        winterizing  why is the pressure vacuum breaker leaking whi...\n",
              "...                                                  ...                                                ...\n",
              "918                               homomorphic-encryption  is there difference between algebraic homomorp...\n",
              "1533                                   japan hotels mail  can you receive mail at hotels weeks before yo...\n",
              "5238                                  doors installation  how do i hang a prehung door i want to put up ...\n",
              "1899                                        baking bread  baking bread     without crust my fiancee does...\n",
              "118    equipment cookware sauteing stainless-steel te...  which is better to saut  with  stainless steel...\n",
              "\n",
              "[900 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TA-udaStEymo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "crosEEvP4qIR",
        "colab_type": "text"
      },
      "source": [
        "### Getting Data Ready for ULMFit"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DC4iDxfRMAoj",
        "colab_type": "text"
      },
      "source": [
        "#### First fit ULMFiT on the dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HkyBsjIv4LTH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lang_model = TextLMDataBunch.from_df(path='/content/ULMFit', train_df=train_df, valid_df=val_df)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kMDWKkzxCxub",
        "colab_type": "code",
        "outputId": "7ab718e1-3353-4e0a-c1c4-23c06421ba3d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447
        }
      },
      "source": [
        "lang_model.show_batch()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>idx</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>seen suggestions on heating the nut but i do not have an xxunk torch handy so i m looking for good advise i am familiar with some of the more obvious techniques such as what s proposed here how to xxunk a nut when bolt and nut move together \\n  any advise \\n \\n  xxbos evolution of xxunk xxunk do the xxunk to create a hive appeared in</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>muscular pieces in the muscular string fiber contract when the fiber contracts if not does that mean the number of the xxunk muscular pieces xxunk to the angle of my joint \\n  xxbos can someone bring a packed phone from us to xxunk of my friend is coming to india from us he bought himself a laptop and a camera which he is using their also i xxunk him</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>they contain xxunk i can t find anything in wikipedia about this \\n  xxbos how to make xxunk powder for candy when i was young someone taught me how to make this powder you find on candy it s acidic and seems to xxunk in the mouth but i don t know the name and so i can t find a recipe \\n  i suppose i need to</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>of subway tiles so i could replace them with custom cut tiles that would fit without a gap \\n  xxbos is it okay to put a hot pan on a granite counter is it okay to put a hot pan pot on a granite counter or will it burn it i don t know if there s a sealer on it \\n  xxbos how to generate many passwords</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>floor framing using joist hangers i have a single story ranch where a corner of the house i believe was a car port the roof line and walls all are framed such that it was always under the roof and considered part of the house however i believe somewhere along the line someone xxunk it up and made a living space out of it they did not raise the floor</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6vixI1_Fz0w3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Initialize the learner\n",
        "learner = language_model_learner(lang_model, AWD_LSTM, drop_mult=0.5)\n",
        "\n",
        "learner.unfreeze()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V67OPWw_z01a",
        "colab_type": "code",
        "outputId": "5c3fb171-85db-4ec6-ad43-67773ee71d8b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 497
        }
      },
      "source": [
        "learner.fit_one_cycle(15, 1e-3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>5.413590</td>\n",
              "      <td>5.112805</td>\n",
              "      <td>0.171875</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>5.195516</td>\n",
              "      <td>4.817350</td>\n",
              "      <td>0.197842</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>4.981871</td>\n",
              "      <td>4.514236</td>\n",
              "      <td>0.227121</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>4.774595</td>\n",
              "      <td>4.198309</td>\n",
              "      <td>0.257924</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>4.551726</td>\n",
              "      <td>3.831174</td>\n",
              "      <td>0.298214</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>4.296237</td>\n",
              "      <td>3.465342</td>\n",
              "      <td>0.349888</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>4.008378</td>\n",
              "      <td>3.083539</td>\n",
              "      <td>0.409189</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>3.722802</td>\n",
              "      <td>2.744548</td>\n",
              "      <td>0.466518</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>3.432054</td>\n",
              "      <td>2.439039</td>\n",
              "      <td>0.522582</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>3.163834</td>\n",
              "      <td>2.225070</td>\n",
              "      <td>0.562500</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>2.932184</td>\n",
              "      <td>2.052544</td>\n",
              "      <td>0.594866</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>2.747534</td>\n",
              "      <td>1.942031</td>\n",
              "      <td>0.616927</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>2.607955</td>\n",
              "      <td>1.887793</td>\n",
              "      <td>0.630506</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>2.503704</td>\n",
              "      <td>1.859704</td>\n",
              "      <td>0.634487</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>2.431058</td>\n",
              "      <td>1.851963</td>\n",
              "      <td>0.635305</td>\n",
              "      <td>00:11</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nwV4v65Ez05W",
        "colab_type": "code",
        "outputId": "c0d55526-d2cb-460f-a666-029762424f98",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "# Checking \n",
        "learner.predict(\"im trying to make an \", n_words=10)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'im trying to make an  identification search advances indian dhabi passports and external embassies weren'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XY5ZCcxj8jLA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learner.save_encoder('ulmfit_enc')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IQnJXw_bMvs5",
        "colab_type": "text"
      },
      "source": [
        "#### The MultiLabel Classifier layer\n",
        " "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tH-ZRvWB8jzz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "classifier = TextClasDataBunch.from_df(path='/content/ULMFit', train_df=train_df, valid_df=val_df, vocab=lang_model.train_ds.vocab, bs=32)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZX6mB3GuM9Bp",
        "colab_type": "text"
      },
      "source": [
        "Full Disclosure : the following class block has been picked up from ref no. 9 but subsequently I learned that the text_classifier_learner function automatically adapts the final layer for multilabel classification so it hasn't been used "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UjdiiXf7nzOA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MultiLabelClassifier(nn.Module):\n",
        "    \n",
        "    def __init__(self, y_range=None):\n",
        "        super().__init__()\n",
        "        self.y_range = y_range\n",
        "    \n",
        "    def forward(self, input):\n",
        "        x, raw_outputs, outputs = input\n",
        "        x = F.sigmoid(x)\n",
        "        if (self.y_range):\n",
        "            x = x * (self.y_range[1] - self.y_range[0])\n",
        "            x = x + self.y_range[0]\n",
        "        \n",
        "        return x, raw_outputs, outputs\n",
        "# del classifier\n",
        "# torch.cuda.empty_cache()\n",
        "\n",
        "# classifier_learn.loss_func.func = F.binary_cross_entropy_with_logits\n",
        "# classifier_learn.summary\n",
        "# classifier_learn.crit = F.binary_cross_entropy_with_logits\n",
        "# classifier_learn.model.add_module('2', MultiLabelClassifier())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bxox2iYY8jrL",
        "colab_type": "code",
        "outputId": "53c10bae-505d-4b24-f46d-f5422fe36286",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "classifier_learner = text_classifier_learner(classifier, AWD_LSTM, drop_mult=0.5)\n",
        "classifier_learner.load_encoder('ulmfit_enc')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RNNLearner(data=TextClasDataBunch;\n",
              "\n",
              "Train: LabelList (900 items)\n",
              "x: TextList\n",
              "xxbos how can i remove this stuck nut on my riding mower i am attempting to repair a shattered housing xxunk on my xxunk riding mower i have it xxunk disassembled however i am facing a common problem a stuck nut on a piece that is moving the only leverage i can get is with the tool on the nut and the blade below i have done some reading and seen suggestions on heating the nut but i do not have an xxunk torch handy so i m looking for good advise i am familiar with some of the more obvious techniques such as what s proposed here how to xxunk a nut when bolt and nut move together \n",
              "  any advise \n",
              " \n",
              " ,xxbos evolution of xxunk xxunk do the xxunk to create a hive appeared in xxunk is there some evidence of \n",
              "  intermediate xxunk with this questions i mean all the factors necessary to build a hive including the social structure the producing of xxunk and architecture \n",
              " ,xxbos does 85 lean ground turkey have the same amount of fat as 85 lean ground beef i m not a cooking expert so i m not exactly sure what the notation means i always thought that turkey was a much xxunk meat than beef so if turkey is labeled as 85 lean does that mean it has the same fat content as 85 lean beef if so does that mean that the fat percentage is simply higher than that of turkey in general \n",
              " ,xxbos wine selection when dining on horse meat i know there is a lot of received wisdom about what xxunk of xxunk to pair with say beef or fish or chicken though i couldn t quote any of it off the top of my own head as i am the xxunk thing from a xxunk in this world nevertheless i am wondering are there the same kind of traditional or standard wine recommendations for pairing with horse meat and what are they \n",
              " ,xxbos why is the pressure vacuum breaker leaking while trying to blow out my sprinkler system i m trying to hook my air compressor to my sprinkler system s pressure vacuum breaker valve \n",
              "  it s a xxunk xxunk 1 \n",
              "  i m hooking it to xxunk 1 in the diagram on page 2 of above the air goes straight out of the top of the xxunk assembly \n",
              "  it is fitted with one of these \n",
              "  check valve and xxunk appear good i don t have a place to hook up the air compressor xxunk \n",
              "  i m at a loss what am i doing wrong \n",
              "  edit i got it to work by xxunk the pressure on my compressor to xxunk xxunk the xxunk on the valve and manually keeping it open while applying air pressure to make an initial seal you ll need an air compressor that can keep the pressure up above xxunk though \n",
              " \n",
              "y: CategoryList\n",
              "repair lawn-mower,evolution sociality,meat turkey fats,wine,winterizing\n",
              "Path: /content/ULMFit;\n",
              "\n",
              "Valid: LabelList (180 items)\n",
              "x: TextList\n",
              "xxbos encrypt user email but be able to find user by xxunk for my dumb question but it s better to ask dumb question than to do dumb things xxunk \n",
              "  i want to encrypt user email in my db so that if someone xxunk the db and not the key he won t be able to restore the email xxunk but i need to be able to find user by email in my db and i can t xxunk over all the emails in the db decrypt each and compare this is too slow minutes hours \n",
              "  if i encrypt emails with aes with random iv then each time i encrypt the same email the encrypted value is different this is great for security but this way i can t just encrypt the given email and search for a value if the iv is the same each time then as far as i understand if attacker have enough encrypted values he can easily find the key right \n",
              "  i was thinking about storing original email hash xxunk the encrypted value but this way attacker will be able to recover original email values by encrypting emails from some xxunk with the same hash algorithm and comparing hash values with values in the db \n",
              "  i thought about storing hash of original email some fixed secret is this secure if not is there a secure solution to my problem \n",
              " ,xxbos robot wire xxunk how to position on wirei m designing my lawn mower robot and i am in the xxunk stage \n",
              "  the electronic part is done and works quite good now comes the software \n",
              "  i need an advice on how to deal with the problem of line following i mean once the robot is on the line parallel to the line that s relatively easy \n",
              "  but how to manage the situation when the robot is driving around and approaches the line wire \n",
              "  i have two sensors left and right turned 45 with respect to the forward direction \n",
              "  the robot could arrive from any angle so the signal amplitude read from the sensor could be completely random \n",
              "  so i don t understand what to do in order to move it in the right position on the wire \n",
              "  what s the usual approach \n",
              " \n",
              "  the idea is the same as here \n",
              " \n",
              "  the wire is all around the yard on the mower there are 2 sensors left and right that sense the signal xxunk from the wire a square wave signal at 34 xxunk the signal amplitude read from the sensors on the mower is about 2 v when it s above the wire \n",
              " ,xxbos where is this castle i know it s somewhere in ireland but where \n",
              "  irish castle i can find the picture everywhere but it never says where it s from \n",
              " ,xxbos is there a map that shows austrian winter road xxunk some xxunk in austria close for the winter season for example the xxunk high xxunk road is closed until sometime in may \n",
              "  where can i find a map that shows which roads are currently open closed \n",
              " ,xxbos aaa benefits as member of a european automobile club ihg rates this travel blog article in german mentions that also members of a european automobile club can use the aaa preferred rate for booking hotel rooms with ihg a hotel group \n",
              " \n",
              "  xxunk xxunk xxunk xxunk xxunk xxunk ist xxunk die aaa rate xxunk xxunk da xxunk xxunk xxunk xxunk \n",
              " \n",
              "  but fails to mention any further details \n",
              "  is there a list of the automobile xxunk that xxunk for this xxunk \n",
              "  in a cursory search i did find mention of a partner program of the german xxunk with aaa that could give some benefits but ihg was not mentioned in an otherwise rather xxunk list \n",
              "  is there any confirmation of this information i e being able to use the aaa rate with ihg as a member of a european automobile club ideally i would like to see some ihg document or repeated personal experience \n",
              "  as far as i know aaa rates can be booked all over the world so my question is not xxunk to bookings in the u s \n",
              "  we have a question about benefits for aaa members while traveling in europe and while it does link to a page that mentions a global discounts program i can not extract any further info from that site \n",
              "  another cursory search for global discounts program send me to this aaa site about discounts for international members coming to us north america it does list a bunch of hotel xxunk but not ihg \n",
              " \n",
              "y: CategoryList\n",
              "encryption hash deterministic-encryption,line-following magnetometer,ireland identify-this architecture locate-this,road-trips automobiles winter austria roads,budget bookings hotels automobiles\n",
              "Path: /content/ULMFit;\n",
              "\n",
              "Test: None, model=SequentialRNN(\n",
              "  (0): MultiBatchEncoder(\n",
              "    (module): AWD_LSTM(\n",
              "      (encoder): Embedding(5736, 400, padding_idx=1)\n",
              "      (encoder_dp): EmbeddingDropout(\n",
              "        (emb): Embedding(5736, 400, padding_idx=1)\n",
              "      )\n",
              "      (rnns): ModuleList(\n",
              "        (0): WeightDropout(\n",
              "          (module): LSTM(400, 1152, batch_first=True)\n",
              "        )\n",
              "        (1): WeightDropout(\n",
              "          (module): LSTM(1152, 1152, batch_first=True)\n",
              "        )\n",
              "        (2): WeightDropout(\n",
              "          (module): LSTM(1152, 400, batch_first=True)\n",
              "        )\n",
              "      )\n",
              "      (input_dp): RNNDropout()\n",
              "      (hidden_dps): ModuleList(\n",
              "        (0): RNNDropout()\n",
              "        (1): RNNDropout()\n",
              "        (2): RNNDropout()\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (1): PoolingLinearClassifier(\n",
              "    (layers): Sequential(\n",
              "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (1): Dropout(p=0.2, inplace=False)\n",
              "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
              "      (3): ReLU(inplace=True)\n",
              "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (5): Dropout(p=0.1, inplace=False)\n",
              "      (6): Linear(in_features=50, out_features=841, bias=True)\n",
              "    )\n",
              "  )\n",
              "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7fed398f6f28>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('/content/ULMFit'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[RNNTrainer\n",
              "learn: RNNLearner(data=TextClasDataBunch;\n",
              "\n",
              "Train: LabelList (900 items)\n",
              "x: TextList\n",
              "xxbos how can i remove this stuck nut on my riding mower i am attempting to repair a shattered housing xxunk on my xxunk riding mower i have it xxunk disassembled however i am facing a common problem a stuck nut on a piece that is moving the only leverage i can get is with the tool on the nut and the blade below i have done some reading and seen suggestions on heating the nut but i do not have an xxunk torch handy so i m looking for good advise i am familiar with some of the more obvious techniques such as what s proposed here how to xxunk a nut when bolt and nut move together \n",
              "  any advise \n",
              " \n",
              " ,xxbos evolution of xxunk xxunk do the xxunk to create a hive appeared in xxunk is there some evidence of \n",
              "  intermediate xxunk with this questions i mean all the factors necessary to build a hive including the social structure the producing of xxunk and architecture \n",
              " ,xxbos does 85 lean ground turkey have the same amount of fat as 85 lean ground beef i m not a cooking expert so i m not exactly sure what the notation means i always thought that turkey was a much xxunk meat than beef so if turkey is labeled as 85 lean does that mean it has the same fat content as 85 lean beef if so does that mean that the fat percentage is simply higher than that of turkey in general \n",
              " ,xxbos wine selection when dining on horse meat i know there is a lot of received wisdom about what xxunk of xxunk to pair with say beef or fish or chicken though i couldn t quote any of it off the top of my own head as i am the xxunk thing from a xxunk in this world nevertheless i am wondering are there the same kind of traditional or standard wine recommendations for pairing with horse meat and what are they \n",
              " ,xxbos why is the pressure vacuum breaker leaking while trying to blow out my sprinkler system i m trying to hook my air compressor to my sprinkler system s pressure vacuum breaker valve \n",
              "  it s a xxunk xxunk 1 \n",
              "  i m hooking it to xxunk 1 in the diagram on page 2 of above the air goes straight out of the top of the xxunk assembly \n",
              "  it is fitted with one of these \n",
              "  check valve and xxunk appear good i don t have a place to hook up the air compressor xxunk \n",
              "  i m at a loss what am i doing wrong \n",
              "  edit i got it to work by xxunk the pressure on my compressor to xxunk xxunk the xxunk on the valve and manually keeping it open while applying air pressure to make an initial seal you ll need an air compressor that can keep the pressure up above xxunk though \n",
              " \n",
              "y: CategoryList\n",
              "repair lawn-mower,evolution sociality,meat turkey fats,wine,winterizing\n",
              "Path: /content/ULMFit;\n",
              "\n",
              "Valid: LabelList (180 items)\n",
              "x: TextList\n",
              "xxbos encrypt user email but be able to find user by xxunk for my dumb question but it s better to ask dumb question than to do dumb things xxunk \n",
              "  i want to encrypt user email in my db so that if someone xxunk the db and not the key he won t be able to restore the email xxunk but i need to be able to find user by email in my db and i can t xxunk over all the emails in the db decrypt each and compare this is too slow minutes hours \n",
              "  if i encrypt emails with aes with random iv then each time i encrypt the same email the encrypted value is different this is great for security but this way i can t just encrypt the given email and search for a value if the iv is the same each time then as far as i understand if attacker have enough encrypted values he can easily find the key right \n",
              "  i was thinking about storing original email hash xxunk the encrypted value but this way attacker will be able to recover original email values by encrypting emails from some xxunk with the same hash algorithm and comparing hash values with values in the db \n",
              "  i thought about storing hash of original email some fixed secret is this secure if not is there a secure solution to my problem \n",
              " ,xxbos robot wire xxunk how to position on wirei m designing my lawn mower robot and i am in the xxunk stage \n",
              "  the electronic part is done and works quite good now comes the software \n",
              "  i need an advice on how to deal with the problem of line following i mean once the robot is on the line parallel to the line that s relatively easy \n",
              "  but how to manage the situation when the robot is driving around and approaches the line wire \n",
              "  i have two sensors left and right turned 45 with respect to the forward direction \n",
              "  the robot could arrive from any angle so the signal amplitude read from the sensor could be completely random \n",
              "  so i don t understand what to do in order to move it in the right position on the wire \n",
              "  what s the usual approach \n",
              " \n",
              "  the idea is the same as here \n",
              " \n",
              "  the wire is all around the yard on the mower there are 2 sensors left and right that sense the signal xxunk from the wire a square wave signal at 34 xxunk the signal amplitude read from the sensors on the mower is about 2 v when it s above the wire \n",
              " ,xxbos where is this castle i know it s somewhere in ireland but where \n",
              "  irish castle i can find the picture everywhere but it never says where it s from \n",
              " ,xxbos is there a map that shows austrian winter road xxunk some xxunk in austria close for the winter season for example the xxunk high xxunk road is closed until sometime in may \n",
              "  where can i find a map that shows which roads are currently open closed \n",
              " ,xxbos aaa benefits as member of a european automobile club ihg rates this travel blog article in german mentions that also members of a european automobile club can use the aaa preferred rate for booking hotel rooms with ihg a hotel group \n",
              " \n",
              "  xxunk xxunk xxunk xxunk xxunk xxunk ist xxunk die aaa rate xxunk xxunk da xxunk xxunk xxunk xxunk \n",
              " \n",
              "  but fails to mention any further details \n",
              "  is there a list of the automobile xxunk that xxunk for this xxunk \n",
              "  in a cursory search i did find mention of a partner program of the german xxunk with aaa that could give some benefits but ihg was not mentioned in an otherwise rather xxunk list \n",
              "  is there any confirmation of this information i e being able to use the aaa rate with ihg as a member of a european automobile club ideally i would like to see some ihg document or repeated personal experience \n",
              "  as far as i know aaa rates can be booked all over the world so my question is not xxunk to bookings in the u s \n",
              "  we have a question about benefits for aaa members while traveling in europe and while it does link to a page that mentions a global discounts program i can not extract any further info from that site \n",
              "  another cursory search for global discounts program send me to this aaa site about discounts for international members coming to us north america it does list a bunch of hotel xxunk but not ihg \n",
              " \n",
              "y: CategoryList\n",
              "encryption hash deterministic-encryption,line-following magnetometer,ireland identify-this architecture locate-this,road-trips automobiles winter austria roads,budget bookings hotels automobiles\n",
              "Path: /content/ULMFit;\n",
              "\n",
              "Test: None, model=SequentialRNN(\n",
              "  (0): MultiBatchEncoder(\n",
              "    (module): AWD_LSTM(\n",
              "      (encoder): Embedding(5736, 400, padding_idx=1)\n",
              "      (encoder_dp): EmbeddingDropout(\n",
              "        (emb): Embedding(5736, 400, padding_idx=1)\n",
              "      )\n",
              "      (rnns): ModuleList(\n",
              "        (0): WeightDropout(\n",
              "          (module): LSTM(400, 1152, batch_first=True)\n",
              "        )\n",
              "        (1): WeightDropout(\n",
              "          (module): LSTM(1152, 1152, batch_first=True)\n",
              "        )\n",
              "        (2): WeightDropout(\n",
              "          (module): LSTM(1152, 400, batch_first=True)\n",
              "        )\n",
              "      )\n",
              "      (input_dp): RNNDropout()\n",
              "      (hidden_dps): ModuleList(\n",
              "        (0): RNNDropout()\n",
              "        (1): RNNDropout()\n",
              "        (2): RNNDropout()\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (1): PoolingLinearClassifier(\n",
              "    (layers): Sequential(\n",
              "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (1): Dropout(p=0.2, inplace=False)\n",
              "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
              "      (3): ReLU(inplace=True)\n",
              "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (5): Dropout(p=0.1, inplace=False)\n",
              "      (6): Linear(in_features=50, out_features=841, bias=True)\n",
              "    )\n",
              "  )\n",
              "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x7fed398f6f28>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('/content/ULMFit'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[...], layer_groups=[Sequential(\n",
              "  (0): Embedding(5736, 400, padding_idx=1)\n",
              "  (1): EmbeddingDropout(\n",
              "    (emb): Embedding(5736, 400, padding_idx=1)\n",
              "  )\n",
              "), Sequential(\n",
              "  (0): WeightDropout(\n",
              "    (module): LSTM(400, 1152, batch_first=True)\n",
              "  )\n",
              "  (1): RNNDropout()\n",
              "), Sequential(\n",
              "  (0): WeightDropout(\n",
              "    (module): LSTM(1152, 1152, batch_first=True)\n",
              "  )\n",
              "  (1): RNNDropout()\n",
              "), Sequential(\n",
              "  (0): WeightDropout(\n",
              "    (module): LSTM(1152, 400, batch_first=True)\n",
              "  )\n",
              "  (1): RNNDropout()\n",
              "), Sequential(\n",
              "  (0): PoolingLinearClassifier(\n",
              "    (layers): Sequential(\n",
              "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (1): Dropout(p=0.2, inplace=False)\n",
              "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
              "      (3): ReLU(inplace=True)\n",
              "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (5): Dropout(p=0.1, inplace=False)\n",
              "      (6): Linear(in_features=50, out_features=841, bias=True)\n",
              "    )\n",
              "  )\n",
              ")], add_time=True, silent=False)\n",
              "alpha: 2.0\n",
              "beta: 1.0], layer_groups=[Sequential(\n",
              "  (0): Embedding(5736, 400, padding_idx=1)\n",
              "  (1): EmbeddingDropout(\n",
              "    (emb): Embedding(5736, 400, padding_idx=1)\n",
              "  )\n",
              "), Sequential(\n",
              "  (0): WeightDropout(\n",
              "    (module): LSTM(400, 1152, batch_first=True)\n",
              "  )\n",
              "  (1): RNNDropout()\n",
              "), Sequential(\n",
              "  (0): WeightDropout(\n",
              "    (module): LSTM(1152, 1152, batch_first=True)\n",
              "  )\n",
              "  (1): RNNDropout()\n",
              "), Sequential(\n",
              "  (0): WeightDropout(\n",
              "    (module): LSTM(1152, 400, batch_first=True)\n",
              "  )\n",
              "  (1): RNNDropout()\n",
              "), Sequential(\n",
              "  (0): PoolingLinearClassifier(\n",
              "    (layers): Sequential(\n",
              "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (1): Dropout(p=0.2, inplace=False)\n",
              "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
              "      (3): ReLU(inplace=True)\n",
              "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (5): Dropout(p=0.1, inplace=False)\n",
              "      (6): Linear(in_features=50, out_features=841, bias=True)\n",
              "    )\n",
              "  )\n",
              ")], add_time=True, silent=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4CyU6Hjx8D_r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "classifier_learner.unfreeze()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p5OalBvwgHRJ",
        "colab_type": "code",
        "outputId": "3473c74a-ca76-4626-d308-f6e633f8f821",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 497
        }
      },
      "source": [
        "classifier_learner.fit_one_cycle(15, 1e-3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>6.918379</td>\n",
              "      <td>6.763258</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>00:21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>6.862895</td>\n",
              "      <td>6.519504</td>\n",
              "      <td>0.011111</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>6.707651</td>\n",
              "      <td>6.140455</td>\n",
              "      <td>0.033333</td>\n",
              "      <td>00:22</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>6.542593</td>\n",
              "      <td>5.684774</td>\n",
              "      <td>0.166667</td>\n",
              "      <td>00:23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>6.315500</td>\n",
              "      <td>5.262594</td>\n",
              "      <td>0.361111</td>\n",
              "      <td>00:23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>6.031075</td>\n",
              "      <td>5.003117</td>\n",
              "      <td>0.511111</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>5.755956</td>\n",
              "      <td>4.395498</td>\n",
              "      <td>0.794444</td>\n",
              "      <td>00:23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>5.431633</td>\n",
              "      <td>4.062148</td>\n",
              "      <td>0.888889</td>\n",
              "      <td>00:21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>5.127717</td>\n",
              "      <td>3.770995</td>\n",
              "      <td>0.927778</td>\n",
              "      <td>00:21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>4.843207</td>\n",
              "      <td>3.493947</td>\n",
              "      <td>0.955556</td>\n",
              "      <td>00:21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>4.606400</td>\n",
              "      <td>3.347330</td>\n",
              "      <td>0.977778</td>\n",
              "      <td>00:21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>4.388585</td>\n",
              "      <td>3.305549</td>\n",
              "      <td>0.994444</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>4.218474</td>\n",
              "      <td>3.263613</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>4.117228</td>\n",
              "      <td>3.175904</td>\n",
              "      <td>0.994444</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>4.050450</td>\n",
              "      <td>3.159141</td>\n",
              "      <td>0.994444</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3HjzPkE1gHMy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# torch.cuda.empty_cache"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nOlmwQofoJYs",
        "colab_type": "code",
        "outputId": "d862f4be-849f-40dc-de0a-4626a1945943",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 278
        }
      },
      "source": [
        "classifier_learner.recorder.plot_losses()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hVVdbA4d9KhxRCGi303lvovTmI\nCCIoUlRURECxj8P4jTo6OmMXsaB0CwJKEaQpUgSkhl6lRkoooSVACGn7++NcMEASAuTm3OSu93nO\nk3tPy7pHzLp773PWFmMMSiml3JeH3QEopZSylyYCpZRyc5oIlFLKzWkiUEopN6eJQCml3JyX3QHc\nrLCwMFOuXDm7w1BKqXxl/fr1J40x4Zlty3eJoFy5ckRHR9sdhlJK5Ssi8mdW27RrSCml3JzTEoGI\nVBWRTRmWBBF59pp9RERGisheEdkiIg2cFY9SSqnMOa1ryBjzB1APQEQ8gSPAzGt2uxOo7FiaAKMc\nP5VSSuWRvBoj6ADsM8Zc20fVHfjaWHUuVotIsIiUMMYczaO4lFI2SklJ4fDhwyQlJdkdSoHh5+dH\nZGQk3t7eOT4mrxLBA8DkTNaXAg5leH/Yse6qRCAig4BBAGXKlHFSiEqpvHb48GECAwMpV64cImJ3\nOPmeMYZTp05x+PBhypcvn+PjnD5YLCI+QDfgh1s9hzFmtDEmyhgTFR6e6d1PSql8KCkpidDQUE0C\nuURECA0NvekWVl7cNXQnsMEYczyTbUeA0hneRzrWKaXchCaB3HUr1zMvuob6kHm3EMBs4CkRmYI1\nSBzviuMD6emG4+eSiDmZSESQL/4+Xqzef4r4iylUKx5IzVJFCPDNd49kKKUU4OREICL+QCfgiQzr\nBgMYY74A5gFdgL1AIvCIM+O5WYnJqUxafZBvVv/JwdOJWe7n6+XBnbWKUyE8gHNJKZxLSiXu3CUe\nbVme5hW12auUqzp16hQdOnQA4NixY3h6enK5+3nt2rX4+Pjc8ByPPPIIw4cPp2rVqk6N1Zkkv01M\nExUVZW71yeLPl+7l+3WHqFIskEA/bzwEnmxXiXJh/lftZ4xh46Gz/GvmNnYcTaBmySAeaFSasqH+\nzN92lGJBfjSvGEaZkMLsPJbA4p0nmLXpCAlJqRTy9sRg8BAhMTmNsABfKoT5ExboQ6ngQkQWLUzF\n8ABqlQoiuPCN/5EpVZDt3LmT6tWr2x0GAP/+978JCAjgxRdfvGq9MQZjDB4e+ef528yuq4isN8ZE\nZba/W/VnvLvgDwBiTv317X7/yQtMG9yMhKRUft1xnD0nzvPL9mPsP3mBsAAfxg+Ion21Ylf2b13l\n6sHq4kX8aFc1gte71eRiShr+ji6ipJQ0Zm+KZdX+U8SevciO2AR+3XGC5LT0K8dWKx5Io3IhlA0t\nzOEzFyla2IfHW5ensI9b/WdRyuXs3buXbt26Ub9+fTZu3MjChQt5/fXX2bBhAxcvXqR37968+uqr\nALRs2ZJPP/2UWrVqERYWxuDBg5k/fz6FCxdm1qxZRERE2Pxpbsxt/+K80b0mArwyazvDp2/lt91x\nHEtIwstDaFYxlMdaladb3ZIE+uXsXlwPD7mSBAD8vD25v1Fp7m9U+qr99p44z/GEJDYePMOaA6eZ\nseEwF5LT8Pfx5EJyGrM2H+GRFuW5r2Ekft6eufmRlXJpr/+0nR2xCbl6zholg3jt7pq3dOyuXbv4\n+uuviYqyvkS//fbbhISEkJqaSrt27ejVqxc1atS46pj4+HjatGnD22+/zfPPP8/48eMZPnz4bX8O\nZ3ObRBCfmHLl9cCW5XmoWTnS0g2Ldp1gavQhigX58t3AJtSKLEJQDv/434pKEQFUigigRaUwngJS\n09I5fymVIoW8WbnvFH//YTOv/LiNb1bFMKJ3fWqUDHJaLEqprFWsWPFKEgCYPHky48aNIzU1ldjY\nWHbs2HFdIihUqBB33nknAA0bNmT58uV5GvOtcptEsDfuHADjHo6iQ3Wrq8fTQxjzUBSbDp2lUngA\nRf3zvs/ey9PjylhBi0ph/D68PUt3x/HStC3c89nvPNW+Ej3ql6J0SOE8j02pvHSr39ydxd//r7HD\nPXv28PHHH7N27VqCg4Pp379/pvfqZxxc9vT0JDU1NU9ivV35Z/TjNh2Lv4SXh1A5IvCq9d6eHjQq\nF2JLEsiMiNCuagQ/P9uadtXC+XDhblq/t4RHJ65j6R8nSE/PX4P7ShUECQkJBAYGEhQUxNGjR/n5\n55/tDilXuU2L4K46JehUoxjenrl4K+fGSVD1TigcknvndAjx9+GL/g3ZF3eB2Ztj+W7NQQZMWEeF\nMH/6NS1L51rFKR7kh6eH3pqqlLM1aNCAGjVqUK1aNcqWLUuLFi3sDilXudXto7nq7CEYWR/8gqDz\n21D7PnDi8wLJqenM33aUiStj2HjwLABhAb50rVOCh5uXo/w1t8AqlR+40u2jBYnePppXgkvDE7/B\nT8/AjMdh82S460MIyXmhp5vh4+VB93ql6F6vFNtj41n/5xlW7TvFd2sO8tWqGCqGB9CgTDD1Shfl\nvqhIvD3dptdPKXWbtEVwu9LTIHo8/Po6pKdC2+HQ7EnwdN6dRxmdOJfE1LWHWHPgNJsPneXcpVQi\nAn2pXaoIjcqH0LdJGafeBaXU7dAWgXPcbItAE0FuSYiFeX+HXXOgWC24eyRENszTEIwxLP0jjmnr\nD7PrWAL74i7g6+XBXXVK8E7POtpKUC5HE4FzaNeQXYJKwgOTYOdPVkIY2wGaPAHt/wW+gTc+PheI\nCO2qRdCumvUk48aDZxi9bD8zNhwhJc3wVo9a2jpQSl1HvyLmtup3w5NrodFAWPMlfNYEds2zJZT6\nZYoyqn9D/tG5Gj9tjqXVO0v4ZNEeTp6/ZEs8SinXpInAGfyC4K734bGF4FcEpvSBqQ9Cgj0Vtoe0\nrcisJ1tQMdyfDxbupvOI5Ww9HG9LLEop16OJwJlKN4InlkGH12DPL/BZY1g3FtLTb3xsLqtbOpgZ\nQ1sw/5lW+Hp5cM/nv9Nr1ErGLt9P3DltISj31K5du+seDhsxYgRDhgzJ8piAgAAAYmNj6dWrV6b7\ntG3blhuNZY4YMYLExL8KYHbp0oWzZ8/mNPRcpYnA2Ty9odXzMGQllKwPc1+ACZ3hxE5bwqleIohZ\nT7XgsZbluZSazptzd9L0f4to/8FSvltzkNS0vE9SStmlT58+TJky5ap1U6ZMoU+fPjc8tmTJkkyb\nNu2Wf/e1iWDevHkEBwff8vluhyaCvBJaER6aBfd8ASf3wBetYNF/IOXm5hbNDWEBvrzcpTo/DWvJ\nr8+34al2lQjy8+blmVtp/vZienz+O098E83nS/eyct9JziWl3PikSuVDvXr1Yu7cuSQnJwMQExND\nbGws9evXp0OHDjRo0IDatWsza9as646NiYmhVq1aAFy8eJEHHniA6tWr06NHDy5evHhlvyFDhhAV\nFUXNmjV57bXXABg5ciSxsbG0a9eOdu3aAVCuXDlOnjwJwIcffkitWrWoVasWI0aMuPL7qlevzuOP\nP07NmjW54447rvo9t0PvGspLIlCvD1S+A375P1j+PmyfCXePgPKtbQmpUkQAz3WqwrMdK7No5wl+\n3HSEs4kp/HHsHD9vt6aZ9hDoUrsEL9xRVZ9gVs4zfzgc25q75yxeG+58O8vNISEhNG7cmPnz59O9\ne3emTJnC/fffT6FChZg5cyZBQUGcPHmSpk2b0q1btyxnGxw1ahSFCxdm586dbNmyhQYNGlzZ9tZb\nbxESEkJaWhodOnRgy5YtPP3003z44YcsWbKEsLCwq861fv16JkyYwJo1azDG0KRJE9q0aUPRokXZ\ns2cPkydPZsyYMdx///1Mnz6d/v373/ZlcmqLQESCRWSaiOwSkZ0i0uya7W1FJF5ENjmWV50Zj8vw\nD4UeX8CDP4JJh6/uhh+HQuJp20ISETrWKManfRvw7cAmLP17Oza+0omvHm3MwFYVWLzrBH/7aBnf\nrIqxLUalnCFj99DlbiFjDC+//DJ16tShY8eOHDlyhOPHj2d5jmXLll35g1ynTh3q1KlzZdv3339P\ngwYNqF+/Ptu3b2fHjh3ZxrNixQp69OiBv78/AQEB3HvvvVfKWZcvX5569eoBVpnrmJiY2/noVzi7\nRfAxsMAY00tEfIDMaikvN8Z0dXIcrqliOxi6Cn57F1aOhN0LoOsIqNHN7sgAKOrvQ5sq4bSpEs7A\nVuUZPn0rr8zazqEzFxnatqJOtalyVzbf3J2pe/fuPPfcc2zYsIHExEQaNmzIxIkTiYuLY/369Xh7\ne1OuXLlMy07fyIEDB3j//fdZt24dRYsWZcCAAbd0nst8fX2vvPb09My1riGntQhEpAjQGhgHYIxJ\nNsbYMyTuyrwLQcfXYNBvEFwGvn8INn1nd1TXiQj0Y/SDDXmgUWlGL9tPu/eX8vP2Y3aHpdRtCwgI\noF27djz66KNXBonj4+OJiIjA29ubJUuW8Oeff2Z7jtatW/Pdd9b/t9u2bWPLli2AVb7a39+fIkWK\ncPz4cebPn3/lmMDAQM6dO3fduVq1asWPP/5IYmIiFy5cYObMmbRq1Sq3Pm6mnNk1VB6IAyaIyEYR\nGSsimXUwNxORzSIyX0QynZlCRAaJSLSIRMfFxTkxZBsVrwUD5kGFNlY30Yav7Y7oOl6eHrzdsw6z\nn2pBqaKFeOKb9bzy4zadI0Hle3369GHz5s1XEkG/fv2Ijo6mdu3afP3111SrVi3b44cMGcL58+ep\nXr06r776Kg0bWuVl6tatS/369alWrRp9+/a9qnz1oEGD6Ny585XB4ssaNGjAgAEDaNy4MU2aNGHg\nwIHUr18/lz/x1ZxWa0hEooDVQAtjzBoR+RhIMMa8kmGfICDdGHNeRLoAHxtjKmd3XpetNZRbUi7C\n1P6w91fo+hFEPWp3RJlKTk3n3QW7GLviAM0rhnJPvVL0bBip8yOom6K1hpzjZmsNObNFcBg4bIxZ\n43g/DWiQcQdjTIIx5rzj9TzAW0SuHkJ3N96FoPckqPw3mPMcrB1jd0SZ8vHy4P/uqs4/76zGoTOJ\nvDR9Cw+OW8PhM4k3Plgp5VKclgiMMceAQyJS1bGqA3DVcLmIFBfH/Vgi0tgRzylnxZRvePtB72+g\n6l0w70VYPcruiDIlIjzRpiLL/t6Od3vVYf2fZ2j97hLeWbCL/FbVVil35uy7hoYBkxx3DO0HHhGR\nwQDGmC+AXsAQEUkFLgIPGP0LYvHyhfsmwvRHYcFwSEuBFk/bHVWmRIT7o0rTvGIoH/+6h1FL9+Hr\n5cGw9pW1q0jdkDEmy/vz1c27lT+hOh+Bq0tLsWZA2z7TqlnU6nm7I8qWMYanJm9k7pajVCseyL/u\nqkHLyu7d26eyduDAAQIDAwkNDdVkkAuMMZw6dYpz585RvvzVsyXqfAT5mac33DsWPLxgkWMWtDYv\n2R1VlkSET/vUp2vtEvx3/k76j1vDvfVL8WaPWhT20X9u6mqRkZEcPnyYAns3oA38/PyIjIy8qWP0\n/8z8wNMLenwJ4glL3nJMiflPq2SFCxIR7qxdgnbVIvh86T5GLtrDsj0neeGOKkSVLUrF8AA8tMtI\nAd7e3td9c1V5TxNBfuHhCfd8brUMfnvHSgbtX3HZZADg5+3J852q0KZKOP/6cRv/nGHVkSlSyJsO\n1SMY2LICNUoG2RylUkoTQX7i4QndPrFaCMs/sMYPOr3h0skAoGHZoswZ1pK9J86z9Ug8a/afYu7W\no8zYcIQ7ahTjvfvqUqSQTqGplF10sDg/Sk+H+X+3Jrlp+iT87S2XTwbXik9M4etVMXy8aA81Sgbx\n3eNNCfDV7yVKOYtdD5QpZ/HwgC7vQ5PBsPozmP8PyGcJvUhhb4Z1qMwX/RuyPTaBwd+s51Jqmt1h\nKeWWNBHkVyLQ+W1o9hSs/RLmPm/LFJi3q2ONYrzTsw4r9p7kuambdIY0pWygbfH8TATueNMaQP59\nhDWA3PVjq8WQj/RqGMnZxGTenLsTD9nEiN718PLMX59BqfxME0F+JwId/209b7DsPUhPswaUPTzt\njuymDGxVgbR0w//m70JEGNG7nj6VrFQe0URQEIhA+3+Bhzcs/a/VMrhnVL5LBk+0qUiaMby74A8i\nAn15pWsNu0NSyi1oIihI2v7D6hZa/KaVDHqMtm41zUeGtKnIiYRLjFtxgNJFCzGghT5spJSz5a+/\nEurGWv/dahn8+pqVDHqOs7qN8gkR4ZWuNYg9e5HX5+wg0M+bng1v7nF5pdTN0RG5gqjls3DHW7Bj\nFvwwANJS7Y7opnh6CCP71KdFxTBenLaZscv3k6azoCnlNJoICqrmT1m3l+6aAz+/bHc0N83P25Mx\nD0XRoVox3py7k3tHreT3vSf19lKlnEATQUHWdMhfzxm46Exn2Snk48mYhxry8QP1OHw6kX5j19Du\ng6VMXntQE4JSuUhLTBR06WnWHMi7F0DfH6ByR7sjuiXnklJYtvsko5ftY/PheCqE+9OzQSR9G5eh\nqL+P3eEp5fKyKzGhicAdXDoPEzrD6Rh47Bcoln9vyzTGsHDHcT5bspctR+Ip7O1Jl9olePFvVSkW\n5Gd3eEq5LNsSgYgEA2OBWoABHjXGrMqwXYCPgS5AIjDAGLMhu3NqIrhF8UdgTHvw8oGBiyEg3O6I\nbtue4+f4fOk+5m45SoCfFz3ql+LJdpUI0RaCUtexs+jcx8ACY0w1oC6w85rtdwKVHcsgwDVnaS8I\nipSCPpPhfBxM6QspSXZHdNsqFwvko971mPJEU+qVDmbiyhg6j1jGjtgEu0NTKl9xWiIQkSJAa2Ac\ngDEm2Rhz9prdugNfG8tqIFhESjgrJrdXqgHcOxoOr4VZT+a7iqVZaVCmKOMHNOKnp1ri6SH0Hr2K\nX3cctzsspfINZ7YIygNxwAQR2SgiY0XE/5p9SgGHMrw/7FinnKVGN+jwGmybBkvftjuaXFWjZBA/\nDG5GuVB/Bn4dzbsLdundRUrlgDMTgRfQABhljKkPXACG38qJRGSQiESLSLROcp0LWj4H9frBb2/D\nlh/sjiZXRRYtzA+Dm9GncRk+X7qPh8av5dT5S3aHpZRLc2YiOAwcNsascbyfhpUYMjoClM7wPtKx\n7irGmNHGmChjTFR4eP4f5LSdCHQdAWVbWF1EB9fc+Jh8xM/bk//dW5v3etVh/Z9n6DtmDWcTk+0O\nSymX5bREYIw5BhwSkaqOVR2AHdfsNht4SCxNgXhjzFFnxaQy8PKB3t9ag8hT+sKZGLsjynX3RZVm\n/IBGHDh5gYfGr+V4Qv4fIFfKGZx919AwYJKIbAHqAf8VkcEiMtixfR6wH9gLjAGGOjkelVHhEOj7\nPaSnwHe9ISne7ohyXYtKYXzerwF7T5znrpHLWbnvpN0hKeVy9IEyBQeWwTc9oHxr6+njfFa6Oif2\nHD/HkEkb2B93nhfuqMqQNhXx0IlvlBvRyetV9sq3hrs+hH2LYcEtjee7vMrFApn1ZAvuqlOS937+\ngy4jl7N8j954oBRoIlCXNXwYmg+DdWNgzZd2R+MU/r5ejHygHh/eX5fk1HQeHLeWgV+t4/yl/FWm\nW6ncpolA/aXj61D1LqtVsPsXu6NxChHh3gaRzHumFS91rsriXSdo+c5iRi7aQ4o+c6DclCYC9RcP\nT+g5BorVgmmPwvHtdkfkNH7engxtW4npQ5rTqFwIHy7cTe8vV3Hk7EW7Q1Mqz2kiUFfz8Ye+U8E3\nwLqT6FzBLtVQv0xRxjwUxSd96rP7+Hm6fLycBduO2R2WUnlKE4G6XlBJ6DMFEk85CtQV/G/Jd9ct\nyZxhLSkdUojB365n4FfRHIvX5w6Ue9BEoDJXsp5VoO7IevhxKKQX/P7zcmH+zBjSguF3VuP3vSfp\n9OFvvDlnB6cv6FPJqmDTRKCyVv1u6Phv2D4Dlv7P7mjyhI+XB4PbVGTu0y1pWy2CiStjGDBhLeeS\nUuwOTSmn0USgstfiGajfH5a9C5un2h1NnqkQHsAnferzRf+G7IhNoPeXqzlxTruKVMGkiUBlTwTu\n+gjKtYLZT8Gfq258TAHSsUYxxj4cRcypC/QZvVormaoCSROBujEvH7j/ayhSGqb2g9P77Y4oT7Wt\nGsHERxpz+MxFHp6gZa1VwaOJQOVM4RDo94M1q9k3PQr8baXXalw+hC/6N2TP8fPc8/nv7Is7b3dI\nSuUaTQQq50IrQr9p1rzH3/YskNVKs9OuWgRTn2hG4qU0Hv8qmviLOoCsCgZNBOrmRDaE3t9A3C6Y\n3AdS3GsAtV7pYD7r14BDZxK5/wt9ElkVDJoI1M2r1AF6fAF/roTpj0GaexVta1ohlImPNCb27EU6\nj1jGL9v1SWSVv2kiULemdi+48x3YNQfmPGuNHbiRFpXCmD2sJRXC/Bk6aQNjl+8nPd29roEqODQR\nqFvX5Alo/RJs/AYWvWF3NHmufJg/3w5sQvtqEbw5dydPfreBpJQ0u8NS6qZpIlC3p93L0PARWPEh\nrPrM7mjyXKCfN18+2JD/61KdBduP0WfMatYeOE1+m/lPuTenJgIRiRGRrSKySUSum19SRNqKSLxj\n+yYRedWZ8SgnEIG7PoDq3eDnl93q6ePLRITHW1fg0z4NiDl5gfu/XMVL07aQpl1FKp/Ii8lp2xlj\nspsxfLkxpmsexKGcxcMTeo6FSb1g1lAoVBSq3GF3VHnurjolaF8tgpGL9zBq6T4upabzbq86+Hl7\n2h2aUtnSriGVO7x8ofckKFYTvn8IDq21OyJbFPLx5B+dq/FS56rM3hzLPZ/9zt4T5+wOS6lsOTsR\nGOAXEVkvIoOy2KeZiGwWkfkiUjOzHURkkIhEi0h0XJxOOO6y/IKg33QIKgGT7oMTO+2OyDZD21Zi\n4iONiDt3ia6frGDcigPaVaRcljhzUEtEShljjohIBLAQGGaMWZZhexCQbow5LyJdgI+NMZWzO2dU\nVJSJjr5uuEG5kjMxMO5vIB7w2M8QXMbuiGxzPCGJ4dO3sOSPOBqUsR5GK1GkkN1hKTckIuuNMVGZ\nbXNqi8AYc8Tx8wQwE2h8zfYEY8x5x+t5gLeIhDkzJpUHipaDB2dAygWrLtGF7IaICrZiQX6MH9CI\nEb3rsfv4efqMXs2O2AS7w1LqKk5LBCLiLyKBl18DdwDbrtmnuIiI43VjRzynnBWTykPFakKfqRB/\n2BpEvuS+/eQiwj31S/HVo41JTE7jvi9WsnKf+yZH5Xqc2SIoBqwQkc3AWmCuMWaBiAwWkcGOfXoB\n2xz7jAQeMHoDdsFRthncNxGOboGp/SHVvcs3NyxblDnDWlKqaCEGTFjHwh3uVcFVuS6njhE4g44R\n5EObvoMfh0DNe63bTD3c+3bKMxeSGTBxHduOxPP+fXXoUT/S7pCUG7BtjEApAOr1hU7/seY+nv8P\nt6tLdK2i/j5MGtiEJuVDeG7qZr5aGWN3SMrNaSJQeaPF09D8aVg3Bn571+5obBfg68X4AY3oVKMY\nr83ezshFe7QshbKNJgKVdzq9AfX6wdL/wrqxdkdjOz9vT0b1a8C9DUrx4cLdvDl3p1YwVbbIixIT\nSllE4O6RkHgK5r4IhUOhZg+7o7KVl6cH7/eqS5CfN+NWHGB7bDyf92tIiL+P3aEpN6ItApW3PL2g\n1wQo3QSmPw77ltgdke08PITX7q7BOz1rs+HgWXqNWsnBU4l2h6XciCYClfd8CkPfKRBW2bqt1E3r\nEmUkIvRuVIZJA5tw6kIyd3+6giV/nLA7LOUmNBEoexQqCv1nWN1DE+6EJf+D1GS7o7Jdo3Ih/PRU\nS0oGF+LRiesYuWiPjhsop9NEoOwTVAIGLYVaPeG3t2Fsezi21e6obFcmtDAzhjSnRz1rEHnQN9Gc\nTdQkqZxHE4GyV+EQuHe0VcL63DEY3Q5+ew/SUuyOzFaFfDz54P66vN6tJr/tjqPLx8tZF3Pa7rBU\nAaWJQLmG6l1h6Bqo0Q2WvAljO7p1GWuwxg0ebl6OGUNa4O3lQe8vV/H+z39w+oK2DlTuylEiEJGK\nIuLreN1WRJ4WkWDnhqbcjn8o9BoP930F8Yfgy9aw4iNIS7U7MlvVjizCnGEt6Va3JJ8u2UunD39j\n2vrD+gCayjU5bRFMB9JEpBIwGigNfOe0qJR7q3mP1Tqo0hl+/TeM/xvE7bY7KlsF+nkz4oH6zHu6\nFZFFC/HiD5t5/Oto4s65dyE/lTtymgjSjTGpQA/gE2PM34ESzgtLub2AcLj/a+g5Dk7vgy9awspP\nID3N7shsVaNkED8+2YLX7q7Bb7vjaPa/RYxbccDusFQ+l9NEkCIifYCHgTmOdd7OCUkpBxGo3ctq\nHVTqAL/8y7rV9NQ+uyOzlYjwSIvyLHi2Ne2qRfCfOTt4/aftOhWmumU5TQSPAM2At4wxB0SkPPCN\n88JSKoPAYvDAd9BjNMTtglEtYPUoSE+3OzJbVQwP4Iv+DXm0RXkm/B7D0EnruZjs3i0mdWtuej4C\nESkKlDbGbHFOSNnT+QjcXMJR+Olp2PMLlG0B3T+DkPJ2R2W78SsO8J+5O6gbGczYh6MIC/C1OyTl\nYm57PgIRWSoiQSISAmwAxojIh7kZpFI5ElQC+n5vJYBjW63Wwdoxbt86eLRleUb1a8jOown0H7uG\n85fc+04rdXNy2jVUxBiTANwLfG2MaQJ0vNFBIhIjIltFZJOIXPc1XiwjRWSviGwRkQY3F75ySyJQ\nvz8MXQVlmsC8F+Gb7nDmT7sjs1XnWsUZ81AUe06cp+fnKzlw8oLdIal8IqeJwEtESgD389dgcU61\nM8bUy6JJcidQ2bEMAkbd5LmVOysSadUruvtjOLIBRjWH9RPdega01lXC+eqRxhw/l0TXkcuZtemI\n3SGpfCCnieAN4GdgnzFmnYhUAPbkwu/vjtXCMMaY1UCwI+EolTMi0HCA1Too1QB+esaqaOrGXUUt\nK4cxZ1hLapQM4tmpm/h2tXu3lNSN5SgRGGN+MMbUMcYMcbzfb4zpmZNDgV9EZL2IDMpkeyngUIb3\nhx3rriIig0QkWkSi4+LichKycjfBZeDBWdD+Fdg1BzZ8ZXdEtoosWphvHmtCu6oR/OvHbTwzZSPx\nF927fpPKWk4HiyNFZKaInHkoViAAAByZSURBVHAs00UkMgeHtjTGNMDqAnpSRFrfSpDGmNHGmChj\nTFR4ePitnEK5Aw8PaPUClGsFC1+ziti5MT9vT758sCHPd6rC3C1Huffz3zl0Wie8UdfLadfQBGA2\nUNKx/ORYly1jzBHHzxPATKDxNbscwSpXcVmkY51St0bEGjNITYL5L9kdje28PT14ukNlJg1swsnz\nyXT/7HdW7Dlpd1jKxeQ0EYQbYyYYY1Idy0Qg26/mIuIvIoGXXwN3ANuu2W028JDj7qGmQLwx5ujN\nfQSlrhFaEdq8BDtmwa55dkfjEppUCGXG0OaEBfjw4Pg1fLRwN/GJ2lWkLDlNBKdEpL+IeDqW/sCp\nGxxTDFghIpuBtcBcY8wCERksIoMd+8wD9gN7gTHA0Fv4DEpdr/nTEFED5r4ASQl2R+MSKoYH8OOT\nLehWtyQfL9pDq3cXM3eLfu9SOXyyWETKAp9glZkwwEpgmDHmULYHOoE+Waxy7HC0Na9B48ehy3t2\nR+MyjDFsORzPv3/azsaDZ3mnZ216Nypjd1jKyW77yWJjzJ/GmG7GmHBjTIQx5h4gJ3cNKWWfyCho\nPMh68vjQOrujcRkiQt3SwUwZ1JTWVcL5x/St9B+7hnlbj5Kc6r633bqzm641dOVAkYPGmDz/GqEt\nAnVTLp2Dz5qAXxEY9Bt4+dgdkUtJSkljwu8xfLMqhtj4JEL9fejbpAyD21TE39fL7vBULrrtFkFW\n572NY5XKG76BcNcHcGIHrPzY7mhcjp+3J0PaVmT5P9ozYUAj6pcpyqdL9nLv5ytZ/+cZu8NTeeR2\nEoH7Psev8peqd0KN7vDbe3Byr93RuCRPD6FdtQjGPhzFxEcaE38xhd5frmLa+sN2h6byQLaJQETO\niUhCJss5rOcJlMof7nwXvPxgzrNuXYsoJ9pUCeeX51vTtEIoL/6wmX/O2MoFrWZaoGWbCIwxgcaY\noEyWQGOMdiCq/COwOHR6HWKWw8Zv7Y7G5QX5eTN+QCOeaF2BKesOcvcnK9h2JN7usJST3E7XkFL5\nS4OHoUxza8rL8yfsjsbl+Xh58M8u1Zn8eFMSk9O49/OVjF9xgFu9wUS5Lk0Eyn14eFjlJ1ISYcFw\nu6PJN5pWCGXeM61oXSWMN+bs4PGvozl8RmsWFSSaCJR7Ca8CrV6EbdNh9y92R5NvhPj7MOahKF67\nuwbLdp+k/Qe/8cmiPSQkaZmKgkATgXI/LZ+D8Gow93m4dN7uaPINEeGRFuVZ+ve2tKoUxgcLd9N3\nzGoOntLWQX6niUC5Hy8fq4so/hAsecvuaPKdksGFGDegEeMejmJ/3AXafbCUJydt0ISQj2kiUO6p\nTFOIegzWfAFH1tsdTb7UoXoxFr/QloEty7NsdxxdP1nOx7/u4Wxist2hqZt0yyUm7KIlJlSuSYq3\nyk8UDoNBS8DT2+6I8q0/T13gP3N28OvOE/h5exBVNoRh7SvRqFwIHh5ahMAVOKvEhFL5m18Rqyrp\n8a2w6lO7o8nXyob6M/bhRix4thUPNCrDjqMJ9B69mu6f/U50zGm7w1M3oC0Cpab0g72/wtBVEFLB\n7mgKhPiLKSzYdpQPF+7meMIlutYpwf/urU2gn7a67KItAqWy0+U98PSBn7T8RG4pUsib3o3KsPiF\ntjzXsQrztx3j7k9WsCNWJwlyRZoIlAoqCR1fgwO/weYpdkdToPj7evFMx8pMGdSUiylp3PP570xZ\ne1CfTnYxTk8EjqktN4rInEy2DRCROBHZ5FgGOjsepTLV8FEo3QR+fhku6OTuua1RuRDmPt2KxuVC\nGD5jKy98v5nEZC1k5yryokXwDLAzm+1TjTH1HMvYPIhHqetdLj9x6ZyVDFSuCwvw5atHG/Nsx8rM\n3HSEFm8v5qfNsXaHpXByIhCRSOAuQP/AK9cXUd166njLVGvwWOU6Tw/h2Y5V+OGJZpQP82fY5I08\nOWmDlqqwmbNbBCOAl4DsJkLtKSJbRGSaiJTObAcRGSQi0SISHRcX55RAlQKg1QsQWhnmPAfJF+yO\npsCKKhfCpIFNebpDZX7efox7P1/J6v2ndOzAJk5LBCLSFThhjMnusc2fgHLGmDrAQuCrzHYyxow2\nxkQZY6LCw8OdEK1SDt5+VhfR2YOw9H92R1OgFfLx5PlOVfjmsSbEX0zhgdGr6fDhb8zadMTu0NyO\nM1sELYBuIhIDTAHai8hVM4IYY04ZYy453o4FGjoxHqVyplwLa+6CVZ9B7Ca7oynwmlUMZflL7fhP\n95r4ennyzJRNdB6xjMW7jmsLIY/kyQNlItIWeNEY0/Wa9SWMMUcdr3sA/zDGNM3uXPpAmcoTF8/C\nZ42tmc0GLgZPnZAvLySlpDFl7UG+WvUnB05eoHnFUIa2rUSLSqGIaKmK2+FSD5SJyBsi0s3x9mkR\n2S4im4GngQF5HY9SmSoUDHe+A0c3w5pRdkfjNvy8PRnQojw/P9ua1+6uwe7j5+g/bg2dRyxn3IoD\n7DyqD6Q5g5aYUCorxsDkPtaDZg/NhtKN7I7I7VxKTWP2pljGrTjArmPnAKhWPJC+TcrQs0Ek/r7a\nUsup7FoEmgiUyk5CLEzoAueOQo8voeY9dkfklowxHDl7kQXbjvHTlqNsPnSWID8v+jcty9MdKuPn\n7Wl3iC5PE4FSt+PCSZjSFw6tgU5vQPOnQfurbbX+zzOMW7GfeVuPEervQ+PyIbzUuRrlw/ztDs1l\naSJQ6nalJMGPg2H7TGj4CHR5XweQXcCKPSeZseEwi3ad4GJyGl3rluClv1WjeBE/u0NzOdklAv2X\nrFROePtBz/FQtBys+Mia5vK+ieAbaHdkbq1l5TBaVg7j0OlERi/bz9ToQ8zdcpTmFUO5L6o0nWoU\nw9tTa2veiLYIlLpZ6yfCnOchogb0nQpFStkdkXI4dDqRcSsOsGjXcQ6dvkiQnxe9G5WmY/ViNC4f\n4ta3oGrXkFK5be8i+P5h8A2Avt9DiTp2R6QySEs3/LrzODM3HGHRruOkpBnqlg7mvoaR9Khfyi3v\nNtJEoJQzHN8Ok+6HpLPQawJUucPuiFQmziWlMHtzLBN+j2HvifOEBfjwVo/adKxeDE83mk9ZE4FS\nzpJwFL67H45vs2Y6a6RTargqYwwbDp7h5Rnb+OP4OYoF+dKrYSTD2rvH7aeaCJRypkvnYfpjsHsB\nNHsKOv3Hmt9AuaSklDR+3XmcHzce4dedJygVXIg7ahajb+MyVC5WcAf/NREo5WzpabBgOKwdDdXv\nhh6jwaew3VGpG1jyxwm+XfUny/eeJDk1nSblQ6hWPJBu9UrRoExwgRpc1kSgVF4wBlaPsmY4K9UA\n+kyBgAi7o1I5cPpCMpPXHmTmxiMcPpNIUko6DcsWpX21CNpVjaBGySC7Q7xtmgiUyks758D0gRAQ\nDv2mQXhVuyNSN+H8pVSmRR+6UgHVQ6ByRCCViwXQtmoErSqHERHom+9aC5oIlMprR9bDd70hLRl6\nfwvlW9sdkboFZy4kM3FlDNtjE9h8+Cxx56zpU3y9PAgq5E2DMsG0rBxOswqhVAz3d+nkoIlAKTuc\n+RMm3Qen90O3T6BeH7sjUrchPd2w42gC62JOczQ+iVPnk1m9/xRHzl4EICzAl2JBvtQvE0zV4kHc\nXacEwYV9bI76L5oIlLLLxbPw/YNwYBm0GQ5th2vBugLEGMPB04ms2neKtQdOc+TsRTYdOsul1HS8\nPYWGZYtSo0QRGpQNpn6ZohQP8rPt2QVNBErZKTUZ5jwLmyZBnQeg20jw8rU7KuVE22Pjmb05luW7\nT3Lg5AUupqQBUDa0MJUjAggP9OPuuiXw9/EixN+HEkX8MODUukiaCJSymzGw7H1Y8iaUbQm9v4HC\nIXZHpfJAWrph48EzbDsSzy87jnPi3KUrdyZl5OkhRBYtRIi/D8UC/ahVKggfLw9aVQ6nWvHA2x5/\n0ESglKvY8gPMGgoBxa3qpZEN7Y5I2SD+Ygqr9p3C21M4ce4SR85Y4wwHTl3gbGIyMScTr4w9ABQP\n8qNzreL8u1vNW/6dtpahFhFPIBo4ksnk9b7A10BD4BTQ2xgT4+yYlLJNnfsgpAL8MADG/w3u+A80\nGazjBm6mSCFvOtcqnuV2YwwJF1NJSk3jtz/iWLr7BKcvJDstHqe3CETkeSAKCMokEQwF6hhjBovI\nA0APY0zv7M6nLQJVICSehh+Hwu751pPI3T6FQsF2R6UKsOxaBE4tiCIikcBdwNgsdukOfOV4PQ3o\nIK58I65SuaVwCPSZDHe8Cbvmweg2ELvJ7qiUm3J2ZawRwEtAehbbSwGHAIwxqUA8EHrtTiIySESi\nRSQ6Li7OWbEqlbdEoPkweGQ+pKXAuE6wdow1sKxUHnJaIhCRrsAJY8z62z2XMWa0MSbKGBMVHh6e\nC9Ep5ULKNIEnlkP5NjDvRZj2KCQl2B2VciPObBG0ALqJSAwwBWgvIt9es88RoDSAiHgBRbAGjZVy\nL/6h1kxnHV6DHbNgdFs4ttXuqJSbcFoiMMb80xgTaYwpBzwALDbG9L9mt9nAw47XvRz7aLtYuScP\nD2j1PDz8EyRfgLEdrfmR9X8J5WR5PnuGiLwhIt0cb8cBoSKyF3geGJ7X8Sjlcsq1gMEroEwz+OkZ\nmDHImvxGKSfRB8qUclXpabD8A1jyXwirDPd9BcVq2B2Vyqdsu31UKXUbPDyhzUvw0CyreN2Y9rBx\nkt1RqQJIE4FSrq5CG6urKDLKKk/x41BITrQ7KlWAaCJQKj8ILGa1DFq/BJu+g7EdIG633VGpAkIT\ngVL5hYcntP8/6D8dzp+wbjHd8r3dUakCQBOBUvlNpQ4weDmUqAszHofZT0PKxRsfp1QWNBEolR8F\nlbSeN2j5HGz4CsZ0gBM77Y5K5VOaCJTKrzy9oOO/od80OH/c6ipaN1YfQFM3TROBUvld5U4wZCWU\nawlzX4ApfeGCVmpROaeJQKmCILAY9P0B/vY/2PsrjGoO+5bYHZXKJzQRKFVQeHhAs6EwcBH4FYFv\n7oFfXoFU581spQoGTQRKFTQl6sCgpRD1KKwcac1zcHKv3VEpF6aJQKmCyKcwdP0Iek+Cs3/Cl61g\nw9c6kKwypYlAqYKseldrIDkyCmYPgx8ehotn7I5KuRhNBEoVdEEl4cFZ0PF12DUXRrWEmN/tjkq5\nEE0ESrkDDw9o+Sw89gt4+cDEu2Dxm9ZcycrtaSJQyp2UamjNj1yvHyx7D8Z3htMH7I5K2UwTgVLu\nxjcA7vkMek2Ak3vgi1aweardUSkbOS0RiIifiKwVkc0isl1EXs9knwEiEicimxzLQGfFo5S6Rq17\nYcgKKF4LZg6C6Y9DUrzdUSkbOLNFcAlob4ypC9QDOotI00z2m2qMqedYxjoxHqXUtYLLwIC50O7/\nYNt0+KIlHFprd1QqjzktERjL5Rm3vR2L3sSslKu5PCXmowus9+M7w+K3IPG0vXGpPOPUMQIR8RSR\nTcAJYKExZk0mu/UUkS0iMk1ESmdxnkEiEi0i0XFxcc4MWSn3VbqxNSVmrZ6w7F34oCp8/xDs/gXS\nUu2OTjmRmDx40lBEgoGZwDBjzLYM60OB88aYSyLyBNDbGNM+u3NFRUWZ6Oho5waslLs7tg02TYIt\nUyHxFASWgLoPQL3+EFbJ7ujULRCR9caYqEy35UUicATxKpBojHk/i+2ewGljTJHszqOJQKk8lJoM\nuxdYSWHPQjBpULoJ1O8PNXuAb6DdEaocyi4ROPOuoXBHSwARKQR0AnZds0+JDG+7ATrFklKuxMsH\nanSDvlPh+R3Q6Q2rRMXsYfB+FZg5GA4sh/R0uyNVt8HLiecuAXzl+KbvAXxvjJkjIm8A0caY2cDT\nItINSAVOAwOcGI9S6nYEFocWz0Dzp+FwNGz6FrZOh82ToWg56yG1un0gONOhPuXC8qxrKLdo15BS\nLiQ5EXb+ZCWFA8sAgQptoP6DUO0u8C5kd4TKIbuuIWe2CJRSBZ1PYajb21rOxMCmybDpO5j+GPgW\ngdo9rfGEkg1AxO5oVRa0RaCUyl3p6RCzHDZ+CztnQ2oShFeD4nWsSqhXLaXAP9x6lkE5lbYIlFJ5\nx8PD6h6q0AaS3odtM2D7TDi0BhJiIf2aiqfiad2eel2ScCSKwBLW4uVjz+dxA5oIlFLO41cEoh6x\nFrBaC4mnIOEInDtq/UyI/Ws5vt26TTXlwvXn8o+AoBJWcricKAIzJIygEuDjn7efr4DQRKCUyjse\nHhAQbi3Uy3wfY6zid5klioRYOPMn/LkSks5ef6xf8PUtiqsSRkkrOel4xVU0ESilXIsIFAq2lojq\nWe+XnHhNsjgCCUf/en10C1w4cf1x3v6OpHBN66JoOSgVZf1eN6OJQCmVP/kUhtCK1pKV1GQ4fyxD\norimdXFguZVMTJrjALGST+nG1hPUpZtASIUC34LQRKCUKri8fKxS28Flst4nPQ3On4CTf1gluA+t\ngW0zYf1Ea3vhMEdScCSHkvUK3PMRmgiUUu7Nw9PRTVQCKrS11qWnOxLDGis5HFwNf8x17O8NJepC\nmaZ/JYfA4nZFnyv0OQKllMqJCyf/ajEcWguxG6xnJMBqcVzuSirdBCJqgKdrfc/W5wiUUup2+YdB\ntS7WAtb4w7EtjsSwxhpv2PqDtc0nAErUg8JFrcFpn8LWra2XX3tffp/xZ2HruMuvvf3z7NkJTQRK\nKXUrvHwgMspamj1p3fYafwgOOhLD0c1wco91d1PKBetn6sWb+x0eXlcnj6hHoflTuf9Rcv2MSinl\njkT+Gpiuc1/m+6SnQ0qitSRfsJbLr1MSr04al38mX/jrdUCEU0LXRKCUUnnFwwN8A6zFhTh1zmKl\nlFKuTxOBUkq5OU0ESinl5pw5Z7GfiKwVkc0isl1EXs9kH18RmSoie0VkjYiUc1Y8SimlMufMFsEl\noL0xpi5WmcHOItL0mn0eA84YYyoBHwHvODEepZRSmXBaIjCW84633o7l2seYuwNfOV5PAzqIFPDq\nTkop5WKcOkYgIp4isgk4ASw0xqy5ZpdSwCEAY0wqEA+EZnKeQSISLSLRcXFxzgxZKaXcjlMTgTEm\nzRhTD4gEGotIrVs8z2hjTJQxJio8PDx3g1RKKTeXJw+UGWPOisgSoDOwLcOmI0Bp4LCIeAFFgFPZ\nnWv9+vUnReTPWwwlDDh5i8e6C71G2dPrc2N6jbJn1/Upm9UGpyUCEQkHUhxJoBDQiesHg2cDDwOr\ngF7AYnODcqjGmFtuEohIdFbV95RFr1H29PrcmF6j7Lni9XFmi6AE8JWIeGJ1QX1vjJkjIm8A0caY\n2cA44BsR2QucBh5wYjxKKaUy4bREYIzZAtTPZP2rGV4nAVlUZ1JKKZUX3O3J4tF2B5AP6DXKnl6f\nG9NrlD2Xuz75boYypZRSucvdWgRKKaWuoYlAKaXcnNskAhHpLCJ/OArcDbc7HruIyHgROSEi2zKs\nCxGRhSKyx/GzqGO9iMhIxzXbIiIN7Is8b4hIaRFZIiI7HMUSn3Gs12tE1sUkRaS8o3DkXkchSR/H\nercsLOmoqrBRROY43rv09XGLROC4hfUz4E6gBtBHRGrYG5VtJmI92JfRcGCRMaYysMjxHqzrVdmx\nDAJG5VGMdkoFXjDG1ACaAk86/q3oNbJkVUzyHeAjRwHJM1gFJcF9C0s+A+zM8N61r48xpsAvQDPg\n5wzv/wn80+64bLwe5YBtGd7/AZRwvC4B/OF4/SXQJ7P93GUBZmE9DKnX6PprUxjYADTBelLWy7H+\nyv9vwM9AM8drL8d+YnfsTr4ukVhfFtoDcwBx9evjFi0CMhS3czjsWKcsxYwxRx2vjwHFHK/d+ro5\nmun1gTXoNbri2mKSwD7grLEKR8LV1yBHhSULmBHAS0C6430oLn593CURqBwy1lcTt7+nWEQCgOnA\ns8aYhIzb3P0amWuKSQLVbA7JZYhIV+CEMWa93bHcDHdJBJeL210W6VinLMdFpASA4+cJx3q3vG4i\n4o2VBCYZY2Y4Vus1uoYx5iywBKurI9hROBKuvgZXrk9OC0vmcy2AbiISA0zB6h76GBe/Pu6SCNYB\nlR0j9z5YNY1m2xyTK7lc/A/Hz1kZ1j/kuDOmKRCfoXukQHJMjDQO2GmM+TDDJr1GWMUkRSTY8fpy\nMcmdWAmhl2O3a6/P5euWo8KS+Zkx5p/GmEhjTDmsvzOLjTH9cPXrY/fASh4O4HQBdmP1Z/6f3fHY\neB0mA0eBFKy+ysew+iQXAXuAX4EQx76CdbfVPmArEGV3/HlwfVpidftsATY5li56ja5cnzrARsf1\n2Qa86lhfAVgL7AV+AHwd6/0c7/c6tlew+zPk4bVqC8zJD9dHS0wopZSbc5euIaWUUlnQRKCUUm5O\nE4FSSrk5TQRKKeXmNBEopZSb00SgXI6IpInIJkeFyw0i0vwG+weLyNAcnHepiLjUpOF2E5GJItLr\nxnuqgkwTgXJFF40x9YxV4fKfwP9usH8wcMNEYJcMT5Qq5ZI0EShXF4RVthcRCRCRRY5WwlYR6e7Y\n522goqMV8Z5j33849tksIm9nON99jnr6u0WklWNfTxF5T0TWOeYUeMKxvoSILHOcd9vl/TMSkRgR\nedfxu9aKSCXH+oki8oWIrAHedcxn8KPj/KtFpE6GzzTBcfwWEenpWH+HiKxyfNYfHLWPEJG3xZor\nYYuIvO9Yd58jvs0isuwGn0lE5FOx5ub4FYjIzf9YKn/SbyrKFRVyVLf0wyr53N6xPgnoYYxJEJEw\nYLWIzMaaG6CWsQqhISJ3At2BJsaYRBEJyXBuL2NMYxHpArwGdMR6ujreGNNIRHyB30XkF+BerHLB\nbznmtCicRbzxxpjaIvIQVuXJro71kUBzY0yaiHwCbDTG3CMi7YGvser5v3L5eEfsRR2f7V9AR2PM\nBRH5B/C8iHwG9ACqGWPM5VIPwKvA34wxRzKsy+oz1QeqYs3LUQzYAYzP0X8VVWBpIlCu6GKGP+rN\ngK9FpBZWOYf/ikhrrBK/pfirHHRGHYEJxphEAGPM6QzbLheRW481LwPAHUCdDH3lRbAmmlkHjBer\nCN2PxphNWcQ7OcPPjzKs/8EYk+Z43RLo6YhnsYiEikiQI9YHLh9gjDkjVgXLGlh/vAF8gFVYJYqT\ngHFizXw1x3HY78BEEfk+w+fL6jO1BiY74ooVkcVZfCblRjQRKJdmjFnl+IYcjlXzJxxoaIxJEavC\no99NnvKS42caf/37F2CYMebna3d2JJ27sP7QfmiM+TqzMLN4feEmY7vya4GFxpg+mcTTGOiAVaDs\nKazZwgaLSBNHnOtFpGFWn8nRElLqKjpGoFyaiFQDPLFK8xbBqvWeIiLtgLKO3c4BgRkOWwg8IiKF\nHefI2DWUmZ+BIY5v/ohIFRHxF5GywHFjzBhgLJDVfMS9M/xclcU+y4F+jvO3BU4aa56DhcCTGT5v\nUWA10CLDeIO/I6YAoIgxZh7wHFDXsb2iMWaNMeZVIA6rrHGmnwlYBvR2jCGUANrd4NooN6AtAuWK\nLo8RgPXN9mFHP/sk4CcR2QpEA7sAjDGnROR3EdkGzDfG/F1E6gHRIpIMzANezub3jcXqJtogVl9M\nHHAPVvXIv4tICnAeeCiL44uKyBas1sZ13+Id/o3VzbQFSOSv0sNvAp85Yk8DXjfGzBCRAcBkR/8+\nWGMG54BZIuLnuC7PO7a9JyKVHesWAZuxqoNm9plmYo257AAOknXiUm5Eq48qdRsc3VNRxpiTdsei\n1K3SriGllHJz2iJQSik3py0CpZRyc5oIlFLKzWkiUEopN6eJQCml3JwmAqWUcnP/D7aOVU5Sbfh3\nAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DRn8uditDqsl",
        "colab_type": "text"
      },
      "source": [
        "#### Outputs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hGNdiXc1oJSg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "classifier_learner.save('/content/multi_label_learner_v1')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j2zaP2_ILcQu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "classifier_learner.export()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GblrPEj7D5hz",
        "colab_type": "text"
      },
      "source": [
        "##### On Val set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cJZj-YPGRY-5",
        "colab_type": "code",
        "outputId": "28250d7f-0e8e-4143-f4ed-c0dc47ee82e8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        }
      },
      "source": [
        "# Some random sample \n",
        "ind = np.random.randint(low=0,high=180)\n",
        "text, categories = classifier_learner.data.valid_ds[ind]\n",
        "print(\"Input :\",text)\n",
        "print(\"Categories :\",categories)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input : xxbos real time simulation model of sensors with xxunk am trying to make a real time simulink model of leddar sensor xxunk sensor xxunk kit in matlab using usb port but i can t use data xxunk toolbox and xxunk control toolbox because the leddar vendor isn t listed in these xxunk i would really appreciate if someone could help me with this because i am new to real time simulink \n",
            " \n",
            "Categories : sensors matlab simulator real-time\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BXcmdD9eHecl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Pass it to trained model\n",
        "\n",
        "p = classifier_learner.predict(text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WMc1xaOeHnC9",
        "colab_type": "code",
        "outputId": "bc183c83-3c84-4ac3-bc65-d3648c24f18d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "p[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Category sensors matlab simulator real-time"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uz1ThwjZJ833",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dEZFGwt5HM0Y",
        "colab_type": "code",
        "outputId": "e1733819-7d82-4327-b5a7-aff5bf1c73d8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 186
        }
      },
      "source": [
        "import matplotlib.cm as cm\n",
        "\n",
        "txt_ci = TextClassificationInterpretation.from_learner(classifier_learner)\n",
        "test_text = text\n",
        "txt_ci.show_intrinsic_attention(test_text,cmap=cm.Purples)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/pytorch/aten/src/ATen/native/cuda/LegacyDefinitions.cpp:19: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead.\n",
            "/pytorch/aten/src/ATen/native/cuda/LegacyDefinitions.cpp:19: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead.\n",
            "/pytorch/aten/src/ATen/native/cuda/LegacyDefinitions.cpp:19: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead.\n",
            "/pytorch/aten/src/ATen/native/cuda/LegacyDefinitions.cpp:19: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"font-family: monospace;\"><span title=\"0.104\" style=\"background-color: rgba(241, 239, 246, 0.5);\">xxbos</span> <span title=\"0.180\" style=\"background-color: rgba(229, 228, 240, 0.5);\">xxbos</span> <span title=\"0.352\" style=\"background-color: rgba(193, 194, 222, 0.5);\">real</span> <span title=\"0.280\" style=\"background-color: rgba(211, 211, 231, 0.5);\">time</span> <span title=\"0.589\" style=\"background-color: rgba(136, 133, 190, 0.5);\">simulation</span> <span title=\"0.229\" style=\"background-color: rgba(221, 221, 236, 0.5);\">model</span> <span title=\"0.089\" style=\"background-color: rgba(243, 241, 247, 0.5);\">of</span> <span title=\"0.609\" style=\"background-color: rgba(132, 128, 187, 0.5);\">sensors</span> <span title=\"0.072\" style=\"background-color: rgba(244, 243, 248, 0.5);\">with</span> <span title=\"0.096\" style=\"background-color: rgba(242, 240, 246, 0.5);\">xxunk</span> <span title=\"0.169\" style=\"background-color: rgba(231, 230, 241, 0.5);\">am</span> <span title=\"0.185\" style=\"background-color: rgba(229, 227, 240, 0.5);\">trying</span> <span title=\"0.065\" style=\"background-color: rgba(245, 243, 248, 0.5);\">to</span> <span title=\"0.245\" style=\"background-color: rgba(219, 219, 235, 0.5);\">make</span> <span title=\"0.112\" style=\"background-color: rgba(240, 238, 245, 0.5);\">a</span> <span title=\"0.428\" style=\"background-color: rgba(175, 174, 211, 0.5);\">real</span> <span title=\"0.228\" style=\"background-color: rgba(221, 221, 236, 0.5);\">time</span> <span title=\"0.541\" style=\"background-color: rgba(148, 144, 195, 0.5);\">simulink</span> <span title=\"0.198\" style=\"background-color: rgba(227, 226, 239, 0.5);\">model</span> <span title=\"0.096\" style=\"background-color: rgba(242, 240, 246, 0.5);\">of</span> <span title=\"0.966\" style=\"background-color: rgba(68, 9, 129, 0.5);\">leddar</span> <span title=\"0.269\" style=\"background-color: rgba(214, 214, 233, 0.5);\">sensor</span> <span title=\"0.050\" style=\"background-color: rgba(247, 245, 249, 0.5);\">xxunk</span> <span title=\"0.177\" style=\"background-color: rgba(230, 229, 240, 0.5);\">sensor</span> <span title=\"0.042\" style=\"background-color: rgba(247, 246, 250, 0.5);\">xxunk</span> <span title=\"0.144\" style=\"background-color: rgba(236, 234, 243, 0.5);\">kit</span> <span title=\"0.056\" style=\"background-color: rgba(246, 244, 249, 0.5);\">in</span> <span title=\"0.387\" style=\"background-color: rgba(184, 185, 217, 0.5);\">matlab</span> <span title=\"0.210\" style=\"background-color: rgba(225, 224, 238, 0.5);\">using</span> <span title=\"1.000\" style=\"background-color: rgba(63, 0, 125, 0.5);\">usb</span> <span title=\"0.100\" style=\"background-color: rgba(241, 240, 246, 0.5);\">port</span> <span title=\"0.055\" style=\"background-color: rgba(246, 244, 249, 0.5);\">but</span> <span title=\"0.051\" style=\"background-color: rgba(247, 245, 249, 0.5);\">i</span> <span title=\"0.111\" style=\"background-color: rgba(240, 238, 245, 0.5);\">can</span> <span title=\"0.084\" style=\"background-color: rgba(243, 241, 247, 0.5);\">t</span> <span title=\"0.081\" style=\"background-color: rgba(243, 242, 247, 0.5);\">use</span> <span title=\"0.170\" style=\"background-color: rgba(231, 230, 241, 0.5);\">data</span> <span title=\"0.075\" style=\"background-color: rgba(244, 242, 248, 0.5);\">xxunk</span> <span title=\"0.290\" style=\"background-color: rgba(208, 208, 230, 0.5);\">toolbox</span> <span title=\"0.033\" style=\"background-color: rgba(248, 247, 250, 0.5);\">and</span> <span title=\"0.059\" style=\"background-color: rgba(245, 244, 249, 0.5);\">xxunk</span> <span title=\"0.152\" style=\"background-color: rgba(234, 233, 243, 0.5);\">control</span> <span title=\"0.292\" style=\"background-color: rgba(208, 208, 230, 0.5);\">toolbox</span> <span title=\"0.102\" style=\"background-color: rgba(241, 239, 246, 0.5);\">because</span> <span title=\"0.085\" style=\"background-color: rgba(243, 241, 247, 0.5);\">the</span> <span title=\"0.992\" style=\"background-color: rgba(63, 1, 125, 0.5);\">leddar</span> <span title=\"0.301\" style=\"background-color: rgba(205, 205, 228, 0.5);\">vendor</span> <span title=\"0.120\" style=\"background-color: rgba(239, 237, 245, 0.5);\">isn</span> <span title=\"0.057\" style=\"background-color: rgba(246, 244, 249, 0.5);\">t</span> <span title=\"0.158\" style=\"background-color: rgba(233, 232, 242, 0.5);\">listed</span> <span title=\"0.051\" style=\"background-color: rgba(246, 245, 249, 0.5);\">in</span> <span title=\"0.103\" style=\"background-color: rgba(241, 239, 246, 0.5);\">these</span> <span title=\"0.045\" style=\"background-color: rgba(247, 246, 250, 0.5);\">xxunk</span> <span title=\"0.059\" style=\"background-color: rgba(245, 244, 249, 0.5);\">i</span> <span title=\"0.102\" style=\"background-color: rgba(241, 239, 246, 0.5);\">would</span> <span title=\"0.136\" style=\"background-color: rgba(237, 235, 244, 0.5);\">really</span> <span title=\"0.193\" style=\"background-color: rgba(227, 226, 239, 0.5);\">appreciate</span> <span title=\"0.086\" style=\"background-color: rgba(243, 241, 247, 0.5);\">if</span> <span title=\"0.165\" style=\"background-color: rgba(232, 230, 241, 0.5);\">someone</span> <span title=\"0.095\" style=\"background-color: rgba(242, 240, 246, 0.5);\">could</span> <span title=\"0.135\" style=\"background-color: rgba(237, 235, 244, 0.5);\">help</span> <span title=\"0.103\" style=\"background-color: rgba(241, 239, 246, 0.5);\">me</span> <span title=\"0.054\" style=\"background-color: rgba(246, 245, 249, 0.5);\">with</span> <span title=\"0.062\" style=\"background-color: rgba(245, 244, 249, 0.5);\">this</span> <span title=\"0.125\" style=\"background-color: rgba(238, 236, 244, 0.5);\">because</span> <span title=\"0.099\" style=\"background-color: rgba(241, 240, 246, 0.5);\">i</span> <span title=\"0.294\" style=\"background-color: rgba(207, 207, 229, 0.5);\">am</span> <span title=\"0.402\" style=\"background-color: rgba(182, 182, 216, 0.5);\">new</span> <span title=\"0.212\" style=\"background-color: rgba(224, 223, 238, 0.5);\">to</span> <span title=\"0.800\" style=\"background-color: rgba(97, 64, 155, 0.5);\">real</span> <span title=\"0.504\" style=\"background-color: rgba(157, 153, 199, 0.5);\">time</span> <span title=\"0.867\" style=\"background-color: rgba(85, 41, 144, 0.5);\">simulink</span></span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U-3tGHayJ8z9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q-KpvkOHT_Od",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "16Yx-UbBDwA8",
        "colab_type": "text"
      },
      "source": [
        "##### rough On Training set:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wRTibarGFdEY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "preds, _ = classifier_learner.get_preds(ds_type=DatasetType.Train)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ybTNk577Fc9o",
        "colab_type": "code",
        "outputId": "86423801-2cfb-43d7-dcaa-173218b83e9e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# np.argmax(preds, axis=0)\n",
        "preds"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.0013, 0.0012, 0.0019,  ..., 0.0004, 0.0012, 0.0011],\n",
              "        [0.0014, 0.0008, 0.0021,  ..., 0.0013, 0.0004, 0.0011],\n",
              "        [0.0027, 0.0020, 0.0011,  ..., 0.0008, 0.0003, 0.0046],\n",
              "        ...,\n",
              "        [0.0008, 0.0006, 0.0015,  ..., 0.0012, 0.0019, 0.0010],\n",
              "        [0.0007, 0.0008, 0.0009,  ..., 0.0004, 0.0009, 0.0004],\n",
              "        [0.0015, 0.0006, 0.0014,  ..., 0.0004, 0.0006, 0.0013]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "88OeJa_NOqkr",
        "colab_type": "code",
        "outputId": "33f5c20f-5a43-4fa8-e6c0-17d713ce7d0d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# np.argmax(preds, axis=1)\n",
        "preds.max()\n",
        "\n",
        "np.average(preds.numpy())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.0011862396"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jAyZwC59ncKU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "thresh = (0.02)\n",
        "labelled_preds = [' '.join([classifier_learner.data.classes[i] for i,p in enumerate(pred) if p > thresh]) for pred in preds]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZIGDlKbWJlLy",
        "colab_type": "code",
        "outputId": "c6df5775-d51a-4c71-a01a-9c99a1acdc79",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "classifier_learner.data.train_ds"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LabelList (900 items)\n",
              "x: TextList\n",
              "xxbos getting a tourist schengen visa after working xxunk s my situation i am currently in france on an internship visa different than a business and tourist visa which ends next month i m interested in doing some traveling afterwards ideally on a 90 day schengen tourist visa i m willing to hop over to a non schengen country like the uk for a couple days and then come back to get the tourist visa if necessary \n",
              "  this post seems similar to my question except the xxunk is trying to get the tourist visa without leaving the schengen area \n",
              "  is this allowed will the border xxunk still give me the tourist visa after my internship visa expires will it work through the uk or should i go through another non schengen country i have a usa passport \n",
              " ,xxbos hashing entropy with only 128 bits of internal xxunk am looking to implement the xxunk algorithm on a system with very xxunk memory constraints using aes 128 as the underlying cipher \n",
              "  as the key size is only 128 bits using a 256 bit entropy pool hashing algorithm to maintain each pool would seem to be excessive and a waste of memory i would like to use a hashing algorithm for which i only need to maintain 128 bits of state information for each pool given this i have a couple of questions \n",
              "  would it be possible to use something like sha 256 but only save and restore 128 bits of the internal state between xxunk would this somehow risk xxunk entropy in the process or produce other undesirable effects \n",
              "  alternately the crypto library i m using also supports md5 which does only require 128 bits of internal state would it be better to use md5 for this application despite its potential weaknesses compared to sha 2 hashes or would sha 256 be stronger even when maintaining only half its internal state between calls \n",
              " ,xxbos can i repair a shower without pulling everything out we had a new shower put in by a contractor who was in over his head during a xxunk a few years ago about a year ago the built in bench starting to collapse when we pulled out the bench we noticed that there was no shower pan under it xxunk we need to add one \n",
              "  here s my question can we simply fill it in instead of starting over every contractor that has looked at it says that they have to pull out the existing floor which includes the tile that i ll have a hard time replacing i don t mind having to replace wall tile i just want to avoid pulling out the floor \n",
              "  i m convinced that there has to be a way even if it s xxunk \n",
              " \n",
              " ,xxbos can i bake pine nuts i was thinking of adding pine nuts to a xxunk filling but wasn t too sure if the heat would make the nuts release anything nasty \n",
              " ,xxbos how to clone a gene xxunk in two direction as sense and antisense in xxunk have a sequence xxunk xxunk xxunk xxunk xxunk xxunk and want to use it as antisense rna in my vector how should i clone it i am confused with the direction of my clone i looking for the direction and sequence of the clones \n",
              " \n",
              "y: CategoryList\n",
              "visas schengen,entropy sha-256 md5 fortuna,repair shower bench,baking nuts,molecular-biology rna-sequencing\n",
              "Path: /content/ULMFit"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eu6juDC3h--T",
        "colab_type": "code",
        "outputId": "276fda34-83fc-4a16-97cf-34c5b9bd882d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "labelled_preds"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['',\n",
              " 'windows installation fitting',\n",
              " 'slam',\n",
              " 'uk legal hotels fees-and-charges',\n",
              " 'hash zero-knowledge-proofs',\n",
              " 'public-transport airport-transfer slovenia ljubljana',\n",
              " 'yeast sourdough-starter',\n",
              " '',\n",
              " 'electrical code-compliance generator',\n",
              " '',\n",
              " 'plaster anchor',\n",
              " '',\n",
              " 'freezing microwave convenience-foods',\n",
              " '',\n",
              " '',\n",
              " 'electrical',\n",
              " 'proteins mrna',\n",
              " 'brushless-motor encoding hall-sensor',\n",
              " '',\n",
              " '',\n",
              " 'budget europe interrail',\n",
              " 'mac poly1305 modular-arithmetic',\n",
              " 'refrigerator',\n",
              " '',\n",
              " 'insulation ventilation',\n",
              " 'sha-256 brute-force-attack rainbow-table',\n",
              " 'budget netherlands amsterdam belgium brussels',\n",
              " '',\n",
              " 'mechanism',\n",
              " 'sensors accelerometer gps',\n",
              " '',\n",
              " 'international-travel money bookings payment-cards',\n",
              " 'repair heating ceiling radiant-heating',\n",
              " 'wood furniture wooden-furniture sanding',\n",
              " 'chicken salt slow-cooking poaching',\n",
              " '',\n",
              " 'cleaning vent dryer ducts',\n",
              " 'studs chimney lath-and-plaster',\n",
              " 'hash implementation stream-cipher encryption salsa20',\n",
              " 'plumbing water',\n",
              " 'health us-citizens insurance',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'aes initialization-vector file-encryption ctr',\n",
              " 'storage-method asian-cuisine stir-fry',\n",
              " 'finishing vanity',\n",
              " 'pathology mycology infection',\n",
              " 'stepper-motor rcservo',\n",
              " 'japan tours',\n",
              " '',\n",
              " '',\n",
              " 'doors carpentry',\n",
              " 'roof waterproofing metal-roof',\n",
              " 'random-number-generator randomness terminology nist',\n",
              " 'electrical insulation basement',\n",
              " 'electrical electrical thermostat thermostat-c-wire',\n",
              " '',\n",
              " 'hvac humidity',\n",
              " 'concrete sealing countertops',\n",
              " 'health tips-and-tricks gadgets',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'resources',\n",
              " '',\n",
              " 'trains japan kyushu jr-pass',\n",
              " 'radiator',\n",
              " '',\n",
              " '',\n",
              " 'steak pan',\n",
              " 'plumbing',\n",
              " 'visas us-citizens singapore',\n",
              " '',\n",
              " 'encryption random-number-generator',\n",
              " 'raw cocoa',\n",
              " 'airport-transfer singapore russian-citizens',\n",
              " 'linear-cryptanalysis',\n",
              " 'evolution genetics dna genomics molecular-genetics plasmids chromosome',\n",
              " 'visas usa b1-b2-visas visa-refusal',\n",
              " '',\n",
              " 'landmarks estonia tallinn',\n",
              " 'hotels toiletries',\n",
              " '',\n",
              " 'electrical',\n",
              " 'pool',\n",
              " 'ceiling mounting',\n",
              " '',\n",
              " 'equipment',\n",
              " 'repair bathroom drywall ceiling water-damage',\n",
              " '',\n",
              " 'leak bathtub',\n",
              " 'encryption',\n",
              " 'homework embryology',\n",
              " '',\n",
              " 'baking please-remove-this-tag cake',\n",
              " '',\n",
              " 'roofing gutters contractors',\n",
              " 'pudding low-fat',\n",
              " 'infection epidemiology antibiotics ant palaeontology',\n",
              " 'baseboard',\n",
              " 'baking',\n",
              " 'hematology',\n",
              " 'baking high-altitude',\n",
              " 'electrical walls receptacle',\n",
              " 'attic venting',\n",
              " 'cleaning utensils chili-peppers',\n",
              " 'human-biology microbiology genomics',\n",
              " '',\n",
              " 'electrical electrical wiring low-voltage ethernet',\n",
              " '',\n",
              " 'human-biology neuroscience',\n",
              " 'accelerometer gyroscope frame',\n",
              " '',\n",
              " '',\n",
              " 'plumbing copper-tubing',\n",
              " 'neuroscience neurophysiology',\n",
              " 'drywall screws',\n",
              " 'electrical',\n",
              " 'accommodation airports london lhr',\n",
              " 'electrical',\n",
              " 'air-travel customs-and-immigration passports airlines dual-nationality',\n",
              " 'evolution',\n",
              " 'molecular-biology rna-sequencing',\n",
              " 'water',\n",
              " 'aes symmetric collision-resistance',\n",
              " 'international-travel netherlands',\n",
              " '',\n",
              " '',\n",
              " 'yeast',\n",
              " 'slam kalman-filter',\n",
              " 'public-key',\n",
              " 'heat glass',\n",
              " 'amsterdam taxis uber',\n",
              " '',\n",
              " 'sauce pasta italian-cuisine mascarpone',\n",
              " '',\n",
              " 'collision-resistance merkle-damgaard encryption',\n",
              " 'electrical repair tools',\n",
              " 'aes known-plaintext-attack biochemistry',\n",
              " 'storage-method refrigerator mexican-cuisine salsa',\n",
              " '',\n",
              " 'tickets australia regulations greyhound',\n",
              " 'walls shelving storage',\n",
              " 'biochemistry organic-chemistry chemistry bioinorganic-chemistry',\n",
              " 'baking',\n",
              " '240v',\n",
              " 'passports stateless-persons',\n",
              " '',\n",
              " 'ceiling',\n",
              " 'mobile-robot design stability wheeled-robot',\n",
              " 'electrical remodeling',\n",
              " '',\n",
              " 'heating pest-control pest',\n",
              " 'plumbing exterior',\n",
              " 'cell-biology mycoplasma',\n",
              " 'additives',\n",
              " 'human-anatomy morphology',\n",
              " 'substitutions slow-cooking beans',\n",
              " '',\n",
              " 'visas uk europe automobiles eurostar',\n",
              " 'public-key diffie-hellman key-exchange key-reuse',\n",
              " '',\n",
              " 'taxis malaysia',\n",
              " 'chicken frozen chicken-breast pressure-cooker',\n",
              " 'air-travel transit',\n",
              " 'public-key tls',\n",
              " 'electrical',\n",
              " 'electrical repair concrete walls brick structural',\n",
              " '',\n",
              " 'evolution zoology physiology ecology ecophysiology',\n",
              " '',\n",
              " 'windows',\n",
              " 'electrical wiring attic',\n",
              " 'remodeling software drafting',\n",
              " 'sauce lemon mistakes hollandaise',\n",
              " 'nutrient-composition soymilk',\n",
              " 'aes authenticated-encryption modes-of-operation gcm ocb',\n",
              " 'water pipe',\n",
              " 'mexican-cuisine',\n",
              " '',\n",
              " 'lighting',\n",
              " '',\n",
              " 'tile',\n",
              " 'encryption',\n",
              " 'food-safety cooking-time',\n",
              " '',\n",
              " 'plumbing usa car-rentals traffic delaware',\n",
              " 'electrical electrical wiring code-compliance dimmer-switch',\n",
              " 'cryptanalysis aes block-cipher serpent twofish',\n",
              " 'authentication discrete-logarithm',\n",
              " 'sensors sensor-fusion ultrasonic-sensors',\n",
              " 'tile',\n",
              " 'quadcopter control pid',\n",
              " 'legal regulations us-citizens cuba',\n",
              " '',\n",
              " 'bread yeast gluten-free rising',\n",
              " 'repair shower bench',\n",
              " 'biochemistry molecular-biology cell-biology bioenergetics',\n",
              " 'usa buses esta us-visa-waiver-program land-borders',\n",
              " 'roof leak',\n",
              " 'baking substitutions cake brown-sugar',\n",
              " 'evolution',\n",
              " 'senses sensory-systems',\n",
              " 'chacha pseudo-random-generator',\n",
              " 'dough pasta',\n",
              " 'transportation tokyo walking locate-this nrt',\n",
              " 'tools wood',\n",
              " 'electrical electrical wiring switch',\n",
              " 'garlic chemistry food-safety botulism',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'equipment',\n",
              " '',\n",
              " 'des avalanche',\n",
              " 'zoology sleep circadian-rhythms melatonin',\n",
              " 'insurance',\n",
              " '',\n",
              " 'air-travel health hygiene',\n",
              " 'japan photography kyoto',\n",
              " '',\n",
              " 'kinematics frame',\n",
              " 'mac',\n",
              " 'public-key diffie-hellman key-exchange safe-prime',\n",
              " 'plumbing water hose',\n",
              " '',\n",
              " 'chicken meat cooking-time pressure-cooker',\n",
              " '',\n",
              " 'electrical electrical wiring circuit',\n",
              " '',\n",
              " 'zoology species-identification',\n",
              " 'rsa elliptic-curves post-quantum-cryptography ed25519',\n",
              " 'group-theory commitments key-generation',\n",
              " 'plumbing water-heater',\n",
              " 'electrical',\n",
              " '',\n",
              " 'molecular-biology dna homework cancer',\n",
              " 'tickets amsterdam brussels klm thalys',\n",
              " 'bathroom flooring toilet subfloor',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'electrical code-compliance',\n",
              " 'bathroom shower',\n",
              " 'medicine',\n",
              " '',\n",
              " 'theoretical-biology sexual-selection sex-chromosome',\n",
              " 'electrical switch motion-sensor',\n",
              " '',\n",
              " 'powertools angle-grinder',\n",
              " '',\n",
              " 'molecular-biology cell-biology ecoli fluorescent-microscopy',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'ceiling noise-reduction',\n",
              " '',\n",
              " 'visas uk us-citizens',\n",
              " 'product-recommendation',\n",
              " 'internet new-zealand auckland',\n",
              " 'industrial-robot',\n",
              " '',\n",
              " 'plumbing',\n",
              " '',\n",
              " 'storage-method vegetables culinary-uses squash',\n",
              " '',\n",
              " 'budget canada accommodation calgary',\n",
              " 'visas georgia-country invitation-letter',\n",
              " 'generator transfer-switch',\n",
              " 'salt peanuts',\n",
              " '',\n",
              " 'corn hominy',\n",
              " '',\n",
              " 'molecular-biology cell-biology ecoli fluorescent-microscopy',\n",
              " 'exterior',\n",
              " 'dough pasta drying',\n",
              " 'australia lakes queensland',\n",
              " 'brick screws anchor',\n",
              " 'transit-visas',\n",
              " 'bread freezing breakfast',\n",
              " 'electrical electrical wiring',\n",
              " 'bioinformatics genomes flow-cytometry',\n",
              " '',\n",
              " 'visas transit hong-kong',\n",
              " 'electrical refrigerator',\n",
              " 'food-safety raw-meat',\n",
              " 'visas uk',\n",
              " 'motor',\n",
              " '',\n",
              " '',\n",
              " 'vietnam luggage-storage train-stations hanoi',\n",
              " 'arduino equipment cleaning oil fryer',\n",
              " 'bell-peppers pressure-canner',\n",
              " 'food-safety food-safety bread proofing',\n",
              " 'hmac key-rotation',\n",
              " '',\n",
              " 'hungarian-cuisine russian-cuisine',\n",
              " '',\n",
              " 'zoology',\n",
              " 'hash',\n",
              " 'floor joists treehouse',\n",
              " 'electrical electrical hvac',\n",
              " '',\n",
              " 'air-travel health hygiene wood woodworking rain',\n",
              " 'brain',\n",
              " '',\n",
              " '',\n",
              " 'money',\n",
              " '',\n",
              " 'transit',\n",
              " 'zoology sleep chickens',\n",
              " '',\n",
              " '',\n",
              " 'biochemistry synthetic-biology abiogenesis visas schengen cyprus saudi-citizens',\n",
              " 'vegetables canning',\n",
              " '',\n",
              " 'electrical',\n",
              " '',\n",
              " 'symmetric keys tls',\n",
              " '',\n",
              " '',\n",
              " 'medicinal-chemistry injury washing-machine',\n",
              " 'ceiling doors carpentry',\n",
              " '',\n",
              " 'human-biology biochemistry physiology hematology',\n",
              " 'walls removal texture',\n",
              " '',\n",
              " '',\n",
              " 'hematology',\n",
              " 'transportation indonesia night-transport komodo lombok',\n",
              " '',\n",
              " 'loyalty-programs star-alliance',\n",
              " 'pressure-cooker',\n",
              " 'drywall kitchens',\n",
              " 'france car-rentals payment-cards driving-licenses lost-documents omelette',\n",
              " 'motor',\n",
              " 'equipment japanese-cuisine',\n",
              " '',\n",
              " 'microbiology',\n",
              " 'electrical outlets',\n",
              " 'genetics terminology',\n",
              " 'fruit drying blueberries',\n",
              " 'evolution',\n",
              " '',\n",
              " 'eggs freezing food-preservation',\n",
              " '',\n",
              " 'electrical sump-pump',\n",
              " '',\n",
              " 'cruising egypt',\n",
              " 'infection epidemiology antibiotics ant zoology',\n",
              " '',\n",
              " 'frame play-structures',\n",
              " '',\n",
              " '',\n",
              " 'population-dynamics ecology theoretical-biology',\n",
              " 'cancer apoptosis',\n",
              " 'electrical electrical receptacle aluminum-wiring',\n",
              " '',\n",
              " 'biochemistry',\n",
              " 'control can',\n",
              " 'legal consulates',\n",
              " '',\n",
              " 'electrical',\n",
              " 'lawn grass',\n",
              " 'food-safety eggs',\n",
              " 'schengen eu',\n",
              " 'food-science coffee tea drinks chemistry',\n",
              " 'encryption public-key',\n",
              " '',\n",
              " 'infection epidemiology antibiotics ant',\n",
              " 'arduino control localization',\n",
              " 'air-travel budget uk africa malawi france tours seasonal surfing',\n",
              " 'repair concrete',\n",
              " 'keys symmetric file-encryption',\n",
              " '',\n",
              " 'resources cookbook',\n",
              " 'plant-physiology',\n",
              " 'electrical led',\n",
              " '',\n",
              " 'electrical',\n",
              " 'hash post-quantum-cryptography',\n",
              " '',\n",
              " 'substitutions yogurt',\n",
              " 'mutations extinction inheritance',\n",
              " 'gfci',\n",
              " 'utensils',\n",
              " 'stream-cipher lightweight',\n",
              " 'molecular-genetics genomics embryology',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'physiology blood-circulation tissue endothelium',\n",
              " 'language-barrier panama spanish-language panama-city',\n",
              " 'outdoor play-structures',\n",
              " 'electrical transfer-switch',\n",
              " '',\n",
              " 'rice packaging',\n",
              " 'basement',\n",
              " 'doors garage garage-door-opener',\n",
              " 'food-safety lemon-juice',\n",
              " 'meat salt seasoning',\n",
              " 'driving automobiles sleeping oregon crater-lake',\n",
              " '',\n",
              " '',\n",
              " 'electrical gas fireplace',\n",
              " 'air-travel',\n",
              " 'substitutions cookies',\n",
              " '',\n",
              " 'baking substitutions sugar',\n",
              " 'food-safety meat freezing',\n",
              " '',\n",
              " 'air-travel luggage bicycles',\n",
              " '',\n",
              " 'hvac ducts',\n",
              " 'electrical electrical ceiling-fan',\n",
              " '',\n",
              " 'rsa public-key',\n",
              " 'botany plant-physiology',\n",
              " 'electrical',\n",
              " 'usa transportation',\n",
              " '',\n",
              " '',\n",
              " 'dessert cream whipped-cream',\n",
              " 'steak marinade lemonade',\n",
              " 'botany taxonomy phylogenetics ethnobiology',\n",
              " '',\n",
              " 'tools battery',\n",
              " 'genetics reproductive-biology species-distribution spices curry',\n",
              " 'visas india business-travel',\n",
              " 'legal driving hitchhiking travel-companions',\n",
              " 'frying fish salmon',\n",
              " 'plumbing',\n",
              " 'food-safety bread proofing',\n",
              " 'baking',\n",
              " 'wood custom-cabinetry',\n",
              " '',\n",
              " 'eggs',\n",
              " 'freezing yeast',\n",
              " 'flooring laminate-floor',\n",
              " '',\n",
              " '',\n",
              " 'europe where-on-earth countries architecture',\n",
              " 'electrical',\n",
              " '',\n",
              " 'foundation waterproofing exterior sealing',\n",
              " 'homework reproduction embryology',\n",
              " '',\n",
              " '',\n",
              " 'electrical',\n",
              " '',\n",
              " 'authentication',\n",
              " 'molecular-biology cell-biology dna molecular-genetics cloning',\n",
              " 'layovers sightseeing day-trips vienna',\n",
              " 'luggage canada airport-transfer vancouver yvr',\n",
              " 'south-africa itineraries namibia swaziland',\n",
              " 'visas transit us-citizens south-africa overstaying',\n",
              " '',\n",
              " 'roasting lamb',\n",
              " '',\n",
              " 'measurements salmon',\n",
              " '',\n",
              " 'shed pest-control',\n",
              " 'electrical led',\n",
              " 'air-travel airports',\n",
              " 'cheese',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 's-boxes linear-cryptanalysis',\n",
              " '',\n",
              " 'roof',\n",
              " '',\n",
              " 'vegetables roasting',\n",
              " 'microbiology uk accommodation rationale zoology sleep chickens',\n",
              " 'freezing seafood tenderizing',\n",
              " 'barbecue smoking',\n",
              " 'human-biology biochemistry medicinal-chemistry',\n",
              " 'walls basement framing',\n",
              " 'germany routes fra',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'sensors accelerometer gps studs',\n",
              " 'electrical furnace water-heater code-compliance',\n",
              " 'windows air-conditioning',\n",
              " '',\n",
              " 'sequence-alignment',\n",
              " 'drain pvc',\n",
              " 'rsa passwords ssh',\n",
              " 'motor electronics power',\n",
              " '',\n",
              " 'electrical electrical ceiling-fan hvac generator',\n",
              " '',\n",
              " 'uk legal driving car-rentals',\n",
              " 'heating steam',\n",
              " 'quadcopter radio-control research',\n",
              " '',\n",
              " 'low-voltage',\n",
              " '',\n",
              " 'socket',\n",
              " '',\n",
              " '',\n",
              " 'wiring',\n",
              " 'rice-cooker',\n",
              " 'washing-machine',\n",
              " 'food-safety rice pressure-cooker kitchen-safety burnt',\n",
              " '',\n",
              " '',\n",
              " 'storage-method utensils',\n",
              " 'electrical',\n",
              " 'electrical',\n",
              " 'heating',\n",
              " 'usa airports airport-security duty-free',\n",
              " 'meat grilling',\n",
              " 'hvac air-conditioning central-air',\n",
              " 'canada photography vancouver vantage-points',\n",
              " 'storage-lifetime refrigerator',\n",
              " '',\n",
              " 'cryptanalysis aes block-cipher',\n",
              " '',\n",
              " '',\n",
              " 'electrical safety',\n",
              " 'implementation xor finite-field',\n",
              " '',\n",
              " 'diet allergies eggs hypersensitivity',\n",
              " 'france france-schengen-visas',\n",
              " '',\n",
              " 'secret-sharing',\n",
              " 'conversion recipe-scaling measuring-scales',\n",
              " 'aes distinguisher',\n",
              " 'toilet septic-tanks',\n",
              " 'trains thailand',\n",
              " 'trains romania passport-stamps serbia land-borders',\n",
              " 'blood-group',\n",
              " 'budget accommodation kuwait',\n",
              " 'hvac thermostat-c-wire',\n",
              " 'pie goat',\n",
              " 'plumbing',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'electrical',\n",
              " 'baking cookies biscuits',\n",
              " 'air-travel health electronic-items geek-travel',\n",
              " 'wood furniture',\n",
              " '',\n",
              " '',\n",
              " 'visas schengen germany long-stay-visas',\n",
              " 'plumbing drain drain-waste-vent',\n",
              " 'equipment cleaning oil fryer',\n",
              " '',\n",
              " 'painting paint texture',\n",
              " 'air-travel transit airlines missed-flights',\n",
              " '',\n",
              " 'hash hmac',\n",
              " '',\n",
              " 'public-key file-encryption broadcast-encryption',\n",
              " 'electrical electrical insulation attic junction-box',\n",
              " 'floor plywood',\n",
              " 'electrical bathroom',\n",
              " '',\n",
              " 'terminology',\n",
              " 'tea',\n",
              " 'aes algorithm-design matrix-multiplication',\n",
              " '',\n",
              " 'beans sprouting',\n",
              " 'microbiology replication',\n",
              " 'hvac attic exhaust-vent',\n",
              " '',\n",
              " 'bread dough rising',\n",
              " 'encryption',\n",
              " 'food-and-drink tips-and-tricks restaurants',\n",
              " '',\n",
              " '',\n",
              " 'europe driving italy taxis tuscany',\n",
              " '',\n",
              " 'electrical electrical wiring switch circuit',\n",
              " 'genetics reproductive-biology species-distribution',\n",
              " '',\n",
              " 'food-safety meat raw-meat packaging',\n",
              " 'dna gene-therapy',\n",
              " 'mobile-robot control robotic-arm motion-planning',\n",
              " 'repair siding',\n",
              " '',\n",
              " 'biochemistry zoology',\n",
              " 'usa india money',\n",
              " 'neuroscience antibody',\n",
              " 'romania locate-this nostalgia',\n",
              " 'genetics homework cancer mutations',\n",
              " '',\n",
              " 'airports ferries hong-kong macau',\n",
              " 'tipping czech-republic',\n",
              " 'cancer ichthyology',\n",
              " 'windows paint oil-based-paint',\n",
              " 'air-travel food-and-drink factoids alcohol',\n",
              " '',\n",
              " 'mounting',\n",
              " '',\n",
              " 'substitutions grapes',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'entomology development',\n",
              " '',\n",
              " '',\n",
              " 'electrical',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'encryption keys diffie-hellman key-exchange',\n",
              " '',\n",
              " 'electrical framing structural construction load-bearing engineering',\n",
              " 'language-barrier languages',\n",
              " 'public-transport transportation airport-transfer belgium crl',\n",
              " 'human-biology neuroscience physiology pain injury',\n",
              " 'insulation basement',\n",
              " '',\n",
              " 'netherlands import-tax',\n",
              " '',\n",
              " 'factoids cities',\n",
              " 'repair stucco',\n",
              " 'france car-rentals payment-cards driving-licenses lost-documents',\n",
              " 'luggage italy rome',\n",
              " 'safety event-based-effects tunisia',\n",
              " 'lighting',\n",
              " 'airports airport-transfer airport-security bahamas nas',\n",
              " '',\n",
              " 'greece denmark copenhagen',\n",
              " 'knives knife-skills chinese-cuisine',\n",
              " '',\n",
              " 'money payment-cards sweden eurozone',\n",
              " '',\n",
              " 'human-biology psychology',\n",
              " 'flavor herbs italian-cuisine seasoning',\n",
              " '',\n",
              " '',\n",
              " 'air-travel usa europe germany standby',\n",
              " '',\n",
              " 'london driving spain tolls',\n",
              " 'customs-and-immigration airport-security singapore sin',\n",
              " 'water filter',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'italy weather-and-climate rome',\n",
              " 'human-biology digestive-system food',\n",
              " 'mobile-robot raspberry-pi',\n",
              " 'air-travel bookings',\n",
              " 'customs-and-immigration uk-citizens singapore nepali-citizens',\n",
              " 'repair dryer',\n",
              " 'quadcopter microcontroller radio-control',\n",
              " 'meat steak grade',\n",
              " '',\n",
              " '',\n",
              " 'trains public-transport tickets netherlands amsterdam',\n",
              " 'visas tickets airlines emirates disputes',\n",
              " '',\n",
              " 'plumbing',\n",
              " 'india local-customs hygiene',\n",
              " 'aes known-plaintext-attack',\n",
              " '',\n",
              " 'visas schengen germany canadian-citizens one-way',\n",
              " 'encryption cryptanalysis classical-cipher vigenere',\n",
              " 'spices beans',\n",
              " 'visas schengen international-travel long-stay-visas',\n",
              " 'fish vegetarian',\n",
              " '',\n",
              " '',\n",
              " 'beef marinade roast-beef pineapple',\n",
              " '',\n",
              " 'norway mail',\n",
              " 'protocol competent-cells',\n",
              " '',\n",
              " '',\n",
              " 'genetics microbiology bacteriology molecular-genetics molecular-biology cell-biology ecoli fluorescent-microscopy',\n",
              " 'electrical',\n",
              " '',\n",
              " 'closet',\n",
              " 'human-genetics embryology growth',\n",
              " 'fish',\n",
              " '',\n",
              " 'repair patio pavers',\n",
              " '',\n",
              " '',\n",
              " 'evolution genetics dna genomics',\n",
              " 'cake melting-chocolate sponge-cake',\n",
              " 'plumbing toilet',\n",
              " 'central-heating boiler',\n",
              " 'painting paint safety',\n",
              " 'storage-lifetime food-preservation',\n",
              " 'cheese goat',\n",
              " 'flavor bell-peppers',\n",
              " '',\n",
              " 'human-biology neuroscience psychology',\n",
              " '',\n",
              " 'cake chinese-cuisine',\n",
              " 'wiring',\n",
              " 'washing-machine',\n",
              " 'transit kul transit-visas',\n",
              " '',\n",
              " 'cellphones indonesia communication mobile-operators phone-networks',\n",
              " 'cheese yogurt cheesecake',\n",
              " 'biochemistry pharmacology digestive-system',\n",
              " 'cheese food-preservation',\n",
              " '',\n",
              " '',\n",
              " 'one-way-function bijection',\n",
              " 'diving water-sports',\n",
              " 'drywall interior finishing',\n",
              " '',\n",
              " 'malaysia locate-this',\n",
              " 'modular-arithmetic finite-field',\n",
              " 'cleaning pan',\n",
              " '',\n",
              " 'dough',\n",
              " 'water-filtration',\n",
              " 'cell-biology alcohol red-blood-cell',\n",
              " 'baking cake apples',\n",
              " 'rsa padding',\n",
              " '',\n",
              " '',\n",
              " 'exterior wiring conduit coax',\n",
              " 'salt cookware',\n",
              " 'insulation heating windows cooling',\n",
              " 'visas uk',\n",
              " 'storage-method cookies',\n",
              " 'human-biology physiology neuroscience',\n",
              " 'biochemistry cell-signaling',\n",
              " 'airport-transfer hotels paris taxis cdg',\n",
              " 'bread yeast gluten-free rising chicken soup yeast sourdough-starter',\n",
              " 'evolution',\n",
              " 'usa regulations road-trips',\n",
              " '',\n",
              " 'cycle pregnancy',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'drywall drill electrical',\n",
              " 'baking',\n",
              " 'cell-biology',\n",
              " 'baking nuts',\n",
              " '',\n",
              " 'garage',\n",
              " 'visas uk transit lax zimbabwe',\n",
              " 'vegetables peeling',\n",
              " '',\n",
              " 'china maps',\n",
              " 'fish cooking-time steaming',\n",
              " '',\n",
              " 'biochemistry synthetic-biology abiogenesis',\n",
              " 'dna telomere cytogenetics',\n",
              " 'equipment electric-stoves salt peanuts',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'homomorphic-encryption',\n",
              " '',\n",
              " 'equipment cream whipped-cream',\n",
              " 'rice-cooker lentils',\n",
              " 'italy milan',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'trains food-and-drink russia trans-siberian siberia',\n",
              " 'customs-and-immigration passports dual-nationality',\n",
              " '',\n",
              " 'fence hardware',\n",
              " 'electrical conduit cinderblock',\n",
              " 'plumbing copper',\n",
              " '',\n",
              " 'electrical wiring receptacle powertools',\n",
              " '',\n",
              " 'equipment grilling barbecue',\n",
              " 'biochemistry cell-membrane',\n",
              " 'desk',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'yeast rising',\n",
              " '',\n",
              " '',\n",
              " 'windows lock deadbolt',\n",
              " '',\n",
              " 'drywall installation cabinets',\n",
              " '',\n",
              " '',\n",
              " 'dryer exhaust-vent',\n",
              " 'air-travel health',\n",
              " 'plumbing roof drainage',\n",
              " '',\n",
              " 'exterior public-transport japan',\n",
              " '',\n",
              " 'usa washington-dc',\n",
              " 'electrical',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'ornithology',\n",
              " 'aes',\n",
              " '',\n",
              " 'human-biology statistics',\n",
              " '',\n",
              " '',\n",
              " 'electrical thermostat thermostat-c-wire',\n",
              " 'gene-expression biotechnology plasmids',\n",
              " 'baking turkey',\n",
              " '',\n",
              " '',\n",
              " 'stop-overs hawaii open-jaw',\n",
              " 'baking dough cake leak bathtub',\n",
              " 'custard',\n",
              " 'plumbing',\n",
              " 'drain pool pump',\n",
              " 'toilet',\n",
              " 'pancakes',\n",
              " 'temperature smoking',\n",
              " '',\n",
              " '',\n",
              " 'radio-control',\n",
              " 'wood',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'substitutions eggs quiche',\n",
              " 'plumbing water shutoff refrigerator',\n",
              " '',\n",
              " '',\n",
              " 'mac molecular-biology cell-biology ecoli fluorescent-microscopy',\n",
              " 'concrete stream-cipher lightweight',\n",
              " 'equipment packaging',\n",
              " 'usa india money visas schengen india indian-citizens',\n",
              " 'food-and-drink austria vienna souvenirs',\n",
              " '',\n",
              " 'staining woodworking',\n",
              " 'hvac air-conditioning central-air',\n",
              " '',\n",
              " 'quadcopter dynamics',\n",
              " 'microbiology pathology epidemiology',\n",
              " 'electrical troubleshooting range',\n",
              " 'hotels expedia',\n",
              " 'bookings payment-cards travel-agents',\n",
              " '',\n",
              " '',\n",
              " '',\n",
              " 'lighting garage',\n",
              " 'electrical ceiling-fan junction-box',\n",
              " 'dough bread rising',\n",
              " 'fruit ice-cream sorbet',\n",
              " 'hash',\n",
              " '',\n",
              " 'drywall interior finishing',\n",
              " 'flavor coffee bananas',\n",
              " 'authentication',\n",
              " 'battery',\n",
              " 'symmetric key-exchange',\n",
              " 'furnace',\n",
              " 'stock',\n",
              " '',\n",
              " 'chacha pseudo-random-generator electrical electrical lighting troubleshooting',\n",
              " 'zoology',\n",
              " 'plumbing drain-waste-vent drains',\n",
              " 'electrical cfl',\n",
              " 'weather-and-climate diving sri-lanka water-sports south-india',\n",
              " '',\n",
              " 'electrical electrical grounding electrical-panel electrical wiring switch',\n",
              " 'gas fireplace hvac',\n",
              " 'concrete walls concrete-block retaining-wall slope',\n",
              " 'electrical electrical ceiling-fan',\n",
              " 'botany taxonomy phylogenetics ethnobiology budget europe camping backpacking',\n",
              " 'trains history',\n",
              " '',\n",
              " 'flooring hardwood polyurethane varnishing engineered-flooring',\n",
              " 'europe schengen working-holiday overstaying',\n",
              " 'gas oven stove',\n",
              " 'electrical electrical wiring',\n",
              " '',\n",
              " '',\n",
              " 'baking cake recipe-scaling',\n",
              " 'bookings hotels price vancouver',\n",
              " 'budget accommodation hotels saudi-arabia europe where-on-earth countries architecture',\n",
              " 'entropy sha-256 md5 fortuna',\n",
              " '',\n",
              " '',\n",
              " 'electrical',\n",
              " 'air-travel tickets children']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RWzY8zlIDi8w",
        "colab_type": "code",
        "outputId": "9ca31396-ddc6-43f0-a72c-5232ef8ad7f5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "len(labelled_preds)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "896"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    }
  ]
}