BERT_Fine_tuning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT_Fine-tuning.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "KAB0G-CA6D-D"
      },
      "source": [
        "import json\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "import openpyxl\n",
        "from pathlib import Path\n",
        "import numpy as np\n",
        "from gensim.models import KeyedVectors\n",
        "\n",
        "def cleaner(text):\n",
        "    final_text = ''\n",
        "    for word in text.split():\n",
        "        if word.startswith('@'):\n",
        "            continue\n",
        "        elif word[-3:] in ['com', 'org']:\n",
        "            continue\n",
        "        elif word.startswith(\"RT\"):\n",
        "            continue\n",
        "        elif word.startswith(\"#\"):\n",
        "            continue\n",
        "        elif word.startswith('pic') or word.startswith('http') or word.startswith('www'):\n",
        "            continue\n",
        "        else:\n",
        "            final_text += word+' '\n",
        "    return final_text\n",
        "\n",
        "df = pd.read_excel(\"/content/drive/MyDrive/Colab Notebooks/models/cleaned_2000.xlsx\") "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "EbXgnQ_MdtEk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y-gTEHlH6KKa"
      },
      "source": [
        "df[\"tweet\"] = df[\"text\"].apply(cleaner)\n",
        "df = df[[\"tweet\",\"label\"]]\n",
        "sentences = df[\"tweet\"].values\n",
        "y = df[\"label\"].values\n",
        "\n",
        "training_texts, test_texts, training_labels, test_labels = train_test_split(sentences, y, test_size=0.25, random_state=1000)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZxeqUymlCpTd"
      },
      "source": [
        "!pip install transformers\n",
        "import transformers\n",
        "from transformers import BertTokenizer\n",
        "from torch.utils.data import TensorDataset, random_split\n",
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "import torch"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gfs__btdCP-t"
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('dbmdz/bert-base-turkish-128k-uncased', do_lower_case=True)\n",
        "sentences = df.tweet.values\n",
        "max_len = 50"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n1yQZfQNDNyN",
        "outputId": "c753c6c7-b15c-48e4-a753-c4787093ea81"
      },
      "source": [
        "input_ids = []\n",
        "attention_masks = []\n",
        "\n",
        "for text in training_texts:\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        text,                     \n",
        "                        add_special_tokens = True,\n",
        "                        max_length = max_len,      \n",
        "                        pad_to_max_length = True,\n",
        "                        return_attention_mask = True, \n",
        "                        return_tensors = 'pt',\n",
        "                   )\n",
        "    \n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "labels = torch.tensor(training_labels)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2232: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rkZVN1PCDkJw"
      },
      "source": [
        "train_dataset = TensorDataset(input_ids, attention_masks, labels)\n",
        "\n",
        "batch_size = 32\n",
        "\n",
        "train_dataloader = DataLoader(\n",
        "            train_dataset,  \n",
        "            sampler = RandomSampler(train_dataset), \n",
        "            batch_size = batch_size \n",
        "        )\n",
        "\n",
        "number_of_categories = 3\n",
        "\n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"dbmdz/bert-base-turkish-128k-uncased\",\n",
        "    num_labels = number_of_categories, \n",
        "    output_attentions = False,\n",
        "    output_hidden_states = False,\n",
        ")\n",
        "\n",
        "model.cuda()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zvcwMMCYEFIb"
      },
      "source": [
        "epochs = 8\n",
        "\n",
        "optimizer = AdamW(model.parameters(),\n",
        "                  lr = 5e-5,\n",
        "                  eps = 1e-8 \n",
        "                )\n",
        "\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0,\n",
        "                                            num_training_steps = total_steps)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "36HQIlY_EQFN"
      },
      "source": [
        "import random\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import time\n",
        "import datetime\n",
        "import tensorflow as tf"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "umPkdiZlE1gI",
        "outputId": "8964f1f4-31fa-4a75-f166-9145d7517ed9"
      },
      "source": [
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name == '/device:GPU:0':\n",
        "    device = torch.device(\"cuda\")\n",
        "    print('GPU:', torch.cuda.get_device_name(0))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "GPU: Tesla T4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gf_faHglI-NS"
      },
      "source": [
        "torch.cuda.empty_cache()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Fine-Tuning the Model"
      ],
      "metadata": {
        "id": "ptB0mKRsh9Bw"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kegoF97vEHgi",
        "outputId": "9c988106-d3ac-42c3-be0e-3fe0e166cd02"
      },
      "source": [
        "def format_time(elapsed):\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))\n",
        "\n",
        "seed_val = 8888\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "training_stats = []\n",
        "total_t0 = time.time()\n",
        "\n",
        "for epoch_i in range(0, epochs):\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    t0 = time.time()\n",
        "    total_train_loss = 0\n",
        "    model.train()\n",
        "\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "        if step % 10 == 0 and not step == 0:\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            print('Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        model.zero_grad()        \n",
        "        loss, logits = model(b_input_ids, \n",
        "                             token_type_ids=None, \n",
        "                             attention_mask=b_input_mask, \n",
        "                             labels=b_labels,\n",
        "                             return_dict=False)\n",
        "        total_train_loss += loss.item()\n",
        "        loss.backward()\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "        optimizer.step()\n",
        "        scheduler.step()\n",
        "\n",
        "    avg_train_loss = total_train_loss / len(train_dataloader)            \n",
        "    training_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"Training epoch took: {:}\".format(training_time))\n",
        "\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch_i + 1,\n",
        "            'Training Loss': avg_train_loss,\n",
        "            'Training Time': training_time,\n",
        "        }\n",
        "    )\n",
        "\n",
        "print(\"Training completed in {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "======== Epoch 1 / 8 ========\n",
            "Batch    10  of     47.    Elapsed: 0:00:13.\n",
            "Batch    20  of     47.    Elapsed: 0:00:26.\n",
            "Batch    30  of     47.    Elapsed: 0:00:39.\n",
            "Batch    40  of     47.    Elapsed: 0:00:53.\n",
            "Average training loss: 1.05\n",
            "Training epoch took: 0:01:02\n",
            "======== Epoch 2 / 8 ========\n",
            "Batch    10  of     47.    Elapsed: 0:00:13.\n",
            "Batch    20  of     47.    Elapsed: 0:00:27.\n",
            "Batch    30  of     47.    Elapsed: 0:00:40.\n",
            "Batch    40  of     47.    Elapsed: 0:00:53.\n",
            "Average training loss: 0.78\n",
            "Training epoch took: 0:01:03\n",
            "======== Epoch 3 / 8 ========\n",
            "Batch    10  of     47.    Elapsed: 0:00:13.\n",
            "Batch    20  of     47.    Elapsed: 0:00:27.\n",
            "Batch    30  of     47.    Elapsed: 0:00:40.\n",
            "Batch    40  of     47.    Elapsed: 0:00:54.\n",
            "Average training loss: 0.52\n",
            "Training epoch took: 0:01:03\n",
            "======== Epoch 4 / 8 ========\n",
            "Batch    10  of     47.    Elapsed: 0:00:13.\n",
            "Batch    20  of     47.    Elapsed: 0:00:27.\n",
            "Batch    30  of     47.    Elapsed: 0:00:40.\n",
            "Batch    40  of     47.    Elapsed: 0:00:53.\n",
            "Average training loss: 0.27\n",
            "Training epoch took: 0:01:03\n",
            "======== Epoch 5 / 8 ========\n",
            "Batch    10  of     47.    Elapsed: 0:00:13.\n",
            "Batch    20  of     47.    Elapsed: 0:00:27.\n",
            "Batch    30  of     47.    Elapsed: 0:00:40.\n",
            "Batch    40  of     47.    Elapsed: 0:00:53.\n",
            "Average training loss: 0.15\n",
            "Training epoch took: 0:01:03\n",
            "======== Epoch 6 / 8 ========\n",
            "Batch    10  of     47.    Elapsed: 0:00:13.\n",
            "Batch    20  of     47.    Elapsed: 0:00:27.\n",
            "Batch    30  of     47.    Elapsed: 0:00:40.\n",
            "Batch    40  of     47.    Elapsed: 0:00:54.\n",
            "Average training loss: 0.09\n",
            "Training epoch took: 0:01:03\n",
            "======== Epoch 7 / 8 ========\n",
            "Batch    10  of     47.    Elapsed: 0:00:13.\n",
            "Batch    20  of     47.    Elapsed: 0:00:27.\n",
            "Batch    30  of     47.    Elapsed: 0:00:40.\n",
            "Batch    40  of     47.    Elapsed: 0:00:54.\n",
            "Average training loss: 0.05\n",
            "Training epoch took: 0:01:03\n",
            "======== Epoch 8 / 8 ========\n",
            "Batch    10  of     47.    Elapsed: 0:00:13.\n",
            "Batch    20  of     47.    Elapsed: 0:00:27.\n",
            "Batch    30  of     47.    Elapsed: 0:00:40.\n",
            "Batch    40  of     47.    Elapsed: 0:00:54.\n",
            "Average training loss: 0.04\n",
            "Training epoch took: 0:01:03\n",
            "Training completed in 0:08:21 (h:mm:ss)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Saving the Model"
      ],
      "metadata": {
        "id": "E18W4v3lhzBm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "path = '/content/drive/MyDrive/Colab Notebooks/models/model_2000_32b_8e_weights.pt'\n",
        "torch.save(model, path)"
      ],
      "metadata": {
        "id": "wFeddmXQd8yO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "path = '/content/drive/MyDrive/Colab Notebooks/models/model_32b_8e_weights.pt'\n",
        "model = torch.load(path)"
      ],
      "metadata": {
        "id": "q4Z4LvDldoAs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "ka3ttVM4Pb1U",
        "outputId": "072350e4-3cc3-4b0b-cc4e-1cc8a63efc12"
      },
      "source": [
        "df_stats = pd.DataFrame(data=training_stats)\n",
        "plt.plot(df_stats['Training Loss'], label=\"Training\")\n",
        "plt.title(\"Training Loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.xticks([1,2,3,4,5,6,7,8])\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9b3G8c83C4RAErZAgIRNNsMmEGnVum9gLXhrteDW3mtLEbVarVsX21qvba21XhWtWmur4EK1tlQp7qjUNWFfFdn3sCXs2b73jzmkMbIEyOTMTJ7365WXM2fOTJ6o8OSc3/n9jrk7IiIiAElhBxARkdihUhARkWoqBRERqaZSEBGRaioFERGpplIQEZFqKgVptMzsX2b2rfreVySemeYpSDwxsx01nqYDe4HK4Pn33H1iw6c6cmZ2GjDB3XPDziICkBJ2AJHD4e4t9j02s+XAd9z99dr7mVmKu1c0ZDaRRKDTR5IQzOw0M1ttZreY2XrgCTNrZWYvmVmxmW0NHufWeM80M/tO8PjbZjbdzO4J9l1mZsOPcN9uZvaOmW03s9fNbLyZTTiCn+nY4PtuM7P5ZjaixmvnmdmC4HusMbMfBtvbBj/nNjPbYmbvmpn+nEud6X8WSSQ5QGugCzCGyP/fTwTPOwO7gQcP8v4vAYuBtsDdwONmZkew79PAR0Ab4OfA5Yf7g5hZKvBP4FWgHXAtMNHMege7PE7kdFkG0A94M9h+I7AayAbaAz8CdI5Y6kylIImkCviZu+91993uvtndX3D3Xe6+Hfhf4NSDvH+Fuz/m7pXAX4AORP5irfO+ZtYZOB643d3L3H06MPkIfpYvAy2AXwef8ybwEjA6eL0cyDezTHff6u4zamzvAHRx93J3f9c1cCiHQaUgiaTY3ffse2Jm6Wb2iJmtMLNS4B2gpZklH+D96/c9cPddwcMWh7lvR2BLjW0Aqw7z5yD4nFXuXlVj2wqgU/D4QuA8YIWZvW1mJwTbfwssAV41s6VmdusRfG9pxFQKkkhq/0Z8I9Ab+JK7ZwKnBNsPdEqoPqwDWptZeo1teUfwOWuBvFrjAZ2BNQDu/rG7jyRyaunvwKRg+3Z3v9HduwMjgBvM7Mwj+P7SSKkUJJFlEBlH2GZmrYGfRfsbuvsKoBD4uZk1CX6D/9qh3mdmaTW/iIxJ7AJuNrPU4NLVrwHPBp97qZlluXs5UErk1Blmdr6Z9QjGN0qIXK5btd9vKrIfKgVJZPcBzYBNwAfA1Ab6vpcCJwCbgTuB54jMpziQTkTKq+ZXHpESGE4k/0PAFe6+KHjP5cDy4LTY2OB7AvQEXgd2AO8DD7n7W/X2k0nC0+Q1kSgzs+eARe4e9SMVkaOlIwWRemZmx5vZMWaWZGbDgJFEzvuLxDzNaBapfznA34jMU1gNXOXuM8ONJFI3On0kIiLVdPpIRESqxd3po7Zt23rXrl3DjiEiEleKioo2uXv2ofaLu1Lo2rUrhYWFYccQEYkrZraiLvvp9JGIiFRTKYiISDWVgoiIVFMpiIhINZWCiIhUUymIiEg1lYKIiFRrFKXw/mebeeLfy8KOISIS8+Ju8tqR+GvRKl6cuYae7TL4Ss+2YccREYlZjeJI4c4L+tGzXQu+/+xM1pXsDjuOiEjMahSlkN4khYcvG8Le8kqunjiDsgrdnVBEZH8aRSkAHJPdgru/MZAZK7dx15SFYccREYlJjaYUAL46oAP/c1I3/vzecv45e23YcUREYk6jKgWA287rw5Aurbj1hTks2bgj7DgiIjGl0ZVCanIS4y8ZTFpqMldNKGLn3oqwI4mIxIyolYKZ/cnMNprZvAO8bmZ2v5ktMbM5ZjY4Wllqy8lK4/7Rg/iseAe3/W0uuiWpiEhENI8U/gwMO8jrw4GewdcY4OEoZvmCk3q05cZzejN59lqe+qBO954QEUl4USsFd38H2HKQXUYCT3rEB0BLM+sQrTz7c9Wpx3Bmn3b88qUFzFy5tSG/tYhITApzTKETsKrG89XBti8wszFmVmhmhcXFxfUWICnJuPfi42ifmcbVE2ewZWdZvX22iEg8iouBZnd/1N0L3L0gO/uQ950+LFnpqTx86RA27SzjumdnUlml8QURabzCLIU1QF6N57nBtgbXPzeLX4zoy7ufbuL+Nz4NI4KISEwIsxQmA1cEVyF9GShx93VhhRl1fB4XDs7l/jc/ZdrijWHFEBEJVTQvSX0GeB/obWarzexKMxtrZmODXaYAS4ElwGPAuGhlqQsz484L+tG7fQbXPzeL1Vt3hRlHRCQUFm/X6BcUFHhhYWHUPn/Zpp2MeGA63bObM2nsCTRNSY7a9xIRaShmVuTuBYfaLy4GmhtSt7bN+e1FA5m9uoQ7X9LCeSLSuKgU9mNYvxzGnNKdpz5YwT9mhTL2LSISCpXCAdx8bm+GdmvNrS/M5ZMN28OOIyLSIFQKB5CSnMSDowfRvGkKYycUsUML54lII6BSOIh2mWk8eMkgVmzexS3Pz9HCeSKS8FQKh/Dl7m246dzevDx3HU/8e3nYcUREokqlUAffO6U75+S3564pCylacbA1/kRE4ptKoQ7MjN9eNJBOrZoxbuIMNu3YG3YkEZGoUCnUUVazyMJ523aVa+E8EUlYKoXDkN8xk19e0I9/L9nM71/7JOw4IiL1TqVwmC4uyGPU8Xk8+NYS3ly0Iew4IiL1SqVwBH4+oi99O2Zy/bOzWLVFC+eJSOJQKRyBtNRkHr50CABXTSxiT3llyIlEROqHSuEIdW6Tzr0XH8e8NaX84p8Lwo4jIlIvVApH4az89ow77Rie+WglzxetDjuOiMhRUykcpRvO7sUJ3dvw4xfnsnBdadhxRESOikrhKKUkJ3H/6EFkNUvlqglFlO4pDzuSiMgRUynUg+yMpoy/dDCrtu7m5r9q4TwRiV8qhXpyfNfW3Da8D1Pnr+eP7y4LO46IyBFRKdSjK7/SjfP65/DrqYv4cOnmsOOIiBw2lUI9MjN+c+EAurRO55pnZrJx+56wI4mIHBaVQj3LSEvlocsGs31POdc+PZOKyqqwI4mI1JlKIQr65GTyq6/358NlW7jnVS2cJyLxQ6UQJf81KJdLv9SZP7z9Ga/OXx92HBGROlEpRNHtX8tnQG4WN/51Nis27ww7jojIIakUoqhpSjLjLxlMkhljJ8zQwnkiEvNUClGW1zqd+0Ydx6L1pdz+j3lhxxEROSiVQgM4vXc7rj29B5MKV/PcxyvDjiMickAqhQZy3Vm9OLlnW376j/nMW1MSdhwRkf1SKTSQ5CTjvm8eR5vmTRg3cQYlu7VwnojEnqiWgpkNM7PFZrbEzG7dz+udzewtM5tpZnPM7Lxo5glbmxaRhfPWlezmxkmzqKrSwnkiEluiVgpmlgyMB4YD+cBoM8uvtdtPgEnuPggYBTwUrTyxYnDnVvz4vGN5feFG/vDOZ2HHERH5nGgeKQwFlrj7UncvA54FRtbax4HM4HEWsDaKeWLGt07sytcGduSeVxbz3mebwo4jIlItmqXQCVhV4/nqYFtNPwcuM7PVwBTg2v19kJmNMbNCMyssLi6ORtYGZWb8+uv96da2Od9/ZiYbSrVwnojEhrAHmkcDf3b3XOA84Ckz+0Imd3/U3QvcvSA7O7vBQ0ZD86Yp/OGyIewqq+Sap2dQroXzRCQGRLMU1gB5NZ7nBttquhKYBODu7wNpQNsoZoopPdtn8OsLB/Dx8q385l+Lwo4jIhLVUvgY6Glm3cysCZGB5Mm19lkJnAlgZscSKYX4Pz90GEYM7Mi3TujCH6cvY8rcdWHHEZFGLmql4O4VwDXAK8BCIlcZzTezO8xsRLDbjcB3zWw28AzwbW+ENzj+8VfzGdy5JTf9dTZLNm4PO46INGIWb38HFxQUeGFhYdgx6t36kj2c/8C7ZDVL5R/XfIUWTVPCjiQiCcTMity94FD7hT3QLIGcrDQeGD2Y5Zt3cfPzs4m3shaRxKBSiCEnHNOGW4b1Zsrc9Tw+fVnYcUSkEVIpxJjvntyd4f1y+NW/FvHB0s1hxxGRRkalEGPMjLu/MYAubdK55ukZrC/RxDYRaTgqhRiUkZbKI8HEtqufnkFZhSa2iUjDUCnEqJ7tM7j7GwMoWrGVu6YsDDuOiDQSKoUYdv6Ajlz5lW78+b3l/GNW7cngIiL1T6UQ424d3oehXVtz6wtzWbxeE9tEJLpUCjEuNTmJBy8dREZaCmMnFFG6R3dsE5HoUSnEgXYZaYy/dDCrtuzixkmzdcc2EYkalUKcOL5ra3503rG8tmCD7tgmIlGjUogj/33Sf+7Y9u8lumObiNQ/lUIc2XfHth7tWnDtMzNZu2132JFEJMGoFOLMvju2lVVUcdXEGeytqAw7kogkEJVCHOqe3YJ7LhrI7FXbuOOfC8KOIyIJRKUQp4b1y2Hsqccw8cOVPF+0Ouw4IpIgVApx7Ifn9OLEY9rw4xfnMn9tSdhxRCQBqBTiWEpyEvePHkSr9CaMnVBEyS5NbBORo6NSiHNtWzTlocsGs75kD9c/N1MT20TkqKgUEsDgzq24/Wt9eWtxMQ+8uSTsOCISx1QKCeKyL3Xm64M7cd8bnzBt8caw44hInFIpJAgz438v6E/v9hlc9+wsVm3ZFXYkEYlDKoUE0qxJMo9cPoQqd66aWMSeck1sE5HDo1JIMF3aNOe+bx7HvDWl3P6PeWHHEZE4o1JIQGce255rz+jBpMLVPPvRyrDjiEgcUSkkqOvP6sXJPdty+z/mM3vVtrDjiEicUCkkqOQk4/5Rg8jOaMq4iTPYsrMs7EgiEgdUCgmsVfMmPHzZYIp37OW6Z2dSqYltInIIKoUENyC3JXeM6Mu7n27ivtc/CTuOiMQ4lUIjMGpoZ75ZkMcDby7h9QUbwo4jIjEsqqVgZsPMbLGZLTGzWw+wz8VmtsDM5pvZ09HM05j9YmRf+nfK4geTZrF8086w44hIjIpaKZhZMjAeGA7kA6PNLL/WPj2B24CT3L0vcH208jR2aanJPHTpYJKTjLETithdpoltIvJF0TxSGAoscfel7l4GPAuMrLXPd4Hx7r4VwN21aE8U5bVO5/9GDWLxhu386MW5uGvgWUQ+L5ql0AlYVeP56mBbTb2AXmb2bzP7wMyG7e+DzGyMmRWaWWFxcXGU4jYOp/bK5gdn9eLFmWuY8MGKsOOISIwJe6A5BegJnAaMBh4zs5a1d3L3R929wN0LsrOzGzhi4rnm9B6c0acdd7y0gBkrt4YdR0RiSDRLYQ2QV+N5brCtptXAZHcvd/dlwCdESkKiKCnJ+P3Fx9EhqxnjJsxg0469YUcSkRhRp1Iws+ZmlhQ87mVmI8ws9RBv+xjoaWbdzKwJMAqYXGufvxM5SsDM2hI5nbT0MPLLEcpKT+XhywazdVcZ1z49k4rKqrAjiUgMqOuRwjtAmpl1Al4FLgf+fLA3uHsFcA3wCrAQmOTu883sDjMbEez2CrDZzBYAbwE3ufvmw/8x5Ej07ZjFXf/Vn/eXbua3ry4OO46IxICUOu5n7r7LzK4EHnL3u81s1qHe5O5TgCm1tt1e47EDNwRfEoILh+Qyc9VWHnl7KYPyWjKsX4ewI4lIiOp6pGBmdgJwKfBysC05OpGkof30/HwG5rXkh3+dw2fFO8KOIyIhqmspXE9kktmLwSmg7kRO90gCaJqSzMOXDqZJShJjnypi596KsCOJSEjqVAru/ra7j3D33wQDzpvc/ftRziYNqGPLZjwwehCfFe/glhfmaGKbSCNV16uPnjazTDNrDswDFpjZTdGNJg3tpB5tuencPrw0Zx1/+vfysOOISAjqevoo391LgQuAfwHdiFyBJAlm7KndOSe/Pb+aspCPlm0JO46INLC6lkJqMC/hAoLJZoDOLyQgM+OeiweS1zqdq5+ewcbSPWFHEpEGVNdSeARYDjQH3jGzLkBptEJJuDLTUvnDZUPYsaeCq5+eQbkmtok0GnUdaL7f3Tu5+3kesQI4PcrZJES9czL49YX9+Xj5Vn79r0VhxxGRBlLXgeYsM7t330qlZvY7IkcNksBGHteJb5/YlcenL+Ofs9eGHUdEGkBdTx/9CdgOXBx8lQJPRCuUxI4fnXcsBV1accsLc1i0XmcMRRJdXUvhGHf/WXDDnKXu/gugezSDSWxokpLEQ5cOpkXTFMY8WcS2XWVhRxKRKKprKew2s6/se2JmJwG7oxNJYk27zDT+cPkQ1pfs4dpntKKqSCKraymMBcab2XIzWw48CHwvaqkk5gzu3Io7L+jHu59u4u5XtKKqSKKq0yqp7j4bGGhmmcHzUjO7HpgTzXASWy4+Po95a0t49J2l5HfI5IJBte+uKiLx7rDuvObupcHMZtBy143ST8/PZ2i31tzywhzmrSkJO46I1LOjuR2n1VsKiRupyZGB5zbNmzDmyULdylMkwRxNKWiZi0aqbYumPHJ5AZt3ljFuomY8iySSg5aCmW03s9L9fG0HOjZQRolB/XOz+M2FA/ho2RbufGlB2HFEpJ4cdKDZ3TMaKojEnwsGdWL+2hIee3cZfTtmcfHxeWFHEpGjdDSnj0S4ZVgfTu7Zlp/8fR4zVm4NO46IHCWVghyVlOQkHhg9iJysNMY+VaSltkXinEpBjlrL9CY8esUQduyt4HsTithbURl2JBE5QioFqRd9cjL53UUDmblyG7f/fb7u8SwSp1QKUm+G9+/ANaf34LnCVUz4YEXYcUTkCKgUpF7dcHYvzujTjl/8cwEfLt0cdhwROUwqBalXSUnGfaOOo3ObdMZNnMHabVpMVySeqBSk3mWmpfLo5QWUVVQx5qlC9pRr4FkkXqgUJCp6tGvBfaOOY/7aUm7721wNPIvECZWCRM2Zx7bnhrN68eLMNTw+fVnYcUSkDlQKElVXn96DYX1zuGvKQqZ/uinsOCJyCFEtBTMbZmaLzWyJmd16kP0uNDM3s4Jo5pGGl5Rk/O7igfRsl8E1z8xg5eZdYUcSkYOIWimYWTIwHhgO5AOjzSx/P/tlANcBH0Yri4SredMUHr1iCO4w5qlCdu6tCDuSiBxANI8UhgJL3H2pu5cBzwIj97PfL4HfAFo0J4F1adOcB0YP4pMN27np+dkaeBaJUdEshU7AqhrPVwfbqpnZYCDP3V8+2AeZ2RgzKzSzwuLi4vpPKg3ilF7Z3Dq8D1PmruehaZ+FHUdE9iO0gWYzSwLuBW481L7u/qi7F7h7QXZ2dvTDSdR89+TujBjYkXteXcybizaEHUdEaolmKawBat51JTfYtk8G0A+YZmbLgS8DkzXYnNjMjN9cOID8Dplc98wsPiveEXYkEakhmqXwMdDTzLqZWRNgFDB534vuXuLubd29q7t3BT4ARrh7YRQzSQxo1iSZRy4fQmpKEmOeLGT7nvKwI4lIIGql4O4VwDXAK8BCYJK7zzezO8xsRLS+r8SH3FbpjL9kMMs37+IHz82iqkoDzyKxwOLtKpCCggIvLNTBRKL4y3vL+dnk+Xz/jB7ccE7vsOOIJCwzK3L3Q56e14xmCdUVJ3ThoiG53P/mEqbOWxd2HJFGT6UgoTIzfnlBP47La8kNk2azeP32sCOJNGoqBQldWmoyf7hsCM2bpjDmqUK27SoLO5JIo6VSkJiQk5XGHy4bzNptu7n2mZlUauBZJBQqBYkZQ7q05pcj+/Hup5u4e+qisOOINEopYQcQqWnU0M7MW1vCI+8sJb9jJiOP63ToN4lIvdGRgsSc28/vy9CurbnlhTnMW1MSdhyRRkWlIDGnSUoS4y8dTKv0JnzvqSI279gbdiSRRkOlIDEpO6Mpj1w+hE079nL10zMor6wKO5JIo6BSkJg1ILclv/p6fz5YuoX/fXlh2HFEGgUNNEtM+/rgXOavLeXx6cvI75jJxQV5h36TiBwxHSlIzLtteB9O6tGGn7w4j5krt4YdRyShqRQk5qUkJ/Hg6MG0y2zK2AlFbCzVnVtFokWlIHGhVfMmPHZFAaW7Kxg7oYi9FZVhRxJJSCoFiRvHdsjknosGMmPlNn4+eT7xtuy7SDxQKUhc+eqADow77Rie+WgVEz9cGXYckYSjUpC4c+M5vTmtdzY/nzyfj5ZtCTuOSEJRKUjcSU4y/m/UIPJapzNuYhFrt+0OO5JIwlApSFzKapbKY1cMYU95Fd97qog95Rp4FqkPKgWJWz3aZfD7bx7H3DUl3Pa3uVRoKQyRo6ZSkLh2dn57bji7Fy/OXMNX75/OO58Uhx1JJK6pFCTuXXtGDx66dDC7yyu54k8f8e0nPuLTDbrXs8iRUClI3DMzzuvfgdduOIUfndeHouVbGfZ/7/LTv8/Tstsih0mlIAmjaUoyY045hmk3ncYlQzvz9EcrOe2eaTz6zmeaAS1SRyoFSThtWjTllxf0Y+p1J1PQpRV3TVnEWfe+zZS56zQLWuQQVAqSsHq2z+CJ/x7Kk/8zlPTUFMZNnMHFj7zP7FXbwo4mErNUCpLwTumVzcvf/wp3/Vd/lm3aycjx/+YHz83SpDeR/VApSKOQkpzEJV/qzFs/PI1xpx3Dy3PXccbvpnHvq4vZubci7HgiMUOlII1KRloqNw/rwxs3nMrZ+Tnc/+YSTrtnGpM+XkVllcYbRFQK0ijltU7ngdGDeOGqE+nUshk3vzCHrz0wnfc+2xR2NJFQRbUUzGyYmS02syVmdut+Xr/BzBaY2Rwze8PMukQzj0htQ7q04sVxJ3L/6EGU7C7nksc+5Dt/KWRp8Y6wo4mEImqlYGbJwHhgOJAPjDaz/Fq7zQQK3H0A8Dxwd7TyiByImTFiYEfeuPFUbh7Wmw+Wbuac37/DL/45n227ysKOJ9KgonmkMBRY4u5L3b0MeBYYWXMHd3/L3XcFTz8AcqOYR+Sg0lKTGXdaD9764WlcVJDHX95bzqm/ncafpi+jrEKL7UnjEM1S6ASsqvF8dbDtQK4E/hXFPCJ1kp3RlF99vT9TrjuZAblZ3PHSAs697x1eW7BBk98k4cXEQLOZXQYUAL89wOtjzKzQzAqLi7UKpjSMPjmZPPk/Q3ni28eTZPDdJwu55LEPmb+2JOxoIlETzVJYA+TVeJ4bbPscMzsL+DEwwt33u3qZuz/q7gXuXpCdnR2VsCL7Y2ac3qcdU68/hTtG9mXR+lLOf2A6Nz8/m42le8KOJ1LvolkKHwM9zaybmTUBRgGTa+5gZoOAR4gUwsYoZhE5KqnJSVxxQlem3XQ63z25Oy/OXMNp90zj/jc+ZXeZFtuTxBG1UnD3CuAa4BVgITDJ3eeb2R1mNiLY7bdAC+CvZjbLzCYf4ONEYkJWs1R+dN6xvH7DqZzaK5t7X/uEM343jRdnrqZKk98kAVi8DZwVFBR4YWFh2DFEAPho2RbufHkBc1aXMCA3i598NZ+h3VqHHUvkC8ysyN0LDrVfTAw0i8Srod1a8/dxJ3HvxQPZWLqXix95n6smFLFy865Dv1kkBqWEHUAk3iUlGV8fnMvwfh147N2lPDztM95YuJFvn9SVq0/vQVaz1LAjitSZjhRE6kmzJsl8/8yeTLvpNEYe15HH3l3K6fdM46n3l1NRqclvEh80piASJfPWlHDnywv4YOkWjsluzsUFeZzbN4eubZuHHU0aobqOKagURKLI3Xl94UYeePNT5qyOTHrrk5PBsH45DOuXQ+/2GZhZyCmlMVApiMSYVVt28eqCDUydt47CFVtxh65t0jm3Xw7D+3VgQKcskpJUEBIdKgWRGLZx+x5eW7CBqfPW8/5nm6mocjpkpXFu3xzO7ZvD8V1bkZKsIT+pPyoFkThRsqucNxZFCuLtT4rZW1FF6+ZNOPvY9gzrl8OJPdrQNCU57JgS51QKInFoV1kF0xYXM3Xeet5ctJEdeyto0TSFM/q0Y3i/HE7tnU16E11JLodPpSAS5/ZWVPLeks1MnbeeVxesZ+uucpqmJHFqr2yG9cvhzD7tyUrXHAipG5WCSAKpqKzi4+VbeWX+eqbOW8/60j2kJBknHNOGYf1yODu/Pe0y0sKOKTFMpSCSoKqqnNmrtzE1KIgVm3dhBgVdWnFu38ilrrmt0sOOKTFGpSDSCLg7izdsZ+q8SEEsWr8dgP6dshjWL3IlU492LUJOKbFApSDSCC3ftLP6CGLWqm0A9GjXgmHBEUTfjpmaLNdIqRREGrl1Jbt5dX7kUtcPl22myiG3VbPqghjcuZUmyzUiKgURqbZ5x17eWLiRqfPXM/3TTZRVVpGd0ZRz8iNzIb7cvQ2pmiyX0FQKIrJfpXvKeWvRRl6Zv563FhWzu7ySzLQUBua1JL9jJvkdMunbMZNubVuQrCOJhKFSEJFD2lNeyTufFPPmoo3MXVPCJxu2U14Z+TshLTWJPjmZ5HeMlER+h0z65GTSrIlmV8cjlYKIHLayiio+K97B/LWlLFhbyoJ1JSxYW0rpngoAkgy6Z7eoPprYd2TRpkXTkJPLodS1FDRfXkSqNUlJ4tgOmRzbIROGRLa5O6u37mbButLqsihasZXJs9dWvy8nM+1zp57yO2aS1ypdA9lxSKUgIgdlZuS1TievdTrn9s2p3r51ZxkL15V+rize/qSYyqrI2YcWTVPI7/Cfo4n8jpn0bN9Ci/vFOJWCiByRVs2bcGKPtpzYo231tj3llXyyYXtw6ilSFpMKV7GrrBKAlCSjR7sW9O2Y9bmy0H2sY4dKQUTqTVpqMgNyWzIgt2X1tqoqZ8WWXcxfW1JdFu98WswLM1ZX75Pbqllw6ikoi46ZdMxK00S7EKgURCSqkpKMbm2b061tc84f0LF6+8bte1i4bvvnyuK1hRvYd+1Ly/TUyJFEcDTRqnkTmqUmk5aaTFpqUo3HkedNkpNUIvVApSAioWiXkUa7jDRO7ZVdvW3n3goWrd/OgnWlLAjK4qkPVrC3ouqQn5dkkSOVfWXRtEZxNAuKI+0Az2tuO/D7I/9smppE05TELSCVgojEjOZNUxjSpRVDurSq3lZRWcXyzbso3VPOnvLK4KuK3WWV7KmIPN63fd+23WVVkdeC57vKKtiys+a2/7z/SK7KN4O0lM+XyCVf6sx3Tu5ej/82wqFSEJGYlpKcFLWVXt2dvRVV7C2vYve+wqkIyqVm2ewrohvGQ/cAAAZ8SURBVOpSqlFOwePsjMSYq6FSEJFGy8yqTx9loSugALQCloiIVFMpiIhINZWCiIhUi2opmNkwM1tsZkvM7Nb9vN7UzJ4LXv/QzLpGM4+IiBxc1ErBzJKB8cBwIB8YbWb5tXa7Etjq7j2A3wO/iVYeERE5tGgeKQwFlrj7UncvA54FRtbaZyTwl+Dx88CZlqgzQkRE4kA0S6ETsKrG89XBtv3u4+4VQAnQpvYHmdkYMys0s8Li4uIoxRURkbgYaHb3R929wN0LsrOzD/0GERE5ItGcvLYGyKvxPDfYtr99VptZCpAFbD7YhxYVFW0ysxX1GTRGtAU2hR3iIJTv6MR6Poj9jMp3dHrXZadolsLHQE8z60bkL/9RwCW19pkMfAt4H/gG8KYf4v6g7p6QhwpmVliXW+WFRfmOTqzng9jPqHxHx8zqdB/jqJWCu1eY2TXAK0Ay8Cd3n29mdwCF7j4ZeBx4ysyWAFuIFIeIiIQkqmsfufsUYEqtbbfXeLwHuCiaGUREpO7iYqC5kXg07ACHoHxHJ9bzQexnVL6jU6d8dohT+CIi0ojoSEFERKqpFEREpJpKIWRm9icz22hm88LOsj9mlmdmb5nZAjObb2bXhZ2pJjNLM7OPzGx2kO8XYWfaHzNLNrOZZvZS2FlqM7PlZjbXzGbV9bLFhmRmLc3seTNbZGYLzeyEsDPtY2a9g39v+75Kzez6sHPVZGY/CP5szDOzZ8ws7aD7a0whXGZ2CrADeNLd+4WdpzYz6wB0cPcZZpYBFAEXuPuCkKMBEKyV1dzdd5hZKjAduM7dPwg52ueY2Q1AAZDp7ueHnacmM1sOFLh7TE68MrO/AO+6+x/NrAmQ7u7bws5VW7AI6BrgS+4eExNszawTkT8T+e6+28wmAVPc/c8Heo+OFELm7u8QmaMRk9x9nbvPCB5vBxbyxTWsQuMRO4KnqcFXTP2mY2a5wFeBP4adJd6YWRZwCpE5Tbh7WSwWQuBM4LNYKYQaUoBmwaoR6cDag+2sUpA6C+53MQj4MNwknxecmpkFbARec/eYygfcB9wMVIUd5AAceNXMisxsTNhhaukGFANPBKff/mhmzcMOdQCjgGfCDlGTu68B7gFWAuuAEnd/9WDvUSlInZhZC+AF4Hp3Lw07T03uXunuxxFZX2uomcXMaTgzOx/Y6O5FYWc5iK+4+2Ai9z65OjilGStSgMHAw+4+CNgJfOGGXWELTmuNAP4adpaazKwVkVsUdAM6As3N7LKDvUelIIcUnKt/AZjo7n8LO8+BBKcV3gKGhZ2lhpOAEcF5+2eBM8xsQriRPi/4bRJ33wi8SOReKLFiNbC6xtHf80RKItYMB2a4+4awg9RyFrDM3YvdvRz4G3Diwd6gUpCDCgZyHwcWuvu9Yeepzcyyzaxl8LgZcDawKNxU/+Hut7l7rrt3JXJ64U13P+hvag3JzJoHFxAQnJY5B4iZK+HcfT2wysz2rfB5JhATFznUMpoYO3UUWAl82czSgz/LZxIZFzwglULIzOwZIqvE9jaz1WZ2ZdiZajkJuJzIb7j7Lrs7L+xQNXQA3jKzOURW5n3N3WPuss8Y1h6YbmazgY+Al919asiZarsWmBj8Nz4OuCvkPJ8TlOnZRH4LjynBEdbzwAxgLpG/8w+63IUuSRURkWo6UhARkWoqBRERqaZSEBGRaioFERGpplIQEZFqKgWRWsysstbKl/U2g9bMusbqirgiEOV7NIvEqd3BshkijY6OFETqKLjvwN3BvQc+MrMewfauZvammc0xszfMrHOwvb2ZvRjc62G2me1bXiDZzB4L1rh/NZiJLRITVAoiX9Ss1umjb9Z4rcTd+wMPEln9FOAB4C/uPgCYCNwfbL8feNvdBxJZr2d+sL0nMN7d+wLbgAuj/POI1JlmNIvUYmY73L3FfrYvB85w96XBIoHr3b2NmW0iciOi8mD7Ondva2bFQK67763xGV2JLMXRM3h+C5Dq7ndG/ycTOTQdKYgcHj/A48Oxt8bjSjS2JzFEpSByeL5Z45/vB4/fI7ICKsClwLvB4zeAq6D6RkBZDRVS5EjpNxSRL2oW3Mltn6nuvu+y1FbBap17iSyXDJFVPJ8ws5uI3CXsv4Pt1wGPBivfVhIpiHVRTy9yFDSmIFJHsX6De5H6oNNHIiJSTUcKIiJSTUcKIiJSTaUgIiLVVAoiIlJNpSAiItVUCiIiUu3/AeQgIJD2mxQ9AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vXJ9ZZdqPihi",
        "outputId": "a76ef4f7-b732-4c68-e8cc-a4d1a9e3a8a1"
      },
      "source": [
        "input_ids = []\n",
        "attention_masks = []\n",
        "max_len = 50\n",
        "\n",
        "for text in test_texts:\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        text,                     \n",
        "                        add_special_tokens = True, \n",
        "                        max_length = max_len,          \n",
        "                        pad_to_max_length = True,\n",
        "                        return_attention_mask = True,  \n",
        "                        return_tensors = 'pt',   \n",
        "                   )\n",
        "    \n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "labels = torch.tensor(test_labels)\n",
        "\n",
        "batch_size = 32\n",
        "prediction_data = TensorDataset(input_ids, attention_masks, labels)\n",
        "prediction_sampler = SequentialSampler(prediction_data)\n",
        "prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2232: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GjZyL2tLQhY-",
        "outputId": "1bb08ae1-fd18-4bd5-8ffd-299473d7a528"
      },
      "source": [
        "print('Prediction started on test data')\n",
        "model.eval()\n",
        "predictions , true_labels = [], []\n",
        "\n",
        "for batch in prediction_dataloader:\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "  b_input_ids, b_input_mask, b_labels = batch\n",
        "\n",
        "  with torch.no_grad():\n",
        "      outputs = model(b_input_ids, token_type_ids=None, \n",
        "                      attention_mask=b_input_mask)\n",
        "\n",
        "  logits = outputs[0]\n",
        "  logits = logits.detach().cpu().numpy()\n",
        "  label_ids = b_labels.to('cpu').numpy()\n",
        "  \n",
        "  predictions.append(logits)\n",
        "  true_labels.append(label_ids)\n",
        "\n",
        "print('Prediction completed')\n",
        "\n",
        "prediction_set = []\n",
        "\n",
        "for i in range(len(true_labels)):\n",
        "  pred_labels_i = np.argmax(predictions[i], axis=1).flatten()\n",
        "  prediction_set.append(pred_labels_i)\n",
        "\n",
        "prediction_scores = [item for sublist in prediction_set for item in sublist]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Prediction started on test data\n",
            "Prediction completed\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FD1PrbPwQ7nl"
      },
      "source": [
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import recall_score\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import classification_report"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227
        },
        "id": "tLqpe80qQwrl",
        "outputId": "1c9db063-ead9-4a6c-b71a-00c25fa9d391"
      },
      "source": [
        "f_score = f1_score(test_labels, prediction_scores, average='macro')\n",
        "precision = precision_score(test_labels, prediction_scores, average='macro')\n",
        "recall = recall_score(test_labels, prediction_scores, average='macro')\n",
        "\n",
        "print(\"F-Score: \", f_score)\n",
        "print(\"Recall: \", recall)\n",
        "print(\"Precision: \", precision)\n",
        "\n",
        "report = pd.DataFrame(classification_report(test_labels, prediction_scores, output_dict=True))\n",
        "report = report.rename(columns={'0':'negative',\n",
        "                          '1':'neutral',\n",
        "                          '2':'positive',\n",
        "                          })\n",
        "\n",
        "report"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F-Score:  0.7456496265697682\n",
            "Recall:  0.7410060580792289\n",
            "Precision:  0.7530655871139099\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-1fab0963-548c-458e-be87-ca7dd0625f6a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>negative</th>\n",
              "      <th>neutral</th>\n",
              "      <th>positive</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>macro avg</th>\n",
              "      <th>weighted avg</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>precision</th>\n",
              "      <td>0.760000</td>\n",
              "      <td>0.733068</td>\n",
              "      <td>0.766129</td>\n",
              "      <td>0.748</td>\n",
              "      <td>0.753066</td>\n",
              "      <td>0.748903</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>recall</th>\n",
              "      <td>0.664336</td>\n",
              "      <td>0.786325</td>\n",
              "      <td>0.772358</td>\n",
              "      <td>0.748</td>\n",
              "      <td>0.741006</td>\n",
              "      <td>0.748000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>f1-score</th>\n",
              "      <td>0.708955</td>\n",
              "      <td>0.758763</td>\n",
              "      <td>0.769231</td>\n",
              "      <td>0.748</td>\n",
              "      <td>0.745650</td>\n",
              "      <td>0.747093</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>support</th>\n",
              "      <td>143.000000</td>\n",
              "      <td>234.000000</td>\n",
              "      <td>123.000000</td>\n",
              "      <td>0.748</td>\n",
              "      <td>500.000000</td>\n",
              "      <td>500.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1fab0963-548c-458e-be87-ca7dd0625f6a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1fab0963-548c-458e-be87-ca7dd0625f6a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1fab0963-548c-458e-be87-ca7dd0625f6a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "             negative     neutral  ...   macro avg  weighted avg\n",
              "precision    0.760000    0.733068  ...    0.753066      0.748903\n",
              "recall       0.664336    0.786325  ...    0.741006      0.748000\n",
              "f1-score     0.708955    0.758763  ...    0.745650      0.747093\n",
              "support    143.000000  234.000000  ...  500.000000    500.000000\n",
              "\n",
              "[4 rows x 6 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "  Testing with Windwoker Data\n",
        "  "
      ],
      "metadata": {
        "id": "pFYrDBifR7Qv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "path = '/content/drive/MyDrive/Colab Notebooks/models/model_32b_8e_weights.pt'\n",
        "model = torch.load(path)"
      ],
      "metadata": {
        "id": "gfktYT9ztoxY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install datasets\n",
        "from datasets import load_dataset\n",
        "\n",
        "dataset = load_dataset(\"winvoker/turkish-sentiment-analysis-dataset\")"
      ],
      "metadata": {
        "id": "nxDp5xFeSAnO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tweets = []\n",
        "for idx in range(0,len(dataset[\"train\"])):\n",
        "  line = dataset[\"train\"][idx]\n",
        "  if line[\"dataset\"] == \"tweet-pn\":\n",
        "    tweets.append(line)\n",
        "for idx in range(0,len(dataset[\"test\"])):\n",
        "  line = dataset[\"test\"][idx]\n",
        "  if line[\"dataset\"] == \"tweet-pn\":\n",
        "    tweets.append(line)\n"
      ],
      "metadata": {
        "id": "njSsO4UjpzcS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "list_dict_tweets = []\n",
        "num_test = len(dataset[\"test\"])\n",
        "for idx in range(0,num_test):\n",
        "  line = dataset[\"test\"][idx]\n",
        "  if line[\"dataset\"] == \"tweet-pn\" :\n",
        "    list_dict_tweets.append(line)\n",
        "num_train = len(dataset[\"train\"])\n",
        "for idx in range(0,num_train):\n",
        "  line = dataset[\"train\"][idx]\n",
        "  if line[\"dataset\"] == \"tweet-pn\" :\n",
        "    list_dict_tweets.append(line)\n",
        "\n",
        "list_dict_wiki = []\n",
        "num_test = len(dataset[\"test\"])\n",
        "for idx in range(0,num_test):\n",
        "  line = dataset[\"test\"][idx]\n",
        "  if line[\"dataset\"] == \"wiki\" :\n",
        "    list_dict_wiki.append(line)\n",
        "num_train = len(dataset[\"train\"])\n",
        "for idx in range(0,num_train):\n",
        "  line = dataset[\"train\"][idx]\n",
        "  if line[\"dataset\"] == \"wiki\" :\n",
        "    list_dict_wiki.append(line)\n",
        "\n"
      ],
      "metadata": {
        "id": "b0sWne27SSqh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "n=9000\n",
        "list_dict_wiki = random.sample(list_dict_wiki, n)\n",
        "list_dict = list_dict_tweets + list_dict_wiki\n",
        "windwoker_df = pd.DataFrame(list_dict)\n",
        "s = pd.Series(windwoker_df[\"label\"])\n",
        "windwoker_df[\"label\"] = s.map({'Negative': 0, \"Notr\": 1,\"Positive\": 2})\n",
        "test_texts = windwoker_df[\"text\"].values\n",
        "test_labels = windwoker_df[\"label\"].values"
      ],
      "metadata": {
        "id": "WlIYFYnslc1F"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_ids = []\n",
        "attention_masks = []\n",
        "\n",
        "for text in test_texts:\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        text,                     \n",
        "                        add_special_tokens = True, \n",
        "                        max_length = max_len,          \n",
        "                        pad_to_max_length = True,\n",
        "                        return_attention_mask = True,  \n",
        "                        return_tensors = 'pt',   \n",
        "                   )\n",
        "    \n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "labels = torch.tensor(test_labels)\n",
        "\n",
        "batch_size = 32\n",
        "\n",
        "prediction_data = TensorDataset(input_ids, attention_masks, labels)\n",
        "prediction_sampler = SequentialSampler(prediction_data)\n",
        "prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uw1WeeGgcSQz",
        "outputId": "5d9b5323-4bf5-4b53-9018-7035dc72598a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2232: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Prediction started on test data')\n",
        "model.eval()\n",
        "predictions , true_labels = [], []\n",
        "\n",
        "for batch in prediction_dataloader:\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "  b_input_ids, b_input_mask, b_labels = batch\n",
        "\n",
        "  with torch.no_grad():\n",
        "      outputs = model(b_input_ids, token_type_ids=None, \n",
        "                      attention_mask=b_input_mask)\n",
        "\n",
        "  logits = outputs[0]\n",
        "  logits = logits.detach().cpu().numpy()\n",
        "  label_ids = b_labels.to('cpu').numpy()\n",
        "  \n",
        "  predictions.append(logits)\n",
        "  true_labels.append(label_ids)\n",
        "\n",
        "print('Prediction completed')\n",
        "\n",
        "prediction_set = []\n",
        "\n",
        "for i in range(len(true_labels)):\n",
        "  pred_labels_i = np.argmax(predictions[i], axis=1).flatten()\n",
        "  prediction_set.append(pred_labels_i)\n",
        "\n",
        "prediction_scores = [item for sublist in prediction_set for item in sublist]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "483NWdiyck0H",
        "outputId": "52305943-5814-4843-ec49-09ebe1e00dfb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Prediction started on test data\n",
            "Prediction completed\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import recall_score\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import classification_report\n",
        "\n",
        "f_score = f1_score(test_labels, prediction_scores, average='macro')\n",
        "precision = precision_score(test_labels, prediction_scores, average='macro')\n",
        "recall = recall_score(test_labels, prediction_scores, average='macro')\n",
        "\n",
        "print(\"F-Score: \", f_score)\n",
        "print(\"Recall: \", recall)\n",
        "print(\"Precision: \", precision)\n",
        "\n",
        "report = pd.DataFrame(classification_report(test_labels, prediction_scores, output_dict=True))\n",
        "report = report.rename(columns={'0':'negative',\n",
        "                          '1':'neutral',\n",
        "                          '2':'positive',\n",
        "                          })\n",
        "\n",
        "report"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227
        },
        "id": "LP6x6sCTcmCr",
        "outputId": "4f0fa46e-aa7e-434a-fa47-6ad4d6d08e99"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F-Score:  0.5684871430880851\n",
            "Recall:  0.6132070273572121\n",
            "Precision:  0.6772099874994443\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-6f9be74d-8855-4d45-ad74-11a6a48e3007\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>negative</th>\n",
              "      <th>neutral</th>\n",
              "      <th>positive</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>macro avg</th>\n",
              "      <th>weighted avg</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>precision</th>\n",
              "      <td>0.637882</td>\n",
              "      <td>0.648084</td>\n",
              "      <td>0.745665</td>\n",
              "      <td>0.65096</td>\n",
              "      <td>0.677210</td>\n",
              "      <td>0.675174</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>recall</th>\n",
              "      <td>0.761272</td>\n",
              "      <td>0.930000</td>\n",
              "      <td>0.148349</td>\n",
              "      <td>0.65096</td>\n",
              "      <td>0.613207</td>\n",
              "      <td>0.650960</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>f1-score</th>\n",
              "      <td>0.694136</td>\n",
              "      <td>0.763860</td>\n",
              "      <td>0.247465</td>\n",
              "      <td>0.65096</td>\n",
              "      <td>0.568487</td>\n",
              "      <td>0.589854</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>support</th>\n",
              "      <td>4968.000000</td>\n",
              "      <td>9000.000000</td>\n",
              "      <td>6087.000000</td>\n",
              "      <td>0.65096</td>\n",
              "      <td>20055.000000</td>\n",
              "      <td>20055.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6f9be74d-8855-4d45-ad74-11a6a48e3007')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6f9be74d-8855-4d45-ad74-11a6a48e3007 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6f9be74d-8855-4d45-ad74-11a6a48e3007');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "              negative      neutral  ...     macro avg  weighted avg\n",
              "precision     0.637882     0.648084  ...      0.677210      0.675174\n",
              "recall        0.761272     0.930000  ...      0.613207      0.650960\n",
              "f1-score      0.694136     0.763860  ...      0.568487      0.589854\n",
              "support    4968.000000  9000.000000  ...  20055.000000  20055.000000\n",
              "\n",
              "[4 rows x 6 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Find the texts that fools the model Error Analysis\n",
        "\n"
      ],
      "metadata": {
        "id": "gNpvg0WFrU0T"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "length = len(prediction_scores)\n",
        "index_list = []\n",
        "for idx in range(0,length):\n",
        "  if prediction_scores[idx] != test_labels[idx]:\n",
        "    index_list.append(idx)\n",
        "\n",
        "miss_predict_df = windwoker_df.iloc[index_list]\n",
        "miss_predict_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "jwzq0HrOrJst",
        "outputId": "38633ccb-6d83-4f71-d063-a8c1f11cb2b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-ba262bef-23e9-4c0b-bd6f-379395044b14\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "      <th>dataset</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Kesin öyle olmuştur. Çocukla kız el ele yolda ...</td>\n",
              "      <td>2</td>\n",
              "      <td>tweet-pn</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>ablacım reis halka üç yüz metre mesafeden sesl...</td>\n",
              "      <td>2</td>\n",
              "      <td>tweet-pn</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>Bir yazar ülkesine tarihine sosyolojisine nas...</td>\n",
              "      <td>2</td>\n",
              "      <td>tweet-pn</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>Öngörün gerçekleşti maalesef</td>\n",
              "      <td>2</td>\n",
              "      <td>tweet-pn</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>ananem ben küçükken sabah kalkınca hemen yüzün...</td>\n",
              "      <td>2</td>\n",
              "      <td>tweet-pn</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20004</th>\n",
              "      <td>Eski çarşısı görülmeye değerdir .</td>\n",
              "      <td>1</td>\n",
              "      <td>wiki</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20014</th>\n",
              "      <td>Mimarisi yoğun bir Osmanlı karakteri taşımakta...</td>\n",
              "      <td>1</td>\n",
              "      <td>wiki</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20040</th>\n",
              "      <td>Aksi halde arabası elinden alınacaktır .</td>\n",
              "      <td>1</td>\n",
              "      <td>wiki</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20048</th>\n",
              "      <td>Partizanlar ve partizan olduğu düşünülen kişil...</td>\n",
              "      <td>1</td>\n",
              "      <td>wiki</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20050</th>\n",
              "      <td>Bununla birlikte bu \"Simurg\" sözcüğünün doğru ...</td>\n",
              "      <td>1</td>\n",
              "      <td>wiki</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>7000 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ba262bef-23e9-4c0b-bd6f-379395044b14')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ba262bef-23e9-4c0b-bd6f-379395044b14 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ba262bef-23e9-4c0b-bd6f-379395044b14');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                    text  label   dataset\n",
              "6      Kesin öyle olmuştur. Çocukla kız el ele yolda ...      2  tweet-pn\n",
              "11     ablacım reis halka üç yüz metre mesafeden sesl...      2  tweet-pn\n",
              "13      Bir yazar ülkesine tarihine sosyolojisine nas...      2  tweet-pn\n",
              "16                          Öngörün gerçekleşti maalesef      2  tweet-pn\n",
              "17     ananem ben küçükken sabah kalkınca hemen yüzün...      2  tweet-pn\n",
              "...                                                  ...    ...       ...\n",
              "20004                  Eski çarşısı görülmeye değerdir .      1      wiki\n",
              "20014  Mimarisi yoğun bir Osmanlı karakteri taşımakta...      1      wiki\n",
              "20040           Aksi halde arabası elinden alınacaktır .      1      wiki\n",
              "20048  Partizanlar ve partizan olduğu düşünülen kişil...      1      wiki\n",
              "20050  Bununla birlikte bu \"Simurg\" sözcüğünün doğru ...      1      wiki\n",
              "\n",
              "[7000 rows x 3 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pos = miss_predict_df[miss_predict_df[\"label\"]==2]\n",
        "len(pos)"
      ],
      "metadata": {
        "id": "CU-d4GXvtBQz"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}