Pytorch_RNN_Toxic_Comment3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import spacy\n",
    "from torchtext.data.utils import get_tokenizer\n",
    "# from nltk import tokenizer\n",
    "from torch.utils.data import DataLoader,Dataset,TensorDataset\n",
    "import torchtext\n",
    "import re\n",
    "import random\n",
    "import os\n",
    "os.environ['KMP_DUPLICATE_LIB_OK']='True'\n",
    "from torchtext.data import Field,BucketIterator,TabularDataset,LabelField"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.read_csv(\"../Scripts/NLP_Toxic-comment-classification-challenge/rnn_train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer=get_tokenizer(\"basic_english\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def spacy_tokenizer(text):\n",
    "#     return [tok.text for tok in tokenize(text)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# text=Field(tokenize=tokenizer,lower=True,include_lengths=True,init_token=\"<sos>\",eos_token=\"<eos>\")\n",
    "# label=LabelField(dtype=torch.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT=Field(tokenize=tokenizer,lower=True,include_lengths=True,batch_first=True, fix_length=200,use_vocab=True)\n",
    "LABEL=LabelField(dtype=torch.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "class DataFrameDataset(Dataset):\n",
    "\n",
    "    def __init__(self, df, fields, is_test=False, **kwargs):\n",
    "        examples = []\n",
    "        for i, row in df.iterrows():\n",
    "            label = row.target if not is_test else None\n",
    "            text = row.text\n",
    "            examples.append(data.Example.fromlist([text, label], fields))\n",
    "\n",
    "        super().__init__(examples, fields, **kwargs)\n",
    "\n",
    "    @staticmethod\n",
    "    def sort_key(ex):\n",
    "        return len(ex.text)\n",
    "\n",
    "    @classmethod\n",
    "    def splits(cls, fields, train_df, val_df=None, test_df=None, **kwargs):\n",
    "        train_data, val_data, test_data = (None, None, None)\n",
    "        data_field = fields\n",
    "\n",
    "        if train_df is not None:\n",
    "            train_data = cls(train_df.copy(), data_field, **kwargs)\n",
    "        if val_df is not None:\n",
    "            val_data = cls(val_df.copy(), data_field, **kwargs)\n",
    "        if test_df is not None:\n",
    "            test_data = cls(test_df.copy(), data_field, True, **kwargs)\n",
    "\n",
    "        return tuple(d for d in (train_data, val_data, test_data) if d is not None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "fields=[(\"text\",TEXT),(\"label\",LABEL)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': ['sr', 'cl', 'you', 'bud', 'bin', 'a', 'way', '.', 'like', 'long', 'time', 'like', 'time', 'outa', 'mind', 'long', '.', 'gooood', 'shit', 'but', 'mo', \"'\", 'latah', 'an', 'dat', '.', 'soon', 'az', 'ah', 'gitz', 'back', 'heah', 'sum', 'a', 'de', 'brotherz', \"'\", 'n', 'sistahs', 'sez', 'hance', 'dead', 'or', 'alive', '?', 'we', 'are', 'about', 'to', 'send', 'a', 'zion', 'ranger', 'to', 'check', 'on', 'his', 'condition', '.', 'ah', 'sez', 'whoa', 'led', 'me', 'gib', 'him', 'a', 'holler', '.', 'dis', 'me', 'hollerin', 'bro', '.', 'de', 'zion', 'rangerz', 'be', 'dem', 'dat', 'do', 'most', 'a', \"'\", 'de', 'walkin', 'tru', 'wallz', '.', 'not', 'all', 'but', 'most', '.', 'de', 'wall', 'walkin', 'dude', 'ain', 'going', 'to', 'do', 'r', 'bro', 'no', 'harm', ',', 'he', 'jes', 'be', 'checkin', '.', 'ee', 'ben', 'so', 'a', 'gentle', 'remindah', 'kum', 'fuhst', '.', 'you', 'be', 'readin', 'i', 'd', '.', 'a', 'leedle', 'postin', 'a', 'dis', \"'\", 'n', 'dat', 'go', 'down', 'good', '.', 'doan', 'fohget', 'to', 'cut', 'sum', 'zzzz', 'aftah', 'a', 'tad', 'a', 'postin', '.', 'sleep', 'good', \"'\", 'n', 'doan', 'fergit', 'ta', 'dream', '.', 'sumptin', 'might', 'kum', 'jes', 'ta', 'bdh', 'all', 'wrapt', 'roun', 'wid', 'delta', 'wave', \"'\", 'n', 'a', 'kupla', 'otha', \"'\", 'carrierz', 'what', 'we', 'diz', 'cuss', 'latah', '.', 'chow', ',', 'bro', '.'], 'label': '1'}\n"
     ]
    }
   ],
   "source": [
    "training_data=TabularDataset(path=\"../Scripts/NLP_Toxic-comment-classification-challenge/rnn_train.csv\",\n",
    "                            format=\"csv\",fields=fields,skip_header=True)\n",
    "\n",
    "print(vars(training_data[4]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_ds, val_ds = DataFrameDataset.splits(fields, train_df=train_df, val_df=valid_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data,valid_data=training_data.split(split_ratio=0.7,random_state=random.seed(2020))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1400, 600)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_data),len(valid_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "device=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT.build_vocab(train_data,min_freq=3,vectors = \"glove.6B.100d\")  \n",
    "LABEL.build_vocab(train_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3121"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(TEXT.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT.vocab.stoi[\"is\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_iter,valid_iter=BucketIterator.splits((train_data,valid_data),\n",
    "                                         sort_key=lambda x:len(x.text),\n",
    "                                         batch_size=64,\n",
    "                                         sort_within_batch=True,\n",
    "                                         device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 73,  23,  12, 910, 193,   3,  27,   4,  84,  28,  17,  78,   7,   0,\n",
      "          0,   0,   2,  33,   4, 441,   7,  23,  41,   3,   4,  37,  29, 223,\n",
      "         54, 171,   2,   0,   0,   0,   1,   1,   1,   1,   1,   1,   1,   1,\n",
      "          1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,\n",
      "          1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,\n",
      "          1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,\n",
      "          1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,\n",
      "          1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,\n",
      "          1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,\n",
      "          1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,\n",
      "          1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,\n",
      "          1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,\n",
      "          1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,\n",
      "          1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1,\n",
      "          1,   1,   1,   1])\n"
     ]
    }
   ],
   "source": [
    "for i in train_iter:\n",
    "    print(i.text[0][0])\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('the', '<unk>', 'people', \"'\", 's')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT.vocab.itos[5],TEXT.vocab.itos[0],TEXT.vocab.itos[83],TEXT.vocab.itos[31],TEXT.vocab.itos[44]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<pad>'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT.vocab.itos[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "i=next(iter(train_iter))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['scum',\n",
       " 'of',\n",
       " 'the',\n",
       " 'earth',\n",
       " 'vandal',\n",
       " '.',\n",
       " 'do',\n",
       " 'not',\n",
       " 'change',\n",
       " 'my',\n",
       " 'properly',\n",
       " 'sourced',\n",
       " 'edits',\n",
       " 'you',\n",
       " 'stupid',\n",
       " 'prick',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>',\n",
       " '<pad>']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b=[TEXT.vocab.itos[x] for x in i.text[0][0]]\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('.', 6665), (',', 3687), ('you', 3496), ('the', 3349), ('i', 2954), ('to', 2373), ('a', 1835), ('and', 1758), ('is', 1735), ('of', 1686)]\n"
     ]
    }
   ],
   "source": [
    "print(TEXT.vocab.freqs.most_common(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3121"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(TEXT.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTM(nn.Module):\n",
    "    def __init__(self,vocab_size,embedding_dim,hidden_dim,n_layers,output_dim,dropout):\n",
    "        #Constructor\n",
    "        super().__init__()          \n",
    "        \n",
    "        self.dropout=nn.Dropout(0.3)\n",
    "        #embedding layer\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        \n",
    "        \n",
    "        #lstm layer\n",
    "        self.lstm = nn.LSTM(embedding_dim, \n",
    "                           hidden_dim, \n",
    "                           num_layers=n_layers, \n",
    "                           bidirectional=True, \n",
    "                           dropout=dropout,\n",
    "                           batch_first=True)\n",
    "        \n",
    "        #dense layer\n",
    "        self.fc = nn.Linear(hidden_dim * 2, output_dim)\n",
    "        \n",
    "        #activation function\n",
    "        self.act = nn.Sigmoid()\n",
    "        \n",
    "    def forward(self, text, text_lengths):\n",
    "        \n",
    "        text=self.dropout(text)\n",
    "        #text = [batch size,sent_length]\n",
    "        embedded = self.embedding(text)\n",
    "        #embedded = [batch size, sent_len, emb dim]\n",
    "      \n",
    "        #packed sequence\n",
    "        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths,batch_first=True)\n",
    "        \n",
    "        packed_output, (hidden, cell) = self.lstm(packed_embedded)\n",
    "        #hidden = [batch size, num layers * num directions,hid dim]\n",
    "        #cell = [batch size, num layers * num directions,hid dim]\n",
    "        \n",
    "        #concat the final forward and backward hidden state\n",
    "        hidden = torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1)\n",
    "                \n",
    "        #hidden = [batch size, hid dim * num directions]\n",
    "        dense_outputs=F.relu(self.fc(hidden))\n",
    "\n",
    "        #Final activation function\n",
    "        outputs=self.act(dense_outputs)\n",
    "        \n",
    "        return outputs\n",
    "        \n",
    "\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LSTM(\n",
       "  (dropout): Dropout(p=0.3, inplace=False)\n",
       "  (embedding): Embedding(3121, 100)\n",
       "  (lstm): LSTM(100, 32, num_layers=2, batch_first=True, dropout=0.2, bidirectional=True)\n",
       "  (fc): Linear(in_features=64, out_features=1, bias=True)\n",
       "  (act): Sigmoid()\n",
       ")"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_size=len(TEXT.vocab)\n",
    "embedding_dim=100\n",
    "hidden_dim=32\n",
    "n_layers=2\n",
    "output_dim=1\n",
    "dropout=0.2\n",
    "model=LSTM(vocab_size,embedding_dim,hidden_dim,n_layers,output_dim,dropout)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model has 371,557 trainable parameters\n",
      "torch.Size([3121, 100])\n"
     ]
    }
   ],
   "source": [
    "#No. of trianable parameters\n",
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "    \n",
    "print(f'The model has {count_parameters(model):,} trainable parameters')\n",
    "pretrained_embeddings=TEXT.vocab.vectors\n",
    "model.embedding.weight.data.copy_(pretrained_embeddings)\n",
    "\n",
    "print(pretrained_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer=torch.optim.Adam(model.parameters())\n",
    "criterion=nn.BCEWithLogitsLoss()\n",
    "\n",
    "def binary_accuracy(preds, y):\n",
    "    #round predictions to the closest integer\n",
    "    rounded_preds = torch.round(torch.sigmoid(preds))\n",
    "    \n",
    "    correct = (rounded_preds == y).float() \n",
    "    acc = correct.sum() / len(correct)\n",
    "    return acc\n",
    "    \n",
    "    \n",
    "model=model.to(device)\n",
    "criterion=criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for batch in train_iter:\n",
    "    text,text_len=batch.text\n",
    "    label=batch.label\n",
    "    \n",
    "    output=model(text,text_len)\n",
    "    rounded_preds = torch.round(torch.sigmoid(output))\n",
    "    \n",
    "    correct = (rounded_preds == label).float()\n",
    "    acc = correct.sum() / len(correct)\n",
    "    \n",
    "    print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion):\n",
    "    \n",
    "    #initialize every epoch \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    #set the model in training phase\n",
    "    model.train()  \n",
    "    \n",
    "    for batch in iterator:\n",
    "        \n",
    "        #resets the gradients after every batch\n",
    "        optimizer.zero_grad()   \n",
    "        \n",
    "        #retrieve text and no. of words\n",
    "        text, text_lengths = batch.text   \n",
    "        \n",
    "        #convert to 1D tensor\n",
    "        predictions = model(text, text_lengths).squeeze(1)  \n",
    "        \n",
    "        #compute the loss\n",
    "        loss = criterion(predictions, batch.label)        \n",
    "        \n",
    "        #compute the binary accuracy\n",
    "        acc = binary_accuracy(predictions, batch.label)   \n",
    "        \n",
    "        #backpropage the loss and compute the gradients\n",
    "        loss.backward()       \n",
    "        \n",
    "        #update the weights\n",
    "        optimizer.step()      \n",
    "        \n",
    "        #loss and accuracy\n",
    "        epoch_loss += loss.item()  \n",
    "        epoch_acc += acc.item()    \n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, criterion):\n",
    "    \n",
    "    #initialize every epoch\n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "\n",
    "    #deactivating dropout layers\n",
    "    model.eval()\n",
    "    \n",
    "    #deactivates autograd\n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for batch in iterator:\n",
    "        \n",
    "            #retrieve text and no. of words\n",
    "            text, text_lengths = batch.text\n",
    "            \n",
    "            #convert to 1d tensor\n",
    "            predictions = model(text, text_lengths).squeeze(1)\n",
    "            \n",
    "            #compute loss and accuracy\n",
    "            loss = criterion(predictions, batch.label)\n",
    "            acc = binary_accuracy(predictions, batch.label)\n",
    "            \n",
    "            #keep track of loss and accuracy\n",
    "            epoch_loss += loss.item()\n",
    "            epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_EPOCHS = 5\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "     \n",
    "    #train the model\n",
    "    train_loss, train_acc = train(model, train_iter, optimizer, criterion)\n",
    "    valid_loss, valid_acc = evaluate(model, valid_iter, criterion)\n",
    "    \n",
    "    #save the best model\n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), '../Scripts/NLP_Toxic-comment-classification-challenge/saved_weights2.pt')\n",
    "    \n",
    "    \n",
    "    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
    "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "path='../Scripts/NLP_Toxic-comment-classification-challenge/saved_weights2.pt'\n",
    "model.load_state_dict(torch.load(path));\n",
    "model.eval();\n",
    "\n",
    "#inference \n",
    "import spacy\n",
    "nlp =spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "def predict(model, sentence):\n",
    "    tokenized = [tok.text for tok in nlp.tokenizer(sentence)]  #tokenize the sentence \n",
    "    indexed = [TEXT.vocab.stoi[t] for t in tokenized]          #convert to integer sequence\n",
    "    length = [len(indexed)]                                    #compute no. of words\n",
    "    tensor = torch.LongTensor(indexed).to(device)              #convert to tensor\n",
    "    tensor = tensor.unsqueeze(1).T                             #reshape in form of batch,no. of words\n",
    "    length_tensor = torch.LongTensor(length)                   #convert to tensor\n",
    "    prediction = model(tensor, length_tensor)                  #prediction \n",
    "    return prediction.item()      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.02778860181570053\n"
     ]
    }
   ],
   "source": [
    "print(predict(model,\"fuck you people . the information of that can be found here\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adding DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\parikalp\\appdata\\local\\programs\\python\\lib\\site-packages\\torch\\nn\\modules\\loss.py:498: UserWarning: Using a target size (torch.Size([64])) that is different to the input size (torch.Size([64, 1])) is deprecated. Please ensure they have the same size.\n",
      "  return F.binary_cross_entropy(input, target, weight=self.weight, reduction=self.reduction)\n",
      "c:\\users\\parikalp\\appdata\\local\\programs\\python\\lib\\site-packages\\torch\\nn\\modules\\loss.py:498: UserWarning: Using a target size (torch.Size([56])) that is different to the input size (torch.Size([56, 1])) is deprecated. Please ensure they have the same size.\n",
      "  return F.binary_cross_entropy(input, target, weight=self.weight, reduction=self.reduction)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 1, train accuracy : 40.25, train loss : 0.6873973608016968,\n",
      "Epoch : 2, train accuracy : 34.5, train loss : 0.6586592197418213,\n",
      "Epoch : 3, train accuracy : 33.09375, train loss : 0.5151001214981079,\n",
      "Epoch : 4, train accuracy : 33.0, train loss : 0.4264102876186371,\n",
      "Epoch : 5, train accuracy : 38.90625, train loss : 0.336434006690979,\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(5):\n",
    "    epoch_loss=0\n",
    "    epoch_acc=0\n",
    "    model.train()\n",
    "    for batch in train_iter:\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        text,text_length=batch.text\n",
    "        \n",
    "        output=model(text,text_length)\n",
    "        \n",
    "        loss=criterion(output,batch.label)\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "#         acc=((output.argmax(1) == batch.label).float().mean())\n",
    "        \n",
    "#         epoch_acc+=acc / len(acc)\n",
    "        rounded_preds = torch.round(output)\n",
    "    \n",
    "        correct = (rounded_preds == batch.label).float() \n",
    "        acc = correct.sum() / len(correct)\n",
    "        \n",
    "        epoch_loss+=loss / len(train_iter)\n",
    "        \n",
    "    print('Epoch : {}, train accuracy : {}, train loss : {},'.format(epoch+1, acc,epoch_loss,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}