Pytorch_RNN_Twitter_Classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import torch\n",
    "from torchtext import data\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import spacy\n",
    "from spacy.tokenizer import Tokenizer\n",
    "from torchtext.data import Field,TabularDataset,BucketIterator\n",
    "import time\n",
    "from torch.nn.utils.rnn import pack_padded_sequence\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>our deeds are the reason of this earthquake ma...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>forest fire near la ronge sask canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>all residents asked to shelter in place are be...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13000 people receive wildfires evacuation orde...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>just got sent this photo from ruby alaska as s...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  target\n",
       "0  our deeds are the reason of this earthquake ma...       1\n",
       "1              forest fire near la ronge sask canada       1\n",
       "2  all residents asked to shelter in place are be...       1\n",
       "3  13000 people receive wildfires evacuation orde...       1\n",
       "4  just got sent this photo from ruby alaska as s...       1"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv(\"nlp-getting-started/real_train.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.drop([\"Unnamed: 0\",\"id\"],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.to_csv(\"nlp-getting-started/real_train.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.drop([\"Unnamed: 0\"],inplace=True,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>our deeds are the reason of this earthquake ma...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>forest fire near la ronge sask canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>all residents asked to shelter in place are be...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13000 people receive wildfires evacuation orde...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>just got sent this photo from ruby alaska as s...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  target\n",
       "0  our deeds are the reason of this earthquake ma...       1\n",
       "1              forest fire near la ronge sask canada       1\n",
       "2  all residents asked to shelter in place are be...       1\n",
       "3  13000 people receive wildfires evacuation orde...       1\n",
       "4  just got sent this photo from ruby alaska as s...       1"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp=spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "tokenizer=Tokenizer(nlp.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "def spacy_tokenizer(x):\n",
    "    return [tok.text for tok in tokenizer(x)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT=Field(sequential=False,use_vocab=False,batch_first=True,dtype=torch.float)\n",
    "LABEL=Field(tokenize=spacy_tokenizer,lower=True,include_lengths=True,batch_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "fields=[(\"index\",LABEL),(\"text\",TEXT),(\"targets\",LABEL)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': '3', 'targets': ['13000', 'people', 'receive', 'wildfires', 'evacuation', 'orders', 'in', 'california']}\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df,valid_df=train_test_split(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT2 = data.Field(tokenize = spacy_tokenizer, include_lengths = True)\n",
    "LABEL2 = data.LabelField(dtype = torch.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "fields2 = [('text',TEXT2), ('label',LABEL2)]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataFrameDataset(data.Dataset):\n",
    "\n",
    "    def __init__(self, df, fields, is_test=False, **kwargs):\n",
    "        examples = []\n",
    "        for i, row in df.iterrows():\n",
    "            label = row.target if not is_test else None\n",
    "            text = row.text\n",
    "            examples.append(data.Example.fromlist([text, label], fields))\n",
    "\n",
    "        super().__init__(examples, fields, **kwargs)\n",
    "\n",
    "    @staticmethod\n",
    "    def sort_key(ex):\n",
    "        return len(ex.text)\n",
    "\n",
    "    @classmethod\n",
    "    def splits(cls, fields, train_df, val_df=None, test_df=None, **kwargs):\n",
    "        train_data, val_data, test_data = (None, None, None)\n",
    "        data_field = fields\n",
    "\n",
    "        if train_df is not None:\n",
    "            train_data = cls(train_df.copy(), data_field, **kwargs)\n",
    "        if val_df is not None:\n",
    "            val_data = cls(val_df.copy(), data_field, **kwargs)\n",
    "        if test_df is not None:\n",
    "            test_data = cls(test_df.copy(), data_field, True, **kwargs)\n",
    "\n",
    "        return tuple(d for d in (train_data, val_data, test_data) if d is not None)\n",
    "# We will first create"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds, val_ds = DataFrameDataset.splits(fields2, train_df=train_df, val_df=valid_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': ['join', 'charity', '10k', 'run', 'event', 'doningtondash', '11am', 'start', 'sun', '20', 'sept', '2015', 'castle', 'donington', 'community', 'first', 'responders'], 'label': 0}\n",
      "<class 'torchtext.data.example.Example'>\n"
     ]
    }
   ],
   "source": [
    "print(vars(train_ds[13]))\n",
    "\n",
    "print(type(train_ds[15]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': ['all', 'residents', 'asked', 'to', 'shelter', 'in', 'place', 'are', 'being', 'notified', 'by', 'officers', 'no', 'other', 'evacuation', 'or', 'shelter', 'in', 'place', 'orders', 'are', 'expected'], 'label': '1'}\n"
     ]
    }
   ],
   "source": [
    "training_data=TabularDataset(path=\"nlp-getting-started/real_train.csv\",format=\"csv\",fields=fields2,skip_header=False)\n",
    "\n",
    "print(vars(training_data.examples[3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': ['our', 'deeds', 'are', 'the', 'reason', 'of', 'this', 'earthquake', 'may', 'allah', 'forgive', 'us', 'all'], 'label': '1'}\n",
      "{'text': ['forest', 'fire', 'near', 'la', 'ronge', 'sask', 'canada'], 'label': '1'}\n",
      "{'text': ['all', 'residents', 'asked', 'to', 'shelter', 'in', 'place', 'are', 'being', 'notified', 'by', 'officers', 'no', 'other', 'evacuation', 'or', 'shelter', 'in', 'place', 'orders', 'are', 'expected'], 'label': '1'}\n"
     ]
    }
   ],
   "source": [
    "for i in range(3):\n",
    "    print(vars(training_data.examples[i+1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>our deeds are the reason of this earthquake ma...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>forest fire near la ronge sask canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>all residents asked to shelter in place are be...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13000 people receive wildfires evacuation orde...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>just got sent this photo from ruby alaska as s...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  target\n",
       "0  our deeds are the reason of this earthquake ma...       1\n",
       "1              forest fire near la ronge sask canada       1\n",
       "2  all residents asked to shelter in place are be...       1\n",
       "3  13000 people receive wildfires evacuation orde...       1\n",
       "4  just got sent this photo from ruby alaska as s...       1"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2=pd.read_csv(\"nlp-getting-started/real_train.csv\")\n",
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT3=Field(sequential=spacy_tokenizer,include_lengths=True)\n",
    "LABEL3=data.LabelField(dtype=torch.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "fields3=[(\"text\",TEXT3),(\"label\",LABEL3)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': ['13000', 'people', 'receive', 'wildfires', 'evacuation', 'orders', 'in', 'california'], 'label': '1'}\n"
     ]
    }
   ],
   "source": [
    "training_data3=TabularDataset(path=\"nlp-getting-started/real_train.csv\",format=\"csv\",fields=fields3,skip_header=False)\n",
    "\n",
    "print(vars(training_data3.examples[4]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_VOCAB_SIZE = 25000\n",
    "\n",
    "TEXT2.build_vocab(train_ds, \n",
    "                 max_size = MAX_VOCAB_SIZE, \n",
    "                 vectors = 'glove.6B.200d',\n",
    "                 unk_init = torch.Tensor.zero_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "LABEL2.build_vocab(train_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "device=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_iterator,valid_iterator=BucketIterator.splits((train_ds,val_ds),batch_size=128,\n",
    "                                                   sort_within_batch=True,device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(45, 15)"
      ]
     },
     "execution_count": 305,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_iterator),len(valid_iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs=25\n",
    "learning_rate=0.001\n",
    "\n",
    "vocab_size=len(TEXT2.vocab)\n",
    "Embedding_dim=200\n",
    "hidden_dim=256\n",
    "output_dim=1\n",
    "n_layers=3\n",
    "biderectional=True\n",
    "dropout=0.2\n",
    "PAD_IDX = TEXT2.vocab.stoi[TEXT2.pad_token]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTM_net(nn.Module):\n",
    "    def __init__(self,vocab_size,Embedding_dim,hidden_dim,output_dim,n_layers,biderectional,dropout,PAD_IDX):\n",
    "        super(LSTM_net,self).__init__()\n",
    "        self.embedding=nn.Embedding(vocab_size,Embedding_dim,padding_idx=PAD_IDX)\n",
    "        self.lstm=nn.LSTM(Embedding_dim,hidden_dim,n_layers,bidirectional=biderectional,dropout=dropout)\n",
    "        self.fc1=nn.Linear(hidden_dim*2,hidden_dim)\n",
    "        self.fc2=nn.Linear(hidden_dim,1)\n",
    "        self.dropout=nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self,text,text_length):\n",
    "        # text = [sent len, batch size]\n",
    "        embedding=self.embedding(text)\n",
    "        # embedded = [sent len, batch size, emb dim]\n",
    "        packed_embedded=pack_padded_sequence(embedding,text_length)\n",
    "        \n",
    "        packed_output,(hidden,cell)=self.lstm(packed_embedded)\n",
    "        #unpack sequence\n",
    "        # output, output_lengths = nn.utils.rnn.pad_packed_sequence(packed_output)\n",
    "\n",
    "        # output = [sent len, batch size, hid dim * num directions]\n",
    "        # output over padding tokens are zero tensors\n",
    "        \n",
    "        # hidden = [num layers * num directions, batch size, hid dim]\n",
    "        # cell = [num layers * num directions, batch size, hid dim]\n",
    "        \n",
    "        # concat the final forward (hidden[-2,:,:]) and backward (hidden[-1,:,:]) hidden layers\n",
    "        # and apply dropout\n",
    "        \n",
    "        hidden=self.dropout(torch.cat((hidden[-2,:,:],hidden[-1,:,:]),dim=1))\n",
    "        output=self.fc1(hidden)\n",
    "        output=self.dropout(self.fc2(output))\n",
    "        \n",
    "        return output\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LSTM_net(\n",
       "  (embedding): Embedding(15002, 200, padding_idx=1)\n",
       "  (lstm): LSTM(200, 256, num_layers=3, dropout=0.2, bidirectional=True)\n",
       "  (fc1): Linear(in_features=512, out_features=256, bias=True)\n",
       "  (fc2): Linear(in_features=256, out_features=1, bias=True)\n",
       "  (dropout): Dropout(p=0.2, inplace=False)\n",
       ")"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model=LSTM_net(vocab_size,Embedding_dim,hidden_dim,output_dim,n_layers,biderectional,dropout,PAD_IDX)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0., 0.]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pretrained_embeddings=TEXT.vocab.vectors\n",
    "\n",
    "print(pretrained_embedding)\n",
    "model.embedding.weight.data.copy_(pretrained_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "The expanded size of the tensor (200) must match the existing size (15002) at non-singleton dimension 0.  Target sizes: [200].  Tensor sizes: [15002]",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-203-cdc8a861224a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0membedding\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mPAD_IDX\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_dim\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0membedding\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: The expanded size of the tensor (200) must match the existing size (15002) at non-singleton dimension 0.  Target sizes: [200].  Tensor sizes: [15002]"
     ]
    }
   ],
   "source": [
    "model.embedding.weight.data[PAD_IDX] = torch.zeros(input_dim)\n",
    "\n",
    "print(model.embedding.weight.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion=nn.BCEWithLogitsLoss()\n",
    "\n",
    "optimizer=torch.optim.Adam(model.parameters(),lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "def binary_accuracy(preds, y):\n",
    "    \"\"\"\n",
    "    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n",
    "    \"\"\"\n",
    "\n",
    "    #round predictions to the closest integer\n",
    "    rounded_preds = torch.round(torch.sigmoid(preds))\n",
    "    correct = (rounded_preds == y).float() #convert into float for division \n",
    "    acc = correct.sum() / len(correct)\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training function \n",
    "def train(model, iterator):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    for batch in iterator:\n",
    "        text, text_lengths = batch.text\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        predictions = model(text, text_lengths).squeeze(1)\n",
    "        loss = criterion(predictions, batch.label)\n",
    "        acc = binary_accuracy(predictions, batch.label)\n",
    "\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "        \n",
    "\n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, iterator):\n",
    "    \n",
    "    epoch_acc = 0\n",
    "    model.eval()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for batch in iterator:\n",
    "            text, text_lengths = batch.text\n",
    "            predictions = model(text, text_lengths).squeeze(1)\n",
    "            acc = binary_accuracy(predictions, batch.label)\n",
    "            \n",
    "            epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_acc / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tTrain Loss: 0.474 | Train Acc: 75.90%\n",
      "\t Val. Acc: 79.41%\n",
      "\tTrain Loss: 0.329 | Train Acc: 84.61%\n",
      "\t Val. Acc: 77.14%\n",
      "\tTrain Loss: 0.254 | Train Acc: 87.47%\n",
      "\t Val. Acc: 76.64%\n",
      "\tTrain Loss: 0.234 | Train Acc: 88.57%\n",
      "\t Val. Acc: 76.20%\n",
      "\tTrain Loss: 0.210 | Train Acc: 88.73%\n",
      "\t Val. Acc: 75.55%\n",
      "\tTrain Loss: 0.186 | Train Acc: 90.62%\n",
      "\t Val. Acc: 74.98%\n",
      "\tTrain Loss: 0.191 | Train Acc: 89.70%\n",
      "\t Val. Acc: 75.33%\n",
      "\tTrain Loss: 0.179 | Train Acc: 90.16%\n",
      "\t Val. Acc: 74.99%\n",
      "\tTrain Loss: 0.178 | Train Acc: 90.54%\n",
      "\t Val. Acc: 74.35%\n",
      "\tTrain Loss: 0.167 | Train Acc: 90.65%\n",
      "\t Val. Acc: 73.14%\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-221-00fff5f09667>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnum_epochs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m     \u001b[0mtrain_loss\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_acc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_iterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      9\u001b[0m     \u001b[0mvalid_acc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalid_iterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-219-bff3618a3153>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(model, iterator)\u001b[0m\n\u001b[0;32m     15\u001b[0m         \u001b[0macc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbinary_accuracy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m         \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     18\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\parikalp\\appdata\\local\\programs\\python\\lib\\site-packages\\torch\\tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[0;32m    116\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[1;33m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    117\u001b[0m         \"\"\"\n\u001b[1;32m--> 118\u001b[1;33m         \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    119\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    120\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\parikalp\\appdata\\local\\programs\\python\\lib\\site-packages\\torch\\autograd\\__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[0;32m     91\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[0;32m     92\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 93\u001b[1;33m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     95\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "t = time.time()\n",
    "loss=[]\n",
    "acc=[]\n",
    "val_acc=[]\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    \n",
    "    train_loss, train_acc = train(model, train_iterator)\n",
    "    valid_acc = evaluate(model, valid_iterator)\n",
    "    \n",
    "    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
    "    print(f'\\t Val. Acc: {valid_acc*100:.2f}%')\n",
    "    \n",
    "    loss.append(train_loss)\n",
    "    acc.append(train_acc)\n",
    "    val_acc.append(valid_acc)\n",
    "    \n",
    "print(f'time:{time.time()-t:.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(pred,y):\n",
    "    ran=torch.rand(4)\n",
    "    print(\"random \",ran)\n",
    "    predction=torch.round(ran)\n",
    "    print(predction)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "random  tensor([0.4145, 0.0182, 0.8509, 0.4250])\n",
      "tensor([0., 0., 1., 0.])\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(accuracy(2,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(pred,y):\n",
    "    prediction=torch.round(torch.sigmoid(pred))\n",
    "    correct=(prediction==y).float()\n",
    "    acc=correct.sum() / len(correct)\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(iteration):\n",
    "    epoch_loss=0\n",
    "    epoch_acc=0\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    for batch in iteration:\n",
    "        text,text_length=batch.text\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        prediction=model(text,text_length).squeeze(1)\n",
    "        \n",
    "        loss=criterion(prediction,batch.label)\n",
    "        \n",
    "        acc=accuracy(prediction,batch.label)\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss+=loss.item()\n",
    "        \n",
    "        epoch_acc+=acc.item()\n",
    "        \n",
    "    return epoch_loss / len(iteration) , epoch_acc / len(iteration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evalution(iteration):\n",
    "    \n",
    "    epoch_acc=0\n",
    "    model.eval()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for batch in iteration:\n",
    "            text,text_length=batch.text\n",
    "            \n",
    "            predcition=model(text,text_length).squeeze(1)\n",
    "            \n",
    "            acc=accuracy(predcition,batch.label)\n",
    "            \n",
    "            epoch_acc+=acc.item()\n",
    "            \n",
    "        return epoch_acc / len(iteration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Loss 0.046 | Train Acc 98.39%\n",
      "Valid Loss 0.76%\n",
      "Train Loss 0.047 | Train Acc 98.24%\n",
      "Valid Loss 0.75%\n",
      "Train Loss 0.041 | Train Acc 98.49%\n",
      "Valid Loss 0.75%\n",
      "Train Loss 0.032 | Train Acc 98.54%\n",
      "Valid Loss 0.73%\n",
      "Train Loss 0.030 | Train Acc 98.75%\n",
      "Valid Loss 0.74%\n",
      "Train Loss 0.026 | Train Acc 98.63%\n",
      "Valid Loss 0.75%\n",
      "Train Loss 0.022 | Train Acc 98.75%\n",
      "Valid Loss 0.75%\n",
      "Train Loss 0.022 | Train Acc 98.80%\n",
      "Valid Loss 0.74%\n",
      "Train Loss 0.023 | Train Acc 98.66%\n",
      "Valid Loss 0.74%\n",
      "Train Loss 0.024 | Train Acc 98.68%\n",
      "Valid Loss 0.72%\n",
      "Train Loss 0.022 | Train Acc 98.59%\n",
      "Valid Loss 0.73%\n",
      "Train Loss 0.021 | Train Acc 98.80%\n",
      "Valid Loss 0.72%\n",
      "Train Loss 0.021 | Train Acc 98.84%\n",
      "Valid Loss 0.71%\n",
      "Train Loss 0.028 | Train Acc 98.72%\n",
      "Valid Loss 0.73%\n",
      "Train Loss 0.033 | Train Acc 98.49%\n",
      "Valid Loss 0.75%\n",
      "Train Loss 0.027 | Train Acc 98.59%\n",
      "Valid Loss 0.73%\n",
      "Train Loss 0.022 | Train Acc 98.82%\n",
      "Valid Loss 0.74%\n",
      "Train Loss 0.027 | Train Acc 98.54%\n",
      "Valid Loss 0.72%\n",
      "Train Loss 0.050 | Train Acc 98.17%\n",
      "Valid Loss 0.72%\n",
      "Train Loss 0.042 | Train Acc 98.14%\n",
      "Valid Loss 0.72%\n"
     ]
    }
   ],
   "source": [
    "loss=[]\n",
    "acc=[]\n",
    "val_acc=[]\n",
    "\n",
    "for i in range(20):\n",
    "    train_loss,train_acc=train(train_iterator)\n",
    "    valid_loss=evalution(valid_iterator)\n",
    "    \n",
    "    print(f\"Train Loss {train_loss:.3f} | Train Acc {train_acc*100:.2f}%\")\n",
    "    print(f\"Valid Loss {valid_loss:.2f}%\")\n",
    "    \n",
    "    loss.append(train_loss)\n",
    "    acc.append(train_acc)\n",
    "    val_acc.append(valid_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), \"nlp-getting-started/saved_weights.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [],
   "source": [
    "path=\"nlp-getting-started/saved_weights.pt\"\n",
    "model.load_state_dict(torch.load(path));\n",
    "model.eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(model, sentence):\n",
    "    tokenized = [tok.text for tok in nlp.tokenizer(sentence)]  #tokenize the sentence \n",
    "    indexed = [TEXT2.vocab.stoi[t] for t in tokenized]          #convert to integer sequence\n",
    "    length = [len(indexed)]                                    #compute no. of words\n",
    "    tensor = torch.LongTensor(indexed).to(device)              #convert to tensor\n",
    "    tensor = tensor.unsqueeze(1).T                             #reshape in form of batch,no. of words\n",
    "    length_tensor = torch.LongTensor(length)                   #convert to tensor\n",
    "    \n",
    "    prediction = model(tensor, length_tensor).squeeze(1)       #prediction \n",
    "\n",
    "    rounded_preds = torch.round(torch.sigmoid(prediction))\n",
    "    predict_class = rounded_preds.tolist()[0]\n",
    "    return predict_class        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'our deeds are the reason of this earthquake may allah forgive us all'"
      ]
     },
     "execution_count": 291,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"text\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Expected `len(lengths)` to be equal to batch_size, but got 1 (batch_size=13)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-292-85d739d446ba>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'our deeds are the reason of this earthquake may allah forgive us all'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-290-3521f7b2a666>\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(model, sentence)\u001b[0m\n\u001b[0;32m      7\u001b[0m     \u001b[0mlength_tensor\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mLongTensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlength\u001b[0m\u001b[1;33m)\u001b[0m                   \u001b[1;31m#convert to tensor\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m     \u001b[0mprediction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlength_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m       \u001b[1;31m#prediction\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m     \u001b[0mrounded_preds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mround\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msigmoid\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprediction\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\parikalp\\appdata\\local\\programs\\python\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    546\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 547\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    548\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-200-43664fd0652a>\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, text, text_length)\u001b[0m\n\u001b[0;32m     12\u001b[0m         \u001b[0membedding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0membedding\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m         \u001b[1;31m# embedded = [sent len, batch size, emb dim]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 14\u001b[1;33m         \u001b[0mpacked_embedded\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpack_padded_sequence\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0membedding\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtext_length\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m         \u001b[0mpacked_output\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcell\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlstm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpacked_embedded\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\parikalp\\appdata\\local\\programs\\python\\lib\\site-packages\\torch\\nn\\utils\\rnn.py\u001b[0m in \u001b[0;36mpack_padded_sequence\u001b[1;34m(input, lengths, batch_first, enforce_sorted)\u001b[0m\n\u001b[0;32m    273\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    274\u001b[0m     \u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m \u001b[1;33m=\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 275\u001b[1;33m         \u001b[0m_VF\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_pack_padded_sequence\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlengths\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_first\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    276\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mPackedSequence\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msorted_indices\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    277\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: Expected `len(lengths)` to be equal to batch_size, but got 1 (batch_size=13)"
     ]
    }
   ],
   "source": [
    "print(predict(model,'our deeds are the reason of this earthquake may allah forgive us all'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/10], Step [22/450], Train Loss: 0.1687, Valid Loss: 1.2051\n",
      "Epoch [1/10], Step [44/450], Train Loss: 0.1618, Valid Loss: 1.2193\n",
      "Epoch [2/10], Step [66/450], Train Loss: 0.1545, Valid Loss: 1.5192\n",
      "Epoch [2/10], Step [88/450], Train Loss: 0.1653, Valid Loss: 1.2656\n",
      "Epoch [3/10], Step [110/450], Train Loss: 0.1583, Valid Loss: 1.3968\n",
      "Epoch [3/10], Step [132/450], Train Loss: 0.1536, Valid Loss: 1.5290\n",
      "Epoch [4/10], Step [154/450], Train Loss: 0.1429, Valid Loss: 1.5995\n",
      "Epoch [4/10], Step [176/450], Train Loss: 0.1555, Valid Loss: 1.5634\n",
      "Epoch [5/10], Step [198/450], Train Loss: 0.1536, Valid Loss: 1.5412\n",
      "Epoch [5/10], Step [220/450], Train Loss: 0.1661, Valid Loss: 1.4665\n",
      "Epoch [6/10], Step [242/450], Train Loss: 0.1451, Valid Loss: 1.3817\n",
      "Epoch [6/10], Step [264/450], Train Loss: 0.1577, Valid Loss: 1.6631\n",
      "Epoch [7/10], Step [286/450], Train Loss: 0.1572, Valid Loss: 1.8010\n",
      "Epoch [7/10], Step [308/450], Train Loss: 0.1654, Valid Loss: 1.7273\n",
      "Epoch [8/10], Step [330/450], Train Loss: 0.1519, Valid Loss: 1.6759\n",
      "Epoch [8/10], Step [352/450], Train Loss: 0.1619, Valid Loss: 1.7098\n",
      "Epoch [9/10], Step [374/450], Train Loss: 0.1412, Valid Loss: 1.7216\n",
      "Epoch [9/10], Step [396/450], Train Loss: 0.1493, Valid Loss: 1.8028\n",
      "Epoch [10/10], Step [418/450], Train Loss: 0.1517, Valid Loss: 1.8236\n",
      "Epoch [10/10], Step [440/450], Train Loss: 0.1555, Valid Loss: 1.7927\n"
     ]
    }
   ],
   "source": [
    "def train(model,\n",
    "          optimizer,\n",
    "          criterion = nn.BCEWithLogitsLoss(),\n",
    "          train_loader = train_iterator,\n",
    "          valid_loader = valid_iterator,\n",
    "          num_epochs = 5,\n",
    "          eval_every = len(train_iterator) // 2,\n",
    "          file_path = \"nlp-getting-started/\",\n",
    "          best_valid_loss = float(\"Inf\")):\n",
    "    \n",
    "    \n",
    "    # initialize running values\n",
    "    running_loss=0.0\n",
    "    valid_running_loss=0.0\n",
    "    global_step=0\n",
    "    train_loss_list=[]\n",
    "    valid_loss_list=[]\n",
    "    global_loss_list=[]\n",
    "    \n",
    "    \n",
    "    #training loop\n",
    "    model.train()\n",
    "    for epoch in range(num_epochs):\n",
    "        for batch in train_iterator:\n",
    "#             labels=labels\n",
    "#             titletext=titletext\n",
    "#             titletext_len=titletext_len\n",
    "            titletext,titletext_len=batch.text\n",
    "            output=model(titletext,titletext_len).squeeze(1)\n",
    "            \n",
    "            loss=criterion(output,batch.label)\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            # update running values\n",
    "            running_loss+=loss.item()\n",
    "            global_step+=1\n",
    "            \n",
    "            \n",
    "            # evaluation step\n",
    "            \n",
    "            if global_step % eval_every == 0:\n",
    "                model.eval()\n",
    "                with torch.no_grad():\n",
    "                    # validation loop\n",
    "                    for batch in valid_loader:\n",
    "#                             labels=labels\n",
    "#                             titletext=titletext\n",
    "#                             titletext_len=titletext_len\n",
    "                            titletext,titletext_len=batch.text\n",
    "                            output=model(titletext,titletext_len).squeeze(1)\n",
    "                            \n",
    "                            loss=criterion(output,batch.label)\n",
    "                            valid_running_loss+=loss.item()\n",
    "                            \n",
    "                            \n",
    "                average_train_loss=running_loss / eval_every\n",
    "                average_valid_loss=valid_running_loss / len(valid_loader)\n",
    "                train_loss_list.append(average_train_loss)\n",
    "                valid_loss_list.append(average_valid_loss)\n",
    "                global_loss_list.append(global_step)\n",
    "                \n",
    "                \n",
    "                running_loss = 0.0                \n",
    "                valid_running_loss = 0.0\n",
    "                model.train()\n",
    "\n",
    "                # print progress\n",
    "                torch.save(model.state_dict(), 'nlp-getting-started/saved_weights2.pt')\n",
    "                print('Epoch [{}/{}], Step [{}/{}], Train Loss: {:.4f}, Valid Loss: {:.4f}'\n",
    "                      .format(epoch+1, num_epochs, global_step, num_epochs*len(train_loader),\n",
    "                              average_train_loss, average_valid_loss))\n",
    "                            \n",
    "                            \n",
    "                            \n",
    "                \n",
    "            \n",
    "            \n",
    "            \n",
    "            \n",
    "    \n",
    "    \n",
    "train(model=model, optimizer=optimizer, num_epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [],
   "source": [
    "path='nlp-getting-started/saved_weights2.pt'\n",
    "model.load_state_dict(torch.load(path));\n",
    "model.eval();\n",
    "\n",
    "#inference \n",
    "import spacy\n",
    "# nlp = spacy.load('e')\n",
    "\n",
    "def predict(model, sentence):\n",
    "    tokenized = [tok.text for tok in nlp.tokenizer(sentence)]  #tokenize the sentence \n",
    "    indexed = [TEXT2.vocab.stoi[t] for t in tokenized]          #convert to integer sequence\n",
    "    length = [len(indexed)]                                    #compute no. of words\n",
    "    tensor = torch.LongTensor(indexed).to(device)              #convert to tensor\n",
    "    tensor = tensor.unsqueeze(1).T                             #reshape in form of batch,no. of words\n",
    "    length_tensor = torch.LongTensor(length)                   #convert to tensor\n",
    "    prediction = model(tensor, length_tensor)                  #prediction \n",
    "    return prediction.item()             "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Expected `len(lengths)` to be equal to batch_size, but got 1 (batch_size=13)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-304-85d739d446ba>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'our deeds are the reason of this earthquake may allah forgive us all'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-302-6226db0e8a35>\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(model, sentence)\u001b[0m\n\u001b[0;32m     14\u001b[0m     \u001b[0mtensor\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtensor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mT\u001b[0m                             \u001b[1;31m#reshape in form of batch,no. of words\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m     \u001b[0mlength_tensor\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mLongTensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlength\u001b[0m\u001b[1;33m)\u001b[0m                   \u001b[1;31m#convert to tensor\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 16\u001b[1;33m     \u001b[0mprediction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlength_tensor\u001b[0m\u001b[1;33m)\u001b[0m                  \u001b[1;31m#prediction\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     17\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mprediction\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\parikalp\\appdata\\local\\programs\\python\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    546\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 547\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    548\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-200-43664fd0652a>\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, text, text_length)\u001b[0m\n\u001b[0;32m     12\u001b[0m         \u001b[0membedding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0membedding\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m         \u001b[1;31m# embedded = [sent len, batch size, emb dim]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 14\u001b[1;33m         \u001b[0mpacked_embedded\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpack_padded_sequence\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0membedding\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtext_length\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m         \u001b[0mpacked_output\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcell\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlstm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpacked_embedded\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\parikalp\\appdata\\local\\programs\\python\\lib\\site-packages\\torch\\nn\\utils\\rnn.py\u001b[0m in \u001b[0;36mpack_padded_sequence\u001b[1;34m(input, lengths, batch_first, enforce_sorted)\u001b[0m\n\u001b[0;32m    273\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    274\u001b[0m     \u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m \u001b[1;33m=\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 275\u001b[1;33m         \u001b[0m_VF\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_pack_padded_sequence\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlengths\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_first\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    276\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mPackedSequence\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msorted_indices\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    277\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: Expected `len(lengths)` to be equal to batch_size, but got 1 (batch_size=13)"
     ]
    }
   ],
   "source": [
    "print(predict(model,'our deeds are the reason of this earthquake may allah forgive us all'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'our deeds are the reason of this earthquake may allah forgive us all'"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"text\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[   17, 14266,    43,  ...,  6591,   118,   118],\n",
      "        [   51,    22,  1320,  ..., 13722,   779,   779],\n",
      "        [ 2501,  1170,  9049,  ...,    22,  1402,  1402],\n",
      "        ...,\n",
      "        [  518,   404,  1462,  ...,  8847,    72,    72],\n",
      "        [   35,     3,  1892,  ...,  9004,   606,   606],\n",
      "        [  198, 14296,   184,  ...,   473,   195,   195]])\n"
     ]
    }
   ],
   "source": [
    "for batch in train_iterator:\n",
    "    text,text_length=batch.text\n",
    "    print(text)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}