1001_Final_Project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "\n",
    "%matplotlib inline \n",
    "import os\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import string\n",
    "import unicodedata\n",
    "from nltk.stem import LancasterStemmer, WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.util import ngrams\n",
    "stopwords = stopwords.words('english')\n",
    "punctuations = string.punctuation\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "hp = pd.read_csv('sample_data_labelled_cleaned.csv',sep = '|')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def article_preprocessing(st):\n",
    "    \n",
    "    st = re.sub(r\"&quot;\", r\" \", st)\n",
    "    st = re.sub(r\"&apos;\", r\" \", st)\n",
    "    st = re.sub(r\"([.!?])\", r\" \", st)\n",
    "    st = re.sub(r\"[^a-zA-Z0-9.!?]+\", r\" \", st)\n",
    "    st = ' '.join(word for word in st.split(' ') if not word.startswith('httpwww'))\n",
    "    st = ' '.join(word for word in st.split(' ') if not word.startswith('OrderedD'))\n",
    "    st = st.replace('type external text','').replace('  ','').replace('In', ' In').replace('externaltype','')\n",
    "    st = st.replace('externalSource','').replace('type internal text','').replace('external Source','').replace('external external text Source','').replace('internal text','').replace('type internal','')\n",
    "    st = st.lower()\n",
    "    \n",
    "    return st\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-b378c0f9e7cc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mhp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Article'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Article'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marticle_preprocessing\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/anaconda2/envs/nlpclass/lib/python3.6/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, convert_dtype, args, **kwds)\u001b[0m\n\u001b[1;32m   3192\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3193\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3194\u001b[0;31m                 \u001b[0mmapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3196\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/src/inference.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m<ipython-input-8-1d5f75b864aa>\u001b[0m in \u001b[0;36marticle_preprocessing\u001b[0;34m(st)\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mr\"([.!?])\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mr\" \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mst\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mr\"[^a-zA-Z0-9.!?]+\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mr\" \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mst\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mword\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mword\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'httpwww'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mword\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mword\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'OrderedD'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'type external text'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'  '\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'In'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' In'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'externaltype'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "hp['Article'] = hp['Article'].apply(article_preprocessing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "hpp = pd.read_csv('sample_data_labelled_cleaned.csv', sep = '|')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train dataset size is 3281\n",
      "Val dataset size is 1094\n"
     ]
    }
   ],
   "source": [
    "# Split train data into actual train and validation sets 0.75/0.25\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_data, val_data = train_test_split(hpp, test_size=0.25, random_state=42)\n",
    "\n",
    "train_index = train_data['ID']\n",
    "train_article = train_data['Article']\n",
    "train_label = train_data['Hyperpartisan']\n",
    "\n",
    "val_index = val_data['ID']\n",
    "val_article = val_data['Article']\n",
    "val_label = val_data['Hyperpartisan']\n",
    " \n",
    "# now, let's build the test dataset. \n",
    "\n",
    "\n",
    "print (\"Train dataset size is {}\".format(len(train_index)))\n",
    "print (\"Val dataset size is {}\".format(len(val_index)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump(train_data, open(\"train_data_split\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data, val_data_final = train_test_split(val_data, test_size=0.5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(test_data, open(\"test_data_split\", \"wb\"))\n",
    "pickle.dump(val_data, open(\"val_data_split\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19650.365437366658"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calcualte mean article length\n",
    "\n",
    "articles = train_article.values\n",
    "length_of_article = [len(article) for article in articles]\n",
    "np.mean(length_of_article)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preproc(train_article, n):\n",
    "    \n",
    "    token_dataset = []\n",
    "    all_tokens = []\n",
    "    values = train_article.values\n",
    "    for i in range(len(values)):\n",
    "        if i % 500 == 0:\n",
    "            print(i)\n",
    "        article = values[i]\n",
    "        article = nlp(article, disable=['tagger', 'parser', 'ner'])\n",
    "        tokenso = [tok.lemma_.strip() for tok in article if tok.lemma_ != '-PRON-']\n",
    "        tokenso = [tok for tok in tokenso if tok not in stopwords and tok not in punctuations]\n",
    "     \n",
    "        # and get a list of all the n-grams,list of n-word tuples \n",
    "        grams = list(ngrams(tokenso, n))\n",
    "        \n",
    "        # token_dataset is a list of list of tuples\n",
    "        token_dataset.append(grams)\n",
    "        \n",
    "        all_tokens += grams\n",
    "\n",
    "    return token_dataset, all_tokens\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "500\n",
      "1000\n",
      "1500\n",
      "2000\n",
      "2500\n",
      "3000\n"
     ]
    }
   ],
   "source": [
    "# tokenize the training set\n",
    "train_article_token_dataset, train_all_tokens = preproc(train_article,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(train_article_token_dataset, open(\"data/train_data_split_preprocessed\", \"wb\"))\n",
    "pickle.dump(train_all_tokens, open(\"data/train_all_tokens\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "500\n"
     ]
    }
   ],
   "source": [
    "val_article = val_data_final[\"Article\"]\n",
    "val_article_labels = val_data_final[\"Hyperpartisan\"]\n",
    "val_article_token_dataset, val_all_tokens = preproc(val_article,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(val_article_token_dataset, open(\"data/val_data_split_preprocessed\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "500\n"
     ]
    }
   ],
   "source": [
    "test_article = test_data[\"Article\"]\n",
    "test_article_labels = test_data[\"Hyperpartisan\"]\n",
    "test_article_token_dataset, test_all_tokens = preproc(val_article,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(test_article_labels, open(\"data/test_labels\", \"wb\"))\n",
    "pickle.dump(val_article_labels, open(\"data/val_labels\", \"wb\"))\n",
    "pickle.dump(train_label, open(\"data/train_labels\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(test_article_token_dataset, open(\"data/test_data_split_preprocessed\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_article = test_data_final[\"Article\"]\n",
    "val_article_labels = val_data_final[\"Hyperpartisan\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Maison',)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_all_tokens[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_data, = train_test_split(val_article_token_dataset, test_size=0.5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "max_vocab_size = 20000\n",
    "# save index 0 for unk and 1 for pad\n",
    "PAD_IDX = 0\n",
    "UNK_IDX = 1\n",
    "\n",
    "def build_vocab(all_tokens):\n",
    "    # Returns:\n",
    "    # id2token: list of tokens, where id2token[i] returns token that corresponds to token i\n",
    "    # token2id: dictionary where keys represent tokens and corresponding values represent indices\n",
    "    token_counter = collections.Counter(all_tokens)\n",
    "    vocab, count = zip(*token_counter.most_common(max_vocab_size))\n",
    "    id2token = list(vocab)\n",
    "    token2id = dict(zip(vocab, range(2,2+len(vocab)))) \n",
    "    id2token = ['<pad>', '<unk>'] + id2token\n",
    "    token2id['<pad>'] = PAD_IDX \n",
    "    token2id['<unk>'] = UNK_IDX\n",
    "    return token2id, id2token\n",
    "\n",
    "token2id, id2token = build_vocab(train_all_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(token2id, open(\"token2id\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20002\n",
      "20002\n"
     ]
    }
   ],
   "source": [
    "print(len(token2id))\n",
    "print(len(id2token))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets check the dictionary by loading random token from it\n",
    "\n",
    "random_token_id = random.randint(0, len(id2token)-1)\n",
    "random_token = id2token[random_token_id]\n",
    "\n",
    "print (\"Token id {} ; token {}\".format(random_token_id, id2token[random_token_id]))\n",
    "print (\"Token {}; token id {}\".format(random_token, token2id[random_token]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train dataset size is 3281\n",
      "Val dataset size is 547\n",
      "Test dataset size is 547\n"
     ]
    }
   ],
   "source": [
    "# convert token to id in the dataset\n",
    "def token2index_dataset(tokens_data):\n",
    "    indices_data = []\n",
    "    for tokens in tokens_data:\n",
    "        index_list = [token2id[token] if token in token2id else UNK_IDX for token in tokens]\n",
    "        indices_data.append(index_list)\n",
    "    return indices_data\n",
    "\n",
    "# get list of list of idxes\n",
    "train_data_indices = token2index_dataset(train_article_token_dataset)\n",
    "val_data_indices = token2index_dataset(val_article_token_dataset)\n",
    "test_data_indices = token2index_dataset(test_article_token_dataset)\n",
    "\n",
    "# double checking\n",
    "print (\"Train dataset size is {}\".format(len(train_data_indices)))\n",
    "print (\"Val dataset size is {}\".format(len(val_data_indices)))\n",
    "print (\"Test dataset size is {}\".format(len(val_data_indices)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(train_data_indices, open(\"train_data_indexed\", \"wb\"))\n",
    "pickle.dump(val_data_indices, open(\"val_data_indexed\", \"wb\"))\n",
    "pickle.dump(test_data_indices, open(\"test_data_indexed\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_ARTICLE_LENGTH = 19650\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "\n",
    "class GroupDataset(Dataset):\n",
    "    \"\"\"\n",
    "    Class that represents a train/validation/test dataset that's readable for PyTorch\n",
    "    Note that this class inherits torch.utils.data.Dataset\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, article_list, label_list):\n",
    "        \"\"\"\n",
    "        @param data_list: list of review tokens indexes\n",
    "        @param target_list: list of review targets \n",
    "\n",
    "        \"\"\"\n",
    "        self.article_list = article_list\n",
    "        self.label_list = label_list\n",
    "        assert (len(self.article_list) == len(self.label_list))\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.article_list)\n",
    "        \n",
    "    def __getitem__(self, key):\n",
    "        \"\"\"\n",
    "        Triggered when you call dataset[i]\n",
    "        \"\"\"\n",
    "        \n",
    "        token_idx = self.article_list[key][:MAX_ARTICLE_LENGTH]\n",
    "        label = self.label_list[key]\n",
    "        return [token_idx, len(token_idx), label]\n",
    "\n",
    "def group_collate_func(batch):\n",
    "    \"\"\"\n",
    "    Customized function for DataLoader that dynamically pads the batch so that all \n",
    "    data have the same length\n",
    "    \"\"\"\n",
    "    article_list = []\n",
    "    label_list = []\n",
    "    length_list = []\n",
    "    \n",
    "\n",
    "    \n",
    "    for datum in batch:\n",
    "        label_list.append(datum[2])\n",
    "        length_list.append(datum[1])\n",
    "    # padding\n",
    "    for datum in batch:\n",
    "        padded_vec = np.pad(np.array(datum[0]), \n",
    "                                pad_width=((0,MAX_SENTENCE_LENGTH-datum[1])), \n",
    "                                mode=\"constant\", constant_values=0)\n",
    "        article_list.append(padded_vec)\n",
    "    return [torch.from_numpy(np.array(article_list)), torch.LongTensor(length_list), torch.LongTensor(label_list)]\n",
    "\n",
    "\n",
    "# batchsize better be power of 2\n",
    "BATCH_SIZE = 128\n",
    "\n",
    "train_dataset = GroupDataset(train_data_indices, train_label.values.tolist())\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n",
    "                                           batch_size=BATCH_SIZE,\n",
    "                                           collate_fn=group_collate_func,\n",
    "                                           shuffle=True)\n",
    "\n",
    "val_dataset = GroupDataset(val_data_indices, val_label.values.tolist())\n",
    "val_loader = torch.utils.data.DataLoader(dataset=val_dataset, \n",
    "                                           batch_size=BATCH_SIZE,\n",
    "                                           collate_fn=group_collate_func,\n",
    "                                           shuffle=True)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}