Twitter Sentiment Analysis with Word2Vec and Convnets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Twitter Sentiment Analysis with Gensim Word2Vec and Keras Convolutional Network\n",
    "Giuseppe Bonaccorso (https://www.bonaccorso.eu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import dill\n",
    "import keras.backend as K\n",
    "import multiprocessing\n",
    "import tensorflow as tf\n",
    "\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Flatten\n",
    "from keras.layers.convolutional import Conv1D\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "from nltk.stem.lancaster import LancasterStemmer\n",
    "from nltk.tokenize import RegexpTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set random seed (for reproducibility)\n",
    "np.random.seed(1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Select whether using Keras with or without GPU support\n",
    "# See: https://stackoverflow.com/questions/40690598/can-keras-with-tensorflow-backend-be-forced-to-use-cpu-or-gpu-at-will\n",
    "use_gpu = True\n",
    "\n",
    "config = tf.ConfigProto(intra_op_parallelism_threads=multiprocessing.cpu_count(), \n",
    "                        inter_op_parallelism_threads=multiprocessing.cpu_count(), \n",
    "                        allow_soft_placement=True, \n",
    "                        device_count = {'CPU' : 1, \n",
    "                                        'GPU' : 1 if use_gpu else 0})\n",
    "\n",
    "session = tf.Session(config=config)\n",
    "K.set_session(session)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download the dataset from: http://thinknook.com/wp-content/uploads/2012/09/Sentiment-Analysis-Dataset.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dataset_location = './dataset.csv'\n",
    "model_location = './model/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parse tweet corpus and sentiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corpus = []\n",
    "labels = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus size: 1578627\n"
     ]
    }
   ],
   "source": [
    "with open(dataset_location, 'r', encoding='utf-8') as df:\n",
    "    for i, line in enumerate(df):\n",
    "        if i == 0:\n",
    "            # Skip the header\n",
    "            continue\n",
    "\n",
    "        parts = line.strip().split(',')\n",
    "        \n",
    "        # Sentiment (0 = Negative, 1 = Positive)\n",
    "        labels.append(int(parts[1].strip()))\n",
    "        \n",
    "        # Tweet\n",
    "        tweet = parts[3].strip()\n",
    "        if tweet.startswith('\"'):\n",
    "            tweet = tweet[1:]\n",
    "        if tweet.endswith('\"'):\n",
    "            tweet = tweet[::-1]\n",
    "        \n",
    "        corpus.append(tweet.strip().lower())\n",
    "        \n",
    "print('Corpus size: {}'.format(len(corpus)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Tokenize and remove stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tkr = RegexpTokenizer('[a-zA-Z0-9@]+')\n",
    "stemmer = LancasterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tokenized_corpus = []\n",
    "\n",
    "for i, tweet in enumerate(corpus):\n",
    "    tokens = [stemmer.stem(t) for t in tkr.tokenize(tweet) if not t.startswith('@')]\n",
    "    tokenized_corpus.append(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save tokenized corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(model_location + 'tokenized_corpus.dill', 'wb') as f:\n",
    "    dill.dump(tokenized_corpus, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load tokenized corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(model_location + 'tokenized_corpus.dill', 'rb') as f:\n",
    "    tokenized_corpus = dill.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gensim Word2Vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vector_size = 512\n",
    "window_size = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create Word2Vec\n",
    "word2vec = Word2Vec(sentences=tokenized_corpus,\n",
    "                    size=vector_size, \n",
    "                    window=window_size, \n",
    "                    negative=20,\n",
    "                    iter=50,\n",
    "                    seed=1000,\n",
    "                    workers=multiprocessing.cpu_count())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save Word2Vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "word2vec.save(model_location + 'word2vec.model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load Word2Vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word2vec = Word2Vec.load(model_location + 'word2vec.model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Copy word vectors and delete Word2Vec model  and original corpus to save memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_vecs = word2vec.wv\n",
    "\n",
    "del word2vec\n",
    "del corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train subset size (0 < size < len(tokenized_corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_size = 1000000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test subset size (0 < size < len(tokenized_corpus) - train_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_size = 100000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Compute average and max tweet length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average tweet length: 11.0288776259\n",
      "Max tweet length: 53\n"
     ]
    }
   ],
   "source": [
    "avg_length = 0.0\n",
    "max_length = 0\n",
    "\n",
    "for tweet in tokenized_corpus:\n",
    "    if len(tweet) > max_length:\n",
    "        max_length = len(tweet)\n",
    "    avg_length += float(len(tweet))\n",
    "    \n",
    "print('Average tweet length: {}'.format(avg_length / float(len(tokenized_corpus))))\n",
    "print('Max tweet length: {}'.format(max_length))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tweet max length (number of tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "max_tweet_length = 15"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create train and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Generate random indexes\n",
    "indexes = np.random.choice(len(tokenized_corpus), train_size + test_size, replace=False)\n",
    "\n",
    "X_train = np.zeros((train_size, max_tweet_length, vector_size), dtype=K.floatx())\n",
    "Y_train = np.zeros((train_size, 2), dtype=np.int32)\n",
    "X_test = np.zeros((test_size, max_tweet_length, vector_size), dtype=K.floatx())\n",
    "Y_test = np.zeros((test_size, 2), dtype=np.int32)\n",
    "\n",
    "for i, index in enumerate(indexes):\n",
    "    for t, token in enumerate(tokenized_corpus[index]):\n",
    "        if t >= max_tweet_length:\n",
    "            break\n",
    "        \n",
    "        if token not in X_vecs:\n",
    "            continue\n",
    "    \n",
    "        if i < train_size:\n",
    "            X_train[i, t, :] = X_vecs[token]\n",
    "        else:\n",
    "            X_test[i - train_size, t, :] = X_vecs[token]\n",
    "            \n",
    "    if i < train_size:\n",
    "        Y_train[i, :] = [1.0, 0.0] if labels[index] == 0 else [0.0, 1.0]\n",
    "    else:\n",
    "        Y_test[i - train_size, :] = [1.0, 0.0] if labels[index] == 0 else [0.0, 1.0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Keras Convolutional model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "nb_epochs = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "\n",
    "model.add(Conv1D(32, kernel_size=3, activation='elu', padding='same', input_shape=(max_tweet_length, vector_size)))\n",
    "model.add(Conv1D(32, kernel_size=3, activation='elu', padding='same'))\n",
    "model.add(Conv1D(32, kernel_size=3, activation='elu', padding='same'))\n",
    "model.add(Conv1D(32, kernel_size=3, activation='elu', padding='same'))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Conv1D(32, kernel_size=2, activation='elu', padding='same'))\n",
    "model.add(Conv1D(32, kernel_size=2, activation='elu', padding='same'))\n",
    "model.add(Conv1D(32, kernel_size=2, activation='elu', padding='same'))\n",
    "model.add(Conv1D(32, kernel_size=2, activation='elu', padding='same'))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Flatten())\n",
    "\n",
    "model.add(Dense(256, activation='tanh'))\n",
    "model.add(Dense(256, activation='tanh'))\n",
    "model.add(Dropout(0.5))\n",
    "\n",
    "model.add(Dense(2, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Compile the model\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=Adam(lr=0.0001, decay=1e-6),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1000000 samples, validate on 100000 samples\n",
      "1000000/1000000 [==============================] - 240s - loss: 0.5171 - acc: 0.7492 - val_loss: 0.4769 - val_acc: 0.7748\n",
      "Epoch 2/100\n",
      "1000000/1000000 [==============================] - 213s - loss: 0.4922 - acc: 0.7643 - val_loss: 0.4640 - val_acc: 0.7814\n",
      "Epoch 3/100\n",
      "1000000/1000000 [==============================] - 230s - loss: 0.4801 - acc: 0.7710 - val_loss: 0.4581 - val_acc: 0.7839\n",
      "Epoch 4/100\n",
      "1000000/1000000 [==============================] - 197s - loss: 0.4729 - acc: 0.7755 - val_loss: 0.4525 - val_acc: 0.7860\n",
      "Epoch 5/100\n",
      "1000000/1000000 [==============================] - 185s - loss: 0.4677 - acc: 0.7785 - val_loss: 0.4493 - val_acc: 0.7887\n",
      "Epoch 6/100\n",
      "1000000/1000000 [==============================] - 183s - loss: 0.4637 - acc: 0.7811 - val_loss: 0.4455 - val_acc: 0.7917\n",
      "Epoch 7/100\n",
      "1000000/1000000 [==============================] - 183s - loss: 0.4605 - acc: 0.7832 - val_loss: 0.4426 - val_acc: 0.7938\n",
      "Epoch 8/100\n",
      "1000000/1000000 [==============================] - 189s - loss: 0.4576 - acc: 0.7848 - val_loss: 0.4422 - val_acc: 0.7934\n",
      "Epoch 9/100\n",
      "1000000/1000000 [==============================] - 193s - loss: 0.4552 - acc: 0.7863 - val_loss: 0.4412 - val_acc: 0.7942\n",
      "Epoch 10/100\n",
      "1000000/1000000 [==============================] - 197s - loss: 0.4530 - acc: 0.7876 - val_loss: 0.4431 - val_acc: 0.7934\n",
      "Epoch 11/100\n",
      "1000000/1000000 [==============================] - 201s - loss: 0.4508 - acc: 0.7889 - val_loss: 0.4415 - val_acc: 0.7947\n",
      "Epoch 12/100\n",
      "1000000/1000000 [==============================] - 204s - loss: 0.4489 - acc: 0.7902 - val_loss: 0.4415 - val_acc: 0.7938\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1207dc6d0>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit the model\n",
    "model.fit(X_train, Y_train,\n",
    "          batch_size=batch_size,\n",
    "          shuffle=True,\n",
    "          epochs=nb_epochs,\n",
    "          validation_data=(X_test, Y_test),\n",
    "          callbacks=[EarlyStopping(min_delta=0.00025, patience=2)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}