Word Embedding NLP.ipynb

{
  "cells": [
    {
      "metadata": {
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "trusted": true
      },
      "cell_type": "code",
      "source": "# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input/keras-imdb/\"))\nprint(os.listdir(\"../input/glove-global-vectors-for-word-representation/\"))\n\n# Any results you write to the current directory are saved as output.",
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": "['imdb.npz', 'imdb_word_index.json', 'aclImdb_v1']\n['glove.6B.200d.txt', 'glove.6B.100d.txt', 'glove.6B.50d.txt']\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
        "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
        "trusted": true
      },
      "cell_type": "code",
      "source": "imdb_dir = '../input/keras-imdb/aclImdb_v1/aclImdb/'\ntraining_dir = os.path.join(imdb_dir, 'train')\nprint(training_dir)\n\nlabels = []\ntexts = []\n\nfor label_type in ['neg', 'pos']:\n    dir_name = os.path.join(training_dir, label_type)\n    for fname in os.listdir(dir_name):\n        if fname[-4:] == \".txt\":\n            f = open(os.path.join(dir_name, fname))\n            texts.append(f.read())\n            f.close()\n        if label_type == 'neg':\n            labels.append(0)\n        else:\n            labels.append(1)\n\n# Lengths of the two arrays\nlen(labels), len(texts)",
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": "../input/keras-imdb/aclImdb_v1/aclImdb/train\n",
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "execution_count": 2,
          "data": {
            "text/plain": "(25000, 25000)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "56ad646def66884ae340b332f3c52f5fbdd5000d"
      },
      "cell_type": "code",
      "source": "from keras.preprocessing.text import Tokenizer\nmax_words = 10000\ntokenizer = Tokenizer(max_words)\ntokenizer.fit_on_texts(texts)\nsequences = tokenizer.texts_to_sequences(texts)",
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": "Using TensorFlow backend.\n",
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "15e80b5223592beeb50891d4a5d673348b974cd6"
      },
      "cell_type": "code",
      "source": "word_index = tokenizer.word_index\nprint(\"Index for : the:\", word_index[\"this\"])\nprint(\"Index for : the:\", word_index[\"good\"])\nprint(\"Index for : the:\", word_index[\"marketing\"])\nprint(\"Sequences:\", sequences[24999][:20])",
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": "Index for : the: 11\nIndex for : the: 49\nIndex for : the: 4892\nSequences: [11, 17, 6, 287, 316, 15, 1, 1110, 932, 2, 725, 113, 581, 18, 47, 79, 6, 32, 218, 2664]\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "c749b1e9914d46d7947f9b53af5e5dd186791495"
      },
      "cell_type": "code",
      "source": "from keras.preprocessing.sequence import pad_sequences\nmaxlen = 100\ndata = pad_sequences(sequences, maxlen)\nprint(data.shape[0], data.shape[1])",
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": "25000 100\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "63df03bdb67539832ab18cd124bf9ecb0e262364"
      },
      "cell_type": "code",
      "source": "labels = np.asarray(labels)\nindices = np.arange(labels.shape[0])\nnp.random.shuffle(indices)\ndata = data[indices]\nlabels = labels[indices]",
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": "[[   5 1342   26 ...  110   67   27]\n [   1 2448  344 ...  219   13  160]\n [ 160   48    6 ...    3 2469 4502]\n ...\n [  28 3629   40 ...   20    1 1108]\n [   8   11  179 ...    6   58 1172]\n [  72   23   61 ...  525  715  229]]\n[1 0 0 ... 0 0 0]\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "1152b14fb38c19acb65c86dcff7400842a136ec5"
      },
      "cell_type": "code",
      "source": "training_samples = 20000\nvalidation_sample = 5000\nx_training_set = data[:training_samples]\ny_training_set = labels[:training_samples]\nx_validation_set = data[training_samples:training_samples + validation_sample]\ny_validation_set = labels[training_samples:training_samples + validation_sample]\nprint(len(x_validation_set))",
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": "5000\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "b5f60f6f56d624ea7397aeee12391683aabf6fad"
      },
      "cell_type": "code",
      "source": "from keras.models import Sequential\nfrom keras.layers import Embedding, Flatten, Dense\n\nembedding_dim = 50\nmodel = Sequential()\nmodel.add(Embedding(max_words, embedding_dim, input_length=maxlen))\nmodel.add(Flatten())\nmodel.add(Dense(32, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.summary()",
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": "_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\nembedding_1 (Embedding)      (None, 100, 50)           500000    \n_________________________________________________________________\nflatten_1 (Flatten)          (None, 5000)              0         \n_________________________________________________________________\ndense_1 (Dense)              (None, 32)                160032    \n_________________________________________________________________\ndense_2 (Dense)              (None, 1)                 33        \n=================================================================\nTotal params: 660,065\nTrainable params: 660,065\nNon-trainable params: 0\n_________________________________________________________________\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "2020dc15c21ca331b2a6a974ed85893892413aa7"
      },
      "cell_type": "code",
      "source": "model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc'])",
      "execution_count": 9,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "a4fc5e20db72b7910b7d9fabb298abf83d0c2485"
      },
      "cell_type": "code",
      "source": "history = model.fit(x_training_set, y_training_set, epochs=10,\n                    batch_size=32, validation_data=(x_validation_set, y_validation_set))",
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": "Train on 20000 samples, validate on 5000 samples\nEpoch 1/10\n20000/20000 [==============================] - 6s 285us/step - loss: 0.4396 - acc: 0.7783 - val_loss: 0.3465 - val_acc: 0.8514\nEpoch 2/10\n20000/20000 [==============================] - 5s 247us/step - loss: 0.1201 - acc: 0.9584 - val_loss: 0.4213 - val_acc: 0.8366\nEpoch 3/10\n20000/20000 [==============================] - 5s 242us/step - loss: 0.0105 - acc: 0.9985 - val_loss: 0.5174 - val_acc: 0.8390\nEpoch 4/10\n20000/20000 [==============================] - 5s 244us/step - loss: 0.0011 - acc: 1.0000 - val_loss: 0.5529 - val_acc: 0.8428\nEpoch 5/10\n20000/20000 [==============================] - 5s 246us/step - loss: 4.4295e-04 - acc: 1.0000 - val_loss: 0.5816 - val_acc: 0.8436\nEpoch 6/10\n20000/20000 [==============================] - 5s 248us/step - loss: 2.5262e-04 - acc: 1.0000 - val_loss: 0.6066 - val_acc: 0.8458\nEpoch 7/10\n20000/20000 [==============================] - 5s 246us/step - loss: 1.5564e-04 - acc: 1.0000 - val_loss: 0.6292 - val_acc: 0.8464\nEpoch 8/10\n20000/20000 [==============================] - 5s 240us/step - loss: 9.9376e-05 - acc: 1.0000 - val_loss: 0.6507 - val_acc: 0.8464\nEpoch 9/10\n20000/20000 [==============================] - 5s 244us/step - loss: 6.5904e-05 - acc: 1.0000 - val_loss: 0.6713 - val_acc: 0.8472\nEpoch 10/10\n20000/20000 [==============================] - 5s 240us/step - loss: 4.4535e-05 - acc: 1.0000 - val_loss: 0.6906 - val_acc: 0.8468\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "09afb3c481d12706dc91496a233efa446f429947"
      },
      "cell_type": "code",
      "source": "glove_dir = '../input/glove-global-vectors-for-word-representation/'\nprint(\"Word Embedding Vectors\")\nembeddings_index = {}\nf = open(os.path.join(glove_dir, 'glove.6B.100d.txt'))",
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": "Word Embedding Vectors\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "84e8ab530e3468c8a8cd1758aa923df22bc7a503"
      },
      "cell_type": "code",
      "source": "for lines in f:\n    values = lines.split()\n    word = values[0]\n    embedding = np.asarray(values[1:], dtype='float32')\n    embeddings_index[word] = embedding\nf.close()",
      "execution_count": 12,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "defd204e0edf5768811844c67c9a33eb764c36b9"
      },
      "cell_type": "code",
      "source": "print(\"Number of Vectors = \",len(embeddings_index))",
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": "Number of Vectors =  400000\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "222b3948751165c2a1de4f7e675fac5e107c2993"
      },
      "cell_type": "code",
      "source": "all_embs = np.stack(embeddings_index.values())\nemb_mean = all_embs.mean()\nemb_std = all_embs.std()\nemb_mean, emb_std",
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:1: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n  \"\"\"Entry point for launching an IPython kernel.\n",
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "execution_count": 14,
          "data": {
            "text/plain": "(0.004451992, 0.4081574)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "ab9b1df884a226f0dda7116747c1843d2e5efad1"
      },
      "cell_type": "code",
      "source": "embedding_dim = 100\nword_index = tokenizer.word_index\nnumber_words = min(max_words, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (number_words, embedding_dim))",
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": "10000 100\n(10000, 100)\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "4c9dc1d5820c23cd286cddd87b95c16694e31377"
      },
      "cell_type": "code",
      "source": "for word, i in word_index.items():\n    if i >= max_words: \n        continue\n    # Get the embedding vector for the word\n    embedding_vector = embeddings_index.get(word)\n    # If there is an embedding vector, put it in the embedding matrix\n    if embedding_vector is not None: \n        embedding_matrix[i] = embedding_vector",
      "execution_count": 18,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "530ea4b865c7e98e43d0589da819b81989f9d709"
      },
      "cell_type": "code",
      "source": "model = Sequential()\nmodel.add(Embedding(max_words, embedding_dim, input_length=maxlen, weights = [embedding_matrix], trainable = False))\nmodel.add(Flatten())\nmodel.add(Dense(32, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.summary()",
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": "_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\nembedding_2 (Embedding)      (None, 100, 100)          1000000   \n_________________________________________________________________\nflatten_2 (Flatten)          (None, 10000)             0         \n_________________________________________________________________\ndense_3 (Dense)              (None, 32)                320032    \n_________________________________________________________________\ndense_4 (Dense)              (None, 1)                 33        \n=================================================================\nTotal params: 1,320,065\nTrainable params: 320,065\nNon-trainable params: 1,000,000\n_________________________________________________________________\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "928a45adc820656125cc4b35ecac07a078492d89"
      },
      "cell_type": "code",
      "source": "model.compile(optimizer='adam',\n              loss='binary_crossentropy',\n              metrics=['acc'])\nhistory = model.fit(x_training_set, y_training_set,\n                    epochs=10,\n                    batch_size=32,\n                    validation_data=(x_validation_set, y_validation_set))",
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": "Train on 20000 samples, validate on 5000 samples\nEpoch 1/10\n20000/20000 [==============================] - 5s 231us/step - loss: 0.6000 - acc: 0.6719 - val_loss: 0.5369 - val_acc: 0.7332\nEpoch 2/10\n20000/20000 [==============================] - 4s 185us/step - loss: 0.4860 - acc: 0.7668 - val_loss: 0.5443 - val_acc: 0.7232\nEpoch 3/10\n20000/20000 [==============================] - 4s 188us/step - loss: 0.4331 - acc: 0.8017 - val_loss: 0.6504 - val_acc: 0.6904\nEpoch 4/10\n20000/20000 [==============================] - 4s 186us/step - loss: 0.3947 - acc: 0.8223 - val_loss: 0.5894 - val_acc: 0.7254\nEpoch 5/10\n20000/20000 [==============================] - 4s 189us/step - loss: 0.3663 - acc: 0.8342 - val_loss: 0.6069 - val_acc: 0.7210\nEpoch 6/10\n20000/20000 [==============================] - 4s 179us/step - loss: 0.3262 - acc: 0.8570 - val_loss: 0.6546 - val_acc: 0.7052\nEpoch 7/10\n20000/20000 [==============================] - 4s 186us/step - loss: 0.2802 - acc: 0.8816 - val_loss: 0.7018 - val_acc: 0.7136\nEpoch 8/10\n20000/20000 [==============================] - 4s 185us/step - loss: 0.2329 - acc: 0.9062 - val_loss: 0.9147 - val_acc: 0.7050\nEpoch 9/10\n20000/20000 [==============================] - 4s 184us/step - loss: 0.1917 - acc: 0.9257 - val_loss: 0.8192 - val_acc: 0.6976\nEpoch 10/10\n20000/20000 [==============================] - 4s 188us/step - loss: 0.1542 - acc: 0.9408 - val_loss: 0.9595 - val_acc: 0.7010\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "5f5f5b1ef2a66d153182d5202555ffebe426d69e"
      },
      "cell_type": "code",
      "source": "my_text = 'I love dogs. Dogs are the best. They are lovely, cuddly animals that only want the best for humans.'\n\nseq = tokenizer.texts_to_sequences([my_text])\nprint('raw seq:',seq)\nseq = pad_sequences(seq, maxlen=maxlen)\nprint('padded seq:',seq)\nprediction = model.predict(seq)\nprint('positivity:',prediction)",
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": "raw seq: [[10, 116, 2520, 2520, 23, 1, 115, 33, 23, 1331, 1386, 12, 61, 178, 1, 115, 15, 1706]]\npadded seq: [[   0    0    0    0    0    0    0    0    0    0    0    0    0    0\n     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n     0    0    0    0    0    0    0    0    0    0    0    0   10  116\n  2520 2520   23    1  115   33   23 1331 1386   12   61  178    1  115\n    15 1706]]\npositivity: [[0.9881958]]\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "74def1fc28a221fcc41ca484c581e1685fb2dcc9"
      },
      "cell_type": "code",
      "source": "my_text_n = 'This place lacks good food, good ambience, good music, good drinks. Would not recommend this place.'\n\nseq = tokenizer.texts_to_sequences([my_text_n])\nprint('raw seq:',seq)\nseq = pad_sequences(seq, maxlen=maxlen)\nprint('padded seq:',seq)\nprediction = model.predict(seq)\nprint('positivity:',prediction)",
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": "raw seq: [[11, 270, 1500, 49, 1642, 49, 49, 225, 49, 6461, 59, 21, 383, 11, 270]]\npadded seq: [[   0    0    0    0    0    0    0    0    0    0    0    0    0    0\n     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n     0   11  270 1500   49 1642   49   49  225   49 6461   59   21  383\n    11  270]]\npositivity: [[0.9585079]]\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "c065aa3a78106756a170cb6c55fd44ac252fa4f5"
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.6.6",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 1
}