logistic_regression_theano.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression for Sentiment Analysis with Theano"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a tutorial on logistic regression using Theano based on the MNST \n",
    "logistic regression code provided at http://deeplearning.net/tutorial/logreg.html.\n",
    "We make some code to read Andrew Mass' distribution of IMBD data\n",
    "and make some changes in the original MNST Theano tutorial as needed. \n",
    "You will need to familiarize yourself with Theano to understand this tutorial well.\n",
    "Take a look here: http://deeplearning.net/software/theano/tutorial/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple Example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It will probably help if you consider this simpler code for logistic regression,\n",
    "This code is provided at: http://deeplearning.net/software/theano/tutorial/examples.html.\n",
    "Maybe try changing values of \"N\" and the number of \"training_steps\" and see what you get.\n",
    "Later, we will use logistic regression too, but with a technique called \"stochastic gradient descent.\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial model:\n",
      "Final model:\n",
      "target values for D:\n",
      "[1 1 1 1 1 0 1 0 1 1 0 1 0 1 0 1 0 1 1 0 1 0 1 1 1 1 1 0 1 1 0 1 1 0 1 1 1\n",
      " 0 1 1 0 0 0 0 1 1 1 1 0 0 1 1 1 0 0 1 1 1 0 0 0 1 1 0 1 1 0 0 1 1 1 1 1 0\n",
      " 1 0 1 0 1 0 1 1 1 0 0 1 1 1 0 0 1 0 0 1 0 0 0 0 0 1 0 0 0 1 0 1 0 0 1 0 1\n",
      " 1 1 1 1 0 0 0 1 0 1 1 0 0 1 0 0 0 1 1 0 1 1 1 0 0 1 0 0 1 1 1 0 0 1 1 1 0\n",
      " 0 1 0 1 0 0 1 1 0 0 1 1 0 0 0 1 0 0 1 0 0 1 1 0 1 1 0 0 0 0 1 1 0 0 0 1 0\n",
      " 1 0 0 0 0 0 1 0 0 1 0 0 1 0 1 0 1 0 0 1 0 0 1 1 0 0 0 1 1 1 1 0 1 1 1 1 1\n",
      " 1 1 0 1 1 0 1 0 1 1 1 1 1 1 0 1 1 1 0 0 1 1 1 0 0 0 1 1 0 0 0 1 0 0 0 0 1\n",
      " 0 0 1 0 1 1 1 1 0 0 1 0 0 1 1 1 0 1 1 1 0 1 0 1 0 1 0 0 0 1 0 0 0 0 1 1 1\n",
      " 1 1 0 1 0 1 0 0 1 0 0 0 0 0 0 0 1 1 1 0 0 0 0 1 1 0 1 1 1 1 1 1 0 1 0 1 1\n",
      " 1 1 0 0 0 0 0 1 0 0 1 1 1 1 1 0 0 1 0 0 0 0 1 1 1 0 1 0 0 0 1 0 0 0 1 1 0\n",
      " 0 0 1 1 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 1 1 1 0 1 0]\n",
      "prediction on D:\n",
      "[0 1 1 1 0 1 1 0 1 1 1 1 1 0 0 0 0 1 1 0 0 1 1 1 1 1 0 0 1 1 1 1 1 1 0 1 1\n",
      " 1 1 0 1 0 0 0 1 0 1 0 0 1 0 0 0 1 1 1 0 1 0 1 1 0 0 0 0 0 1 0 1 0 1 1 0 0\n",
      " 1 0 1 1 0 0 0 1 1 1 0 0 0 1 1 1 0 0 1 1 0 0 0 1 1 0 1 1 0 1 0 1 0 1 0 0 0\n",
      " 0 1 1 1 0 1 0 1 1 0 1 1 0 1 1 0 1 0 1 0 1 1 1 1 1 1 1 0 1 0 1 1 0 1 0 1 0\n",
      " 1 0 1 0 1 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 1 0 0 1 1 1 1 0 1 0 0 0 0 1\n",
      " 1 0 0 0 0 0 1 1 0 0 0 1 1 1 1 0 1 1 1 1 0 1 1 1 1 0 1 0 0 0 1 0 1 0 1 0 0\n",
      " 0 0 0 1 0 0 0 1 1 1 0 0 1 0 0 1 1 1 0 0 0 0 1 1 1 1 0 1 1 0 0 0 0 0 0 1 0\n",
      " 1 1 0 1 1 1 1 1 1 1 1 0 0 0 0 0 1 0 0 1 0 0 0 1 0 1 0 1 1 0 0 1 1 0 1 0 0\n",
      " 1 1 0 0 1 1 0 0 0 1 1 0 0 0 0 1 1 1 0 1 0 1 0 1 1 0 0 1 0 0 0 0 1 0 0 0 1\n",
      " 0 1 1 1 1 0 0 0 1 1 0 0 1 0 1 1 0 0 1 0 1 1 0 1 1 0 1 0 1 1 1 1 0 1 0 0 1\n",
      " 1 0 0 0 0 0 0 1 0 0 0 1 0 1 0 1 0 0 0 1 0 0 0 1 1 0 1 1 1 0]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: GeForce GT 750M\n"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "rng = numpy.random\n",
    "##############\n",
    "N = 400                                   # training sample size\n",
    "feats = 784                               # number of input variables\n",
    "\n",
    "# generate a dataset: D = (input_values, target_class)\n",
    "D = (rng.rand(N, feats).astype(theano.config.floatX), rng.randint(size=N, low=0, high=2))\n",
    "training_steps = 100\n",
    "#np.asarray(your_data, dtype=theano.config.floatX)\n",
    "\n",
    "# Declare Theano symbolic variables\n",
    "x = T.matrix(\"x\")\n",
    "y = T.vector(\"y\")\n",
    "\n",
    "# initialize the weight vector w randomly\n",
    "#\n",
    "# this and the following bias variable b\n",
    "# are shared so they keep their values\n",
    "# between training iterations (updates)\n",
    "w = theano.shared(rng.randn(feats), name=\"w\")\n",
    "\n",
    "# initialize the bias term\n",
    "b = theano.shared(0., name=\"b\")\n",
    "#print b.eval()\n",
    "print(\"Initial model:\")\n",
    "#print(w.get_value())\n",
    "#print(b.get_value())\n",
    "\n",
    "# Construct Theano expression graph\n",
    "p_1 = 1 / (1 + T.exp(-T.dot(x, w) - b))   # Probability that target = 1\n",
    "prediction = p_1 > 0.5                    # The prediction thresholded\n",
    "xent = -y * T.log(p_1) - (1-y) * T.log(1-p_1) # Cross-entropy loss function\n",
    "cost = xent.mean() + 0.01 * (w ** 2).sum()# The cost to minimize\n",
    "gw, gb = T.grad(cost, [w, b])             # Compute the gradient of the cost\n",
    "                                          # w.r.t weight vector w and\n",
    "                                          # bias term b\n",
    "                                          # (we shall return to this in a\n",
    "                                          # following section of this tutorial)\n",
    "\n",
    "# Compile\n",
    "train = theano.function(\n",
    "          inputs=[x,y],\n",
    "          outputs=[prediction, xent],\n",
    "          updates=((w, w - 0.1 * gw), (b, b - 0.1 * gb)),\n",
    "          allow_input_downcast=True) # added downcasting...\n",
    "predict = theano.function(inputs=[x], outputs=prediction)\n",
    "\n",
    "# Train\n",
    "for i in range(training_steps):\n",
    "    pred, err = train(D[0], D[1])\n",
    "\n",
    "print(\"Final model:\")\n",
    "#print(w.get_value())\n",
    "#print(b.get_value())\n",
    "print(\"target values for D:\")\n",
    "print(D[1])\n",
    "print(\"prediction on D:\")\n",
    "print(predict(D[0]))\n",
    "#----------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Now try the code with different values of \"training_steps\" and see what you get.\n",
    "# For example, you can try:\n",
    "# training_steps= 100, training_steps=500, training_steps=10000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Logistic Regression with Theano, using Stochastic Gradient Descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "space_len: 3047\n",
      "train_vecs.shape: 25000, 3047\n",
      "dev_vecs.shape: 5000, 3047\n",
      "test_vecs.shape: 20000, 3047\n",
      "Subtensor{int64}.0\n",
      "... building the model\n",
      "... training the model\n",
      "epoch 1, minibatch 41/41, validation error 52.083333 %\n",
      "     epoch 1, minibatch 41/41, test error of best model 50.505051 %\n",
      "epoch 2, minibatch 41/41, validation error 52.083333 %\n",
      "epoch 3, minibatch 41/41, validation error 52.062500 %\n",
      "     epoch 3, minibatch 41/41, test error of best model 50.505051 %\n",
      "epoch 4, minibatch 41/41, validation error 51.895833 %\n",
      "     epoch 4, minibatch 41/41, test error of best model 50.297980 %\n",
      "epoch 5, minibatch 41/41, validation error 50.937500 %\n",
      "     epoch 5, minibatch 41/41, test error of best model 49.560606 %\n",
      "epoch 6, minibatch 41/41, validation error 49.416667 %\n",
      "     epoch 6, minibatch 41/41, test error of best model 48.267677 %\n",
      "epoch 7, minibatch 41/41, validation error 47.708333 %\n",
      "     epoch 7, minibatch 41/41, test error of best model 46.616162 %\n",
      "epoch 8, minibatch 41/41, validation error 45.958333 %\n",
      "     epoch 8, minibatch 41/41, test error of best model 44.853535 %\n",
      "epoch 9, minibatch 41/41, validation error 44.395833 %\n",
      "     epoch 9, minibatch 41/41, test error of best model 43.176768 %\n",
      "epoch 10, minibatch 41/41, validation error 42.875000 %\n",
      "     epoch 10, minibatch 41/41, test error of best model 41.651515 %\n",
      "epoch 11, minibatch 41/41, validation error 41.458333 %\n",
      "     epoch 11, minibatch 41/41, test error of best model 40.393939 %\n",
      "epoch 12, minibatch 41/41, validation error 39.979167 %\n",
      "     epoch 12, minibatch 41/41, test error of best model 39.333333 %\n",
      "epoch 13, minibatch 41/41, validation error 38.520833 %\n",
      "     epoch 13, minibatch 41/41, test error of best model 38.378788 %\n",
      "epoch 14, minibatch 41/41, validation error 37.708333 %\n",
      "     epoch 14, minibatch 41/41, test error of best model 37.489899 %\n",
      "epoch 15, minibatch 41/41, validation error 36.583333 %\n",
      "     epoch 15, minibatch 41/41, test error of best model 36.686869 %\n",
      "epoch 16, minibatch 41/41, validation error 35.562500 %\n",
      "     epoch 16, minibatch 41/41, test error of best model 36.025253 %\n",
      "epoch 17, minibatch 41/41, validation error 34.937500 %\n",
      "     epoch 17, minibatch 41/41, test error of best model 35.272727 %\n",
      "epoch 18, minibatch 41/41, validation error 34.250000 %\n",
      "     epoch 18, minibatch 41/41, test error of best model 34.631313 %\n",
      "epoch 19, minibatch 41/41, validation error 33.604167 %\n",
      "     epoch 19, minibatch 41/41, test error of best model 33.989899 %\n",
      "epoch 20, minibatch 41/41, validation error 33.166667 %\n",
      "     epoch 20, minibatch 41/41, test error of best model 33.479798 %\n",
      "epoch 21, minibatch 41/41, validation error 32.562500 %\n",
      "     epoch 21, minibatch 41/41, test error of best model 33.010101 %\n",
      "epoch 22, minibatch 41/41, validation error 32.208333 %\n",
      "     epoch 22, minibatch 41/41, test error of best model 32.555556 %\n",
      "epoch 23, minibatch 41/41, validation error 31.770833 %\n",
      "     epoch 23, minibatch 41/41, test error of best model 32.171717 %\n",
      "epoch 24, minibatch 41/41, validation error 31.375000 %\n",
      "     epoch 24, minibatch 41/41, test error of best model 31.772727 %\n",
      "epoch 25, minibatch 41/41, validation error 31.083333 %\n",
      "     epoch 25, minibatch 41/41, test error of best model 31.500000 %\n",
      "epoch 26, minibatch 41/41, validation error 30.770833 %\n",
      "     epoch 26, minibatch 41/41, test error of best model 31.267677 %\n",
      "epoch 27, minibatch 41/41, validation error 30.437500 %\n",
      "     epoch 27, minibatch 41/41, test error of best model 31.065657 %\n",
      "epoch 28, minibatch 41/41, validation error 30.062500 %\n",
      "     epoch 28, minibatch 41/41, test error of best model 30.792929 %\n",
      "epoch 29, minibatch 41/41, validation error 29.875000 %\n",
      "     epoch 29, minibatch 41/41, test error of best model 30.570707 %\n",
      "epoch 30, minibatch 41/41, validation error 29.479167 %\n",
      "     epoch 30, minibatch 41/41, test error of best model 30.328283 %\n",
      "epoch 31, minibatch 41/41, validation error 29.291667 %\n",
      "     epoch 31, minibatch 41/41, test error of best model 30.050505 %\n",
      "epoch 32, minibatch 41/41, validation error 29.083333 %\n",
      "     epoch 32, minibatch 41/41, test error of best model 29.858586 %\n",
      "epoch 33, minibatch 41/41, validation error 29.000000 %\n",
      "     epoch 33, minibatch 41/41, test error of best model 29.691919 %\n",
      "epoch 34, minibatch 41/41, validation error 28.770833 %\n",
      "     epoch 34, minibatch 41/41, test error of best model 29.474747 %\n",
      "epoch 35, minibatch 41/41, validation error 28.625000 %\n",
      "     epoch 35, minibatch 41/41, test error of best model 29.222222 %\n",
      "epoch 36, minibatch 41/41, validation error 28.437500 %\n",
      "     epoch 36, minibatch 41/41, test error of best model 28.989899 %\n",
      "epoch 37, minibatch 41/41, validation error 28.333333 %\n",
      "     epoch 37, minibatch 41/41, test error of best model 28.813131 %\n",
      "epoch 38, minibatch 41/41, validation error 28.104167 %\n",
      "     epoch 38, minibatch 41/41, test error of best model 28.656566 %\n",
      "epoch 39, minibatch 41/41, validation error 27.770833 %\n",
      "     epoch 39, minibatch 41/41, test error of best model 28.484848 %\n",
      "epoch 40, minibatch 41/41, validation error 27.645833 %\n",
      "     epoch 40, minibatch 41/41, test error of best model 28.343434 %\n",
      "epoch 41, minibatch 41/41, validation error 27.458333 %\n",
      "     epoch 41, minibatch 41/41, test error of best model 28.252525 %\n",
      "epoch 42, minibatch 41/41, validation error 27.312500 %\n",
      "     epoch 42, minibatch 41/41, test error of best model 28.166667 %\n",
      "epoch 43, minibatch 41/41, validation error 27.312500 %\n",
      "epoch 44, minibatch 41/41, validation error 27.125000 %\n",
      "     epoch 44, minibatch 41/41, test error of best model 27.964646 %\n",
      "epoch 45, minibatch 41/41, validation error 26.979167 %\n",
      "     epoch 45, minibatch 41/41, test error of best model 27.803030 %\n",
      "epoch 46, minibatch 41/41, validation error 26.791667 %\n",
      "     epoch 46, minibatch 41/41, test error of best model 27.717172 %\n",
      "epoch 47, minibatch 41/41, validation error 26.729167 %\n",
      "     epoch 47, minibatch 41/41, test error of best model 27.606061 %\n",
      "epoch 48, minibatch 41/41, validation error 26.541667 %\n",
      "     epoch 48, minibatch 41/41, test error of best model 27.555556 %\n",
      "epoch 49, minibatch 41/41, validation error 26.395833 %\n",
      "     epoch 49, minibatch 41/41, test error of best model 27.500000 %\n",
      "epoch 50, minibatch 41/41, validation error 26.354167 %\n",
      "     epoch 50, minibatch 41/41, test error of best model 27.378788 %\n",
      "epoch 51, minibatch 41/41, validation error 26.291667 %\n",
      "     epoch 51, minibatch 41/41, test error of best model 27.287879 %\n",
      "epoch 52, minibatch 41/41, validation error 26.187500 %\n",
      "     epoch 52, minibatch 41/41, test error of best model 27.267677 %\n",
      "epoch 53, minibatch 41/41, validation error 26.041667 %\n",
      "     epoch 53, minibatch 41/41, test error of best model 27.196970 %\n",
      "epoch 54, minibatch 41/41, validation error 25.937500 %\n",
      "     epoch 54, minibatch 41/41, test error of best model 27.106061 %\n",
      "epoch 55, minibatch 41/41, validation error 25.791667 %\n",
      "     epoch 55, minibatch 41/41, test error of best model 27.060606 %\n",
      "epoch 56, minibatch 41/41, validation error 25.750000 %\n",
      "     epoch 56, minibatch 41/41, test error of best model 26.979798 %\n",
      "epoch 57, minibatch 41/41, validation error 25.645833 %\n",
      "     epoch 57, minibatch 41/41, test error of best model 26.944444 %\n",
      "epoch 58, minibatch 41/41, validation error 25.666667 %\n",
      "epoch 59, minibatch 41/41, validation error 25.562500 %\n",
      "     epoch 59, minibatch 41/41, test error of best model 26.823232 %\n",
      "epoch 60, minibatch 41/41, validation error 25.562500 %\n",
      "epoch 61, minibatch 41/41, validation error 25.541667 %\n",
      "     epoch 61, minibatch 41/41, test error of best model 26.636364 %\n",
      "epoch 62, minibatch 41/41, validation error 25.520833 %\n",
      "     epoch 62, minibatch 41/41, test error of best model 26.595960 %\n",
      "epoch 63, minibatch 41/41, validation error 25.479167 %\n",
      "     epoch 63, minibatch 41/41, test error of best model 26.520202 %\n",
      "epoch 64, minibatch 41/41, validation error 25.500000 %\n",
      "epoch 65, minibatch 41/41, validation error 25.416667 %\n",
      "     epoch 65, minibatch 41/41, test error of best model 26.409091 %\n",
      "epoch 66, minibatch 41/41, validation error 25.375000 %\n",
      "     epoch 66, minibatch 41/41, test error of best model 26.343434 %\n",
      "epoch 67, minibatch 41/41, validation error 25.354167 %\n",
      "     epoch 67, minibatch 41/41, test error of best model 26.287879 %\n",
      "epoch 68, minibatch 41/41, validation error 25.333333 %\n",
      "     epoch 68, minibatch 41/41, test error of best model 26.202020 %\n",
      "epoch 69, minibatch 41/41, validation error 25.312500 %\n",
      "     epoch 69, minibatch 41/41, test error of best model 26.202020 %\n",
      "epoch 70, minibatch 41/41, validation error 25.291667 %\n",
      "     epoch 70, minibatch 41/41, test error of best model 26.156566 %\n",
      "epoch 71, minibatch 41/41, validation error 25.250000 %\n",
      "     epoch 71, minibatch 41/41, test error of best model 26.106061 %\n",
      "epoch 72, minibatch 41/41, validation error 25.270833 %\n",
      "epoch 73, minibatch 41/41, validation error 25.250000 %\n",
      "epoch 74, minibatch 41/41, validation error 25.250000 %\n",
      "epoch 75, minibatch 41/41, validation error 25.187500 %\n",
      "     epoch 75, minibatch 41/41, test error of best model 25.904040 %\n",
      "epoch 76, minibatch 41/41, validation error 25.208333 %\n",
      "epoch 77, minibatch 41/41, validation error 25.208333 %\n",
      "epoch 78, minibatch 41/41, validation error 25.187500 %\n",
      "epoch 79, minibatch 41/41, validation error 25.229167 %\n",
      "epoch 80, minibatch 41/41, validation error 25.187500 %\n",
      "epoch 81, minibatch 41/41, validation error 25.145833 %\n",
      "     epoch 81, minibatch 41/41, test error of best model 25.696970 %\n",
      "epoch 82, minibatch 41/41, validation error 25.125000 %\n",
      "     epoch 82, minibatch 41/41, test error of best model 25.656566 %\n",
      "epoch 83, minibatch 41/41, validation error 25.041667 %\n",
      "     epoch 83, minibatch 41/41, test error of best model 25.636364 %\n",
      "epoch 84, minibatch 41/41, validation error 25.041667 %\n",
      "epoch 85, minibatch 41/41, validation error 25.062500 %\n",
      "epoch 86, minibatch 41/41, validation error 25.062500 %\n",
      "epoch 87, minibatch 41/41, validation error 25.041667 %\n",
      "epoch 88, minibatch 41/41, validation error 25.041667 %\n",
      "epoch 89, minibatch 41/41, validation error 25.000000 %\n",
      "     epoch 89, minibatch 41/41, test error of best model 25.474747 %\n",
      "epoch 90, minibatch 41/41, validation error 25.041667 %\n",
      "epoch 91, minibatch 41/41, validation error 24.958333 %\n",
      "     epoch 91, minibatch 41/41, test error of best model 25.434343 %\n",
      "epoch 92, minibatch 41/41, validation error 24.958333 %\n",
      "     epoch 92, minibatch 41/41, test error of best model 25.414141 %\n",
      "epoch 93, minibatch 41/41, validation error 24.895833 %\n",
      "     epoch 93, minibatch 41/41, test error of best model 25.409091 %\n",
      "epoch 94, minibatch 41/41, validation error 24.895833 %\n",
      "epoch 95, minibatch 41/41, validation error 24.895833 %\n",
      "epoch 96, minibatch 41/41, validation error 24.854167 %\n",
      "     epoch 96, minibatch 41/41, test error of best model 25.318182 %\n",
      "epoch 97, minibatch 41/41, validation error 24.854167 %\n",
      "epoch 98, minibatch 41/41, validation error 24.833333 %\n",
      "     epoch 98, minibatch 41/41, test error of best model 25.303030 %\n",
      "epoch 99, minibatch 41/41, validation error 24.833333 %\n",
      "epoch 100, minibatch 41/41, validation error 24.812500 %\n",
      "     epoch 100, minibatch 41/41, test error of best model 25.257576 %\n",
      "epoch 101, minibatch 41/41, validation error 24.812500 %\n",
      "epoch 102, minibatch 41/41, validation error 24.791667 %\n",
      "     epoch 102, minibatch 41/41, test error of best model 25.227273 %\n",
      "epoch 103, minibatch 41/41, validation error 24.770833 %\n",
      "     epoch 103, minibatch 41/41, test error of best model 25.207071 %\n",
      "epoch 104, minibatch 41/41, validation error 24.770833 %\n",
      "epoch 105, minibatch 41/41, validation error 24.770833 %\n",
      "epoch 106, minibatch 41/41, validation error 24.770833 %\n",
      "epoch 107, minibatch 41/41, validation error 24.750000 %\n",
      "     epoch 107, minibatch 41/41, test error of best model 25.161616 %\n",
      "epoch 108, minibatch 41/41, validation error 24.750000 %\n",
      "epoch 109, minibatch 41/41, validation error 24.750000 %\n",
      "epoch 110, minibatch 41/41, validation error 24.729167 %\n",
      "     epoch 110, minibatch 41/41, test error of best model 25.070707 %\n",
      "epoch 111, minibatch 41/41, validation error 24.729167 %\n",
      "epoch 112, minibatch 41/41, validation error 24.708333 %\n",
      "     epoch 112, minibatch 41/41, test error of best model 25.030303 %\n",
      "epoch 113, minibatch 41/41, validation error 24.666667 %\n",
      "     epoch 113, minibatch 41/41, test error of best model 25.010101 %\n",
      "epoch 114, minibatch 41/41, validation error 24.625000 %\n",
      "     epoch 114, minibatch 41/41, test error of best model 24.984848 %\n",
      "epoch 115, minibatch 41/41, validation error 24.625000 %\n",
      "epoch 116, minibatch 41/41, validation error 24.604167 %\n",
      "     epoch 116, minibatch 41/41, test error of best model 24.934343 %\n",
      "epoch 117, minibatch 41/41, validation error 24.583333 %\n",
      "     epoch 117, minibatch 41/41, test error of best model 24.924242 %\n",
      "epoch 118, minibatch 41/41, validation error 24.562500 %\n",
      "     epoch 118, minibatch 41/41, test error of best model 24.914141 %\n",
      "epoch 119, minibatch 41/41, validation error 24.562500 %\n",
      "epoch 120, minibatch 41/41, validation error 24.541667 %\n",
      "     epoch 120, minibatch 41/41, test error of best model 24.893939 %\n",
      "epoch 121, minibatch 41/41, validation error 24.541667 %\n",
      "Optimization complete with best validation score of 24.541667 %,with test performance 24.893939 %\n",
      "The code run for 122 epochs, with 4.817208 epochs/sec\n",
      "The code for file best_model.pkl ran for 25.3s\n",
      "Now predicting...\n",
      "Predicted values for the first 10 examples in test set:\n",
      "[0 0 0 1 1 1 1 0 0 0]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "This tutorial introduces logistic regression using Theano and stochastic\n",
    "gradient descent.\n",
    "\n",
    "Logistic regression is a probabilistic, linear classifier. It is parametrized\n",
    "by a weight matrix :math:`W` and a bias vector :math:`b`. Classification is\n",
    "done by projecting data points onto a set of hyperplanes, the distance to\n",
    "which is used to determine a class membership probability.\n",
    "\n",
    "Mathematically, this can be written as:\n",
    "\n",
    ".. math::\n",
    "  P(Y=i|x, W,b) &= softmax_i(W x + b) \\\\\n",
    "                &= \\frac {e^{W_i x + b_i}} {\\sum_j e^{W_j x + b_j}}\n",
    "\n",
    "\n",
    "The output of the model or prediction is then done by taking the argmax of\n",
    "the vector whose i'th element is P(Y=i|x).\n",
    "\n",
    ".. math::\n",
    "\n",
    "  y_{pred} = argmax_i P(Y=i|x,W,b)\n",
    "\n",
    "\n",
    "This tutorial presents a stochastic gradient descent optimization method\n",
    "suitable for large datasets.\n",
    "\n",
    "\n",
    "References:\n",
    "\n",
    "    - textbooks: \"Pattern Recognition and Machine Learning\" -\n",
    "                 Christopher M. Bishop, section 4.3.2\n",
    "\n",
    "\"\"\"\n",
    "from collections import namedtuple, defaultdict\n",
    "from random import shuffle, randint\n",
    "#----------------------------------------------------\n",
    "__docformat__ = 'restructedtext en'\n",
    "\n",
    "import cPickle\n",
    "import gzip\n",
    "import os\n",
    "import sys\n",
    "import timeit\n",
    "\n",
    "import numpy\n",
    "import numpy as np\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "#----------------------------------------------------\n",
    "\n",
    "def get_data():\n",
    "    '''\n",
    "    \n",
    "    '''\n",
    "    all_data = []  \n",
    "    DataDoc= namedtuple('DataDoc', 'tag words')\n",
    "    with open('/Users/mam/CORE/RESEARCH/DEEPLEARNING/Doc2Vec/data/aclImdb/alldata-id.txt') as alldata:\n",
    "        for line_no, line in enumerate(alldata):\n",
    "            label=line.split()[0]\n",
    "            word_list=line.lower().split()[1:]\n",
    "            all_data.append(DataDoc(label, word_list))\n",
    "    train_data = all_data[:25000]\n",
    "    dev_data = all_data[25000:27500]+all_data[47500:50000]\n",
    "    test_data=all_data[27500:47500]\n",
    "    # labels\n",
    "    train_tags=[ 1.0 for i in range(12500)] + [ 0.0 for i in range(12500)]\n",
    "    dev_tags= [ 1.0 for i in range(2500)] + [ 0.0 for i in range(2500)]\n",
    "    test_tags= [ 1.0 for i in range(10000)] + [ 0.0 for i in range(10000)]\n",
    "    return train_data, train_tags, dev_data, dev_tags, test_data, test_tags\n",
    "    #--------------------------------------------------\n",
    "#train_data, train_tags, dev_data, dev_tags, test_data, test_tags=get_data()\n",
    "########################\n",
    "\n",
    "\n",
    "# Let's get a dictionary of all the words in training data\n",
    "# These will be our bag-of-words features\n",
    "# We won't need this function, since we will use gensim's built-in method \"Dictionary\" from the corpus module\n",
    "# --> corpora.Dictionary, but we provide this so that you are clear on one way of how to do this.\n",
    "def get_space(train_data):\n",
    "    \"\"\"\n",
    "    input is a list of namedtuples\n",
    "    get a dict of word space\n",
    "    key=word\n",
    "    value=len of the dict at that point \n",
    "    (that will be the index of the word and it is unique since the dict grows as we loop)\n",
    "    \"\"\"\n",
    "    word_space=defaultdict(int)\n",
    "    for doc in train_data:\n",
    "        for w in doc.words:\n",
    "            # indexes of words won't be in sequential order as they occur in data (can you tell why?), \n",
    "            # but that doesn't matter.\n",
    "            word_space[w]+=1\n",
    "    return word_space\n",
    "\n",
    "# train_data, train_tags, dev_data, dev_tags, test_data, test_tags=get_data()\n",
    "# word_space=get_space(train_data)\n",
    "# word_space={w: word_space[w] for w in word_space if word_space[w] > 500}\n",
    "# space_len=len(word_space)\n",
    "# print \"space_len: \", space_len\n",
    "def get_sparse_vec(data_point, space):\n",
    "    # create empty vector\n",
    "    sparse_vec = np.zeros((len(space)))\n",
    "    for w in set(data_point.words):\n",
    "        # use exception handling such that this function can also be used to vectorize \n",
    "        # data with words not in train (i.e., test and dev data)\n",
    "        try:\n",
    "            sparse_vec[space[w]]=1\n",
    "        except:\n",
    "            continue\n",
    "    return sparse_vec\n",
    "\n",
    "   \n",
    "# train_vecs= [get_sparse_vec(data_point, word_space) for data_point in train_data]\n",
    "# test_vecs= [get_sparse_vec(data_point, word_space) for data_point in test_data]\n",
    "# dev_vecs= [get_sparse_vec(data_point, word_space) for data_point in dev_data]\n",
    "# #---------------------------\n",
    "# train_vecs=np.array(train_vecs)\n",
    "# train_tags=np.array(train_tags)\n",
    "# dev_vecs=np.array(dev_vecs)\n",
    "# dev_tags=np.array(dev_tags)\n",
    "# test_vecs=np.array(test_vecs)\n",
    "# test_tags=np.array(test_tags)\n",
    "# print train_vecs.shape\n",
    "# print dev_vecs.shape\n",
    "# print test_vecs.shape\n",
    "\n",
    "\n",
    "def load_data(train_vecs, train_tags, dev_vecs, dev_tags, test_vecs, test_tags):\n",
    "    #------------------------------\n",
    "    # Modified from Theano tutorial.\n",
    "    # I basically pass data_x, data_y instead of data_xy\n",
    "    def shared_dataset(data_x, data_y, borrow=True):\n",
    "        \"\"\" Function that loads the dataset into shared variables\n",
    "\n",
    "        The reason we store our dataset in shared variables is to allow\n",
    "        Theano to copy it into the GPU memory (when code is run on GPU).\n",
    "        Since copying data into the GPU is slow, copying a minibatch everytime\n",
    "        is needed (the default behaviour if the data is not in a shared\n",
    "        variable) would lead to a large decrease in performance.\n",
    "        \"\"\"\n",
    "        shared_x = theano.shared(numpy.asarray(data_x,\n",
    "                                               dtype=theano.config.floatX), borrow=borrow)\n",
    "        shared_y = theano.shared(numpy.asarray(data_y,\n",
    "                                               dtype=theano.config.floatX),\n",
    "                                 borrow=borrow)\n",
    "        # When storing data on the GPU it has to be stored as floats\n",
    "        # therefore we will store the labels as ``floatX`` as well\n",
    "        # (``shared_y`` does exactly that). But during our computations\n",
    "        # we need them as ints (we use labels as index, and if they are\n",
    "        # floats it doesn't make sense) therefore instead of returning\n",
    "        # ``shared_y`` we will have to cast it to int. This little hack\n",
    "        # lets ous get around this issue\n",
    "        return shared_x, T.cast(shared_y, 'int32')\n",
    "    #-----------------------------------------------------------------\n",
    "    train_set_x, train_set_y = shared_dataset(train_vecs, train_tags)\n",
    "    valid_set_x, valid_set_y = shared_dataset(dev_vecs, dev_tags)\n",
    "    test_set_x, test_set_y = shared_dataset(test_vecs, test_tags)\n",
    "\n",
    "    rval = [(train_set_x, train_set_y), (valid_set_x, valid_set_y),\n",
    "            (test_set_x, test_set_y)]\n",
    "    return rval\n",
    "\n",
    "#rval=load_data(train_vecs, train_tags)\n",
    "#print rval\n",
    "\n",
    "\n",
    "class LogisticRegression(object):\n",
    "    \"\"\"Multi-class Logistic Regression Class\n",
    "\n",
    "    The logistic regression is fully described by a weight matrix :math:`W`\n",
    "    and bias vector :math:`b`. Classification is done by projecting data\n",
    "    points onto a set of hyperplanes, the distance to which is used to\n",
    "    determine a class membership probability.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, input, n_in, n_out):\n",
    "        \"\"\" Initialize the parameters of the logistic regression\n",
    "\n",
    "        :type input: theano.tensor.TensorType\n",
    "        :param input: symbolic variable that describes the input of the\n",
    "                      architecture (one minibatch)\n",
    "\n",
    "        :type n_in: int\n",
    "        :param n_in: number of input units, the dimension of the space in\n",
    "                     which the datapoints lie\n",
    "\n",
    "        :type n_out: int\n",
    "        :param n_out: number of output units, the dimension of the space in\n",
    "                      which the labels lie\n",
    "\n",
    "        \"\"\"\n",
    "        # start-snippet-1\n",
    "        # initialize with 0 the weights W as a matrix of shape (n_in, n_out)\n",
    "        self.W = theano.shared(\n",
    "            value=numpy.zeros(\n",
    "                (n_in, n_out),\n",
    "                dtype=theano.config.floatX\n",
    "            ),\n",
    "            name='W',\n",
    "            borrow=True\n",
    "        )\n",
    "        # initialize the biases b as a vector of n_out 0s\n",
    "        self.b = theano.shared(\n",
    "            value=numpy.zeros(\n",
    "                (n_out,),\n",
    "                dtype=theano.config.floatX\n",
    "            ),\n",
    "            name='b',\n",
    "            borrow=True\n",
    "        )\n",
    "\n",
    "        # symbolic expression for computing the matrix of class-membership\n",
    "        # probabilities\n",
    "        # Where:\n",
    "        # W is a matrix where column-k represent the separation hyperplane for\n",
    "        # class-k\n",
    "        # x is a matrix where row-j  represents input training sample-j\n",
    "        # b is a vector where element-k represent the free parameter of\n",
    "        # hyperplane-k\n",
    "        self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W) + self.b)\n",
    "\n",
    "        # symbolic description of how to compute prediction as class whose\n",
    "        # probability is maximal\n",
    "        self.y_pred = T.argmax(self.p_y_given_x, axis=1)\n",
    "        # end-snippet-1\n",
    "\n",
    "        # parameters of the model\n",
    "        self.params = [self.W, self.b]\n",
    "\n",
    "        # keep track of model input\n",
    "        self.input = input\n",
    "\n",
    "    def negative_log_likelihood(self, y):\n",
    "        \"\"\"Return the mean of the negative log-likelihood of the prediction\n",
    "        of this model under a given target distribution.\n",
    "\n",
    "        .. math::\n",
    "\n",
    "            \\frac{1}{|\\mathcal{D}|} \\mathcal{L} (\\theta=\\{W,b\\}, \\mathcal{D}) =\n",
    "            \\frac{1}{|\\mathcal{D}|} \\sum_{i=0}^{|\\mathcal{D}|}\n",
    "                \\log(P(Y=y^{(i)}|x^{(i)}, W,b)) \\\\\n",
    "            \\ell (\\theta=\\{W,b\\}, \\mathcal{D})\n",
    "\n",
    "        :type y: theano.tensor.TensorType\n",
    "        :param y: corresponds to a vector that gives for each example the\n",
    "                  correct label\n",
    "\n",
    "        Note: we use the mean instead of the sum so that\n",
    "              the learning rate is less dependent on the batch size\n",
    "        \"\"\"\n",
    "        # start-snippet-2\n",
    "        # y.shape[0] is (symbolically) the number of rows in y, i.e.,\n",
    "        # number of examples (call it n) in the minibatch\n",
    "        # T.arange(y.shape[0]) is a symbolic vector which will contain\n",
    "        # [0,1,2,... n-1] T.log(self.p_y_given_x) is a matrix of\n",
    "        # Log-Probabilities (call it LP) with one row per example and\n",
    "        # one column per class LP[T.arange(y.shape[0]),y] is a vector\n",
    "        # v containing [LP[0,y[0]], LP[1,y[1]], LP[2,y[2]], ...,\n",
    "        # LP[n-1,y[n-1]]] and T.mean(LP[T.arange(y.shape[0]),y]) is\n",
    "        # the mean (across minibatch examples) of the elements in v,\n",
    "        # i.e., the mean log-likelihood across the minibatch.\n",
    "        return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]), y])\n",
    "        # end-snippet-2\n",
    "\n",
    "    def errors(self, y):\n",
    "        \"\"\"Return a float representing the number of errors in the minibatch\n",
    "        over the total number of examples of the minibatch ; zero one\n",
    "        loss over the size of the minibatch\n",
    "\n",
    "        :type y: theano.tensor.TensorType\n",
    "        :param y: corresponds to a vector that gives for each example the\n",
    "                  correct label\n",
    "        \"\"\"\n",
    "\n",
    "        # check if y has same dimension of y_pred\n",
    "        if y.ndim != self.y_pred.ndim:\n",
    "            raise TypeError(\n",
    "                'y should have the same shape as self.y_pred',\n",
    "                ('y', y.type, 'y_pred', self.y_pred.type)\n",
    "            )\n",
    "        # check if y is of the correct datatype\n",
    "        if y.dtype.startswith('int'):\n",
    "            # the T.neq operator returns a vector of 0s and 1s, where 1\n",
    "            # represents a mistake in prediction\n",
    "            return T.mean(T.neq(self.y_pred, y))\n",
    "        else:\n",
    "            raise NotImplementedError()\n",
    "\n",
    "\n",
    "def sgd_optimization(learning_rate=0.13, n_epochs=1000,\n",
    "                           batch_size=600):\n",
    "    \"\"\"\n",
    "    Demonstrate stochastic gradient descent optimization of a log-linear\n",
    "    model\n",
    "\n",
    "    This is demonstrated on MNIST.\n",
    "\n",
    "    :type learning_rate: float\n",
    "    :param learning_rate: learning rate used (factor for the stochastic\n",
    "                          gradient)\n",
    "\n",
    "    :type n_epochs: int\n",
    "    :param n_epochs: maximal number of epochs to run the optimizer\n",
    "\n",
    "    :type dataset: string\n",
    "    :param dataset: the path of the MNIST dataset file from\n",
    "                 http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz\n",
    "\n",
    "    \"\"\"\n",
    "    datasets=load_data(train_vecs, train_tags, dev_vecs, dev_tags, test_vecs, test_tags)\n",
    "    train_set_x, train_set_y = datasets[0]\n",
    "    valid_set_x, valid_set_y = datasets[1]\n",
    "    test_set_x, test_set_y = datasets[2]\n",
    "    print train_set_x.shape[0]\n",
    "    # compute number of minibatches for training, validation and testing\n",
    "    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size\n",
    "    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size\n",
    "    n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size\n",
    "\n",
    "    ######################\n",
    "    # BUILD ACTUAL MODEL #\n",
    "    ######################\n",
    "    print '... building the model'\n",
    "\n",
    "    # allocate symbolic variables for the data\n",
    "    index = T.lscalar()  # index to a [mini]batch\n",
    "\n",
    "    # generate symbolic variables for input (x and y represent a\n",
    "    # minibatch)\n",
    "    x = T.matrix('x')  # data, presented as rasterized images\n",
    "    y = T.ivector('y')  # labels, presented as 1D vector of [int] labels\n",
    "\n",
    "    # construct the logistic regression class\n",
    "    # Each MNIST image has size 28*28\n",
    "    # MAM: We change size: n_in=space_len\n",
    "    classifier = LogisticRegression(input=x, n_in=space_len, n_out=2)\n",
    "\n",
    "    # the cost we minimize during training is the negative log likelihood of\n",
    "    # the model in symbolic format\n",
    "    cost = classifier.negative_log_likelihood(y)\n",
    "\n",
    "    # compiling a Theano function that computes the mistakes that are made by\n",
    "    # the model on a minibatch\n",
    "    test_model = theano.function(\n",
    "        inputs=[index],\n",
    "        outputs=classifier.errors(y),\n",
    "        givens={\n",
    "            x: test_set_x[index * batch_size: (index + 1) * batch_size],\n",
    "            y: test_set_y[index * batch_size: (index + 1) * batch_size]\n",
    "        }\n",
    "    )\n",
    "\n",
    "    validate_model = theano.function(\n",
    "        inputs=[index],\n",
    "        outputs=classifier.errors(y),\n",
    "        givens={\n",
    "            x: valid_set_x[index * batch_size: (index + 1) * batch_size],\n",
    "            y: valid_set_y[index * batch_size: (index + 1) * batch_size]\n",
    "        }\n",
    "    )\n",
    "\n",
    "    # compute the gradient of cost with respect to theta = (W,b)\n",
    "    g_W = T.grad(cost=cost, wrt=classifier.W)\n",
    "    g_b = T.grad(cost=cost, wrt=classifier.b)\n",
    "\n",
    "    # start-snippet-3\n",
    "    # specify how to update the parameters of the model as a list of\n",
    "    # (variable, update expression) pairs.\n",
    "    updates = [(classifier.W, classifier.W - learning_rate * g_W),\n",
    "               (classifier.b, classifier.b - learning_rate * g_b)]\n",
    "\n",
    "    # compiling a Theano function `train_model` that returns the cost, but in\n",
    "    # the same time updates the parameter of the model based on the rules\n",
    "    # defined in `updates`\n",
    "    train_model = theano.function(\n",
    "        inputs=[index],\n",
    "        outputs=cost,\n",
    "        updates=updates,\n",
    "        givens={\n",
    "            x: train_set_x[index * batch_size: (index + 1) * batch_size],\n",
    "            y: train_set_y[index * batch_size: (index + 1) * batch_size]\n",
    "        }\n",
    "    )\n",
    "    # end-snippet-3\n",
    "\n",
    "    ###############\n",
    "    # TRAIN MODEL #\n",
    "    ###############\n",
    "    print '... training the model'\n",
    "    # early-stopping parameters\n",
    "    patience = 5000  # look as this many examples regardless\n",
    "    patience_increase = 2  # wait this much longer when a new best is\n",
    "                                  # found\n",
    "    improvement_threshold = 0.995  # a relative improvement of this much is\n",
    "                                  # considered significant\n",
    "    validation_frequency = min(n_train_batches, patience / 2)\n",
    "                                  # go through this many\n",
    "                                  # minibatche before checking the network\n",
    "                                  # on the validation set; in this case we\n",
    "                                  # check every epoch\n",
    "\n",
    "    best_validation_loss = numpy.inf\n",
    "    test_score = 0.\n",
    "    start_time = timeit.default_timer()\n",
    "\n",
    "    done_looping = False\n",
    "    epoch = 0\n",
    "    while (epoch < n_epochs) and (not done_looping):\n",
    "        epoch = epoch + 1\n",
    "        for minibatch_index in xrange(n_train_batches):\n",
    "\n",
    "            minibatch_avg_cost = train_model(minibatch_index)\n",
    "            # iteration number\n",
    "            iter = (epoch - 1) * n_train_batches + minibatch_index\n",
    "\n",
    "            if (iter + 1) % validation_frequency == 0:\n",
    "                # compute zero-one loss on validation set\n",
    "                validation_losses = [validate_model(i)\n",
    "                                     for i in xrange(n_valid_batches)]\n",
    "                this_validation_loss = numpy.mean(validation_losses)\n",
    "\n",
    "                print(\n",
    "                    'epoch %i, minibatch %i/%i, validation error %f %%' %\n",
    "                    (\n",
    "                        epoch,\n",
    "                        minibatch_index + 1,\n",
    "                        n_train_batches,\n",
    "                        this_validation_loss * 100.\n",
    "                    )\n",
    "                )\n",
    "\n",
    "                # if we got the best validation score until now\n",
    "                if this_validation_loss < best_validation_loss:\n",
    "                    #improve patience if loss improvement is good enough\n",
    "                    if this_validation_loss < best_validation_loss *  \\\n",
    "                       improvement_threshold:\n",
    "                        patience = max(patience, iter * patience_increase)\n",
    "\n",
    "                    best_validation_loss = this_validation_loss\n",
    "                    # test it on the test set\n",
    "\n",
    "                    test_losses = [test_model(i)\n",
    "                                   for i in xrange(n_test_batches)]\n",
    "                    test_score = numpy.mean(test_losses)\n",
    "\n",
    "                    print(\n",
    "                        (\n",
    "                            '     epoch %i, minibatch %i/%i, test error of'\n",
    "                            ' best model %f %%'\n",
    "                        ) %\n",
    "                        (\n",
    "                            epoch,\n",
    "                            minibatch_index + 1,\n",
    "                            n_train_batches,\n",
    "                            test_score * 100.\n",
    "                        )\n",
    "                    )\n",
    "\n",
    "                    # save the best model\n",
    "                    with open('best_model.pkl', 'w') as f:\n",
    "                        cPickle.dump(classifier, f)\n",
    "\n",
    "            if patience <= iter:\n",
    "                done_looping = True\n",
    "                break\n",
    "\n",
    "    end_time = timeit.default_timer()\n",
    "    print(\n",
    "        (\n",
    "            'Optimization complete with best validation score of %f %%,'\n",
    "            'with test performance %f %%'\n",
    "        )\n",
    "        % (best_validation_loss * 100., test_score * 100.)\n",
    "    )\n",
    "    print 'The code run for %d epochs, with %f epochs/sec' % (\n",
    "        epoch, 1. * epoch / (end_time - start_time))\n",
    "    print ('The code for file ' +\n",
    "                          'best_model.pkl' +\n",
    "                          ' ran for %.1fs' % ((end_time - start_time)))\n",
    "\n",
    "\n",
    "def predict():\n",
    "    \"\"\"\n",
    "    An example of how to load a trained model and use it\n",
    "    to predict labels.\n",
    "    \"\"\"\n",
    "\n",
    "    # load the saved model\n",
    "    classifier = cPickle.load(open('best_model.pkl'))\n",
    "\n",
    "    # compile a predictor function\n",
    "    predict_model = theano.function(\n",
    "        inputs=[classifier.input],\n",
    "        outputs=classifier.y_pred)\n",
    "\n",
    "    # We can test it on some examples from test test\n",
    "    datasets=load_data(train_vecs, train_tags, dev_vecs, dev_tags, test_vecs, test_tags)\n",
    "    #train_set_x, train_set_y = datasets[0]\n",
    "    #valid_set_x, valid_set_y = datasets[1]\n",
    "    test_set_x, test_set_y = datasets[2]\n",
    "    test_set_x = test_set_x.get_value()\n",
    "    predicted_values = predict_model(test_set_x[:10])\n",
    "    print (\"Predicted values for the first 10 examples in test set:\")\n",
    "    print predicted_values\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    train_data, train_tags, dev_data, dev_tags, test_data, test_tags=get_data()\n",
    "    word_space=get_space(train_data)\n",
    "    # We only use words with a give frequency threshold. This also helps much with memory.\n",
    "    # Note that I am using a *very* high threshold here, since the goal is to demonstrate \n",
    "    # the technique and its utility, rather than higher accuracy. I also had some memory issues.\n",
    "    word_space={w: word_space[w] for w in word_space if word_space[w] > 150} \n",
    "    space_len=len(word_space)\n",
    "    print(\"space_len: %d\" % space_len)\n",
    "    train_vecs= [get_sparse_vec(data_point, word_space) for data_point in train_data]\n",
    "    test_vecs= [get_sparse_vec(data_point, word_space) for data_point in test_data]\n",
    "    dev_vecs= [get_sparse_vec(data_point, word_space) for data_point in dev_data]\n",
    "    #del word_space\n",
    "    #---------------------------\n",
    "    train_vecs=np.array(train_vecs)\n",
    "    train_tags=np.array(train_tags)\n",
    "    dev_vecs=np.array(dev_vecs)\n",
    "    dev_tags=np.array(dev_tags)\n",
    "    test_vecs=np.array(test_vecs)\n",
    "    test_tags=np.array(test_tags)\n",
    "    #del train_data, train_tags, dev_data, dev_tags, test_data, test_tags\n",
    "    print('train_vecs.shape: %d, %d' % train_vecs.shape)\n",
    "    print('dev_vecs.shape: %d, %d' % dev_vecs.shape)\n",
    "    print('test_vecs.shape: %d, %d' % test_vecs.shape)\n",
    "    sgd_optimization()\n",
    "    #------------------------------------------------------\n",
    "    print('Now predicting...')\n",
    "    predict()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}