RunTrainedModel.ipynb

{
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  },
  "name": "",
  "signature": "sha256:d3f074604e9d2ba6f8f306feb529e430150a0116035539d3b7025be87dde39eb"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pickle\n",
      "import CTMParallel\n",
      "import numpy as np\n",
      "from sklearn.svm import SVC\n",
      "from sklearn.metrics import accuracy_score\n",
      "from sklearn.linear_model import LogisticRegression\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Vendor:  Continuum Analytics, Inc.\n",
        "Package: mkl\n",
        "Message: trial mode EXPIRED 4 days ago\n",
        "\n",
        "    Your usage of mkl is now out of compliance with the license agreement.\n",
        "    A license for mkl can be purchased at: http://continuum.io\n",
        "    \n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# Returns the min, max, and average accuracy across the folds\n",
      "def KFoldCrossValidation(X, Y, num_folds, print_outputs=True):\n",
      "    accuracies = []\n",
      "    for train, test in KFold(X.shape[0], num_folds, shuffle=True):\n",
      "        X_train, X_test, Y_train, Y_test = X[train], X[test], Y[train], Y[test]\n",
      "        logistic_classifier = LogisticRegression()\n",
      "        logistic_classifier.fit(X_train, Y_train)\n",
      "        accuracies.append(accuracy_score(logistic_classifier.predict(X_test), Y_test))\n",
      "    if print_outputs:\n",
      "        print \"Max Accuracy: \", max(accuracies), \"\\nMin Accuracy: \", min(accuracies), \"\\nAverage Accuracy: \", sum(accuracies)/len(accuracies)\n",
      "    return max(accuracies), min(accuracies), sum(accuracies)/len(accuracies)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 70
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Rat Data - 5, 15, 25, 35, 45 Topics"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "rat_topics_options = [5, 15, 25, 35, 45]\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# source can be either rat or neuron\n",
      "def loadSavedModel(numTopics, source):\n",
      "    # Load the trained model\n",
      "    model = pickle.load(open(\"pickled_objects/\" + source + \"/model_\" + source + \"_\" + str(numTopics) + \"topics.p\", \"rb\"))\n",
      "    classes = pickle.load(open(\"pickled_objects/\" + source + \"/classes_\" + source + \"_\" + str(numTopics) + \"topics.p\", \"rb\"))\n",
      "    vocab = pickle.load(open(\"pickled_objects/\" + source + \"/vocab_\" + source + \"_\" + str(numTopics) + \"topics.p\", \"rb\"))\n",
      "    counts = pickle.load(open(\"pickled_objects/\" + source + \"/counts_\" + source + \"_\" + str(numTopics) + \"topics.p\", \"rb\"))\n",
      "    return model, classes, vocab, counts\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "def getClassificationAccuracy(model,classes, vocab, counts):\n",
      "    return KFoldCrossValidation(model.saved_lambdas, classes, 5)   "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Test for classification accuracy on the rat\n",
      "print \"Rat Classification accuracy on raw counts\"\n",
      "model, classes, vocab, counts = loadSavedModel(5, \"rat\")\n",
      "KFoldCrossValidation(counts, classes, 5)\n",
      "print\n",
      "print\n",
      "print\n",
      "\n",
      "for topic in rat_topics_options:\n",
      "    model, classes, vocab, counts = loadSavedModel(topic, \"rat\")\n",
      "    print \"Rat Classification with numTopics = \", topic\n",
      "    print \"Accuracy with original features\"\n",
      "    getClassificationAccuracy(model, classes, vocab, counts)\n",
      "    print\n",
      "    print \"Accuracy with PCA Features\"\n",
      "    pca_rat = PCA(n_components=topic)\n",
      "    pca_features = pca_rat.fit_transform(counts)\n",
      "    min_acc, max_acc, avg_acc = KFoldCrossValidation(pca_features, classes, 5)\n",
      "    print\n",
      "    print\n",
      "    print"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Rat Classification accuracy on raw counts\n",
        "Max Accuracy:  1.0 \n",
        "Min Accuracy:  0.984375 \n",
        "Average Accuracy:  0.996875\n",
        "\n",
        "\n",
        "\n",
        "Rat Classification with numTopics =  5\n",
        "Accuracy with original features\n",
        "Max Accuracy:  0.890625 \n",
        "Min Accuracy:  0.828125 \n",
        "Average Accuracy:  0.86875\n",
        "\n",
        "Accuracy with PCA Features\n",
        "Max Accuracy:  0.671875 \n",
        "Min Accuracy:  0.625 \n",
        "Average Accuracy:  0.640625\n",
        "\n",
        "\n",
        "\n",
        "Rat Classification with numTopics =  15\n",
        "Accuracy with original features\n",
        "Max Accuracy:  1.0 \n",
        "Min Accuracy:  0.984375 \n",
        "Average Accuracy:  0.996875\n",
        "\n",
        "Accuracy with PCA Features\n",
        "Max Accuracy:  0.984375 \n",
        "Min Accuracy:  0.953125 \n",
        "Average Accuracy:  0.975\n",
        "\n",
        "\n",
        "\n",
        "Rat Classification with numTopics =  25\n",
        "Accuracy with original features\n",
        "Max Accuracy:  1.0 \n",
        "Min Accuracy:  0.984375 \n",
        "Average Accuracy:  0.99375\n",
        "\n",
        "Accuracy with PCA Features\n",
        "Max Accuracy:  1.0 \n",
        "Min Accuracy:  0.984375 \n",
        "Average Accuracy:  0.99375\n",
        "\n",
        "\n",
        "\n",
        "Rat Classification with numTopics =  35\n",
        "Accuracy with original features\n",
        "Max Accuracy:  1.0 \n",
        "Min Accuracy:  1.0 \n",
        "Average Accuracy:  1.0\n",
        "\n",
        "Accuracy with PCA Features\n",
        "Max Accuracy:  1.0 \n",
        "Min Accuracy:  0.984375 \n",
        "Average Accuracy:  0.99375\n",
        "\n",
        "\n",
        "\n",
        "Rat Classification with numTopics =  45\n",
        "Accuracy with original features\n",
        "Max Accuracy:  1.0 \n",
        "Min Accuracy:  0.953125 \n",
        "Average Accuracy:  0.990625\n",
        "\n",
        "Accuracy with PCA Features\n",
        "Max Accuracy:  1.0 \n",
        "Min Accuracy:  0.984375 \n",
        "Average Accuracy:  0.996875\n",
        "\n",
        "\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Neuron Dataset - 5, 15, 25, 35 topics"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "neuron_topics_options = [5, 15, 25, 35]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print \"Rat Classification accuracy on raw counts\"\n",
      "model, classes, vocab, counts = loadSavedModel(5, \"neuron\")\n",
      "KFoldCrossValidation(counts, classes, 5)\n",
      "print\n",
      "print\n",
      "print\n",
      "\n",
      "for topic in neuron_topics_options:\n",
      "    model, classes, vocab, counts = loadSavedModel(topic, \"neuron\")\n",
      "    print \"Neuron Classification with numTopics = \", topic\n",
      "    print \"Accuracy with original features\"\n",
      "    getClassificationAccuracy(model, classes, vocab, counts)\n",
      "    print\n",
      "    print \"Accuracy with PCA Features\"\n",
      "    pca_neuron = PCA(n_components=topic)\n",
      "    pca_features = pca_rat.fit_transform(counts)\n",
      "    min_acc, max_acc, avg_acc = KFoldCrossValidation(pca_features, classes, 5)\n",
      "    print\n",
      "    print\n",
      "    print"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Rat Classification accuracy on raw counts\n",
        "Max Accuracy:  0.97 \n",
        "Min Accuracy:  0.91 \n",
        "Average Accuracy:  0.934\n",
        "\n",
        "\n",
        "\n",
        "Neuron Classification with numTopics =  5\n",
        "Accuracy with original features\n",
        "Max Accuracy:  0.84 \n",
        "Min Accuracy:  0.72 \n",
        "Average Accuracy:  0.76\n",
        "\n",
        "Accuracy with PCA Features\n",
        "Max Accuracy:  0.93 \n",
        "Min Accuracy:  0.87 \n",
        "Average Accuracy:  0.91\n",
        "\n",
        "\n",
        "\n",
        "Neuron Classification with numTopics =  15\n",
        "Accuracy with original features\n",
        "Max Accuracy:  0.94 \n",
        "Min Accuracy:  0.9 \n",
        "Average Accuracy:  0.922\n",
        "\n",
        "Accuracy with PCA Features\n",
        "Max Accuracy:  0.94 \n",
        "Min Accuracy:  0.87 \n",
        "Average Accuracy:  0.904\n",
        "\n",
        "\n",
        "\n",
        "Neuron Classification with numTopics =  25\n",
        "Accuracy with original features\n",
        "Max Accuracy:  0.93 \n",
        "Min Accuracy:  0.9 \n",
        "Average Accuracy:  0.916\n",
        "\n",
        "Accuracy with PCA Features\n",
        "Max Accuracy:  0.98 \n",
        "Min Accuracy:  0.85 \n",
        "Average Accuracy:  0.906\n",
        "\n",
        "\n",
        "\n",
        "Neuron Classification with numTopics =  35\n",
        "Accuracy with original features\n",
        "Max Accuracy:  0.96 \n",
        "Min Accuracy:  0.87 \n",
        "Average Accuracy:  0.912\n",
        "\n",
        "Accuracy with PCA Features\n",
        "Max Accuracy:  0.94 \n",
        "Min Accuracy:  0.88 \n",
        "Average Accuracy:  0.912\n",
        "\n",
        "\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "def explore_relationships(numTopics, source):\n",
      "    model, classes, vocab, counts = loadSavedModel(numTopics, source)\n",
      "    # Get the top genes in each topic\n",
      "    n_top_words = 25\n",
      "    for i, topic_dist in enumerate(model.beta):\n",
      "        topic_words = np.array(vocab)[np.argsort(topic_dist)][:-n_top_words:-1]\n",
      "        print('Topic {}: {}'.format(i+1, ' '.join(topic_words)))\n",
      "        print \"\"\n",
      "    print \"Covariance matrix\"\n",
      "    print model.sigma\n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "explore_relationships(15, \"neuron\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Topic 1: Cst3 Sparcl1 Apoe Slc1a2 Mt1 Atp1a2 Clu Dbi Cpe Rps29 Slc1a3 Prdx6 Sparc Ckb Mt2 Sepp1 Lars2 Rplp1 Aqp4 Actg1 Ptn Prdx1 Glul Itm2b\n",
        "\n",
        "Topic 2: Ttr Usmg5 Cox7c Mdh1 Rps29 Dynll1 Gstp1 Ndufa4 Pcsk2 Slc25a4 Cox6c Atp5e Enpp2 Cox7a2 Atp5g1 Camk2n1 Snca Snhg11 Uqcrh Cycs Scg5 Ttc3 Atp5g3 Atp5j\n",
        "\n",
        "Topic 3: Rps29 Npy Rplp1 Ftl1 Rps28 Rpl32 Rps3 Ndufa4 H3f3b Cox7c Akr1a1 Rpl18a Rpl35 Mdh1 Itm2b Atp6v1a Cox8a Tubb2a Fau Ddx5 Eef1a1 Rpl23 2010107E04Rik Rplp2\n",
        "\n",
        "Topic 4: Slc25a4 Usmg5 Scg5 Stmn3 Mdh1 Cox6b1 Eif4a2 Cox8a Atpif1 Cox4i1 Mif Atp5b Cycs Uqcrq Atp5j2 Tpi1 Atp6v1a Fau Zwint Tecr Rpl4 Atp5f1 Oaz1-ps Ndufa13\n",
        "\n",
        "Topic 5: Nrgn Snhg11 Rtn1 Ppp3ca Aplp1 Prkcb Prnp Camk2n1 Camk2b Nrxn1 Cpe Kif1b Itm2b Camk2a Celf2 Chn1 Lars2 Grin2b Rgs4 Gpm6b 2900097C17Rik Ttc3 Rtn4 Hpca\n",
        "\n",
        "Topic 6: Ndufa4 Mdh1 Cox6c Sub1 Aldoa Cox7a2 Actg1 Cox7b Pcp4 Stmn2 Chn1 Cycs Syt1 Slc25a4 2010107E04Rik Atp5l Uqcrh Lars2 Tagln3 Atp5e Atp6v1d Ndufa5 Atp5c1 Uqcrq\n",
        "\n",
        "Topic 7: Snca Scg5 Dynll1 Fkbp1a Calm3 Sub1 Itm2b Prnp Serinc1 Mdh1 Nsf Ywhah Atp5b Stmn3 Nrgn Atp6v0c-ps2 Prkcb Tubb2a Actg1 Olfm1 Ociad2 Chn1 Pja2 Snurf\n",
        "\n",
        "Topic 8: Snca Ywhaz Actg1 Rtn1 Tubb2a Chn1 3110035E14Rik Dynll1 Aldoa Stmn3 Atp6v1e1 Snurf Hspa8 Serinc1 Sub1 Hpca Ywhah Tpi1 Crym Itm2b Atp6v0c-ps2 Pja2 Syt1 Prnp\n",
        "\n",
        "Topic 9: Sst Vip Ttc3 Syt1 Npy Ndrg4 Aldoa Hspa8 Gad1 Gnas Scg5 Rtn1 Htr3a Atp6v0c-ps2 Tubb2a Serinc1 Snhg11 Slc25a4 Atp6v1a Mdh1 Syt11 Tcf4 Atp5b Rtn3\n",
        "\n",
        "Topic 10: Syt1 Cnr1 Scg2 Mdh1 Npy Slc25a4 Rtn1 Rab3c Htr3a 2900097C17Rik Gad2 Atp6v0c-ps2 Gnas Dner Nrxn3 Calm3 Nap1l5 Snhg11 Cck Atp5b Map1b Ndufa4 Snurf Atp1a3\n",
        "\n",
        "Topic 11: Ppp3ca Snhg11 Olfm1 Map1b Chn1 Cpe Gria1 Basp1 Aldoa Ywhaz Prnp Ttc3 Rtn3 Syt1 Nrgn Rtn1 Nptn Tcf4 Sub1 Calm3 Ppp3r1 Prkcb Tspan13 Dnm1\n",
        "\n",
        "Topic 12: Apod Mbp Ptgds Mog Enpp2 Car2 Gstp1 Cnp Scd2 Ugt8a Cd81 Mobp Ermn Tspan2 Cryab Mag Syt11 Qdpr Qk Ptma Cldn11 Ptn Hmgcs1 Aplp1\n",
        "\n",
        "Topic 13: Prkcb Cpe Ttc3 Rtn1 Ppp3ca D3Bwg0562e Dlgap1 Grm5 Map1b Scn2a1 Slc8a1 Syt1 Grin2b Atp1a3 Rasgrp1 Tcf4 Hpca Serinc1 Calm3 Syt11 Ywhah Trim2 Cadm2 Actg1\n",
        "\n",
        "Topic 14: Dynll1 Itm2a Myl12b Tubb2a Sub1 Cox7c Usmg5 Rps3a1 Stmn2 Tpi1 Rplp1 Rps6 Myl6 Cox7a2 Rpl35 Atp5j Atp5k Cox6b1 Sptbn1 Rps29 Ndufa4 Aldoa Stmn3 Cox4i1\n",
        "\n",
        "Topic 15: Atp6v1a Prkcb Aldoa Serinc1 Atp6v0c-ps2 Actg1 Tubb2a Pja2 Hpca Ywhaz Gria1 Slc8a1 Itm2b 2900097C17Rik Celf2 Tspan13 Snhg11 Syt1 Scg5 Nrgn Ppp3r1 Camta1 Kif1b Thy1\n",
        "\n",
        "Covariance matrix\n",
        "[[ 3.18479332 -0.3975801   0.36339261 -0.45979814 -1.14714679 -0.55513867\n",
        "  -1.24082048 -1.13248272 -1.06754839 -1.31563542 -1.49237668  0.92105309\n",
        "  -1.44531993  0.16712976 -1.23714884]\n",
        " [-0.3975801   1.69097926  0.49997025  1.07649534  0.57871876  1.07040716\n",
        "   1.02691432  1.00262713  0.7651779   0.81803315  0.81213905 -0.45965309\n",
        "   0.68627235  0.82895321  0.65653264]\n",
        " [ 0.36339261  0.49997025  1.97216849  0.89437066 -0.75887488  0.67788236\n",
        "  -0.14826092  0.0260237   0.16410782 -0.07569005 -0.71500011  0.2005211\n",
        "  -0.86159864  0.86769922 -0.47485062]\n",
        " [-0.45979814  1.07649534  0.89437066  2.16520742  0.16239861  1.35508705\n",
        "   0.98233696  1.07282278  0.91474147  0.8412617   0.49175295 -0.37180961\n",
        "   0.25537374  1.04782612  0.41848782]\n",
        " [-1.14714679  0.57871876 -0.75887488  0.16239861  3.33694716  0.66667358\n",
        "   2.17476266  1.76677303  1.48957416  2.11693083  3.05791164 -1.78796518\n",
        "   3.20819414  0.09031946  2.52331948]\n",
        " [-0.55513867  1.07040716  0.67788236  1.35508705  0.66667358  2.0320041\n",
        "   1.29151179  1.28892418  1.21375365  1.27832611  1.04773486 -0.8899285\n",
        "   0.87928793  1.05162968  0.90249006]\n",
        " [-1.24082048  1.02691432 -0.14826092  0.98233696  2.17476266  1.29151179\n",
        "   2.99896898  2.08430544  1.72365632  2.20878292  2.76903095 -1.86349836\n",
        "   2.72258683  0.64647328  2.30000918]\n",
        " [-1.13248272  1.00262713  0.0260237   1.07282278  1.76677303  1.28892418\n",
        "   2.08430544  2.56193985  1.59183765  1.97127455  2.35007614 -1.58927043\n",
        "   2.26363234  0.6993098   1.95122534]\n",
        " [-1.06754839  0.7651779   0.16410782  0.91474147  1.48957416  1.21375365\n",
        "   1.72365632  1.59183765  2.81907428  2.37243663  2.02914353 -1.70458339\n",
        "   2.01271021  0.56963199  1.78090337]\n",
        " [-1.31563542  0.81803315 -0.07569005  0.8412617   2.11693083  1.27832611\n",
        "   2.20878292  1.97127455  2.37243663  3.51234808  2.77827284 -2.29615319\n",
        "   2.80552679  0.54613933  2.4089911 ]\n",
        " [-1.49237668  0.81213905 -0.71500011  0.49175295  3.05791164  1.04773486\n",
        "   2.76903095  2.35007614  2.02914353  2.77827284  4.40871653 -2.52219152\n",
        "   3.84446844  0.32298488  3.10829761]\n",
        " [ 0.92105309 -0.45965309  0.2005211  -0.37180961 -1.78796518 -0.8899285\n",
        "  -1.86349836 -1.58927043 -1.70458339 -2.29615319 -2.52219152  4.58232158\n",
        "  -2.55697857 -0.58593004 -2.22615947]\n",
        " [-1.44531993  0.68627235 -0.86159864  0.25537374  3.20819414  0.87928793\n",
        "   2.72258683  2.26363234  2.01271021  2.80552679  3.84446844 -2.55697857\n",
        "   4.69736129  0.18517678  3.20662541]\n",
        " [ 0.16712976  0.82895321  0.86769922  1.04782612  0.09031946  1.05162968\n",
        "   0.64647328  0.6993098   0.56963199  0.54613933  0.32298488 -0.58593004\n",
        "   0.18517678  1.96839727  0.33033851]\n",
        " [-1.23714884  0.65653264 -0.47485062  0.41848782  2.52331948  0.90249006\n",
        "   2.30000918  1.95122534  1.78090337  2.4089911   3.10829761 -2.22615947\n",
        "   3.20662541  0.33033851  3.28929169]]\n"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": null
    }
   ],
   "metadata": {}
  }
 ]
}