Intro_spaCy_NLP.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:6842b637870ccb1c99f940c90e75c511dc193e15ea69d132c85630a4dc6ce0df"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Set up spaCy\n",
      "from spacy.en import English\n",
      "parser = English()\n",
      "\n",
      "# Test Data\n",
      "multiSentence = \"There is an art, it says, or rather, a knack to flying.\" \\\n",
      "                 \"The knack lies in learning how to throw yourself at the ground and miss.\" \\\n",
      "                 \"In the beginning the Universe was created. This has made a lot of people \"\\\n",
      "                 \"very angry and been widely regarded as a bad move.\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "spaCy does tokenization, sentence recognition, part of speech tagging, lemmatization, dependency parsing, and named entity recognition all at once!"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# all you have to do to parse text is this:\n",
      "#note: the first time you run spaCy in a file it takes a little while to load up its modules\n",
      "parsedData = parser(multiSentence)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 59
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Let's look at the tokens\n",
      "# All you have to do is iterate through the parsedData\n",
      "# Each token is an object with lots of different properties\n",
      "# A property with an underscore at the end returns the string representation\n",
      "# while a property without the underscore returns an index (int) into spaCy's vocabulary\n",
      "# The probability estimate is based on counts from a 3 billion word corpus, smoothed using the Simple Good-Turing method.\n",
      "for i, token in enumerate(parsedData):\n",
      "    print(\"original:\", token.orth, token.orth_)\n",
      "    print(\"lowercased:\", token.lower, token.lower_)\n",
      "    print(\"lemma:\", token.lemma, token.lemma_)\n",
      "    print(\"shape:\", token.shape, token.shape_)\n",
      "    print(\"prefix:\", token.prefix, token.prefix_)\n",
      "    print(\"suffix:\", token.suffix, token.suffix_)\n",
      "    print(\"log probability:\", token.prob)\n",
      "    print(\"Brown cluster id:\", token.cluster)\n",
      "    print(\"----------------------------------------\")\n",
      "    if i > 10:\n",
      "        break"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "original: 300 There\n",
        "lowercased: 144 there\n",
        "lemma: 300 There\n",
        "shape: 187 Xxxxx\n",
        "prefix: 32 T\n",
        "suffix: 66 ere\n",
        "log probability: -7.663576126098633\n",
        "Brown cluster id: 1918\n",
        "----------------------------------------\n",
        "original: 29 is\n",
        "lowercased: 29 is\n",
        "lemma: 52 be\n",
        "shape: 7 xx\n",
        "prefix: 14 i\n",
        "suffix: 29 is\n",
        "log probability: -5.002371311187744\n",
        "Brown cluster id: 762\n",
        "----------------------------------------\n",
        "original: 59 an\n",
        "lowercased: 59 an\n",
        "lemma: 59 an\n",
        "shape: 7 xx\n",
        "prefix: 11 a\n",
        "suffix: 59 an\n",
        "log probability: -5.829381465911865\n",
        "Brown cluster id: 3\n",
        "----------------------------------------\n",
        "original: 334 art\n",
        "lowercased: 334 art\n",
        "lemma: 334 art\n",
        "shape: 3 xxx\n",
        "prefix: 11 a\n",
        "suffix: 334 art\n",
        "log probability: -9.482678413391113\n",
        "Brown cluster id: 633\n",
        "----------------------------------------\n",
        "original: 1 ,\n",
        "lowercased: 1 ,\n",
        "lemma: 1 ,\n",
        "shape: 1 ,\n",
        "prefix: 1 ,\n",
        "suffix: 1 ,\n",
        "log probability: -3.0368354320526123\n",
        "Brown cluster id: 4\n",
        "----------------------------------------\n",
        "original: 44 it\n",
        "lowercased: 44 it\n",
        "lemma: 906264 -PRON-\n",
        "shape: 7 xx\n",
        "prefix: 14 i\n",
        "suffix: 44 it\n",
        "log probability: -5.498129367828369\n",
        "Brown cluster id: 474\n",
        "----------------------------------------\n",
        "original: 274 says\n",
        "lowercased: 274 says\n",
        "lemma: 253 say\n",
        "shape: 20 xxxx\n",
        "prefix: 27 s\n",
        "suffix: 275 ays\n",
        "log probability: -7.604108810424805\n",
        "Brown cluster id: 244\n",
        "----------------------------------------\n",
        "original: 1 ,\n",
        "lowercased: 1 ,\n",
        "lemma: 1 ,\n",
        "shape: 1 ,\n",
        "prefix: 1 ,\n",
        "suffix: 1 ,\n",
        "log probability: -3.0368354320526123\n",
        "Brown cluster id: 4\n",
        "----------------------------------------\n",
        "original: 79 or\n",
        "lowercased: 79 or\n",
        "lemma: 79 or\n",
        "shape: 7 xx\n",
        "prefix: 8 o\n",
        "suffix: 79 or\n",
        "log probability: -6.262600898742676\n",
        "Brown cluster id: 404\n",
        "----------------------------------------\n",
        "original: 1400 rather\n",
        "lowercased: 1400 rather\n",
        "lemma: 1400 rather\n",
        "shape: 20 xxxx\n",
        "prefix: 357 r\n",
        "suffix: 131 her\n",
        "log probability: -9.074186325073242\n",
        "Brown cluster id: 6698\n",
        "----------------------------------------\n",
        "original: 1 ,\n",
        "lowercased: 1 ,\n",
        "lemma: 1 ,\n",
        "shape: 1 ,\n",
        "prefix: 1 ,\n",
        "suffix: 1 ,\n",
        "log probability: -3.0368354320526123\n",
        "Brown cluster id: 4\n",
        "----------------------------------------\n",
        "original:"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 11 a\n",
        "lowercased: 11 a\n",
        "lemma: 11 a\n",
        "shape: 12 x\n",
        "prefix: 11 a\n",
        "suffix: 11 a\n",
        "log probability: -4.003841400146484\n",
        "Brown cluster id: 19\n",
        "----------------------------------------\n"
       ]
      }
     ],
     "prompt_number": 60
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Let's look at the sentences\n",
      "sents = []\n",
      "# the \"sents\" property returns spans\n",
      "# spans have indices into the original string\n",
      "# where each index value represents a token\n",
      "for span in parsedData.sents:\n",
      "    # go from the start to the end of each span, returning each token in the sentence\n",
      "    # combine each token using join()\n",
      "    sent = ''.join(parsedData[i].string for i in range(span.start, span.end)).strip()\n",
      "    sents.append(sent)\n",
      "\n",
      "for sentence in sents:\n",
      "    print(sentence)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "There is an art, it says, or rather, a knack to flying.\n",
        "The knack lies in learning how to throw yourself at the ground and miss.\n",
        "In the beginning the Universe was created.\n",
        "This has made a lot of people very angry and been widely regarded as a bad move.\n"
       ]
      }
     ],
     "prompt_number": 61
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Let's look at the part of speech tags of the first sentence\n",
      "for span in parsedData.sents:\n",
      "    sent = [parsedData[i] for i in range(span.start, span.end)]\n",
      "    break\n",
      "\n",
      "for token in sent:\n",
      "    print(token.orth_, token.pos_)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "There DET\n",
        "is VERB\n",
        "an DET\n",
        "art NOUN\n",
        ", PUNCT\n",
        "it PRON\n",
        "says VERB\n",
        ", PUNCT\n",
        "or CONJ\n",
        "rather ADV\n",
        ", PUNCT\n",
        "a DET\n",
        "knack NOUN\n",
        "to ADP\n",
        "flying NOUN\n",
        ". PUNCT\n"
       ]
      }
     ],
     "prompt_number": 62
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Let's look at the dependencies of this example:\n",
      "example = \"The boy with the spotted dog quickly ran after the firetruck.\"\n",
      "parsedEx = parser(example)\n",
      "# shown as: original token, dependency tag, head word, left dependents, right dependents\n",
      "for token in parsedEx:\n",
      "    print(token.orth_, token.dep_, token.head.orth_, [t.orth_ for t in token.lefts], [t.orth_ for t in token.rights])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "The det boy [] []\n",
        "boy nsubj ran ['The'] ['with']\n",
        "with prep boy [] ['dog']\n",
        "the det dog [] []\n",
        "spotted amod dog [] []\n",
        "dog pobj with ['the', 'spotted'] []\n",
        "quickly advmod ran [] []\n",
        "ran ROOT ran ['boy', 'quickly'] ['after', '.']\n",
        "after prep ran [] ['firetruck']\n",
        "the det firetruck [] []\n",
        "firetruck pobj after ['the'] []\n",
        ". punct ran [] []\n"
       ]
      }
     ],
     "prompt_number": 63
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Let's look at the named entities of this example:\n",
      "example = \"Apple's stocks dropped dramatically after the death of Steve Jobs in October.\"\n",
      "parsedEx = parser(example)\n",
      "for token in parsedEx:\n",
      "    print(token.orth_, token.ent_type_ if token.ent_type_ != \"\" else \"(not an entity)\")\n",
      "\n",
      "print(\"-------------- entities only ---------------\")\n",
      "# if you just want the entities and nothing else, you can do access the parsed examples \"ents\" property like this:\n",
      "ents = list(parsedEx.ents)\n",
      "for entity in ents:\n",
      "    print(entity.label, entity.label_, ' '.join(t.orth_ for t in entity))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Apple ORG\n",
        "'s (not an entity)\n",
        "stocks (not an entity)\n",
        "dropped (not an entity)\n",
        "dramatically (not an entity)\n",
        "after (not an entity)\n",
        "the (not an entity)\n",
        "death (not an entity)\n",
        "of (not an entity)\n",
        "Steve PERSON\n",
        "Jobs (not an entity)\n",
        "in (not an entity)\n",
        "October DATE\n",
        ". (not an entity)\n",
        "-------------- entities only ---------------\n",
        "274530 ORG Apple\n",
        "112504 PERSON Steve Jobs\n",
        "71288 DATE October\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "spaCy is trained to attempt to handle messy data, including emoticons and other web-based features"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "messyData = \"lol that is rly funny :) This is gr8 i rate it 8/8!!!\"\n",
      "parsedData = parser(messyData)\n",
      "for token in parsedData:\n",
      "    print(token.orth_, token.pos_, token.lemma_)\n",
      "    \n",
      "# it does pretty well! Note that it does fail on the token \"gr8\", taking it as a verb rather than an adjective meaning \"great\"\n",
      "# and \"lol\" probably isn't a noun...it's more like an interjection"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "lol NOUN lol\n",
        "that DET that\n",
        "is VERB be\n",
        "rly ADV rly\n",
        "funny ADJ funny\n",
        ":) PUNCT :)\n",
        "This DET This\n",
        "is VERB be\n",
        "gr8 VERB gr8\n",
        "i PRON i\n",
        "rate VERB rate\n",
        "it PRON -PRON-\n",
        "8/8 NUM 8/8\n",
        "! PUNCT !\n",
        "! PUNCT !\n",
        "! PUNCT !\n"
       ]
      }
     ],
     "prompt_number": 65
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "spaCy has word vector representations built in!"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from numpy import dot\n",
      "from numpy.linalg import norm\n",
      "\n",
      "# you can access known words from the parser's vocabulary\n",
      "nasa = parser.vocab['NASA']\n",
      "\n",
      "# cosine similarity\n",
      "cosine = lambda v1, v2: dot(v1, v2) / (norm(v1) * norm(v2))\n",
      "\n",
      "# gather all known words, take only the lowercased versions\n",
      "allWords = list({w for w in parser.vocab if w.has_repvec and w.orth_.islower() and w.lower_ != \"nasa\"})\n",
      "\n",
      "# sort by similarity to NASA\n",
      "allWords.sort(key=lambda w: cosine(w.repvec, nasa.repvec))\n",
      "allWords.reverse()\n",
      "print(\"Top 20 most similar words to NASA:\")\n",
      "for word in allWords[:20]:   \n",
      "    print(word.orth_)\n",
      "    \n",
      "# Let's see if it can figure out this analogy\n",
      "# Man is to King as Woman is to ??\n",
      "king = parser.vocab['king']\n",
      "man = parser.vocab['man']\n",
      "woman = parser.vocab['woman']\n",
      "\n",
      "result = king.repvec - man.repvec + woman.repvec\n",
      "\n",
      "# gather all known words, take only the lowercased versions\n",
      "allWords = list({w for w in parser.vocab if w.has_repvec and w.orth_.islower() and w.lower_ != \"king\" and w.lower_ != \"man\" and w.lower_ != \"woman\"})\n",
      "# sort by similarity to the result\n",
      "allWords.sort(key=lambda w: cosine(w.repvec, result))\n",
      "allWords.reverse()\n",
      "print(\"\\n----------------------------\\nTop 3 closest results for king - man + woman:\")\n",
      "for word in allWords[:3]:   \n",
      "    print(word.orth_)\n",
      "    \n",
      "# it got it! Queen!"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Top 20 most similar words to NASA:\n",
        "jpl\n",
        "noaa\n",
        "esa\n",
        "cern\n",
        "nih\n",
        "norad\n",
        "fema\n",
        "isro\n",
        "usaid\n",
        "nsf\n",
        "nsa\n",
        "dod\n",
        "usda\n",
        "caltech\n",
        "defra\n",
        "raytheon\n",
        "cia\n",
        "unhcr\n",
        "fermilab\n",
        "cdc\n",
        "\n",
        "----------------------------\n",
        "Top 3 closest results for king - man + woman:"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "queen\n",
        "monarch\n",
        "princess\n"
       ]
      }
     ],
     "prompt_number": 66
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "You can do cool things like extract Subject, Verb, Object triples from the dependency parse if you use my code in subject_object_extraction.py. Note: Doesn't work on complicated sentences. Fails if the dependency parse is incorrect."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from subject_object_extraction import findSVOs\n",
      "\n",
      "# can still work even without punctuation\n",
      "parse = parser(\"he and his brother shot me and my sister\")\n",
      "print(findSVOs(parse))\n",
      "\n",
      "# very complex sample. Only some are correct. Some are missed.\n",
      "parse = parser(\"Far out in the uncharted backwaters of the unfashionable end of the Western Spiral arm of the Galaxy lies a small unregarded yellow sun. \"\n",
      "                \"Orbiting this at a distance of roughly ninety-two million miles is an utterly insignificant little blue green planet whose ape-descended \"\n",
      "                \"life forms are so amazingly primitive that they still think digital watches are a pretty neat idea. \"\n",
      "                \"This planet has \u2013 or rather had \u2013 a problem, which was this: most of the people living on it were unhappy for pretty much of the time. \"\n",
      "                \"Many solutions were suggested for this problem, but most of these were largely concerned with the movements of small green pieces of paper, \"\n",
      "                \"which is odd because on the whole it wasn\u2019t the small green pieces of paper that were unhappy. And so the problem remained; lots of the \"\n",
      "                \"people were mean, and most of them were miserable, even the ones with digital watches.\")\n",
      "print(findSVOs(parse))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[('he', 'shot', 'me'), ('he', 'shot', 'sister'), ('brother', 'shot', 'me'), ('brother', 'shot', 'sister')]\n",
        "[('orbiting', 'is', 'planet'), ('watches', 'are', 'idea'), ('problem', 'was', 'this'), ('it', 'wasn\u2019t', 'pieces'), ('most', 'were', 'ones')]\n"
       ]
      }
     ],
     "prompt_number": 67
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "If you want to include spaCy in your machine learning it is not too difficult"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.feature_extraction.text import CountVectorizer\n",
      "from sklearn.base import TransformerMixin\n",
      "from sklearn.pipeline import Pipeline\n",
      "from sklearn.svm import LinearSVC\n",
      "from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS\n",
      "from sklearn.metrics import accuracy_score\n",
      "from nltk.corpus import stopwords\n",
      "import string\n",
      "import re\n",
      "\n",
      "# A custom stoplist\n",
      "STOPLIST = set(stopwords.words('english') + [\"n't\", \"'s\", \"'m\", \"ca\"] + list(ENGLISH_STOP_WORDS))\n",
      "# List of symbols we don't care about\n",
      "SYMBOLS = \" \".join(string.punctuation).split(\" \") + [\"-----\", \"---\", \"...\", \"\u201c\", \"\u201d\", \"'ve\"]\n",
      "\n",
      "# Every step in a pipeline needs to be a \"transformer\". Define a custom transformer to clean text using spaCy\n",
      "class CleanTextTransformer(TransformerMixin):\n",
      "    \"\"\"\n",
      "    Convert text to cleaned text\n",
      "    \"\"\"\n",
      "\n",
      "    def transform(self, X, **transform_params):\n",
      "        return [cleanText(text) for text in X]\n",
      "\n",
      "    def fit(self, X, y=None, **fit_params):\n",
      "        return self\n",
      "\n",
      "    def get_params(self, deep=True):\n",
      "        return {}\n",
      "    \n",
      "# A custom function to clean the text before sending it into the vectorizer\n",
      "def cleanText(text):\n",
      "    # get rid of newlines\n",
      "    text = text.strip().replace(\"\\n\", \" \").replace(\"\\r\", \" \")\n",
      "    \n",
      "    # replace twitter @mentions\n",
      "    mentionFinder = re.compile(r\"@[a-z0-9_]{1,15}\", re.IGNORECASE)\n",
      "    text = mentionFinder.sub(\"@MENTION\", text)\n",
      "    \n",
      "    # replace HTML symbols\n",
      "    text = text.replace(\"&amp;\", \"and\").replace(\"&gt;\", \">\").replace(\"&lt;\", \"<\")\n",
      "    \n",
      "    # lowercase\n",
      "    text = text.lower()\n",
      "\n",
      "    return text\n",
      "\n",
      "# A custom function to tokenize the text using spaCy\n",
      "# and convert to lemmas\n",
      "def tokenizeText(sample):\n",
      "\n",
      "    # get the tokens using spaCy\n",
      "    tokens = parser(sample)\n",
      "\n",
      "    # lemmatize\n",
      "    lemmas = []\n",
      "    for tok in tokens:\n",
      "        lemmas.append(tok.lemma_.lower().strip() if tok.lemma_ != \"-PRON-\" else tok.lower_)\n",
      "    tokens = lemmas\n",
      "\n",
      "    # stoplist the tokens\n",
      "    tokens = [tok for tok in tokens if tok not in STOPLIST]\n",
      "\n",
      "    # stoplist symbols\n",
      "    tokens = [tok for tok in tokens if tok not in SYMBOLS]\n",
      "\n",
      "    # remove large strings of whitespace\n",
      "    while \"\" in tokens:\n",
      "        tokens.remove(\"\")\n",
      "    while \" \" in tokens:\n",
      "        tokens.remove(\" \")\n",
      "    while \"\\n\" in tokens:\n",
      "        tokens.remove(\"\\n\")\n",
      "    while \"\\n\\n\" in tokens:\n",
      "        tokens.remove(\"\\n\\n\")\n",
      "\n",
      "    return tokens\n",
      "\n",
      "def printNMostInformative(vectorizer, clf, N):\n",
      "    \"\"\"Prints features with the highest coefficient values, per class\"\"\"\n",
      "    feature_names = vectorizer.get_feature_names()\n",
      "    coefs_with_fns = sorted(zip(clf.coef_[0], feature_names))\n",
      "    topClass1 = coefs_with_fns[:N]\n",
      "    topClass2 = coefs_with_fns[:-(N + 1):-1]\n",
      "    print(\"Class 1 best: \")\n",
      "    for feat in topClass1:\n",
      "        print(feat)\n",
      "    print(\"Class 2 best: \")\n",
      "    for feat in topClass2:\n",
      "        print(feat)\n",
      "\n",
      "# the vectorizer and classifer to use\n",
      "# note that I changed the tokenizer in CountVectorizer to use a custom function using spaCy's tokenizer\n",
      "vectorizer = CountVectorizer(tokenizer=tokenizeText, ngram_range=(1,1))\n",
      "clf = LinearSVC()\n",
      "# the pipeline to clean, tokenize, vectorize, and classify\n",
      "pipe = Pipeline([('cleanText', CleanTextTransformer()), ('vectorizer', vectorizer), ('clf', clf)])\n",
      "\n",
      "# data\n",
      "train = [\"I love space. Space is great.\", \"Planets are cool. I am glad they exist in space\", \"lol @twitterdude that is gr8\", \n",
      "        \"twitter &amp; reddit are fun.\", \"Mars is a planet. It is red.\", \"@Microsoft: y u skip windows 9?\", \"Rockets launch from Earth and go to other planets.\",\n",
      "        \"twitter social media &gt; &lt;\", \"@someguy @somegirl @twitter #hashtag\", \"Orbiting the sun is a little blue-green planet.\"]\n",
      "labelsTrain = [\"space\", \"space\", \"twitter\", \"twitter\", \"space\", \"twitter\", \"space\", \"twitter\", \"twitter\", \"space\"]\n",
      "\n",
      "test = [\"i h8 riting comprehensibly #skoolsux\", \"planets and stars and rockets and stuff\"]\n",
      "labelsTest = [\"twitter\", \"space\"]\n",
      "\n",
      "# train\n",
      "pipe.fit(train, labelsTrain)\n",
      "\n",
      "# test\n",
      "preds = pipe.predict(test)\n",
      "print(\"----------------------------------------------------------------------------------------------\")\n",
      "print(\"results:\")\n",
      "for (sample, pred) in zip(test, preds):\n",
      "    print(sample, \":\", pred)\n",
      "print(\"accuracy:\", accuracy_score(labelsTest, preds))\n",
      "\n",
      "print(\"----------------------------------------------------------------------------------------------\")\n",
      "print(\"Top 10 features used to predict: \")\n",
      "# show the top features\n",
      "printNMostInformative(vectorizer, clf, 10)\n",
      "\n",
      "print(\"----------------------------------------------------------------------------------------------\")\n",
      "print(\"The original data as it appeared to the classifier after tokenizing, lemmatizing, stoplisting, etc\")\n",
      "# let's see what the pipeline was transforming the data into\n",
      "pipe = Pipeline([('cleanText', CleanTextTransformer()), ('vectorizer', vectorizer)])\n",
      "transform = pipe.fit_transform(train, labelsTrain)\n",
      "\n",
      "# get the features that the vectorizer learned (its vocabulary)\n",
      "vocab = vectorizer.get_feature_names()\n",
      "\n",
      "# the values from the vectorizer transformed data (each item is a row,column index with value as # times occuring in the sample, stored as a sparse matrix)\n",
      "for i in range(len(train)):\n",
      "    s = \"\"\n",
      "    indexIntoVocab = transform.indices[transform.indptr[i]:transform.indptr[i+1]]\n",
      "    numOccurences = transform.data[transform.indptr[i]:transform.indptr[i+1]]\n",
      "    for idx, num in zip(indexIntoVocab, numOccurences):\n",
      "        s += str((vocab[idx], num))\n",
      "    print(\"Sample {}: {}\".format(i, s))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "----------------------------------------------------------------------------------------------\n",
        "results:\n",
        "i h8 riting comprehensibly #skoolsux : twitter\n",
        "planets and stars and rockets and stuff : space\n",
        "accuracy: 1.0\n",
        "----------------------------------------------------------------------------------------------\n",
        "Top 10 features used to predict: \n",
        "Class 1 best: \n",
        "(-0.52882810587037121, 'planet')\n",
        "(-0.35193565503626856, 'space')\n",
        "(-0.2182987490483107, 'mar')\n",
        "(-0.2182987490483107, 'red')\n",
        "(-0.15592826214493352, 'earth')\n",
        "(-0.15592826214493352, 'launch')\n",
        "(-0.15592826214493352, 'rocket')\n",
        "(-0.1482804579342584, 'great')\n",
        "(-0.1482804579342584, 'love')\n",
        "(-0.099226355509375405, 'blue')\n",
        "Class 2 best: \n",
        "(0.41129938045689757, 'twitter')\n",
        "(0.34038557663231445, '@mention')\n",
        "(0.23401502570811406, 'lol')\n",
        "(0.23401502570811406, 'gr8')\n",
        "(0.20564996854629114, 'social')\n",
        "(0.20564996854629114, 'medium')\n",
        "(0.20564941191060651, 'reddit')\n",
        "(0.20564941191060651, 'fun')\n",
        "(0.10637055092420053, 'y')\n",
        "(0.10637055092420053, 'window')\n",
        "----------------------------------------------------------------------------------------------\n",
        "The original data as it appeared to the classifier after tokenizing, lemmatizing, stoplisting, etc\n",
        "Sample 0: ('love', 1)('space', 2)('great', 1)\n",
        "Sample 1: ('space', 1)('planet', 1)('cool', 1)('glad', 1)('exist', 1)\n",
        "Sample 2: ('lol', 1)('@mention', 1)('gr8', 1)\n",
        "Sample 3: ('twitter', 1)('reddit', 1)('fun', 1)\n",
        "Sample 4: ('planet', 1)('mar', 1)('red', 1)\n",
        "Sample 5: ('@mention', 1)('y', 1)('u', 1)('skip', 1)('window', 1)('9', 1)\n",
        "Sample 6: ('planet', 1)('rocket', 1)('launch', 1)('earth', 1)\n",
        "Sample 7: ('twitter', 1)('social', 1)('medium', 1)\n",
        "Sample 8: ('@mention', 3)('hashtag', 1)\n",
        "Sample 9: ('planet', 1)('orbit', 1)('sun', 1)('little', 1)('blue', 1)('green', 1)\n"
       ]
      }
     ],
     "prompt_number": 68
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}