P4DS4D; 07; Shaping Data.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:540797cc171af0d23e3949e207ad71458d07638c506bf0f6032d345a0c25f097"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Working with HTML Pages"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Parsing XML and HTML"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from lxml import objectify\n",
      "import pandas as pd\n",
      "from distutils import util\n",
      "\n",
      "xml = objectify.parse(open('XMLData.xml'))\n",
      "root = xml.getroot()\n",
      "df = pd.DataFrame(columns=('Number', 'Boolean'))\n",
      "\n",
      "for i in range(0,4):\n",
      "    obj = root.getchildren()[i].getchildren()\n",
      "    row = dict(zip(['Number', 'Boolean'], \n",
      "                   [obj[0].pyval, \n",
      "                    bool(util.strtobool(obj[2].text))]))\n",
      "    row_s = pd.Series(row)\n",
      "    row_s.name = obj[1].text\n",
      "    df = df.append(row_s)\n",
      "    \n",
      "print type(df.ix['First']['Number'])\n",
      "print type(df.ix['First']['Boolean'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "<type 'numpy.float64'>\n",
        "<type 'bool'>\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Using XPath for data extraction"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from lxml import objectify\n",
      "import pandas as pd\n",
      "from distutils import util\n",
      "\n",
      "xml = objectify.parse(open('XMLData.xml'))\n",
      "root = xml.getroot()\n",
      "\n",
      "data = zip(map(int, root.xpath('Record/Number')), \n",
      "           map(bool, map(util.strtobool, \n",
      "                map(str, root.xpath('Record/Boolean')))))\n",
      "\n",
      "df = pd.DataFrame(data, \n",
      "                  columns=('Number', 'Boolean'), \n",
      "                  index=map(str, \n",
      "                        root.xpath('Record/String')))\n",
      "\n",
      "print df\n",
      "print type(df.ix['First']['Number'])\n",
      "print type(df.ix['First']['Boolean'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "        Number Boolean\n",
        "First        1    True\n",
        "Second       2   False\n",
        "Third        3    True\n",
        "Fourth       4   False\n",
        "<type 'numpy.int64'>\n",
        "<type 'numpy.bool_'>\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Working with Raw Text"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Stemming and removing stop words"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import sklearn.feature_extraction.text as ext\n",
      "from nltk import word_tokenize          \n",
      "from nltk.stem.porter import PorterStemmer\n",
      "\n",
      "stemmer = PorterStemmer()\n",
      "\n",
      "def stem_tokens(tokens, stemmer):\n",
      "    stemmed = []\n",
      "    for item in tokens:\n",
      "        stemmed.append(stemmer.stem(item))\n",
      "    return stemmed\n",
      "\n",
      "def tokenize(text):\n",
      "    tokens = word_tokenize(text)\n",
      "    stems = stem_tokens(tokens, stemmer)\n",
      "    return stems\n",
      "\n",
      "vocab = ['Sam loves swimming so he swims all the time']\n",
      "vect = ext.CountVectorizer(tokenizer=tokenize, \n",
      "                           stop_words='english')\n",
      "vec = vect.fit(vocab)\n",
      "\n",
      "sentence1 = vec.transform(['George loves swimming too!'])\n",
      "\n",
      "print vec.get_feature_names()\n",
      "print sentence1.toarray()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[u'love', u'sam', u'swim', u'time']\n",
        "[[1 0 1 0]]\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Introducing regular expressions"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import re\n",
      "\n",
      "data1 = 'My phone number is: 800-555-1212.'\n",
      "data2 = '800-555-1234 is my phone number.'\n",
      "\n",
      "pattern = re.compile(r'(\\d{3})-(\\d{3})-(\\d{4})')\n",
      "\n",
      "dmatch1 = pattern.search(data1).groups()\n",
      "dmatch2 = pattern.search(data2).groups()\n",
      "\n",
      "print dmatch1\n",
      "print dmatch2"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "('800', '555', '1212')\n",
        "('800', '555', '1234')\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Using the Bag of Words Model and Beyond"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Understanding the bag of words model"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.datasets import fetch_20newsgroups\n",
      "import sklearn.feature_extraction.text as ext\n",
      "\n",
      "categories = ['comp.graphics', 'misc.forsale', \n",
      "              'rec.autos', 'sci.space']\n",
      "twenty_train = fetch_20newsgroups(subset='train',\n",
      "                                  categories=categories, \n",
      "                                  shuffle=True, \n",
      "                                  random_state=42)\n",
      "\n",
      "count_vect = ext.CountVectorizer()\n",
      "X_train_counts = count_vect.fit_transform(\n",
      "    twenty_train.data)\n",
      "\n",
      "print X_train_counts.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(2356, 34750)\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Working with n-grams"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.datasets import fetch_20newsgroups\n",
      "import sklearn.feature_extraction.text as ext\n",
      "\n",
      "categories = ['sci.space']\n",
      "\n",
      "twenty_train = fetch_20newsgroups(subset='train', \n",
      "        categories=categories, \n",
      "        remove=('headers', 'footers', 'quotes'),\n",
      "        shuffle=True, \n",
      "        random_state=42)\n",
      "\n",
      "count_chars = ext.CountVectorizer(analyzer='char_wb', \n",
      "        ngram_range=(3,3), \n",
      "        max_features=10).fit(twenty_train['data'])\n",
      "count_words = ext.CountVectorizer(analyzer='word', \n",
      "        ngram_range=(2,2),\n",
      "        max_features=10,\n",
      "        stop_words='english').fit(twenty_train['data'])\n",
      "X = count_chars.transform(twenty_train.data)\n",
      "\n",
      "print count_words.get_feature_names()\n",
      "print X[1].todense()\n",
      "print count_words.get_feature_names()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[u'ax ax', u'ax max', u'distribution world', u'don know', u'edu organization', u'max ax', u'nntp posting', u'organization university', u'posting host', u'writes article']\n",
        "[[0 0 5 1 0 0 4 2 5 1]]\n",
        "[u'ax ax', u'ax max', u'distribution world', u'don know', u'edu organization', u'max ax', u'nntp posting', u'organization university', u'posting host', u'writes article']\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Implementing TF-IDF transformations"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.datasets import fetch_20newsgroups\n",
      "import sklearn.feature_extraction.text as ext\n",
      "\n",
      "categories = ['sci.space']\n",
      "\n",
      "twenty_train = fetch_20newsgroups(subset='train', \n",
      "        categories=categories, \n",
      "        remove=('headers', 'footers', 'quotes'),                  \n",
      "        shuffle=True, \n",
      "        random_state=42)\n",
      "\n",
      "count_vect = ext.CountVectorizer()\n",
      "X_train_counts = count_vect.fit_transform(\n",
      "    twenty_train.data)\n",
      "\n",
      "tfidf = ext.TfidfTransformer().fit(X_train_counts)\n",
      "X_train_tfidf = tfidf.transform(X_train_counts)\n",
      "\n",
      "print X_train_tfidf.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(593, 13564)\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Working with Graph Data"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Using NetworkX basics"
     ]
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Creating the initial graph"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import networkx as nx\n",
      "\n",
      "G = nx.cycle_graph(10)\n",
      "A = nx.adjacency_matrix(G)\n",
      "\n",
      "print(A.todense())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[0 1 0 0 0 0 0 0 0 1]\n",
        " [1 0 1 0 0 0 0 0 0 0]\n",
        " [0 1 0 1 0 0 0 0 0 0]\n",
        " [0 0 1 0 1 0 0 0 0 0]\n",
        " [0 0 0 1 0 1 0 0 0 0]\n",
        " [0 0 0 0 1 0 1 0 0 0]\n",
        " [0 0 0 0 0 1 0 1 0 0]\n",
        " [0 0 0 0 0 0 1 0 1 0]\n",
        " [0 0 0 0 0 0 0 1 0 1]\n",
        " [1 0 0 0 0 0 0 0 1 0]]\n"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Visualizing the graph"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import matplotlib.pyplot as plt\n",
      "nx.draw_networkx(G)\n",
      "plt.show()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Adding to the Graph"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "G.add_edge(1,5)\n",
      "nx.draw_networkx(G)\n",
      "plt.show()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}