P4DS4D; 05; Dataset Load.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:79f0fceff69d00329295b568c31d6ce2be8a3becae840788589d1b1d93b9cb20"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Uploading, Streaming, and Sampling Data"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Uploading small amounts of data into memory"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "with open(\"Colors.txt\", 'rb') as open_file:\n",
      "    print 'Colors.txt content:\\n' + open_file.read()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Colors.txt content:\n",
        "Color\tValue\r\n",
        "Red\t1\r\n",
        "Orange\t2\r\n",
        "Yellow\t3\r\n",
        "Green\t4\r\n",
        "Blue\t5\r\n",
        "Purple\t6\r\n",
        "Black\t7\r\n",
        "White\t8\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Streaming large amounts of data into memory"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "with open(\"Colors.txt\", 'rb') as open_file:\n",
      "    for observation in open_file:\n",
      "        print 'Reading Data: ' + observation"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Reading Data: Color\tValue\r\n",
        "\n",
        "Reading Data: Red\t1\r\n",
        "\n",
        "Reading Data: Orange\t2\r\n",
        "\n",
        "Reading Data: Yellow\t3\r\n",
        "\n",
        "Reading Data: Green\t4\r\n",
        "\n",
        "Reading Data: Blue\t5\r\n",
        "\n",
        "Reading Data: Purple\t6\r\n",
        "\n",
        "Reading Data: Black\t7\r\n",
        "\n",
        "Reading Data: White\t8\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Sampling data"
     ]
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Fixed samples"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "n = 3\n",
      "with open(\"Colors.txt\", 'rb') as open_file:\n",
      "    for j, observation in enumerate(open_file):\n",
      "        if j % n==0:\n",
      "            print('Reading Line: ' + str(j) + \n",
      "            ' Content: ' + observation)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Reading Line: 0 Content: Color\tValue\r\n",
        "\n",
        "Reading Line: 3 Content: Yellow\t3\r\n",
        "\n",
        "Reading Line: 6 Content: Purple\t6\r\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Random samples"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from random import random\n",
      "sample_size = 0.25\n",
      "with open(\"Colors.txt\", 'rb') as open_file:\n",
      "    for j, observation in enumerate(open_file):\n",
      "        if random()<=sample_size:\n",
      "            print('Reading Line: ' + str(j) + \n",
      "            ' Content: ' + observation)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Reading Line: 2 Content: Orange\t2\r\n",
        "\n",
        "Reading Line: 3 Content: Yellow\t3\r\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Accessing Data in Structured Flat File Form"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Reading from a text file"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "color_table = pd.io.parsers.read_table(\"Colors.txt\")\n",
      "print color_table"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "    Color  Value\n",
        "0     Red      1\n",
        "1  Orange      2\n",
        "2  Yellow      3\n",
        "3   Green      4\n",
        "4    Blue      5\n",
        "5  Purple      6\n",
        "6   Black      7\n",
        "7   White      8\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Reading CSV delimited format"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "titanic = pd.io.parsers.read_csv(\"Titanic.csv\")\n",
      "X = titanic[['age']]\n",
      "#X = titanic[['age']].values\n",
      "print X"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "            age\n",
        "0       29.0000\n",
        "1        0.9167\n",
        "2        2.0000\n",
        "3       30.0000\n",
        "4       25.0000\n",
        "5       48.0000\n",
        "6       63.0000\n",
        "7       39.0000\n",
        "8       53.0000\n",
        "9       71.0000\n",
        "10      47.0000\n",
        "11      18.0000\n",
        "12      24.0000\n",
        "13      26.0000\n",
        "14      80.0000\n",
        "15    9999.0000\n",
        "16      24.0000\n",
        "17      50.0000\n",
        "18      32.0000\n",
        "19      36.0000\n",
        "20      37.0000\n",
        "21      47.0000\n",
        "22      26.0000\n",
        "23      42.0000\n",
        "24      29.0000\n",
        "25      25.0000\n",
        "26      25.0000\n",
        "27      19.0000\n",
        "28      35.0000\n",
        "29      28.0000\n",
        "...         ...\n",
        "1279    14.0000\n",
        "1280    22.0000\n",
        "1281    22.0000\n",
        "1282  9999.0000\n",
        "1283  9999.0000\n",
        "1284  9999.0000\n",
        "1285    32.5000\n",
        "1286    38.0000\n",
        "1287    51.0000\n",
        "1288    18.0000\n",
        "1289    21.0000\n",
        "1290    47.0000\n",
        "1291  9999.0000\n",
        "1292  9999.0000\n",
        "1293  9999.0000\n",
        "1294    28.5000\n",
        "1295    21.0000\n",
        "1296    27.0000\n",
        "1297  9999.0000\n",
        "1298    36.0000\n",
        "1299    27.0000\n",
        "1300    15.0000\n",
        "1301    45.5000\n",
        "1302  9999.0000\n",
        "1303  9999.0000\n",
        "1304    14.5000\n",
        "1305  9999.0000\n",
        "1306    26.5000\n",
        "1307    27.0000\n",
        "1308    29.0000\n",
        "\n",
        "[1309 rows x 1 columns]\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Reading Excel and other Microsoft Office files"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "xls = pd.ExcelFile(\"Values.xls\")\n",
      "trig_values = xls.parse('Sheet1', index_col=None, \n",
      "                        na_values=['NA'])\n",
      "#trig_values = pd.read_excel(\"Values.xls\", 'Sheet1', index_col=None, na_values=['NA'])\n",
      "print trig_values"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "    Angle (Degrees)      Sine    Cosine    Tangent\n",
        "0        138.550574  0.661959 -0.749540  -0.883153\n",
        "1        305.535745 -0.813753  0.581211  -1.400100\n",
        "2        280.518695 -0.983195  0.182556  -5.385709\n",
        "3        216.363795 -0.592910 -0.805269   0.736289\n",
        "4         36.389247  0.593268  0.805005   0.736974\n",
        "5         31.474311  0.522116  0.852874   0.612184\n",
        "6        120.121669  0.864962 -0.501838  -1.723588\n",
        "7        293.947055 -0.913921  0.405892  -2.251634\n",
        "8        179.882632  0.002048 -0.999998  -0.002048\n",
        "9        120.927562  0.857818 -0.513954  -1.669056\n",
        "10        71.349485  0.947487  0.319795   2.962796\n",
        "11       241.971082 -0.882711 -0.469917   1.878439\n",
        "12       297.208817 -0.889346  0.457235  -1.945053\n",
        "13       142.004551  0.615599 -0.788060  -0.781158\n",
        "14       173.770696  0.108508 -0.994096  -0.109152\n",
        "15       229.232002 -0.757360 -0.652998   1.159820\n",
        "16        67.926976  0.926706  0.375788   2.466033\n",
        "17       261.866575 -0.989941 -0.141479   6.997102\n",
        "18        59.185450  0.858830  0.512261   1.676547\n",
        "19        98.029275  0.990197 -0.139679  -7.089086\n",
        "20         9.336088  0.162225  0.986754   0.164403\n",
        "21        90.746371  0.999915 -0.013026 -76.761483\n",
        "22       217.798087 -0.612881 -0.790175   0.775626\n",
        "23        58.616049  0.853697  0.520771   1.639295\n",
        "24       197.196367 -0.295647 -0.955297   0.309482\n",
        "25       331.194892 -0.481832  0.876264  -0.549871\n",
        "26         6.509875  0.113374  0.993552   0.114110\n",
        "27       266.390707 -0.998017 -0.062952  15.853513\n",
        "28       230.323819 -0.769665 -0.638448   1.205525\n",
        "29       314.224257 -0.716615  0.697469  -1.027452\n",
        "..              ...       ...       ...        ...\n",
        "42       324.080564 -0.586647  0.809843  -0.724396\n",
        "43       140.893727  0.630761 -0.775977  -0.812860\n",
        "44       128.226889  0.785567 -0.618777  -1.269547\n",
        "45        82.770054  0.992049  0.125852   7.882680\n",
        "46       303.112455 -0.837600  0.546284  -1.533268\n",
        "47       164.824273  0.261780 -0.965127  -0.271239\n",
        "48       218.829827 -0.627009 -0.779012   0.804878\n",
        "49        28.649593  0.479452  0.877568   0.546341\n",
        "50       349.336296 -0.185044  0.982730  -0.188296\n",
        "51        84.889713  0.996025  0.089073  11.182105\n",
        "52       197.935862 -0.307952 -0.951402   0.323683\n",
        "53       303.049380 -0.838201  0.545362  -1.536963\n",
        "54       183.737235 -0.065181 -0.997873   0.065320\n",
        "55       346.153919 -0.239314  0.970942  -0.246477\n",
        "56       218.822745 -0.626913 -0.779089   0.804674\n",
        "57       243.969070 -0.898557 -0.438856   2.047498\n",
        "58       115.600771  0.901827 -0.432098  -2.087089\n",
        "59       125.906606  0.809974 -0.586466  -1.381111\n",
        "60       200.094363 -0.343567 -0.939128   0.365836\n",
        "61       337.860807 -0.376858  0.926271  -0.406855\n",
        "62       168.176975  0.204889 -0.978785  -0.209330\n",
        "63       305.708155 -0.812000  0.583657  -1.391229\n",
        "64       162.656078  0.298107 -0.954533  -0.312306\n",
        "65       219.007899 -0.629428 -0.777059   0.810012\n",
        "66       222.830465 -0.679831 -0.733368   0.926998\n",
        "67       324.199562 -0.584964  0.811059  -0.721234\n",
        "68       187.948172 -0.138277 -0.990394   0.139619\n",
        "69       270.678249 -0.999930  0.011837 -84.472139\n",
        "70       270.779159 -0.999908  0.013598 -73.530885\n",
        "71       200.213513 -0.345520 -0.938412   0.368196\n",
        "\n",
        "[72 rows x 4 columns]\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Sending Data in Unstructured File Form"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Rendering the image"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from skimage.io import imread\n",
      "from skimage.transform import resize \n",
      "from matplotlib import pyplot as plt\n",
      "import matplotlib.cm as cm\n",
      "\n",
      "example_file = (\"http://upload.wikimedia.org/\" +\n",
      "    \"wikipedia/commons/7/7d/Dog_face.png\")\n",
      "image = imread(example_file, as_grey=True)\n",
      "plt.imshow(image, cmap=cm.gray)\n",
      "plt.show()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Displaying the image information"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print(\"data type: %s, shape: %s\" % \n",
      "      (type(image), image.shape))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "data type: <type 'numpy.ndarray'>, shape: (90L, 90L)\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Cropping the image"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "image2 = image[5:70,0:70]\n",
      "plt.imshow(image2, cmap=cm.gray)\n",
      "plt.show()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Resizing the image"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "image3 = resize(image2, (30, 30), mode='nearest')\n",
      "plt.imshow(image3, cmap=cm.gray)\n",
      "print(\"data type: %s, shape: %s\" % \n",
      "      (type(image3), image3.shape))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "data type: <type 'numpy.ndarray'>, shape: (30L, 30L)\n"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Flatening the image"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "image_row = image3.flatten()\n",
      "print(\"data type: %s, shape: %s\" % \n",
      "      (type(image_row), image_row.shape))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "data type: <type 'numpy.ndarray'>, shape: (900L,)\n"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Accessing Data from the Web"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from lxml import objectify\n",
      "import pandas as pd\n",
      "\n",
      "xml = objectify.parse(open('XMLData.xml'))\n",
      "root = xml.getroot()\n",
      "df = pd.DataFrame(columns=('Number', 'String', 'Boolean'))\n",
      "\n",
      "for i in range(0,4):\n",
      "    obj = root.getchildren()[i].getchildren()\n",
      "    row = dict(zip(['Number', 'String', 'Boolean'], \n",
      "                   [obj[0].text, obj[1].text, \n",
      "                    obj[2].text]))\n",
      "    row_s = pd.Series(row)\n",
      "    row_s.name = i\n",
      "    df = df.append(row_s)\n",
      "    \n",
      "print df"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "  Number  String Boolean\n",
        "0      1   First    True\n",
        "1      2  Second   False\n",
        "2      3   Third    True\n",
        "3      4  Fourth   False\n"
       ]
      }
     ],
     "prompt_number": 13
    }
   ],
   "metadata": {}
  }
 ]
}