pandas_tutorial_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pandas:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Pandas has two core data structures: Series & DataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    2\n",
      "1    4\n",
      "2    6\n",
      "3    8\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas import Series, DataFrame\n",
    "counts= Series([2, 4, 6, 8])\n",
    "print(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 4 6 8]\n"
     ]
    }
   ],
   "source": [
    "print(counts.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Int64Index([0, 1, 2, 3], dtype='int64')\n"
     ]
    }
   ],
   "source": [
    "print(counts.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a    2\n",
      "b    4\n",
      "c    6\n",
      "d    8\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# We can create customized indexes:\n",
    "counts= Series([2, 4, 6, 8], index=[\"a\", \"b\", \"c\", \"d\"])\n",
    "print(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    }
   ],
   "source": [
    "# We can use the indexes to access values:\n",
    "# Note: We need to use quotes around an index:\n",
    "print(counts[\"a\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a     2\n",
      "b     4\n",
      "c    22\n",
      "d     8\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Reassign:\n",
    "counts[\"c\"]=22\n",
    "print(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a    2\n",
      "b    4\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(counts[[\"a\", \"b\"]]) # Note the double square brackets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a    7.389056e+00\n",
      "b    5.459815e+01\n",
      "c    3.584913e+09\n",
      "d    2.980958e+03\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# We can perform operations, similar to Numpy, while preserving the index values\n",
    "print(np.exp(counts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a    0.6\n",
      "b    1.2\n",
      "c    6.6\n",
      "d    2.4\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print(counts*0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a     2\n",
      "b     4\n",
      "c    22\n",
      "d     8\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Alex    10\n",
      "Evan    20\n",
      "Gabi    15\n",
      "John    12\n",
      "Juan    20\n",
      "Mary    13\n",
      "Noha     9\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# We can create a Series from a Python dictionary:\n",
    "d={\"Alex\": 10, \"John\": 12, \"Mary\": 13, \"Gabi\": 15, \"Noha\": 9,\\\n",
    "   \"Juan\": 20, \"Evan\": 20}\n",
    "grades=Series(d)\n",
    "print(grades)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Alex    False\n",
      "Evan     True\n",
      "Gabi     True\n",
      "John    False\n",
      "Juan     True\n",
      "Mary     True\n",
      "Noha    False\n",
      "dtype: bool\n"
     ]
    }
   ],
   "source": [
    "print(grades > 12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Alex    12.0\n",
      "Evan    24.0\n",
      "Gabi    18.0\n",
      "John    14.4\n",
      "Juan    24.0\n",
      "Mary    15.6\n",
      "Noha    10.8\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "raised= grades * 1.2\n",
    "print(raised)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   age                 courses names\n",
      "0   25                  Python  Alex\n",
      "1   27                    Perl  John\n",
      "2   32           Deep Learning  Mary\n",
      "3   19     Pattern Recognition  Gabi\n",
      "4   23             Data Mining  Noha\n",
      "5   20  Computational Archives  Juan\n",
      "6   21      Health Informatics  Evan\n"
     ]
    }
   ],
   "source": [
    "# The DataFrame is a (possibly heterogeneous) spreadsheet-like (think Excel) data structure\n",
    "# that enables both row and column indexing. Intutively, we can think about a DataFrame as \n",
    "# a dict of Series\n",
    "\n",
    "data= {\"courses\": [\"Python\", \"Perl\", \"Deep Learning\", \"Pattern Recognition\", \"Data Mining\",\\\n",
    "           \"Computational Archives\", \"Health Informatics\"],\n",
    "       \"age\": [25, 27, 32, 19, 23, 20, 21],\n",
    "       \"names\": [\"Alex\", \"John\", \"Mary\", \"Gabi\", \"Noha\", \"Juan\", \"Evan\"]}\n",
    "\n",
    "       \n",
    "frame=DataFrame(data)\n",
    "print(frame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    age                 courses names\n",
      "s1   25                  Python  Alex\n",
      "s2   27                    Perl  John\n",
      "s3   32           Deep Learning  Mary\n",
      "s4   19     Pattern Recognition  Gabi\n",
      "s5   23             Data Mining  Noha\n",
      "s6   20  Computational Archives  Juan\n",
      "s7   21      Health Informatics  Evan\n"
     ]
    }
   ],
   "source": [
    "frame=DataFrame(data, index=[\"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\", \"s7\" ])\n",
    "print(frame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s1    Alex\n",
      "s2    John\n",
      "s3    Mary\n",
      "s4    Gabi\n",
      "s5    Noha\n",
      "s6    Juan\n",
      "s7    Evan\n",
      "Name: names, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(frame[\"names\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s1    Alex\n",
      "s2    John\n",
      "s3    Mary\n",
      "s4    Gabi\n",
      "s5    Noha\n",
      "s6    Juan\n",
      "s7    Evan\n",
      "Name: names, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(frame.names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "age            25\n",
      "courses    Python\n",
      "names        Alex\n",
      "Name: s1, dtype: object\n"
     ]
    }
   ],
   "source": [
    "# Rows can be retrieved by e.g., the \"ix\" indexing field:\n",
    "print(frame.ix[\"s1\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "age          27\n",
      "courses    Perl\n",
      "names      John\n",
      "Name: s2, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(frame.ix[\"s2\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    age                 courses names School\n",
      "s1   25                  Python  Alex    UBC\n",
      "s2   27                    Perl  John    UBC\n",
      "s3   32           Deep Learning  Mary    UBC\n",
      "s4   19     Pattern Recognition  Gabi    UBC\n",
      "s5   23             Data Mining  Noha    UBC\n",
      "s6   20  Computational Archives  Juan    UBC\n",
      "s7   21      Health Informatics  Evan    UBC\n"
     ]
    }
   ],
   "source": [
    "# add a coulmn\n",
    "frame[\"School\"]=\"UBC\"\n",
    "print(frame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index([u'age', u'courses', u'names', u'School'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(frame.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index([u's1', u's2', u's3', u's4', u's5', u's6', u's7'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(frame.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "amazement     544395\n",
      "loathing       74923\n",
      "admiration     65759\n",
      "grief          42947\n",
      "terror         35705\n",
      "ecstasy        30206\n",
      "rage            8738\n",
      "vigilance        695\n",
      "Name: label, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "import statsmodels.api as sm\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from patsy import dmatrices\n",
    "from random import shuffle, randint, sample\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "\n",
    "emotion = pd.read_csv('emotions_p1_extended_lang_id_noduplic_denoised.csv', delimiter=',', header=0)\n",
    "#----------------------------------------\n",
    "print(pd.value_counts(emotion[\"label\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>message</th>\n",
       "      <th>label</th>\n",
       "      <th>lang_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>596908062054555648</td>\n",
       "      <td>Last week , Yuki Kawauchi ran 3 HM in 3 consec...</td>\n",
       "      <td>admiration</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>257202468386115584</td>\n",
       "      <td>Had a Turkish bath today . #amazing</td>\n",
       "      <td>amazement</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>223865330487930880</td>\n",
       "      <td>Taking my 6yo niece shopping #imintrouble #goi...</td>\n",
       "      <td>ecstasy</td>\n",
       "      <td>nl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>411617825149566976</td>\n",
       "      <td>&lt;USER&gt; &lt;USER&gt; &lt;USER&gt; Britt and I tried for so ...</td>\n",
       "      <td>grief</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>267380735453835264</td>\n",
       "      <td>I love this new song of one direction gotta ad...</td>\n",
       "      <td>amazement</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           message_id                                            message  \\\n",
       "0  596908062054555648  Last week , Yuki Kawauchi ran 3 HM in 3 consec...   \n",
       "1  257202468386115584                Had a Turkish bath today . #amazing   \n",
       "2  223865330487930880  Taking my 6yo niece shopping #imintrouble #goi...   \n",
       "3  411617825149566976  <USER> <USER> <USER> Britt and I tried for so ...   \n",
       "4  267380735453835264  I love this new song of one direction gotta ad...   \n",
       "\n",
       "        label lang_id  \n",
       "0  admiration      en  \n",
       "1   amazement      en  \n",
       "2     ecstasy      nl  \n",
       "3       grief      en  \n",
       "4   amazement      en  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emotion.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}