DecisionTree.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Decison Trees"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "First we'll load some fake data on past hires I made up. Note how we use pandas to convert a csv file into a DataFrame:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import tree\n",
    "\n",
    "input_file = \"d:/7.DataScience/DataScience-Python3/PastHires.csv\"\n",
    "df = pd.read_csv(input_file, header = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Years Experience</th>\n",
       "      <th>Employed?</th>\n",
       "      <th>Previous employers</th>\n",
       "      <th>Level of Education</th>\n",
       "      <th>Top-tier school</th>\n",
       "      <th>Interned</th>\n",
       "      <th>Hired</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>Y</td>\n",
       "      <td>4</td>\n",
       "      <td>BS</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>N</td>\n",
       "      <td>0</td>\n",
       "      <td>BS</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>N</td>\n",
       "      <td>6</td>\n",
       "      <td>BS</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>Y</td>\n",
       "      <td>1</td>\n",
       "      <td>MS</td>\n",
       "      <td>Y</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>N</td>\n",
       "      <td>2</td>\n",
       "      <td>PhD</td>\n",
       "      <td>Y</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Years Experience Employed?  Previous employers Level of Education  \\\n",
       "0                10         Y                   4                 BS   \n",
       "1                 0         N                   0                 BS   \n",
       "2                 7         N                   6                 BS   \n",
       "3                 2         Y                   1                 MS   \n",
       "4                20         N                   2                PhD   \n",
       "\n",
       "  Top-tier school Interned Hired  \n",
       "0               N        N     Y  \n",
       "1               Y        Y     Y  \n",
       "2               N        N     N  \n",
       "3               Y        N     Y  \n",
       "4               Y        N     N  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "scikit-learn needs everything to be numerical for decision trees to work. So, we'll map Y,N to 1,0 and levels of education to some scale of 0-2. In the real world, you'd need to think about how to deal with unexpected or missing data! By using map(), we know we'll get NaN for unexpected values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Years Experience</th>\n",
       "      <th>Employed?</th>\n",
       "      <th>Previous employers</th>\n",
       "      <th>Level of Education</th>\n",
       "      <th>Top-tier school</th>\n",
       "      <th>Interned</th>\n",
       "      <th>Hired</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Years Experience  Employed?  Previous employers  Level of Education  \\\n",
       "0                10          1                   4                   0   \n",
       "1                 0          0                   0                   0   \n",
       "2                 7          0                   6                   0   \n",
       "3                 2          1                   1                   1   \n",
       "4                20          0                   2                   2   \n",
       "\n",
       "   Top-tier school  Interned  Hired  \n",
       "0                0         0      1  \n",
       "1                1         1      1  \n",
       "2                0         0      0  \n",
       "3                1         0      1  \n",
       "4                1         0      0  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = {'Y': 1, 'N': 0}\n",
    "df['Hired'] = df['Hired'].map(d)\n",
    "df['Employed?'] = df['Employed?'].map(d)\n",
    "df['Top-tier school'] = df['Top-tier school'].map(d)\n",
    "df['Interned'] = df['Interned'].map(d)\n",
    "d = {'BS': 0, 'MS': 1, 'PhD': 2}\n",
    "df['Level of Education'] = df['Level of Education'].map(d)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Next we need to separate the features from the target column that we're trying to bulid a decision tree for."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Years Experience',\n",
       " 'Employed?',\n",
       " 'Previous employers',\n",
       " 'Level of Education',\n",
       " 'Top-tier school',\n",
       " 'Interned']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features = list(df.columns[:6])\n",
    "features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now actually construct the decision tree:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y = df[\"Hired\"]\n",
    "X = df[features]\n",
    "clf = tree.DecisionTreeClassifier()\n",
    "clf = clf.fit(X,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "... and display it. Note you need to have pydotplus installed for this to work. (!pip install pydotplus)\n",
    "\n",
    "To read this decision tree, each condition branches left for \"true\" and right for \"false\". When you end up at a value, the value array represents how many samples exist in each target value. So value = [0. 5.] mean there are 0 \"no hires\" and 5 \"hires\" by the tim we get to that point. value = [3. 0.] means 3 no-hires and 0 hires."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "ename": "InvocationException",
     "evalue": "GraphViz's executables not found",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mInvocationException\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-daeca59d9a5c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      7\u001b[0m                          feature_names=features)  \n\u001b[0;32m      8\u001b[0m \u001b[0mgraph\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpydotplus\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgraph_from_dot_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdot_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetvalue\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m \u001b[0mImage\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgraph\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreate_png\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\Users\\ACER\\AppData\\Local\\Enthought\\Canopy\\edm\\envs\\User\\lib\\site-packages\\pydotplus\\graphviz.py\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(f, prog)\u001b[0m\n\u001b[0;32m   1795\u001b[0m             self.__setattr__(\n\u001b[0;32m   1796\u001b[0m                 \u001b[1;34m'create_'\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mfrmt\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1797\u001b[1;33m                 \u001b[1;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfrmt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprog\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprog\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprog\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mprog\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1798\u001b[0m             )\n\u001b[0;32m   1799\u001b[0m             \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__dict__\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'create_'\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mfrmt\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\ACER\\AppData\\Local\\Enthought\\Canopy\\edm\\envs\\User\\lib\\site-packages\\pydotplus\\graphviz.py\u001b[0m in \u001b[0;36mcreate\u001b[1;34m(self, prog, format)\u001b[0m\n\u001b[0;32m   1958\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprogs\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1959\u001b[0m                 raise InvocationException(\n\u001b[1;32m-> 1960\u001b[1;33m                     'GraphViz\\'s executables not found')\n\u001b[0m\u001b[0;32m   1961\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1962\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mprog\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprogs\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInvocationException\u001b[0m: GraphViz's executables not found"
     ]
    }
   ],
   "source": [
    "from IPython.display import Image  \n",
    "from sklearn.externals.six import StringIO  \n",
    "import pydotplus\n",
    "\n",
    "dot_data = StringIO()  \n",
    "tree.export_graphviz(clf, out_file=dot_data,  \n",
    "                         feature_names=features)  \n",
    "graph = pydotplus.graph_from_dot_data(dot_data.getvalue())  \n",
    "Image(graph.create_png())  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Ensemble learning: using a random forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "We'll use a random forest of 10 decision trees to predict employment of specific candidate profiles:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=10)\n",
    "clf = clf.fit(X, y)\n",
    "\n",
    "#Predict employment of an employed 10-year veteran\n",
    "print (clf.predict([[10, 1, 4, 0, 0, 0]]))\n",
    "#...and an unemployed 10-year veteran\n",
    "print (clf.predict([[10, 0, 4, 0, 0, 0]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Activity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Modify the test data to create an alternate universe where everyone I hire everyone I normally wouldn't have, and vice versa. Compare the resulting decision tree to the one from the original data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import tree\n",
    "\n",
    "input_file = \"d:/7.DataScience/DataScience-Python3/PastHires2.csv\"\n",
    "df = pd.read_csv(input_file, header = 0)\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "d = {'Y': 1, 'N': 0}\n",
    "df['Hired'] = df['Hired'].map(d)\n",
    "df['Employed?'] = df['Employed?'].map(d)\n",
    "df['Top-tier school'] = df['Top-tier school'].map(d)\n",
    "df['Interned'] = df['Interned'].map(d)\n",
    "d = {'BS': 0, 'MS': 1, 'PhD': 2}\n",
    "df['Level of Education'] = df['Level of Education'].map(d)\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "features = list(df.columns[:6])\n",
    "y = df[\"Hired\"]\n",
    "X = df[features]\n",
    "clf = tree.DecisionTreeClassifier()\n",
    "clf = clf.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import graphviz\n",
    "from IPython.display import Image  \n",
    "from sklearn.externals.six import StringIO  \n",
    "import pydotplus\n",
    "\n",
    "dot_data = StringIO()  \n",
    "tree.export_graphviz(clf, out_file=dot_data,  feature_names=features)  \n",
    "graph = pydotplus.graph_from_dot_data(dot_data.getvalue())  \n",
    "Image(graph.create_png()) "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}