P4DS4D; 20; Understanding the Power of the Many.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:ffbfe6bc3518f72a0378c2bb256062e1854e898c5410f51d731a9adb498ccdfa"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Starting with a Plain Decision Tree"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Understanding the idea behind the decision tree"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# http://scikit-learn.org/stable/modules/tree.html"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Creating classification and regression trees"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.datasets import load_iris\n",
      "iris = load_iris()\n",
      "X, y = iris.data, iris.target\n",
      "features = iris.feature_names"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.cross_validation import cross_val_score\n",
      "from sklearn.cross_validation import KFold\n",
      "crossvalidation = KFold(n=X.shape[0], n_folds=5, shuffle=True, random_state=1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn import tree\n",
      "for depth in range(1,10):\n",
      "    tree_classifier = tree.DecisionTreeClassifier(max_depth=depth, random_state=0)\n",
      "    if tree_classifier.fit(X,y).tree_.max_depth < depth:\n",
      "        break\n",
      "    score = np.mean(cross_val_score(tree_classifier, X, y, scoring='accuracy', cv=crossvalidation, n_jobs=1))\n",
      "    print 'Depth: %i Accuracy: %.3f' % (depth,score)\n",
      "\n",
      "# min_samples_leaf=3\n",
      "# min_samples_split"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Depth: 1 Accuracy: 0.580\n",
        "Depth: 2 Accuracy: 0.913\n",
        "Depth: 3 Accuracy: 0.920\n",
        "Depth: 4 Accuracy: 0.940\n",
        "Depth: 5 Accuracy: 0.920\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "### GRAPHIC OUTPUT\n",
      "from sklearn.externals.six import StringIO  \n",
      "import pydot \n",
      "dot_data = StringIO()\n",
      "tree_classifier = tree.DecisionTreeClassifier(max_depth=4, random_state=0)\n",
      "tree_classifier.fit(X,y)\n",
      "tree.export_graphviz(tree_classifier, out_file=dot_data, feature_names=features, max_depth=4) \n",
      "graph = pydot.graph_from_dot_data(dot_data.getvalue()) \n",
      "graph.write_pdf(\"Iris.pdf\")\n",
      "graph.write_png(\"9781118844182 fg2001.png\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "True"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "from sklearn.preprocessing import OneHotEncoder\n",
      "titanic = pd.io.parsers.read_csv(\"Titanic.csv\")\n",
      "encoder = OneHotEncoder(sparse=False)\n",
      "X = titanic[['sex','age','sibsp','parch','pclass']].values\n",
      "y = titanic['survived'].values == 'survived'\n",
      "tree_classifier = tree.DecisionTreeClassifier(max_depth=2, min_samples_split=467, min_samples_leaf=10, random_state=0)\n",
      "\n",
      "df = pd.concat([\n",
      "pd.get_dummies(titanic['sex']),\n",
      "titanic[['age']],\n",
      "], join='outer', axis=1)\n",
      "\n",
      "tree_classifier.fit(df,y)\n",
      "\n",
      "### GRAPHIC OUTPUT\n",
      "from sklearn.externals.six import StringIO  \n",
      "import pydot \n",
      "dot_data = StringIO() \n",
      "tree.export_graphviz(tree_classifier, out_file=dot_data, feature_names=df.columns, max_depth=2) \n",
      "graph = pydot.graph_from_dot_data(dot_data.getvalue()) \n",
      "graph.write_pdf(\"Titanic.pdf\") \n",
      "graph.write_png(\"9781118844182 fg2001.png\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 52,
       "text": [
        "True"
       ]
      }
     ],
     "prompt_number": 52
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tree_classifier = tree.DecisionTreeClassifier(min_samples_split=30, min_samples_leaf=10, random_state=0)\n",
      "tree_classifier.fit(X,y)\n",
      "score = np.mean(cross_val_score(tree_classifier, X, y, scoring='accuracy', cv=crossvalidation, n_jobs=1))\n",
      "print 'Accuracy: %.3f' % score"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Accuracy: 0.913\n"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "### GRAPHIC OUTPUT\n",
      "from sklearn.externals.six import StringIO  \n",
      "import pydot \n",
      "dot_data = StringIO() \n",
      "tree.export_graphviz(tree_classifier, out_file=dot_data, feature_names=features, max_depth=4) \n",
      "graph = pydot.graph_from_dot_data(dot_data.getvalue()) \n",
      "graph.write_pdf(\"Iris simpler.pdf\") "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 20,
       "text": [
        "True"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.datasets import load_boston\n",
      "boston = load_boston()\n",
      "X, y = boston.data, boston.target\n",
      "features = boston.feature_names"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.tree import DecisionTreeRegressor\n",
      "regression_tree = tree.DecisionTreeRegressor(min_samples_split=30, min_samples_leaf=10, random_state=0)\n",
      "regression_tree.fit(X,y)\n",
      "score = np.mean(cross_val_score(regression_tree, X, y, scoring='mean_squared_error', cv=crossvalidation, n_jobs=1))\n",
      "print 'Mean squared error: %.3f' % abs(score)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Mean squared error: 22.593\n"
       ]
      }
     ],
     "prompt_number": 56
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "### GRAPHIC OUTPUT\n",
      "from sklearn.externals.six import StringIO  \n",
      "import pydot \n",
      "dot_data = StringIO() \n",
      "tree.export_graphviz(regression_tree, out_file=dot_data, feature_names=features, max_depth=4) \n",
      "graph = pydot.graph_from_dot_data(dot_data.getvalue()) \n",
      "graph.write_pdf(\"Boston.pdf\") "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 23,
       "text": [
        "True"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Getting Lost in a Random Forest"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Making machine learning accessible to all"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.ensemble import BaggingClassifier\n",
      "from sklearn import tree\n",
      "tree_classifier = tree.DecisionTreeClassifier(random_state=0)\n",
      "crossvalidation = KFold(n=X.shape[0], n_folds=5, shuffle=True, random_state=1)\n",
      "bagging = BaggingClassifier(tree_classifier, max_samples=0.7, max_features=0.7, n_estimators=300)\n",
      "scores = np.mean(cross_val_score(bagging, X, y, scoring='accuracy', cv=crossvalidation, n_jobs=1))\n",
      "print 'Accuracy: %.3f' % score"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Mean squared error: 0.966\n"
       ]
      }
     ],
     "prompt_number": 52
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Working with a random forest classifier"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.datasets import load_digits\n",
      "digit = load_digits()\n",
      "X, y = digit.data, digit.target\n",
      "print X.shape, y.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(1797L, 64L) (1797L,)\n"
       ]
      }
     ],
     "prompt_number": 47
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.ensemble import RandomForestClassifier\n",
      "from sklearn.cross_validation import cross_val_score\n",
      "from sklearn.cross_validation import KFold\n",
      "crossvalidation = KFold(n=X.shape[0], n_folds=5, shuffle=True, random_state=1)\n",
      "RF_cls = RandomForestClassifier(n_estimators=300)\n",
      "score = np.mean(cross_val_score(RF_cls, X, y, scoring='accuracy', cv=crossvalidation, n_jobs=1))\n",
      "print 'Accuracy: %.3f' % score"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Accuracy: 0.974\n"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.learning_curve import validation_curve\n",
      "train_scores, test_scores = validation_curve(RF_cls, X, y,\n",
      "                                             'n_estimators', param_range=[10,50,100,200,300,500,800,1000,1500], \n",
      "                                             cv=crossvalidation, scoring='accuracy', n_jobs=1)\n",
      "print 'mean cv accuracy %s' % np.mean(test_scores,axis=1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "mean cv accuracy [ 0.94768802  0.96939338  0.97384865  0.97495667  0.97607088  0.97718353\n",
        "  0.97607088  0.97718508  0.97662643]\n"
       ]
      }
     ],
     "prompt_number": 30
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import matplotlib.pyplot as plt\n",
      "mean_test = np.mean(test_scores,axis=1)\n",
      "#plt.plot([10,50,100,200,300,500,1000],mean_train,'ro--', label='Training')\n",
      "plt.plot([10,50,100,200,300,500,800,1000,1500],mean_test,'bD-.', label='Cross-validation')\n",
      "plt.grid()\n",
      "plt.xlabel('Number of estimators') # adds label to x axis\n",
      "plt.ylabel('accuracy') # adds label to y axis\n",
      "plt.legend(loc='lower right', numpoints= 1)\n",
      "plt.show()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEPCAYAAACOU4kjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtcVHX+P/AXCGYJKJiiAjbIZUDBYWzQtPwpq0iUoqgZ\nagZq5dd1W/OCWq6Jlgjb1cu6GmlqrWhbKppKLIZdvCEqrouEqIzJJUJmBlDUkeH9+2NiYhwU0DPM\nGXg/H495yLm/zhyYt/P5nIsNEREYY4wxAdlaOgBjjLHWh4sLY4wxwXFxYYwxJjguLowxxgTHxYUx\nxpjguLgwxhgTnFmLS2pqKvz8/ODj44PExEST6Wq1GpGRkZDJZBg4cCBycnIM01atWoW+ffsiMDAQ\nkydPxu3btwEAKpUKoaGh8PX1xciRI6HRaMy5C4wxxh6A2YqLTqfDX/7yF6SmpuL8+fNITk5Gbm6u\n0Tzx8fHo378/zp49i23btmHOnDkAAKVSiaSkJJw+fRrnzp2DTqfDjh07AAAJCQkIDQ3FhQsXMHz4\ncCQkJJhrFxhjjD0gsxWXzMxMeHt7QyKRwN7eHlFRUUhJSTGaJzc3FyEhIQAAqVQKpVKJsrIyODk5\nwd7eHtXV1aipqUF1dTXc3NwAAHv37kV0dDQAIDo6Gnv27DHXLjDGGHtAZisuRUVF8PDwMAy7u7uj\nqKjIaB6ZTIZdu3YB0BejK1euoLCwEC4uLpg/fz569eqFnj17olOnThgxYgQAoLS0FK6urgAAV1dX\nlJaWmmsXGGOMPSCzFRcbG5tG51m8eDE0Gg3kcjnWrVsHuVyOdu3a4dKlS/j444+hVCpRXFyMGzdu\n4F//+leD22jKdhhjjLUsO3Ot2M3NDVevXjUMX716Fe7u7kbzODo6YvPmzYZhT09P9O7dG/v378fg\nwYPRpUsXAMC4ceNw9OhRTJkyBa6urvj111/RvXt3lJSUoFu3bvfcfnFxsRn2jDHGWicvLy9cvHhR\nkHWZ7ZuLQqFAfn4+lEoltFotdu7ciYiICKN5KioqoNVqAQBJSUkYOnQoHBwcIJVKcfz4cdy8eRNE\nhPT0dPTp0wcAEBERga1btwIAtm7dirFjxza4/eLiYhCRqF/Lli2zeAbOyTk5J+ese126dEmwGmC2\nby52dnZYt24dwsLCoNPpMGPGDPj7+2Pjxo0AgJkzZ+L8+fOIiYmBjY0NAgICsGnTJgBAUFAQXn75\nZSgUCtja2qJ///547bXXAOib0iZOnIhNmzZBIpHgyy+/NNcumJ1SqbR0hCbhnMLinMLinOJktuIC\nAOHh4QgPDzcaN3PmTMPPgwYNQl5eXoPLLly4EAsXLjQZ7+LigvT0dGGDMsYYExRfoW9BMTExlo7Q\nJJxTWJxTWJxTnGyIqFU+LMzGxgatdNcYY8wshPzc5G8uFnT48GFLR2gSzikszikszilOXFwYY4wJ\njpvFGGOMAeBmMcYYYyLHxcWCrKUNlnMKi3MKi3OKExcXxhhjguM+F8YYYwC4z4UxxpjIcXGxIGtp\ngxU6p1qtxpgxf4ZarRZ0vW31/TQHtVqNp58eI/gxMgdreD8B68kpFC4urEWp1WqMHLkEe/fGYuTI\nJVbx4dXW1B2jo0ej+BixB8Z9LqzF1H1oZWWtBOAMQA2FYgnS0lbC2dnZ0vEY+Bi1ddznwqxOVpYa\nAQH1P7QAwBlZWSsxcuQSHD+uxpEjlkzIjhxRIzi44WPUs+cSyOVqxMebLnf8OLB9u+n4/Hz9+N27\ngf/+13R6dTVQWSnkHrQd5mpaFhIXFwuyljbY5uQsLAT27DEdv2DBEhQXx+KPD606zsjKisX8+Utw\n+bLpcl9/DbzwAjBvHtDYkxZa4/sppFu3gJMngS1bgM8/N52+dOkSXLpU/xgd/v1fZ9y6FQtHxyV4\n4QXT5RwcgK5dTccXFQH79gHbtgEZGabTd+8Gli0zHf/FF4BEAvj7AytWmE7//nv9OuvUvZ85OcAn\nn+j3LSvLdLmKCuDaNdPxLUWo424tTctmfZ4La32IgKtXgf/9T/+/zqgo4+m3bgEFBabL7d698q7m\nljpqKBTv4ZtvVqKhVpcnnwRqavTb1OlMp//jH0C7dsD//Z/x+LIyfdauXQEbm+bupXW7c0f/fvXu\nbTw+Jwd47TWgb18gJMR0ua+/vv8xSklp+BgFBOhfdxs2TP+6lylT9K+7RUYCTz+t/11ycDCd7uqq\nP+Z3Ky/XF89bt4DbtwGFwnj6N98AubnAu+8aj//0U2DxYuDRR4Hp04Hly42nf/utvlBOn248/vRp\n4PBh/XIyGTB4sPH0sjJAqwXc3Brc/Qdyd7Nl3Td/UTZbUivVinetRdTWEpWVmY6/dImoRw+iESOI\nli1r3jpVKhUpFLMIUJH+o18/rFKpHjhnaSlRcbHp+L//nahLF6IOHYj++U/T6Xl5DS9nTW7fJrp4\n0XS8Uqk/Pg/CHMdI7G7dIvrtN6IrV/T/3u3CBaKsLNPxP/1E9MYbRDNnEn3xhen07duJ4uNNx69Z\nQ/TII0SdOxMtWWI6fc8eog0bTMenpqqoZ8/6x4YEP0ZCfm622k/gtlhcVCoVRUQ0/5fs2jXTcRUV\nRE88oS8yQvrjw+tyi3xo3bhBVFVlOv6DD4h27TId//nnRG+/TZSURFRQIHyeBz1G9VVXE/Xtqy+c\nMpn1H6O2RqfTH8Pycv3f2d0uXyb63/9Mxw8Zoj8mxoWl7nWZIiJmPXQ2Li5NYA3FJSMjQ7B1NfcD\nobaWKCyMqGtXImdn/S97S+Ssy/qwH7ANESLnt98SLV1KFBND9MMPptP//neiY8dMx9fUNL7uP47R\n9iYfo4ED9QXybmfPEt282fg2H5RKpaLBgyOsorAI/ftpLg+b0/RbJX9zsYi2VFwaa8qYNKnhJqAf\nfyQqKmr8f75t5Y+3KY4cISosNB3//PP6ZrigIKITJ0ynX7igoiefrDtGGYZj9P77KoqJIVIoiK5e\nNV3u3DmiO3eE34+m4OMuLCFymrvZkotLE1hDcRGCSlX/Q6vh/80cO9bw/36ZcHQ6ff9PVhaRWm08\nTaVSUadODR8jD49Z9NFHKjpyRN/2z1hjzNlsKeTnJl9EaeXGjPkz9u6NBeDZwNQCRES8h5SU9S0d\ni9XDx4gJTa1WIyZmCbZsEfYsMb6IspVo7nnvRMDdp7Rv2bIS/fu/B+Duc931p49u2bLyYSIC4OtH\nHtaWLSuhUNQ/Rod//1e4Y2QOYn0/79YWczo7OyMlZb34Tj+ux6zFJTU1FX5+fvDx8UFiYqLJdLVa\njcjISMhkMgwcOBA5OTkAgLy8PMjlcsOrU6dOWLNmDQAgLi4O7u7uhmmpqanm3AVR2btXfzFhfc7O\nzkhPXwmFYgn++PDiW3aIibOzM9LS+BixNkawBra71NTUkJeXFxUUFJBWqyWZTEbnz583mmfBggW0\nYsUKIiL6+eefafjw4Sbr0el01L17d/rll1+IiCguLo4++OCDRrdvxl0TRGNnTDV0GqxOd+/Odz59\nVPz4GDGxE/Jz02zfXDIzM+Ht7Q2JRAJ7e3tERUUhJSXFaJ7c3FyE/H6psFQqhVKpRFlZmdE86enp\n8PLygoeHR/2CaK7YLeJet28gAt55548rqG/fNl7O1vbeV5vX/e84IuI9/t+wSPExYm2J2YpLUVGR\nUUFwd3dHUVGR0TwymQy7du0CoC9GV65cQWFhodE8O3bswOTJk43GrV27FjKZDDNmzIBGozHTHpiH\n8e0brhhu36BWq2Fjo7/dRVIScOkS8MgjzVu3udph22Kbtrk4Oztj7tyJVlFYrOH9BDinWJnt3mI2\nTbih0+LFizFnzhzI5XIEBgZCLpejXb2bBmm1Wuzbt8+ov2bWrFl4++23AQBLly7F/PnzsWnTpgbX\nHxMTA4lEAgDo3LkzgoKCMOz3mx3VHeiWHK6qqsKKFQd/LyxnAWQDGIasrJV46qkYvP/+K5g7d7TF\n8ln7cHZ2tqjyWPswv5+t//2s+1mpVEJwgjWw3eXYsWMUFhZmGI6Pj6eEhIT7LiORSKiq3r069uzZ\nY7SOuxUUFFBAQECD08y4aw8sIsL8t29gjLEHJeTnptmaxRQKBfLz86FUKqHVarFz505EREQYzVNR\nUQGtVgsASEpKwtChQ+FQ7zaoycnJmDRpktEyJSUlhp93796NwMBAc+3CQysuBtasAVb+fqap6Smp\ndcR9SipjjDWbYGWqAQcOHCBfX1/y8vKi+N9vD7phwwba8PstP48ePUq+vr4klUpp/PjxpNFoDMte\nv36dunTpQpWVlUbrnDp1KgUGBlK/fv1ozJgx9Ouvvza4bTPvWqMuXNDfsys6migt7Y/xxrdvyCBr\nuOtsW7q9RkvgnMLinMIR8nPTrM9zCQ8PR3h4uNG4mTNnGn4eNGgQ8vLyGly2Y8eOuNbAk3221X9K\nUAu63xWxZWXA448bn8nl7Q2UlJh2ytedMaTv1B8CheJLPnOIMdbq8O1fmuCPM7xioVCYnkY6cKD+\nca5eXs1bpzlu38AYYw9KyM9NLi6NuPvJbw1dWU3U9p52yBhrffjeYi3EtLAA9R8tWnfx44MWlvqn\nA4oZ5xQW5xQW5xQnLi73EROjbwozfp44oC8wsYiJWWKJWIwxJnrcLHYfDX9zAfimg4yx1oibxVoI\n382WMcYeDBeXRhgXmAJBC4u1tMFyTmFxTmFxTnHi4tIEfDdbxhhrHu5zuY+CAmDLFmD5cmEyMcaY\nmHGfSwvp2BF46ilLp2CMMevDxeU+unUD7rp7jaCspQ2WcwqLcwqLc4oTFxfGGGOC4z4XxhhjALjP\nxexqavQvxhhjD4aLSwN27wYmTzb/dqylDZZzCotzCotzihMXlwZs2waMHm3pFIwxZr24z+Uut28D\ncjmQmQnUe+IyY4y1evw8lyZ4mDeJn8/CGGuLuEPfzFqqsFhLGyznFBbnFBbnFCcuLowxxgTHzWKM\nMcYAcLOYWdTUAImJgE5n6SSMMWb9uLj8rrpa/2+7di23TWtpg+WcwuKcwuKc4mTW4pKamgo/Pz/4\n+PggMTHRZLparUZkZCRkMhkGDhyInJwcAEBeXh7kcrnh1alTJ6xZswYAoFKpEBoaCl9fX4wcORIa\njUaQrE5OwKJFgqyKMcbaPLP1ueh0OkilUqSnp8PNzQ3BwcFITk6Gv7+/YZ7Y2Fg4OTlh6dKlyMvL\nw+zZs5Genm60ntraWri5uSEzMxMeHh5YuHAhHn/8cSxcuBCJiYlQq9VISEgw3THuc2GMsWaxij6X\nzMxMeHt7QyKRwN7eHlFRUUhJSTGaJzc3FyEhIQAAqVQKpVKJsrIyo3nS09Ph5eUFDw8PAMDevXsR\nHR0NAIiOjsaePXvMtQuMMcYekNmKS1FRkaEgAIC7uzuKioqM5pHJZNi1axcAfTG6cuUKCgsLjebZ\nsWMHJte70VdpaSlcXV0BAK6urigtLTXXLpidtbTBck5hcU5hcU5xsjPXim2acCXi4sWLMWfOHMjl\ncgQGBkIul6NdvR51rVaLffv2NdhfU7eN+20nJiYGEokEANC5c2cEBQVh2LBhAPQHuqqqCmvWHMSN\nGyuxcuVZ2NjAaDpg3uHs7OwW3V5rH+b3k99PMQ+L8f2s+1mpVEJwZCbHjh2jsLAww3B8fDwlJCTc\ndxmJREJVVVWG4T179hitg4hIKpVSSUkJEREVFxeTVCptcF2N7ZpKpSKFYhYBl6lr11mkUqnuOz9j\njLV2QpYEszWLKRQK5OfnQ6lUQqvVYufOnYiIiDCap6KiAlqtFgCQlJSEoUOHwqHe3SKTk5MxadIk\no2UiIiKwdetWAMDWrVsxduzYZmdTq9UYOXIJsrJWAvBEWdlKjBy5BGq1utnrYowx1gDBylQDDhw4\nQL6+vuTl5UXx8fFERLRhwwbasGEDEREdPXqUfH19SSqV0vjx40mj0RiWvX79OnXp0oUqKyuN1lle\nXk7Dhw8nHx8fCg0NJbVa3eC277Vrf3xjUZH+FpV1L/34lvwGk5GR0WLbehicU1icU1icUzhClgSz\n9bkAQHh4OMLDw43GzZw50/DzoEGDkJeX1+CyHTt2xLVr10zGu7i4mJyu3BwxMUuQlRULwPmuKc7I\nyopFTMwSpKSsf+D1M8YYa4P3FjNuEqtfYNRQKJYgLW0lnJ3vLjyMMdb6WcV1LmLl7OyMtLSVUCiW\nAKjrY+HCwhhjQmpzxQX4o8AEBS0BUGCxwlL/dEAx45zC4pzC4pziZNY+FzFzdnaGt/dKPProEuzf\nz99YGGNMSG2uz4UxxljDuM+FMcaYqHFxsSBraYPlnMLinMLinOLExYUxxpjg2myfS20tUFICuLm1\nYCjGGBMx7nMRQFkZEBRk6RSMMdY6tdniUlEBdOpk2QzW0gbLOYXFOYXFOcWpzRaX27cBb29Lp2CM\nsdapzfa5MMYYM8Z9LowxxkSNi4sFWUsbLOcUFucUFucUJy4ujDHGBNdm+1zKy4H27QFHxxYMxRhj\nIsZ9LgJYvx7YudPSKRhjrHVqs8Vl6VLglVcsm8Fa2mA5p7A4p7A4pzi12eLCGGPMfNpsnwtjjDFj\n3OfCGGNM1NpscVEqAZ3OshmspQ2WcwqLcwqLc4pTo8Vl3Lhx2L9/P2pra5u98tTUVPj5+cHHxweJ\niYkm09VqNSIjIyGTyTBw4EDk5OQYpmk0GkyYMAH+/v7o06cPTpw4AQCIi4uDu7s75HI55HI5UlNT\nm50LAORy/c0rGWOMCa/RPpf//Oc/+Oyzz3D8+HFMnDgR06ZNg1QqbXTFOp0OUqkU6enpcHNzQ3Bw\nMJKTk+Hv72+YJzY2Fk5OTli6dCny8vIwe/ZspKenAwCio6MxdOhQTJ8+HTU1Nbhx4wY6deqE5cuX\nw9HREfPmzbv/jt2n7bC2Vn+Ny61bgJ1do7vCGGNtQov2uYSGhmL79u04ffo0JBIJhg8fjsGDB+Oz\nzz7DnTt37rlcZmYmvL29IZFIYG9vj6ioKKSkpBjNk5ubi5CQEACAVCqFUqlEWVkZKioq8OOPP2L6\n9OkAADs7O3Sqd3/8h935mzcBf38uLIwxZi5N6nMpLy/Hli1b8Omnn6J///7461//ilOnTiE0NPSe\nyxQVFcHDw8Mw7O7ujqKiIqN5ZDIZdu3aBUBfjK5cuYLCwkIUFBSga9eumDZtGvr3749XX30V1dXV\nhuXWrl0LmUyGGTNmQKPRNGuHAaBjR+DcuWYvJjhraYPlnMLinMLinOLUaHGJjIzEM888g+rqauzb\ntw979+5FVFQU1q1bh6qqqnsuZ2Nj0+jGFy9eDI1GA7lcjnXr1kEul6Ndu3aoqanB6dOn8ec//xmn\nT59Gx44dkZCQAACYNWsWCgoKkJ2djR49emD+/Pn3XH9MTAzi4uIQFxeHjz/+2OjgHj582OLD2dnZ\nospj7cP8fgo7zO+nsMNifD8PHz6MuLg4xMTEICYmBkJqtM8lIyPD0HTVHMePH0dcXJyhw33VqlWw\ntbXFokWL7rmMp6cnzp07h+vXr2PQoEEoKCgAAPz0009ISEjAN998YzS/UqnE6NGjca6BryF8nQtj\njDVPi/a55OTkQK1WG4bVajXWr1/f6IoVCgXy8/OhVCqh1Wqxc+dOREREGM1TUVEBrVYLAEhKSsLQ\noUPh4OCA7t27w8PDAxcuXAAApKeno2/fvgCAkpISw/K7d+9GYGBgE3aTMcZYi6JG9OvXz2ScTCZr\nbDEiIjpw4AD5+vqSl5cXxcfHExHRhg0baMOGDUREdPToUfL19SWpVErjx48njUZjWDY7O5sUCgX1\n69ePIiMjDdOmTp1KgYGB1K9fPxozZgz9+uuvDW77frumUhGVljZpF8wqIyPD0hGahHMKi3MKi3MK\npwklockaPV+qtrYWtbW1sLXVf8nR6XT3PUusvvDwcISHhxuNmzlzpuHnQYMGIS8vr8FlZTIZTp48\naTJ+27ZtTdr2/Xz7LZCTA7zzzkOvijHGWAMa7XNZsGABfvnlF8ycORNEhI0bN6JXr1744IMPWirj\nA+E+F8YYax4hPzcbLS46nQ6ffPIJDh06BEB/3csrr7yCdu3aCRLAXLi4MMZY87Roh367du0wa9Ys\nfPXVV/jqq68wc+ZM0RcWa1H/1EAx45zC4pzC4pzi1Gify4ULF/DWW2/h/PnzuHnzJgB9dbt8+bLZ\nwzHGGLNOjTaLPf3001i+fDnmzZuHvXv3YsuWLdDpdHhH5L3h9/t6p1QC3boBjz3WspkYY0zMWrRZ\n7ObNmxgxYgSICBKJBHFxcdi/f78gG7eUSZOAs2ctnYIxxlqvRotLhw4doNPp4O3tjXXr1mHXrl24\nceNGS2QzG40GqHcfTIuxljZYzikszikszilOjfa5rF69GtXV1VizZg2WLl2KyspKbN26tSWymU33\n7oCLi6VTMMZY63XfPhedTodFixbh/fffb8lMguBTkRljrHlarM+lXbt2+Omnn/hDmjHGWLM02ucS\nFBSEMWPG4PPPP8fXX3+Nr7/+2vAMFvZwrKUNlnMKi3MKi3OKU6N9Lrdu3YKLiwu+++47o/Hjxo0z\nWyjGGGPWrdHrXKzVvdoOr18HysoAT08LhGKMMRETss+l0W8u06ZNM9k4AGzevFmQAC0tNxf4xz+A\nLVssnYQxxlqvRvtcnn/+eYwaNQqjRo3C8OHDUVFRgY4dO7ZENrMIDhZPYbGWNljOKSzOKSzOKU6N\nfnOZMGGC0fDkyZPx9NNPmy0QY4wx69fsPpeff/4Zo0aNwsWLF82VSRB8nQtjjDVPi/a5ODg4GPpZ\nbGxs4OrqisTEREE2zhhjrHVqtM/l+vXrqKqqQlVVFSorK5Gfn4/x48e3RDaz+OUXQKWydAo9a2mD\n5ZzC4pzC4pzi1Ghx2b17NzQajWFYo9Fgz549Zg1lTkuXAikplk7BGGOtW6N9LjKZDGfvuj99UFAQ\nsrOzzRrsYd2r7XDsWCA6GoiMtEAoxhgTsRZ9nktDG9LpdIJs3BJcXYEePSydgjHGWrdGi8uTTz6J\nefPm4dKlS7h48SLmzp2LJ598siWymcXGjcBTT1k6hZ61tMFyTmFxTmFxTnFqtLisXbsW9vb2ePHF\nFxEVFYUOHTrgH//4R5NWnpqaCj8/P/j4+DR4hplarUZkZCRkMhkGDhyInJwcwzSNRoMJEybA398f\nffr0wfHjxwEAKpUKoaGh8PX1xciRI436gxhjjImD2e4tptPpIJVKkZ6eDjc3NwQHByM5ORn+/v6G\neWJjY+Hk5ISlS5ciLy8Ps2fPRnp6OgAgOjoaQ4cOxfTp01FTU4MbN26gU6dOWLhwIR5//HEsXLgQ\niYmJUKvVSEhIMN0xvs6FMcaapUX7XEaMGGH07UClUiEsLKzRFWdmZsLb2xsSiQT29vaIiopCyl2n\naeXm5iIkJAQAIJVKoVQqUVZWhoqKCvz444+YPn06AMDOzg6dfn8u8d69exEdHQ1AX4Cs+cw1xhhr\nrRotLteuXUPnzp0Nwy4uLigtLW10xUVFRfDw8DAMu7u7o6ioyGgemUxmeDZMZmYmrly5gsLCQhQU\nFKBr166YNm0a+vfvj1dffRXV1dUAgNLSUri6ugIAXF1dm5Slzp07wF0nvlmUtbTBck5hcU5hcU5x\navQK/Xbt2uHKlSt44oknAABKpRK2to3WJMNV/fezePFizJkzB3K5HIGBgZDL5WjXrh20Wi1Onz6N\ndevWITg4GG+88QYSEhKwYsUKk23cbzsxMTGQSCQAgM6dO0MiCcL69cOQlvbHgR42bBgAywxnZ2db\ndPutbZjfT34/xTwsxvez7melUgnBUSMOHjxIHh4e9NJLL9GUKVPIw8ODDh482NhidOzYMQoLCzMM\nx8fHU0JCwn2XkUgkVFVVRSUlJSSRSAzjf/jhB3r++eeJiEgqlVJJSQkRERUXF5NUKm1wXU3YNcYY\nY/UI+bnZ6FeQZ599FllZWZBKpYiKisKHH36Ixx57rNGipVAokJ+fD6VSCa1Wi507dyIiIsJonoqK\nCmi1WgBAUlIShg4dCgcHB3Tv3h0eHh64cOECAODQoUPo27cvACAiIgJbt24FAGzduhVjx45tTi1l\njDHWEhqrPp988gkFBARQp06daNiwYdShQwcKCQlpUuU6cOAA+fr6kpeXF8XHxxMR0YYNG2jDhg1E\nRHT06FHy9fUlqVRK48ePJ41GY1g2OzubFAoF9evXjyIjIw3TysvLafjw4eTj40OhoaGkVqsb3HYT\nds3iMjIyLB2hSTinsDinsDincIT83Gy0z2X16tU4efIkBg0ahIyMDPz888948803m1S4wsPDER4e\nbjRu5syZhp8HDRqEvLy8BpeVyWQ4efKkyXgXFxfD6cqMMcbEqdHrXBQKBbKyshAUFITjx4+jQ4cO\n6NOnD86fP99SGR9IQ+drFxcDOh1Q7yQ2xhhjv2vR57l4eHhArVZj7NixCA0NhbOzs+EMLGuzdStQ\nUQE0cM0lY4wxATXplvvOzs6Ii4vDO++8g1deecVqL1zUaIB6l+xYXP3TAcWMcwqLcwqLc4pTo99c\n6qs7R9pade0K9Opl6RSMMdb6me3eYpbG9xZjjLHmadF7izHGGGPNxcXFgqylDZZzCotzCotzihMX\nF8YYY4JrU30umZmAXA7Y21soFGOMiRj3uTygt94Cbt+2dArGGGv92lRxSU8HHBwsneIP1tIGyzmF\nxTmFxTnFqU0VF8YYYy2jTfW5MMYYuzfuc2GMMSZqbaa4lJcDYruRs7W0wXJOYXFOYXFOcWozxeX7\n74ElSyydgjHG2oY20+fy2Wf6ArNli+UyMcaYmHGfywNwcgJkMkunYIyxtqHNFJfx44G5cy2dwpi1\ntMFyTmFxTmFxTnFqM8WFMcZYy2kzfS6MMcbuj/tcGGOMiVqbKS6nTgGVlZZOYcxa2mA5p7A4p7A4\npziZtbikpqbCz88PPj4+SExMNJmuVqsRGRkJmUyGgQMHIicnxzBNIpGgX79+kMvlGDBggGF8XFwc\n3N3dIZfLIZfLkZqa2qQsSUlAYeHD7xNjjLHGma3PRafTQSqVIj09HW5ubggODkZycjL8/f0N88TG\nxsLJyQlxM42cAAAfPklEQVRLly5FXl4eZs+ejfT0dACAp6cnTp06BRcXF6P1Ll++HI6Ojpg3b959\nt899Lowx1jxW0eeSmZkJb29vSCQS2NvbIyoqCikpKUbz5ObmIiQkBAAglUqhVCpRVlZmmH6vneSi\nwRhj4ma24lJUVAQPDw/DsLu7O4qKiozmkclk2LVrFwB9Mbpy5QoKf2+7srGxwYgRI6BQKJCUlGS0\n3Nq1ayGTyTBjxgxoNBpz7YLZWUsbLOcUFucUFucUJztzrdjGxqbReRYvXow5c+ZALpcjMDAQcrkc\n7dq1AwD89NNP6NmzJ8rKyhAaGgo/Pz8MGTIEs2bNwttvvw0AWLp0KebPn49NmzY1uP6YmBhIJBIA\nQOfOnREUFIRhw4YB+ONAW3I4OztbVHmsfZjfT34/xTwsxvez7melUgmhma3P5fjx44iLizN0uK9a\ntQq2trZYtGjRPZfx9PTEuXPn4HDX4yKXL18OBwcHzJ8/32i8UqnE6NGjce7cOZN11W87rK4GsrOB\nwYMfdq8YY6z1soo+F4VCgfz8fCiVSmi1WuzcuRMRERFG81RUVECr1QIAkpKSMHToUDg4OKC6uhpV\nVVUAgBs3biAtLQ2BgYEAgJKSEsPyu3fvNoy/n4ICYMYMofaMMcZYY8xWXOzs7LBu3TqEhYWhT58+\nePHFF+Hv74+NGzdi48aNAIDz588jMDAQfn5++Pbbb7F69WoAQGlpKYYMGYKgoCAMHDgQo0aNwsiR\nIwEAixYtQr9+/SCTyfD999/jo48+ajRLRQXQubO59vTB1f9qKmacU1icU1icU5zM1ucCAOHh4QgP\nDzcaN3PmTMPPgwYNQl5enslynp6eyM7ObnCd27Zta3aORx4Bhgxp9mKMMcYeEN9bjDHGGAAr6XNh\njDHWdnFxsSBraYPlnMLinMLinOLExYUxxpjg2kSfy7lzQJcuQM+eFg7FGGMixn0uzZSaCtS74TJj\njDEzaxPFJTYWCA21dApT1tIGyzmFxTmFxTnFqU0UF8YYYy2rTfS5MMYYaxz3uTDGGBO1NlFcUlMB\nnc7SKUxZSxss5xQW5xQW5xSnVl9camqA558HmvB4GcYYYwJp9X0u5eWAjw+gUlk6EWOMiRv3uTSD\nTgeMHm3pFIwx1ra0+uLSrRuwdaulUzTMWtpgOaewOKewOKc4tfriwhhjrOW1+j4XxhhjTcN9Lowx\nxkSt1ReX/HwgN9fSKRpmLW2wnFNYnFNYnFOc7CwdwNyys4GbNwF/f0snYYyxtoP7XBhjjAHgPhfG\nGGMix8XFgqylDZZzCotzCotzipNZi0tqair8/Pzg4+ODxMREk+lqtRqRkZGQyWQYOHAgcuo9LlIi\nkaBfv36Qy+UYMGCAYbxKpUJoaCh8fX0xcuRIaDQac+4CY4yxB2C2PhedTgepVIr09HS4ubkhODgY\nycnJ8K/Xsx4bGwsnJycsXboUeXl5mD17NtLT0wEAnp6eOHXqFFxcXIzWu3DhQjz++ONYuHAhEhMT\noVarkZCQYLpjv7cdHjoEBAcDTk7m2EvGGGs9rKLPJTMzE97e3pBIJLC3t0dUVBRSUlKM5snNzUVI\nSAgAQCqVQqlUoqyszDC9oZ3cu3cvoqOjAQDR0dHYs2fPfXPMnAmUlj7s3jDGGGsOsxWXoqIieHh4\nGIbd3d1RVFRkNI9MJsOuXbsA6IvRlStXUFhYCEBfQUeMGAGFQoGkpCTDMqWlpXB1dQUAuLq6orSR\nyqHRAJ07C7JLgrOWNljOKSzOKSzOKU5mu87FpgkPUFm8eDHmzJkDuVyOwMBAyOVytGvXDgDw008/\noWfPnigrK0NoaCj8/PwwZMgQk23cbzsxMTFwdZVgzRqgS5fOCAoKwrBhwwD8caAtOZydnS2qPNY+\nzO8nv59iHhbj+1n3s1KphNDM1udy/PhxxMXFITU1FQCwatUq2NraYtGiRfdcxtPTE+fOnYODg4PR\n+OXLl8PR0RHz5s2Dn58fDh8+jO7du6OkpAQhISH4+eefTdbF17kwxljzWEWfi0KhQH5+PpRKJbRa\nLXbu3ImIiAijeSoqKqDVagEASUlJGDp0KBwcHFBdXY2qqioAwI0bN5CWloaAgAAAQEREBLb+fg/9\nrVu3YuzYsebaBcYYYw+KzOjAgQPk6+tLXl5eFB8fT0REGzZsoA0bNhAR0dGjR8nX15ekUimNHz+e\nNBoNERFdvnyZZDIZyWQy6tu3r2FZIqLy8nIaPnw4+fj4UGhoKKnV6ga3beZdE0RGRoalIzQJ5xSW\nOXM6OzsTAH7x674vZ2fnBn9/AOE+N/n2LxZ0+PBhQxuomHFOYZkzpzX83jPLu9fviZC/P626uBQW\nEq5cAQYPtnQaxloGFxfWFC1RXFr17V+uXgV++snSKRhjrO1p1cXlqaeAhQstneLe6p8OKGacU1jW\nkpOxh9GqiwtjjDHLaNV9Lq101xi7J/69bxlKpRK9e/dGTU0NbG1t8dxzz2HSpEmYOnVqo/M216pV\nq3D58mWjO5U8LO5zYYy1KLVajTFj/gy1Wm22bWzfvh0KhQKOjo7o2bMnnnvuORw5csRs22sJBw4c\naLCwNNfhw4eNbpsFAG+++aaghaWltOricvCgGmFh5v1DeRjW0vbOOYUl1pxqtRojRy7B3r2xGDly\niVn+bj788EPMnTsXf/vb3/Dbb7/h6tWrmD17Nvbu3Wsyr06nE3z7rAUJdsWMyAAgJ6dZBFwmhWIW\nqVQqS0cywRf9CYtzPvhFcCqVihSKWQSoCCACVIL/3Wg0GnJwcKCvvvqqwenLli2j8ePH00svvURO\nTk60adMmKioqotGjR5OLiwt5e3tTUlKSYf4TJ07Qk08+SU5OTuTq6krz5s0jIqKbN2/SlClTqEuX\nLtS5c2cKDg6m0tJSk+3t2LGDFAqF0bgPP/yQIiIiiIjom2++oaCgIHJyciIPDw+Ki4szzFdQUEA2\nNjak0+mIiGjo0KH06aefEhFRTU0NzZ8/nx5//HHq3bs3rVu3zmjezZs3k7+/Pzk6OlLv3r1p48aN\nRER0/fp16tChA9na2pKDgwM5OjpScXExLVu2jF566SXDtlNSUqhPnz7UuXNnGjZsGOXm5hqmPfHE\nE/T+++9Tv379qFOnTvTiiy/SrVu3TPb9Xr8nQpaEVl1czPmHwpgYPciHg2lhIbP83Rw8eJDs7OwM\nH7J3W7ZsGdnb21NKSgoR6YvEkCFDaPbs2XT79m3Kzs6mrl270nfffUdERE899RR98cUXRER048YN\nOnHiBBHp7wIyevRounnzJtXW1tLp06epsrLSZHvV1dXk6OhI+fn5hnEKhYJ27txJRESHDx+m//3v\nf0RE9N///pdcXV1pz549RGRaXIYNG0abNm0iIqJ//vOf5OfnR4WFhaRSqWjYsGFka2trmHf//v10\n+fJlIiL6/vvv6bHHHqPTp08btunu7m6UMy4uzlBc8vLyqGPHjpSenk41NTX097//nby9venOnTtE\nRCSRSGjgwIFUUlJCKpWK/P39DXdEqa8likurbhYDnA3/ZmWtNNtXfcbELC4OsLHRv+LiTKcPGrQE\nWVmx+OPvpY4zsrJi4eKypMHl4uIaXt+9lJeX4/HHH79vp/bgwYMN9yAsKyvD0aNHkZiYiPbt20Mm\nk+GVV17Btm3bAADt27dHfn4+rl27hscee8zwxNr27dujvLwc+fn5sLGxgVwuh6Ojo8m2Hn30UYwZ\nMwbJyckAgPz8fOTl5Rm2P3ToUPTt2xcAEBgYiKioKHz//feN7ueXX36JuXPnws3NDc7OznjrrbeM\nOsmfe+45eHp6AgD+3//7fxg5ciR+/PFHAGiwM73+uJ07d2LUqFEYPnw42rVrhwULFuDmzZs4evSo\nYZ6//vWv6N69O5ydnTF69GhkZ2c3mtkcWnlxqU//hxITs8TSQQzE2vZ+N84prJbOGRf3x3eRhorB\nsWMroVC8B+Du/3ipoVC8B5VqpSDFpUuXLrh27Rpqa2vvOY+7u7vh5+LiYri4uKBjx46Gcb169TI8\nF2rTpk24cOEC/P39MWDAAOzfvx8AMHXqVISFhSEqKgpubm5YtGgRampq8OOPP8LR0RGOjo4IDAwE\nAEyePNlQXLZv347IyEh06NABAHDixAmEhISgW7du6Ny5MzZu3Ijy8vJG97OkpMSoU75Xr15G0w8e\nPIinnnoKXbp0gbOzMw4cONCk9da9J/XXZ2NjAw8PD6NnZXXv3t3w86OPPorr1683ad1Ca0PFRf+H\nsmXLSksHYUxUnJ2dkZa2EgrFEvxRYNRQKJYgLW0lnJ3v/kbzYAYNGoRHHnkEu3fvbnD63c9n6tmz\nJ1QqldGH4y+//GIoQN7e3ti+fTvKysqwaNEiTJgwATdv3oSdnR3efvtt5OTk4OjRo/jmm2+wbds2\nDBkyBFVVVaiqqsK5c+cAACNGjEBZWRnOnj2LHTt2YPLkyYZtTZ48GWPHjkVhYSE0Gg3+7//+776F\nsU6PHj3wyy+/GGWuc/v2bYwfPx4LFy7Eb7/9BrVajeeee87w7aSx52C5ubnhypUrhmEiwtWrV+Hm\n5nbP99RSWnlxMd8fihCs4SaLAOcUmhhzGheYArP8vXTq1AkrVqzA7NmzkZKSgurqaty5cwcHDx5s\n8DlPHh4eGDx4MN58803cvn0b//3vf7F582a89NJLAIAvvvjC8Fj0Tp06wcbGBra2tsjIyMC5c+eg\n0+ng6OgIe3t7w0MI72Zvb48XXngBCxYsgFqtRmhoqGHa9evX4ezsjPbt2yMzMxPbt29v0of1xIkT\nsWbNGhQVFUGtViMhIcEwTavVQqvVGpoHDx48iLS0NMN0V1dXlJeXo7KyssF1v/DCC9i/fz++++47\n3LlzBx988AE6dOiAwfe4gWJDzWwtpVUXF3P+oTDW2tQVmIiI98z29zJv3jx8+OGHePfdd9GtWzf0\n6tUL69evR2RkJADT/2knJydDqVSiZ8+eGDduHFasWIE//elPAIBvv/0WAQEBcHR0xNy5c7Fjxw48\n8sgjKC0txQsvvIBOnTqhT58+GDZs2H2vQZk8eTIOHTqEF154wag/aP369Xj77bfh5OSEd955By++\n+KLRcvcqNK+++irCwsIgk8mgUCgwfvx4w7yOjo5Ys2YNJk6cCBcXFyQnJ2PMmDGGZf38/DBp0iT0\n7t0bLi4uKCkpMfpGJ5VK8cUXX+D1119H165dsX//fuzbtw92dg0/VLixp/WaU6u+Ql+lUiEmZgm2\nbBFnYeFbxAuLc/IV+qxpWuIK/YbLXSvh7OyMlJT1lo7BGGNtTqv+5tJKd42xe+Lfe9YUfG8xxhhj\nVomLiwXxdRnC4pyMiQcXF8YYY4LjPhfGWhH+vWdNwWeLMcaaxdnZ2aJXZTPr0BKXZpi1WSw1NRV+\nfn7w8fFBYmKiyXS1Wo3IyEjIZDIMHDgQOTk5RtN1Oh3kcjlGjx5tGBcXFwd3d3fI5XLI5XKkpqaa\ncxfMylra3jmnsMyZU6VSgfR3O3/oV0ZGhmDrMueLczb/pVKpzPY7WMdsxUWn0+Evf/kLUlNTcf78\neSQnJyM3N9donvj4ePTv3x9nz57Ftm3bMGfOHKPpq1evRp8+fYz+J2ZjY4N58+bhzJkzOHPmDJ59\n9llz7YLZWepupc3FOYXFOYXFOcXJbMUlMzMT3t7ekEgksLe3R1RUFFJSUozmyc3NRUhICAD9bQ2U\nSqXhXkGFhYU4cOAAXnnlFRAZtwHePWytNBqNpSM0CecUFucUFucUJ7MVl6KiIqPbTru7uxvdFhoA\nZDIZdu3aBUBfjK5cuYLCwkIAwNy5c/Hee+81+OyHtWvXQiaTYcaMGW3ugDHGmDUwW3FpSqfi4sWL\nodFoIJfLsW7dOsjlctja2uKbb75Bt27dIJfLTb6lzJo1CwUFBcjOzkaPHj0wf/58c+2C2SmVSktH\naBLOKSzOKSzOKVJkJseOHaOwsDDDcHx8PCUkJNx3GYlEQpWVlfTmm2+Su7s7SSQS6t69Oz322GM0\ndepUk/kLCgooICCgwXV5eXn9/qhjfvGLX/ziV1NeXl5eD/fBX4/ZrnOpqamBVCrFoUOH0LNnTwwY\nMADJycnw9/c3zFNRUYFHH30U7du3R1JSEo4cOYItW7YYref777/H+++/j3379gHQP+WtR48eAICP\nPvoIJ0+exPbt282xC4wxxh6Q2a5zsbOzw7p16xAWFgadTocZM2bA398fGzduBADMnDkT58+fR0xM\nDGxsbBAQEIBNmzY1uK76TWyLFi1CdnY2bGxs4OnpaVgfY4wx8Wi1V+gzxhiznFZ3b7HGLtxsSVev\nXkVISAj69u2LgIAArFmzBoD+QrfQ0FD4+vpi5MiRRme8rVq1Cj4+PvDz8zN6/Km53X3BqhgzajQa\nTJgwAf7+/ujTpw9OnDghypyrVq1C3759ERgYiMmTJ+P27duiyDl9+nS4uroiMDDQMO5Bcp06dQqB\ngYHw8fExuTbNXDljY2Ph7+8PmUyGcePGoaKiQpQ563zwwQewtbU1ulhRbDnXrl0Lf39/BAQEGD1m\nWrCcgvXeiEBNTQ15eXlRQUEBabVakslkdP78eYvlKSkpoTNnzhARUVVVFfn6+tL58+cpNjaWEhMT\niYgoISGBFi1aREREOTk5JJPJSKvVUkFBAXl5eZFOp2uRrB988AFNnjyZRo8eTUQkyowvv/wybdq0\niYiI7ty5QxqNRnQ5CwoKyNPTk27dukVERBMnTqQtW7aIIucPP/xAp0+fNjoJpjm5amtriYgoODiY\nTpw4QURE4eHhdPDgQbPnTEtLM7wvixYtEm1OIqJffvmFwsLCSCKRUHl5uShzfvfddzRixAjSarVE\nRPTbb78JnrNVFZejR48anaG2atUqWrVqlQUTGRszZgz95z//IalUSr/++isR6QuQVColItMz6sLC\nwujYsWNmz3X16lUaPnw4fffddzRq1CgiItFl1Gg05OnpaTJebDnLy8vJ19eXVCoV3blzh0aNGkVp\naWmiyXn3GZbNzVVcXEx+fn6G8cnJyTRz5kyz56xv165dNGXKFNHmnDBhAp09e9aouIgt5wsvvECH\nDh0ymU/InK2qWawpF25ailKpxJkzZzBw4ECUlpbC1dUVAODq6orS0lIAQHFxMdzd3Q3LtFT+hi5Y\nFVvGgoICdO3aFdOmTUP//v3x6quv4saNG6LL6eLigvnz56NXr17o2bMnOnfujNDQUNHlrNPcXHeP\nd3Nza/G/sc2bN+O5554TZc6UlBS4u7ujX79+RuPFljM/Px8//PADnnrqKQwbNgxZWVmC52xVxUWs\nd4O9fv06xo8fj9WrV8PR0dFomo2NzX1zm3uf7nfBav0MlswI6E9tP336NP785z/j9OnT6NixIxIS\nEkxyWDrnpUuX8PHHH0OpVKK4uBjXr1/HF198YZLD0jnvtV2x/g3VWblyJdq3b4/JkydbOoqJ6upq\nxMfHY/ny5YZx9/qbsrSamhqo1WocP34c7733HiZOnCj4NlpVcXFzc8PVq1cNw1evXjWqtpZw584d\njB8/HlOnTsXYsWMB6P+H+OuvvwLQX7fTrVs3AKb5CwsL4ebmZtZ8R48exd69e+Hp6YlJkybhu+++\nw9SpU0WVEdD/D8rd3R3BwcEAgAkTJuD06dPo3r27qHJmZWVh8ODB6NKlC+zs7DBu3DgcO3ZMdDnr\nNOc4u7u7w83NzXCLppbOu2XLFhw4cAD/+te/DOPElPPSpUtQKpWQyWTw9PREYWEhnnzySZSWlooq\nJ6D/exo3bhwAIDg4GLa2trh27ZqwOR+6MU9E7ty5Q71796aCggK6ffu2xTv0a2traerUqfTGG28Y\njY+NjTW0a65atcqkc/L27dt0+fJl6t27t6EzrSUcPnzY0OcixoxDhgyhvLw8IiJatmwZxcbGii5n\ndnY29e3bl6qrq6m2tpZefvllWrdunWhy3t32/iC5BgwYQMePH6fa2lqzdEA3lPPgwYPUp08fKisr\nM5pPbDnra6hDXyw5N2zYQG+//TYREeXl5ZGHh4fgOVtVcSEiOnDgAPn6+pKXlxfFx8dbNMuPP/5I\nNjY2JJPJKCgoiIKCgujgwYNUXl5Ow4cPJx8fHwoNDSW1Wm1YZuXKleTl5UVSqZRSU1NbNO/hw4cN\nZ4uJMWN2djYpFArq168fRUZGkkajEWXOxMRE6tOnDwUEBNDLL79MWq1WFDmjoqKoR48eZG9vT+7u\n7rR58+YHypWVlUUBAQHk5eVFr7/+utlzbtq0iby9valXr16Gv6NZs2aJJmf79u0N72d9np6ehuIi\ntpxarZZeeuklCggIoP79+1NGRobgOfkiSsYYY4JrVX0ujDHGxIGLC2OMMcFxcWGMMSY4Li6MMcYE\nx8WFMcaY4Li4MMYYExwXFyZqtra2WLBggWH4/fffN7q9xsOIiYnB119/Lci67uff//43+vTpg+HD\nhz/Uej7++GPcvHnTMPz888+jsrLyYePh7NmzOHjw4EOvh7H6uLgwUWvfvj12796N8vJyAMLed+th\n1lVTU9PkeTdt2oRPP/0Uhw4deuDtAcDq1atRXV1tGN6/fz+cnJweap0AcObMGRw4cKBZyzRn/1nb\nxMWFiZq9vT1ee+01fPTRRybT7v7m4eDgAAA4fPgwhg4dirFjx8LLywuLFy/G559/jgEDBqBfv364\nfPmyYZn09HQEBwdDKpVi//79APQPTouNjcWAAQMgk8nwySefGNY7ZMgQjBkzBn379jXJk5ycjH79\n+iEwMBCLFy8GAKxYsQJHjhzB9OnTsXDhQpNl3nvvPcN24uLiAAA3btzA888/j6CgIAQGBuLLL7/E\n2rVrUVxcjJCQEMM3IIlEApVKBaVSCT8/P0ybNg1SqRRTpkxBWloann76afj6+uLkyZMAgMzMTAwe\nPBj9+/fH008/jQsXLkCr1eLtt9/Gzp07IZfL8e9//xsqlQpjx46FTCbDoEGDcO7cOQBAXFwcpk6d\nimeeeQbR0dHIycnBgAEDIJfLIZPJcPHixeYdXNa6CX6vAcYE5ODgQJWVlSSRSKiiooLef/99iouL\nIyKimJgY+uqrr4zmJSLKyMigzp0706+//kq3b9+mnj170rJly4iIaPXq1YZ7vUVHR1N4eDgREeXn\n55O7uzvdunWLNm7cSO+++y4REd26dYsUCgUVFBRQRkYGdezYkZRKpUnOoqIi6tWrF127do1qamro\nT3/6E+3Zs4eIiIYNG0anTp0yWebbb7+l1157jYiIdDodjRo1in744Qf6+uuv6dVXXzXMV1lZSUTG\n96qqP1xQUEB2dnb0v//9j2pra+nJJ5+k6dOnExFRSkoKjR071rCempoaIiL6z3/+Q+PHjycioi1b\nthjdzuMvf/kLrVixgoj0D5UKCgoiIv393BQKheFBaK+//jr961//IiL9ff1u3rzZ0CFkbZSdpYsb\nY41xdHTEyy+/jDVr1uDRRx9t0jLBwcGG55R4e3sjLCwMABAQEICMjAwA+maxuluNe3t7o3fv3vj5\n55+RlpaGc+fO4auvvgIAVFZW4uLFi7Czs8OAAQPwxBNPmGzv5MmTCAkJQZcuXQAAU6ZMwQ8//IAx\nY8YAaPjW62lpaUhLS4NcLgeg/8Zy8eJFPPPMM5g/fz4WL16MUaNG4Zlnnml0fz09PQ3fpvr27YsR\nI0YY9lepVALQPyb65ZdfxsWLF2FjY2No2iL9PQYN6zpy5Ah27doFAAgJCUF5eTmqqqpgY2ODiIgI\nPPLIIwCAQYMGYeXKlSgsLMS4cePg7e3daE7WdnCzGLMKb7zxBjZt2oQbN24YxtnZ2aG2thYAUFtb\nC61Wa5hW9wEI6E8KqBu2tbW9b39BXT/MunXrcObMGZw5cwaXLl0yfFh37NjxnsvV/4AmIqM+nXv1\n77z55puG7Vy4cAHTpk2Dj48Pzpw5g8DAQPztb3/DO++8c8+899rf9u3bm+zv0qVLMXz4cJw7dw77\n9u0zOjngbg0VQwB47LHHDD9PmjQJ+/btw6OPPornnnvOULQZA7i4MCvh7OyMiRMnYtOmTYYPaolE\nglOnTgEA9u7dizt37jRrnUSEf//73yAiXLp0CZcvX4afnx/CwsKwfv16w4fyhQsXjDrSGxIcHIzv\nv/8e5eXl0Ol02LFjB4YOHXrfZcLCwrB582ZDwSwqKkJZWRlKSkrQoUMHTJkyBQsWLMCZM2cA6L/B\nPczZYZWVlejZsycA4LPPPjOMd3JyQlVVlWF4yJAhhmemHD58GF27doWjo6NJwSkoKICnpydef/11\njBkzxtA3wxgAcLMYE7X6/+OfP38+1q1bZxh+9dVXMWbMGAQFBeHZZ581dOjfvdzd66ubZmNjg169\nemHAgAGorKzExo0b0b59e7zyyitQKpXo378/iAjdunXD7t277/ukxh49eiAhIQEhISEgIowaNQqj\nR4++776FhoYiNzcXgwYNAqAvHp9//jkuXryI2NhY2Nrawt7eHhs2bAAAvPbaa3j22Wfh5uZmcubZ\n3bka+ta0cOFCREdH491338Xzzz9vGB8SEoKEhATI5XK89dZbiIuLw/Tp0yGTydCxY0ds3brV5L0D\ngC+//BKff/457O3t0aNHDyxZsuS++8vaFr7lPmOMMcFxsxhjjDHBcXFhjDEmOC4ujDHGBMfFhTHG\nmOC4uDDGGBMcFxfGGGOC4+LCGGNMcFxcGGOMCe7/A45qPvwzy2uEAAAAAElFTkSuQmCC\n",
       "text": [
        "<matplotlib.figure.Figure at 0x207230b8>"
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Working with a random forest regressor"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X, y = boston.data, boston.target"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 64
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.ensemble import RandomForestRegressor\n",
      "RF_rg = RandomForestRegressor (n_estimators=300, random_state=1)\n",
      "crossvalidation = KFold(n=X.shape[0], n_folds=5, shuffle=True, random_state=1)\n",
      "score = np.mean(cross_val_score(RF_rg, X, y, scoring='mean_squared_error', cv=crossvalidation, n_jobs=1))\n",
      "print 'Mean squared error: %.3f' % abs(score)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Mean squared error: 11.920\n"
       ]
      }
     ],
     "prompt_number": 65
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Optimizing a random forest"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.ensemble import RandomForestClassifier\n",
      "X, y = digit.data, digit.target\n",
      "crossvalidation = KFold(n=X.shape[0], n_folds=5, shuffle=True, random_state=1)\n",
      "RF_cls = RandomForestClassifier(n_estimators=300)\n",
      "scorer = 'accuracy'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 54
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "RF_cls = RandomForestClassifier(n_estimators=300).fit(X,y)\n",
      "X = RF_cls.transform(X)\n",
      "print X.shape\n",
      "print RF_cls.feature_importances_"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(1797L, 31L)\n",
        "[  0.00000000e+00   1.82344118e-03   2.04688900e-02   9.95683552e-03\n",
        "   9.44594976e-03   1.93138834e-02   8.35963956e-03   5.90175680e-04\n",
        "   4.98793019e-05   1.02607026e-02   2.76353664e-02   6.52640963e-03\n",
        "   1.55687575e-02   2.72906222e-02   5.53425609e-03   5.89013006e-04\n",
        "   9.29651339e-05   7.68002597e-03   2.08135918e-02   2.61134283e-02\n",
        "   2.96625199e-02   5.09749049e-02   9.16296341e-03   3.63989944e-04\n",
        "   5.50785439e-05   1.28918236e-02   4.44117979e-02   2.44743870e-02\n",
        "   3.22417158e-02   2.22773452e-02   3.01759074e-02   5.54781617e-05\n",
        "   0.00000000e+00   2.96271669e-02   2.76171029e-02   1.69894228e-02\n",
        "   4.06851465e-02   1.98866575e-02   2.56347728e-02   0.00000000e+00\n",
        "   4.35552108e-05   9.44618146e-03   3.80107080e-02   4.42118665e-02\n",
        "   2.11806555e-02   1.81103954e-02   1.89029826e-02   1.03823283e-04\n",
        "   5.81485235e-05   2.22007956e-03   1.55943037e-02   2.19845064e-02\n",
        "   1.37198844e-02   2.37288974e-02   2.60834791e-02   1.68087865e-03\n",
        "   1.06655788e-05   1.69595476e-03   2.23411600e-02   1.06587987e-02\n",
        "   2.42659800e-02   2.96634295e-02   1.74635106e-02   3.51814114e-03]\n"
       ]
      }
     ],
     "prompt_number": 64
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn import grid_search\n",
      "search_grid =  {'max_features': [Xr.shape[1]/3, 'sqrt', 'log2', 'auto'], 'min_samples_leaf': [1, 2, 10, 30]}\n",
      "search_func = grid_search.GridSearchCV(estimator=RF_cls, param_grid=search_grid, scoring=scorer, n_jobs=1, cv=crossvalidation)\n",
      "search_func.fit(Xr,y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 62,
       "text": [
        "GridSearchCV(cv=sklearn.cross_validation.KFold(n=1797, n_folds=5, shuffle=True, random_state=1),\n",
        "       estimator=RandomForestClassifier(bootstrap=True, compute_importances=None,\n",
        "            criterion='gini', max_depth=None, max_features='auto',\n",
        "            max_leaf_nodes=None, min_density=None, min_samples_leaf=1,\n",
        "            min_samples_split=2, n_estimators=300, n_jobs=1,\n",
        "            oob_score=False, random_state=None, verbose=0),\n",
        "       fit_params={}, iid=True, loss_func=None, n_jobs=1,\n",
        "       param_grid={'max_features': [11L, 'sqrt', 'log2', 'auto'], 'min_samples_leaf': [1, 2, 10, 30]},\n",
        "       pre_dispatch='2*n_jobs', refit=True, score_func=None,\n",
        "       scoring='accuracy', verbose=0)"
       ]
      }
     ],
     "prompt_number": 62
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print 'Best parameters: %s' % search_func.best_params_\n",
      "print 'Best accuracy: %.3f' % search_func.best_score_"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Best parameters: {'max_features': 'log2', 'min_samples_leaf': 1}\n",
        "Best accuracy: 0.977\n"
       ]
      }
     ],
     "prompt_number": 63
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Boosting predictions"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X, y = digit.data, digit.target"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 49
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Knowing that many weak predictors win "
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.ensemble import AdaBoostClassifier\n",
      "ada = AdaBoostClassifier(n_estimators=1000, learning_rate=0.01, random_state=1)\n",
      "crossvalidation = KFold(n=X.shape[0], n_folds=5, shuffle=True, random_state=1)\n",
      "score = np.mean(cross_val_score(ada, X, y, scoring='accuracy', cv=crossvalidation, n_jobs=1))\n",
      "print 'Accuracy: %.3f' % score"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Accuracy: 0.826\n"
       ]
      }
     ],
     "prompt_number": 81
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Creating a gradient boosting classifier"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X, y = digit.data, digit.target\n",
      "crossvalidation = KFold(n=X.shape[0], n_folds=5, shuffle=True, random_state=1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(1797L, 64L) (1797L,)\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.ensemble import GradientBoostingClassifier\n",
      "GBC = GradientBoostingClassifier(n_estimators=300, subsample=1.0, max_depth=2, learning_rate=0.1, random_state=1)\n",
      "score = np.mean(cross_val_score(GBC, X, y, scoring='accuracy', cv=crossvalidation, n_jobs=1))\n",
      "print 'Accuracy: %.3f' % score"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Accuracy: 0.972\n"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Creating a gradient boosting regressor"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X, y = boston.data, boston.target"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 87
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.ensemble import GradientBoostingRegressor\n",
      "GBR = GradientBoostingRegressor(n_estimators=1000, subsample=1.0, max_depth=3, learning_rate=0.01, random_state=1)\n",
      "crossvalidation = KFold(n=X.shape[0], n_folds=5, shuffle=True, random_state=1)\n",
      "score = np.mean(cross_val_score(GBR, X, y, scoring='mean_squared_error', cv=crossvalidation, n_jobs=1))\n",
      "print 'Mean squared error: %.3f' % abs(score)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Mean squared error: 10.105\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Using GBM hyper-parameters"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X, y = boston.data, boston.target\n",
      "crossvalidation = KFold(n=X.shape[0], n_folds=5, shuffle=True, random_state=1)\n",
      "GBR = GradientBoostingRegressor(n_estimators=1000, subsample=1.0, max_depth=3, learning_rate=0.01, random_state=1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn import grid_search\n",
      "search_grid =  {'subsample': [1.0, 0.9], 'max_depth': [2, 3, 5], 'n_estimators': [500 , 1000, 2000]}\n",
      "search_func = grid_search.GridSearchCV(estimator=GBR, param_grid=search_grid, scoring='mean_squared_error', n_jobs=1, cv=crossvalidation)\n",
      "search_func.fit(X,y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "GridSearchCV(cv=sklearn.cross_validation.KFold(n=506, n_folds=5, shuffle=True, random_state=1),\n",
        "       estimator=GradientBoostingRegressor(alpha=0.9, init=None, learning_rate=0.01, loss='ls',\n",
        "             max_depth=3, max_features=None, max_leaf_nodes=None,\n",
        "             min_samples_leaf=1, min_samples_split=2, n_estimators=1000,\n",
        "             random_state=1, subsample=1.0, verbose=0, warm_start=False),\n",
        "       fit_params={}, iid=True, loss_func=None, n_jobs=1,\n",
        "       param_grid={'n_estimators': [500, 1000, 2000], 'subsample': [1.0, 0.9], 'max_depth': [2, 3, 5]},\n",
        "       pre_dispatch='2*n_jobs', refit=True, score_func=None,\n",
        "       scoring='mean_squared_error', verbose=0)"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print 'Best parameters: %s' % search_func.best_params_\n",
      "print 'Best mean squared error: %.3f' % abs(search_func.best_score_)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Best parameters: {'n_estimators': 2000, 'subsample': 0.9, 'max_depth': 3}\n",
        "Best mean squared error: 9.263\n"
       ]
      }
     ],
     "prompt_number": 10
    }
   ],
   "metadata": {}
  }
 ]
}