test_jmlr.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Simon/Dev/anaconda/lib/python3.6/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import smtplib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from email.mime.application import MIMEApplication\n",
    "from email.mime.multipart import MIMEMultipart\n",
    "from email.mime.text import MIMEText\n",
    "from email.utils import COMMASPACE\n",
    "from sys import stdout, argv\n",
    "from prettytable import PrettyTable\n",
    "from time import time\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from tick.inference import LogisticRegression\n",
    "from tick.preprocessing import FeaturesBinarizer\n",
    "from sklearn.utils.validation import indexable\n",
    "from sklearn.model_selection import check_cv\n",
    "from sklearn.metrics.scorer import check_scoring\n",
    "from sklearn.model_selection._validation import _fit_and_score\n",
    "from sklearn.externals.joblib import Parallel, delayed\n",
    "import seaborn.apionly as sns\n",
    "import pylab as pl\n",
    "from prettytable import PrettyTable\n",
    "from sklearn.metrics import roc_auc_score, roc_curve, precision_recall_curve\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"default_cb\" #\"ionosphere\" default_cb adult\n",
    "header = \"infer\"  #\"infer\" None\n",
    "\n",
    "os.chdir('./datasets/%s' % filename)\n",
    "df = pd.read_csv('./%s' % filename, header=header)\n",
    "\n",
    "K = 5\n",
    "selection = \"min\"\n",
    "test = False\n",
    "\n",
    "# default\n",
    "n_cuts_min = 10\n",
    "n_cuts_max = 80\n",
    "n_cuts_grid_size = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>LIMIT_BAL</th>\n",
       "      <th>SEX</th>\n",
       "      <th>EDUCATION</th>\n",
       "      <th>MARRIAGE</th>\n",
       "      <th>AGE</th>\n",
       "      <th>PAY_0</th>\n",
       "      <th>PAY_2</th>\n",
       "      <th>PAY_3</th>\n",
       "      <th>PAY_4</th>\n",
       "      <th>...</th>\n",
       "      <th>BILL_AMT4</th>\n",
       "      <th>BILL_AMT5</th>\n",
       "      <th>BILL_AMT6</th>\n",
       "      <th>PAY_AMT1</th>\n",
       "      <th>PAY_AMT2</th>\n",
       "      <th>PAY_AMT3</th>\n",
       "      <th>PAY_AMT4</th>\n",
       "      <th>PAY_AMT5</th>\n",
       "      <th>PAY_AMT6</th>\n",
       "      <th>default payment next month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>20000</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>689</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>120000</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>26</td>\n",
       "      <td>-1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>3272</td>\n",
       "      <td>3455</td>\n",
       "      <td>3261</td>\n",
       "      <td>0</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>90000</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>14331</td>\n",
       "      <td>14948</td>\n",
       "      <td>15549</td>\n",
       "      <td>1518</td>\n",
       "      <td>1500</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>5000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>50000</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>28314</td>\n",
       "      <td>28959</td>\n",
       "      <td>29547</td>\n",
       "      <td>2000</td>\n",
       "      <td>2019</td>\n",
       "      <td>1200</td>\n",
       "      <td>1100</td>\n",
       "      <td>1069</td>\n",
       "      <td>1000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>50000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>57</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>20940</td>\n",
       "      <td>19146</td>\n",
       "      <td>19131</td>\n",
       "      <td>2000</td>\n",
       "      <td>36681</td>\n",
       "      <td>10000</td>\n",
       "      <td>9000</td>\n",
       "      <td>689</td>\n",
       "      <td>679</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  LIMIT_BAL  SEX  EDUCATION  MARRIAGE  AGE  PAY_0  PAY_2  PAY_3  PAY_4  \\\n",
       "0   1      20000    2          2         1   24      2      2     -1     -1   \n",
       "1   2     120000    2          2         2   26     -1      2      0      0   \n",
       "2   3      90000    2          2         2   34      0      0      0      0   \n",
       "3   4      50000    2          2         1   37      0      0      0      0   \n",
       "4   5      50000    1          2         1   57     -1      0     -1      0   \n",
       "\n",
       "              ...              BILL_AMT4  BILL_AMT5  BILL_AMT6  PAY_AMT1  \\\n",
       "0             ...                      0          0          0         0   \n",
       "1             ...                   3272       3455       3261         0   \n",
       "2             ...                  14331      14948      15549      1518   \n",
       "3             ...                  28314      28959      29547      2000   \n",
       "4             ...                  20940      19146      19131      2000   \n",
       "\n",
       "   PAY_AMT2  PAY_AMT3  PAY_AMT4  PAY_AMT5  PAY_AMT6  \\\n",
       "0       689         0         0         0         0   \n",
       "1      1000      1000      1000         0      2000   \n",
       "2      1500      1000      1000      1000      5000   \n",
       "3      2019      1200      1100      1069      1000   \n",
       "4     36681     10000      9000       689       679   \n",
       "\n",
       "   default payment next month  \n",
       "0                           1  \n",
       "1                           1  \n",
       "2                           0  \n",
       "3                           0  \n",
       "4                           0  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cross_val_score_(estimators, X, y=None, groups=None, scoring=None,\n",
    "                     cv=None, n_jobs=1, verbose=0, fit_params=None):\n",
    "    X, y, groups = indexable(X, y, groups)\n",
    "    cv = check_cv(cv, y, classifier=True)\n",
    "    cv_iter = list(cv.split(X, y, groups))\n",
    "\n",
    "    parallel = Parallel(n_jobs=n_jobs, verbose=0)\n",
    "\n",
    "    scores = parallel(delayed(_fit_and_score)(estimators[i], X, y,\n",
    "                                              check_scoring(estimators[i],\n",
    "                                                            scoring=scoring),\n",
    "                                              train, test, verbose, None,\n",
    "                                              fit_params)\n",
    "                      for i, (train, test) in enumerate(cv_iter))\n",
    "\n",
    "    return np.array(scores)[:, 0]\n",
    "\n",
    "\n",
    "def compute_score(clf, X, y, K, verbose=True, fit_params=None):\n",
    "    scores = cross_val_score_(clf, X, y, cv=K, verbose=0,\n",
    "                              n_jobs=1, scoring=\"roc_auc\",\n",
    "                              fit_params=fit_params)\n",
    "    score_mean = scores.mean()\n",
    "    score_std = 2 * scores.std()\n",
    "    if verbose:\n",
    "        print(\"\\n AUC: %0.3f (+/- %0.3f)\" % (score_mean, score_std))\n",
    "    return score_mean, score_std\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training:\n",
      "(20100, 24)\n",
      "Test:\n",
      "(9900, 24)\n",
      "data centered and reduced\n"
     ]
    }
   ],
   "source": [
    "with_categorical = False\n",
    "\n",
    "# drop lines with NaN values\n",
    "df.dropna(axis=0, how='any', inplace=True)\n",
    "\n",
    "# if dataset churn: drop phone feature\n",
    "if filename == 'churn':\n",
    "    df = df.drop(df.columns[[3]], axis=1)\n",
    "\n",
    "# get label (have to be the last column!)\n",
    "idx_label_column = -1\n",
    "labels = df.iloc[:, idx_label_column]\n",
    "labels = 2 * (labels.values != labels.values[0]) - 1\n",
    "# drop it from df\n",
    "df = df.drop(df.columns[[idx_label_column]], axis=1)\n",
    "\n",
    "# shuffle and split training and test sets\n",
    "X, X_test, y, y_test = train_test_split(\n",
    "    df, labels, test_size=.33, random_state=0, stratify=labels)\n",
    "\n",
    "del df\n",
    "\n",
    "# speed up restriction\n",
    "# n_restrict = 1000000  # 200k examples max\n",
    "if test:\n",
    "    n_restrict = 200\n",
    "    C_grid_size = 4\n",
    "    n_cuts_grid_size = 3\n",
    "    X = X.iloc[:n_restrict, :]\n",
    "    y = y[:n_restrict]\n",
    "    X_test = X_test.iloc[:n_restrict, :]\n",
    "    y_test = y_test[:n_restrict]\n",
    "else:\n",
    "    C_grid_size = 25\n",
    "\n",
    "# get categorical features index\n",
    "cate_feat_idx = []\n",
    "for i in range(X.shape[1]):\n",
    "    feature_type = FeaturesBinarizer._detect_feature_type(X.ix[:, i])\n",
    "    if feature_type == 'discrete':\n",
    "        cate_feat_idx.append(i)\n",
    "\n",
    "if (len(cate_feat_idx) == 0):\n",
    "    with_categorical = False\n",
    "\n",
    "original_feature_names = X.columns\n",
    "\n",
    "if not with_categorical:\n",
    "    feature_names_cont = list()\n",
    "    for i, name in enumerate(original_feature_names):\n",
    "        if i not in cate_feat_idx:\n",
    "            feature_names_cont.append(name)\n",
    "else:\n",
    "    feature_names_cont = original_feature_names\n",
    "\n",
    "n_cuts_grid = np.linspace(n_cuts_min, n_cuts_max, n_cuts_grid_size, dtype=int)\n",
    "\n",
    "# separate continuous and categorical features\n",
    "X_cat = X[X.columns[cate_feat_idx]]\n",
    "X_test_cat = X_test[X_test.columns[cate_feat_idx]]\n",
    "X_cat.reset_index(drop=True, inplace=True)\n",
    "X_test_cat.reset_index(drop=True, inplace=True)\n",
    "\n",
    "if with_categorical:\n",
    "    binarizer = FeaturesBinarizer()\n",
    "    binarizer.fit(pd.concat([X_cat, X_test_cat], axis=0))\n",
    "    X_cat_bin = pd.DataFrame(binarizer.transform(X_cat).toarray())\n",
    "    X_test_cat_bin = pd.DataFrame(binarizer.transform(X_test_cat).toarray())\n",
    "\n",
    "#del X_cat, X_test_cat\n",
    "\n",
    "X_cont = X.drop(X.columns[cate_feat_idx], axis=1)\n",
    "X_test_cont = X_test.drop(X_test.columns[cate_feat_idx], axis=1)\n",
    "X_cont.reset_index(drop=True, inplace=True)\n",
    "X_test_cont.reset_index(drop=True, inplace=True)\n",
    "\n",
    "print(\"Training:\")\n",
    "print(X.shape)\n",
    "print(\"Test:\")\n",
    "print(X_test.shape)\n",
    "\n",
    "# Center and reduce continuous data\n",
    "standardscaler = StandardScaler()\n",
    "X_std = pd.DataFrame(standardscaler.fit_transform(X_cont))\n",
    "X_test_std = pd.DataFrame(standardscaler.transform(X_test_cont))\n",
    "print(\"data centered and reduced\")\n",
    "\n",
    "# use only 10k examples max for Cross-Val\n",
    "n_restrict_cv = 8000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " launch bina_pen_bin_feat\n"
     ]
    }
   ],
   "source": [
    "# logistic regression on binarized features, binarsity penalization\n",
    "model = \"bina_pen_bin_feat\"\n",
    "print(\"\\n launch %s\" % model)\n",
    "\n",
    "if with_categorical:\n",
    "    X_final = pd.concat([X_cont, X_cat], axis=1)\n",
    "    X_test_final = pd.concat([X_test_cont, X_test_cat], axis=1)\n",
    "else:\n",
    "    X_final = X_cont\n",
    "    X_test_final = X_test_cont\n",
    "\n",
    "# prendre une gde valeur de n_cut puis cross valider sur C\n",
    "n_cuts_chosen = 30\n",
    "\n",
    "binarizer = FeaturesBinarizer(n_cuts=n_cuts_chosen)\n",
    "binarizer.fit(pd.concat([X_final, X_test_final], axis=0))\n",
    "\n",
    "if with_categorical:\n",
    "    X_final = pd.concat([X_cont.iloc[:n_restrict_cv, :],\n",
    "                         X_cat.iloc[:n_restrict_cv, :]],\n",
    "                        axis=1)\n",
    "    X_test_final = pd.concat([X_test_cont.iloc[:n_restrict_cv, :],\n",
    "                              X_test_cat.iloc[:n_restrict_cv, :]],\n",
    "                             axis=1)\n",
    "\n",
    "else:\n",
    "    X_final = X_cont.iloc[:n_restrict_cv, :]\n",
    "    X_test_final = X_test_cont.iloc[:n_restrict_cv, :]\n",
    "\n",
    "X_bin = binarizer.transform(X_final)\n",
    "X_test_bin = binarizer.transform(X_test_final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# logistic regression on binarized features, binarsity penalization\n",
    "if with_categorical:\n",
    "    X_final = pd.concat([X_cont, X_cat], axis=1)\n",
    "    X_test_final = pd.concat([X_test_cont, X_test_cat], axis=1)\n",
    "else:\n",
    "    X_final = X_cont\n",
    "    X_test_final = X_test_cont\n",
    "\n",
    "# prendre une gde valeur de n_cut puis cross valider sur C\n",
    "n_cuts_chosen = 30\n",
    "\n",
    "binarizer = FeaturesBinarizer(n_cuts=n_cuts_chosen)\n",
    "binarizer.fit(pd.concat([X_final, X_test_final], axis=0))\n",
    "\n",
    "if with_categorical:\n",
    "    X_final = pd.concat([X_cont.iloc[:n_restrict_cv, :],\n",
    "                         X_cat.iloc[:n_restrict_cv, :]],\n",
    "                        axis=1)\n",
    "    X_test_final = pd.concat([X_test_cont.iloc[:n_restrict_cv, :],\n",
    "                              X_test_cat.iloc[:n_restrict_cv, :]],\n",
    "                             axis=1)\n",
    "\n",
    "else:\n",
    "    X_final = X_cont.iloc[:n_restrict_cv, :]\n",
    "    X_test_final = X_test_cont.iloc[:n_restrict_cv, :]\n",
    "\n",
    "X_bin = binarizer.transform(X_final)\n",
    "X_test_bin = binarizer.transform(X_test_final)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bina or Group-TV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "C_grid = np.logspace(1, 4, C_grid_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " launch bina_pen_bin_feat\n",
      "CV bina_pen_bin_feat: 4%\n",
      "CV bina_pen_bin_feat: 8%\n",
      "CV bina_pen_bin_feat: 12%\n",
      "CV bina_pen_bin_feat: 16%\n",
      "CV bina_pen_bin_feat: 20%\n",
      "CV bina_pen_bin_feat: 24%\n",
      "CV bina_pen_bin_feat: 28%\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-9d505dbc196e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m         for _ in range(K)]\n\u001b[1;32m     20\u001b[0m     auc = compute_score(learners, X_bin, y[:n_restrict_cv], K,\n\u001b[0;32m---> 21\u001b[0;31m                         verbose=False)[0]\n\u001b[0m\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m     \u001b[0mavg_scores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mavg_scores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mauc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mauc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-4-394a93c8edff>\u001b[0m in \u001b[0;36mcompute_score\u001b[0;34m(clf, X, y, K, verbose, fit_params)\u001b[0m\n\u001b[1;32m     20\u001b[0m     scores = cross_val_score_(clf, X, y, cv=K, verbose=0,\n\u001b[1;32m     21\u001b[0m                               \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscoring\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"roc_auc\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m                               fit_params=fit_params)\n\u001b[0m\u001b[1;32m     23\u001b[0m     \u001b[0mscore_mean\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscores\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0mscore_std\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mscores\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-4-394a93c8edff>\u001b[0m in \u001b[0;36mcross_val_score_\u001b[0;34m(estimators, X, y, groups, scoring, cv, n_jobs, verbose, fit_params)\u001b[0m\n\u001b[1;32m     12\u001b[0m                                               \u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m                                               fit_params)\n\u001b[0;32m---> 14\u001b[0;31m                       for i, (train, test) in enumerate(cv_iter))\n\u001b[0m\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Simon/Dev/anaconda/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m    756\u001b[0m             \u001b[0;31m# was dispatched. In particular this covers the edge\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    757\u001b[0m             \u001b[0;31m# case of Parallel used with an exhausted iterator.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 758\u001b[0;31m             \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    759\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    760\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Simon/Dev/anaconda/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[0;34m(self, iterator)\u001b[0m\n\u001b[1;32m    606\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    607\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 608\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    609\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    610\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Simon/Dev/anaconda/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    569\u001b[0m         \u001b[0mdispatch_timestamp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    570\u001b[0m         \u001b[0mcb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBatchCompletionCallBack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdispatch_timestamp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 571\u001b[0;31m         \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    572\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    573\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Simon/Dev/anaconda/lib/python3.6/site-packages/sklearn/externals/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[0;34m(self, func, callback)\u001b[0m\n\u001b[1;32m    107\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m         \u001b[0;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 109\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    110\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    111\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Simon/Dev/anaconda/lib/python3.6/site-packages/sklearn/externals/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    324\u001b[0m         \u001b[0;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    325\u001b[0m         \u001b[0;31m# arguments in memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 326\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    327\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Simon/Dev/anaconda/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 131\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Simon/Dev/anaconda/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 131\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Simon/Dev/anaconda/lib/python3.6/site-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36m_fit_and_score\u001b[0;34m(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, error_score)\u001b[0m\n\u001b[1;32m    236\u001b[0m             \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 238\u001b[0;31m             \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    239\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    240\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Simon/Dev/tick/tick/inference/logistic_regression.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    184\u001b[0m         \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_encode_labels_vector\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    185\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 186\u001b[0;31m         \u001b[0mLearnerGLM\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    187\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    188\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mdecision_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Simon/Dev/tick/tick/inference/base/learner_glm.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m         \u001b[0;31m# Launch the solver\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 209\u001b[0;31m         \u001b[0mcoeffs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msolver_obj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msolve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcoeffs_start\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    210\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    211\u001b[0m         \u001b[0;31m# Get the learned coefficients\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Simon/Dev/tick/tick/optim/solver/base/first_order.py\u001b[0m in \u001b[0;36msolve\u001b[0;34m(self, x0, step)\u001b[0m\n\u001b[1;32m    248\u001b[0m             raise ValueError('You must first set the prox using '\n\u001b[1;32m    249\u001b[0m                              '``set_prox``.')\n\u001b[0;32m--> 250\u001b[0;31m         \u001b[0msolution\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSolver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msolve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    251\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0msolution\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    252\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Simon/Dev/tick/tick/optim/solver/base/solver.py\u001b[0m in \u001b[0;36msolve\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    107\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msolve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start_solve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 109\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_solve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    110\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_end_solve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    111\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msolution\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Simon/Dev/tick/tick/optim/solver/base/first_order_sto.py\u001b[0m in \u001b[0;36m_solve\u001b[0;34m(self, x0, step)\u001b[0m\n\u001b[1;32m    175\u001b[0m             \u001b[0mprev_obj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m             \u001b[0;31m# Launch one epoch using the wrapped C++ solver\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 177\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_solver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msolve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    178\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_solver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_minimizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mminimizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m             \u001b[0;31m# The step might be modified by the C++ solver\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/Simon/Dev/tick/tick/optim/solver/build/solver.py\u001b[0m in \u001b[0;36msolve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    250\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msolve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;34m\"void\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    251\u001b[0m         \u001b[0;34m\"\"\"solve(SVRG self)\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 252\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_solver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSVRG_solve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    253\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    254\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = \"bina_pen_bin_feat\"\n",
    "print(\"\\n launch %s\" % model)\n",
    "\n",
    "\n",
    "# cross validation on C\n",
    "avg_scores, score_test = np.empty(0), []\n",
    "tmp = 0\n",
    "for i, C_ in enumerate(C_grid):\n",
    "    tmp += 1\n",
    "    print(\"CV %s: %d%%\" % (\n",
    "        model, tmp * 100 / C_grid_size))\n",
    "    stdout.flush()\n",
    "\n",
    "    learners = [\n",
    "        LogisticRegression(penalty='binarsity', solver='svrg', C=C_,\n",
    "                      verbose=False, step=1e-3,\n",
    "                      blocks_start=binarizer.feature_indices[:-1, ],\n",
    "                      blocks_length=binarizer.n_values)\n",
    "        for _ in range(K)]\n",
    "    auc = compute_score(learners, X_bin, y[:n_restrict_cv], K,\n",
    "                        verbose=False)[0]\n",
    "\n",
    "    avg_scores = np.append(avg_scores, max(auc, 1 - auc))\n",
    "\n",
    "    learner = LogisticRegression(penalty='binarsity', solver='svrg',\n",
    "                            C=C_, verbose=False, step=1e-3,\n",
    "                            blocks_start=binarizer.feature_indices[\n",
    "                                         :-1, ],\n",
    "                            blocks_length=binarizer.n_values)\n",
    "    learner.fit(X_bin, y[:n_restrict_cv])\n",
    "    y_pred = learner.predict_proba(X_test_bin)[:, 1]\n",
    "    score_test.append(roc_auc_score(y_test[:n_restrict_cv], y_pred))\n",
    "\n",
    "idx_best = np.unravel_index(avg_scores.argmax(),\n",
    "                            avg_scores.shape)[0]\n",
    "C_best = C_grid[idx_best]\n",
    "if selection == 'min':\n",
    "    C_chosen = C_best\n",
    "if selection == '1st':\n",
    "    max_ = avg_scores.max()\n",
    "    min_ = avg_scores.min()\n",
    "    idx = [i for i, is_up in enumerate(\n",
    "        list(avg_scores >= max_ - .05 * (max_ - min_)))\n",
    "           if is_up]\n",
    "    idx_chosen = min(idx) if len(idx) > 0 else idx_best\n",
    "    C_chosen = C_grid[idx_chosen]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learning curves\n",
    "learning_curves = np.column_stack((C_grid, avg_scores, score_test))\n",
    "\n",
    "fig = pl.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "C_grid = learning_curves[:, 0]\n",
    "C_grid_ = C_grid\n",
    "avg_scores = learning_curves[:, 1]\n",
    "score_test = learning_curves[:, 2]\n",
    "\n",
    "idx_best = np.unravel_index(avg_scores.argmax(), \n",
    "                            avg_scores.shape)[0]\n",
    "C_best = C_grid[idx_best]\n",
    "if selection == 'min':\n",
    "    C_chosen = C_best\n",
    "if selection == '1st':\n",
    "    max_ = avg_scores.max()\n",
    "    min_ = avg_scores.min()\n",
    "    idx = [i for i, is_up in enumerate(\n",
    "        list(avg_scores >= max_ - .05 * (max_ - min_)))\n",
    "           if is_up]\n",
    "    idx_chosen = min(idx) if len(idx) > 0 else idx_best\n",
    "    C_chosen = C_grid[idx_chosen]\n",
    "\n",
    "pl.xscale('log')\n",
    "ax.plot(C_grid, avg_scores, label=\"AUC on CV\")\n",
    "ax.plot(C_grid, score_test , '-r', \n",
    "        label=\"AUC on test set\")\n",
    "y_min = ax.get_ylim()[0]\n",
    "ax.plot(C_best,y_min,'g^',ms=20, label=\"best C on CV\")\n",
    "ax.plot(C_chosen,y_min,'r^',ms=20, label=\"C chosen\")\n",
    "pl.suptitle(\"Learning curves bina\", \n",
    "            fontsize=14, fontweight=\"bold\")\n",
    "pl.xlabel(\"C\")\n",
    "pl.ylabel(\"AUC\")\n",
    "pl.legend(bbox_to_anchor=(1.15,1), loc=2, borderaxespad=0.,\n",
    "          numpoints=1, markerscale=.5)\n",
    "pl.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if with_categorical:\n",
    "    X_final = pd.concat([X_cont, X_cat], axis=1)\n",
    "    X_test_final = pd.concat([X_test_cont, X_test_cat], axis=1)\n",
    "else:\n",
    "    X_final = X_cont\n",
    "    X_test_final = X_test_cont\n",
    "\n",
    "binarizer = FeaturesBinarizer(n_cuts=n_cuts_chosen)\n",
    "binarizer.fit(pd.concat([X_final, X_test_final], axis=0))\n",
    "X_bin = binarizer.transform(X_final)\n",
    "X_test_bin = binarizer.transform(X_test_final)\n",
    "\n",
    "blocks_start = binarizer.feature_indices[:-1, ]\n",
    "blocks_length = binarizer.n_values\n",
    "\n",
    "learner = LogisticRegression(penalty='binarsity', solver='svrg', C=C_chosen,\n",
    "                        verbose=False, step=1e-3,\n",
    "                        blocks_start=blocks_start,\n",
    "                        blocks_length=blocks_length)\n",
    "start = time()\n",
    "learner.fit(X_bin, y)\n",
    "y_pred = learner.predict_proba(X_test_bin)[:, 1]\n",
    "\n",
    "auc = roc_auc_score(y_test, y_pred)\n",
    "auc = max(auc, 1 - auc)\n",
    "\n",
    "print(\"\\n %s done, AUC: %.3f\" % (model, auc))\n",
    "\n",
    "coeffs = learner.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fig = pl.figure(figsize=(13,5))\n",
    "ax = fig.add_subplot(111)\n",
    "for val in blocks_start:\n",
    "    ax.axvline(val, color='g', linestyle='--') \n",
    "#pl.suptitle(\"Beta, %s\" % model.replace('_',' '), \n",
    "#            fontsize=14, fontweight=\"bold\")\n",
    "pl.xlabel(\"Coefs values\", fontsize=12)\n",
    "pl.ylabel(\"Beta coeffs\", fontsize=12)\n",
    "ax.stem(coeffs, 'b', markerfmt='ob')\n",
    "\n",
    "ax.set_xlim([-5, len(coeffs)+5])\n",
    "pl.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Group-L1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# logistic regression on binarized features, binarsity penalization\n",
    "model = \"bina_pen_bin_feat\"\n",
    "print(\"\\n launch %s\" % model)\n",
    "\n",
    "if with_categorical:\n",
    "    X_final = pd.concat([X_cont, X_cat], axis=1)\n",
    "    X_test_final = pd.concat([X_test_cont, X_test_cat], axis=1)\n",
    "else:\n",
    "    X_final = X_cont\n",
    "    X_test_final = X_test_cont\n",
    "\n",
    "# prendre une gde valeur de n_cut puis cross valider sur C\n",
    "n_cuts_chosen = 30\n",
    "\n",
    "binarizer = FeaturesBinarizer(n_cuts=n_cuts_chosen)\n",
    "binarizer.fit(pd.concat([X_final, X_test_final], axis=0))\n",
    "\n",
    "if with_categorical:\n",
    "    X_final = pd.concat([X_cont.iloc[:n_restrict_cv, :],\n",
    "                         X_cat.iloc[:n_restrict_cv, :]],\n",
    "                        axis=1)\n",
    "    X_test_final = pd.concat([X_test_cont.iloc[:n_restrict_cv, :],\n",
    "                              X_test_cat.iloc[:n_restrict_cv, :]],\n",
    "                             axis=1)\n",
    "\n",
    "else:\n",
    "    X_final = X_cont.iloc[:n_restrict_cv, :]\n",
    "    X_test_final = X_test_cont.iloc[:n_restrict_cv, :]\n",
    "\n",
    "X_bin = binarizer.transform(X_final)\n",
    "X_test_bin = binarizer.transform(X_test_final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cross validation on C\n",
    "\n",
    "C_grid = np.logspace(1, 4, C_grid_size)\n",
    "\n",
    "avg_scores, score_test = np.empty(0), []\n",
    "tmp = 0\n",
    "for i, C_ in enumerate(C_grid):\n",
    "    tmp += 1\n",
    "    print(\"CV %s: %d%%\" % (\n",
    "        model, tmp * 100 / C_grid_size))\n",
    "    stdout.flush()\n",
    "\n",
    "    learners = [\n",
    "        LogisticRegression(penalty='group-L1', solver='svrg', C=C_,\n",
    "                      verbose=False, step=1e-3,\n",
    "                      blocks_start=binarizer.feature_indices[:-1, ],\n",
    "                      blocks_length=binarizer.n_values)\n",
    "        for _ in range(K)]\n",
    "    auc = compute_score(learners, X_bin, y[:n_restrict_cv], K,\n",
    "                        verbose=False)[0]\n",
    "\n",
    "    avg_scores = np.append(avg_scores, max(auc, 1 - auc))\n",
    "\n",
    "    learner = LogisticRegression(penalty='group-L1', solver='svrg',\n",
    "                            C=C_, verbose=False, step=1e-3,\n",
    "                            blocks_start=binarizer.feature_indices[\n",
    "                                         :-1, ],\n",
    "                            blocks_length=binarizer.n_values)\n",
    "    learner.fit(X_bin, y[:n_restrict_cv])\n",
    "    y_pred = learner.predict_proba(X_test_bin)[:, 1]\n",
    "    score_test.append(roc_auc_score(y_test[:n_restrict_cv], y_pred))\n",
    "\n",
    "idx_best = np.unravel_index(avg_scores.argmax(),\n",
    "                            avg_scores.shape)[0]\n",
    "C_best = C_grid[idx_best]\n",
    "if selection == 'min':\n",
    "    C_chosen = C_best\n",
    "if selection == '1st':\n",
    "    max_ = avg_scores.max()\n",
    "    min_ = avg_scores.min()\n",
    "    idx = [i for i, is_up in enumerate(\n",
    "        list(avg_scores >= max_ - .05 * (max_ - min_)))\n",
    "           if is_up]\n",
    "    idx_chosen = min(idx) if len(idx) > 0 else idx_best\n",
    "    C_chosen = C_grid[idx_chosen]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn.apionly as sns\n",
    "import pylab as pl\n",
    "from prettytable import PrettyTable\n",
    "from sklearn.metrics import roc_auc_score, roc_curve, precision_recall_curve\n",
    "\n",
    "# learning curves\n",
    "learning_curves = np.column_stack((C_grid, avg_scores, score_test))\n",
    "\n",
    "fig = pl.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "C_grid = learning_curves[:, 0]\n",
    "C_grid_ = C_grid\n",
    "avg_scores = learning_curves[:, 1]\n",
    "score_test = learning_curves[:, 2]\n",
    "\n",
    "idx_best = np.unravel_index(avg_scores.argmax(), \n",
    "                            avg_scores.shape)[0]\n",
    "C_best = C_grid[idx_best]\n",
    "if selection == 'min':\n",
    "    C_chosen = C_best\n",
    "if selection == '1st':\n",
    "    max_ = avg_scores.max()\n",
    "    min_ = avg_scores.min()\n",
    "    idx = [i for i, is_up in enumerate(\n",
    "        list(avg_scores >= max_ - .05 * (max_ - min_)))\n",
    "           if is_up]\n",
    "    idx_chosen = min(idx) if len(idx) > 0 else idx_best\n",
    "    C_chosen = C_grid[idx_chosen]\n",
    "\n",
    "pl.xscale('log')\n",
    "ax.plot(C_grid, avg_scores, label=\"AUC on CV\")\n",
    "ax.plot(C_grid, score_test , '-r', \n",
    "        label=\"AUC on test set\")\n",
    "y_min = ax.get_ylim()[0]\n",
    "ax.plot(C_best,y_min,'g^',ms=20, label=\"best C on CV\")\n",
    "ax.plot(C_chosen,y_min,'r^',ms=20, label=\"C chosen\")\n",
    "pl.suptitle(\"Learning curves bina\", \n",
    "            fontsize=14, fontweight=\"bold\")\n",
    "pl.xlabel(\"C\")\n",
    "pl.ylabel(\"AUC\")\n",
    "pl.legend(bbox_to_anchor=(1.15,1), loc=2, borderaxespad=0.,\n",
    "          numpoints=1, markerscale=.5)\n",
    "pl.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if with_categorical:\n",
    "    X_final = pd.concat([X_cont, X_cat], axis=1)\n",
    "    X_test_final = pd.concat([X_test_cont, X_test_cat], axis=1)\n",
    "else:\n",
    "    X_final = X_cont\n",
    "    X_test_final = X_test_cont\n",
    "\n",
    "binarizer = FeaturesBinarizer(n_cuts=n_cuts_chosen)\n",
    "binarizer.fit(pd.concat([X_final, X_test_final], axis=0))\n",
    "X_bin = binarizer.transform(X_final)\n",
    "X_test_bin = binarizer.transform(X_test_final)\n",
    "\n",
    "blocks_start = binarizer.feature_indices[:-1, ]\n",
    "blocks_length = binarizer.n_values\n",
    "\n",
    "learner = LogisticRegression(penalty='group-L1', solver='svrg', C=C_chosen,\n",
    "                        verbose=False, step=1e-3,\n",
    "                        blocks_start=blocks_start,\n",
    "                        blocks_length=blocks_length)\n",
    "start = time()\n",
    "learner.fit(X_bin, y)\n",
    "y_pred = learner.predict_proba(X_test_bin)[:, 1]\n",
    "\n",
    "auc = roc_auc_score(y_test, y_pred)\n",
    "auc = max(auc, 1 - auc)\n",
    "\n",
    "print(\"\\n %s done, AUC: %.3f\" % (model, auc))\n",
    "\n",
    "coeffs = learner.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = pl.figure(figsize=(13,5))\n",
    "ax = fig.add_subplot(111)\n",
    "for val in blocks_start:\n",
    "    ax.axvline(val, color='g', linestyle='--') \n",
    "#pl.suptitle(\"Beta, %s\" % model.replace('_',' '), \n",
    "#            fontsize=14, fontweight=\"bold\")\n",
    "pl.xlabel(\"Coefs values\", fontsize=12)\n",
    "pl.ylabel(\"Beta coeffs\", fontsize=12)\n",
    "ax.stem(coeffs, 'b', markerfmt='ob')\n",
    "\n",
    "ax.set_xlim([-5, len(coeffs)+5])\n",
    "pl.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}