001_dude_horizontal.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import KFold, StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.stats import pearsonr, spearmanr, kendalltau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import oddt\n",
    "from oddt.scoring.functions import rfscore\n",
    "from oddt.scoring import ensemble_model, scorer\n",
    "from oddt.scoring.models.regressors import randomforest, svm, neuralnetwork\n",
    "from oddt.metrics import enrichment_factor, roc_auc, roc_log_auc\n",
    "from sklearn.metrics import accuracy_score, precision_score, mean_squared_error, recall_score\n",
    "from sklearn.utils import compute_sample_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# from cluster_helper import cluster\n",
    "# cluster.dill = dill"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from cluster_helper.cluster import cluster_view"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dude_ids = ['aa2ar', 'abl1', 'ace', 'aces', 'ada', 'ada17', 'adrb1', 'adrb2', 'akt1', 'akt2', 'aldr', 'ampc', 'andr', 'aofb', 'bace1', 'braf', 'cah2', 'casp3', 'cdk2', 'comt', 'cp2c9', 'cp3a4', 'csf1r', 'cxcr4', 'def', 'dhi1', 'dpp4', 'drd3', 'dyr', 'egfr', 'esr1', 'esr2', 'fa10', 'fa7', 'fabp4', 'fak1', 'fgfr1', 'fkb1a', 'fnta', 'fpps', 'gcr', 'glcm', 'gria2', 'grik1', 'hdac2', 'hdac8', 'hivint', 'hivpr', 'hivrt', 'hmdh', 'hs90a', 'hxk4', 'igf1r', 'inha', 'ital', 'jak2', 'kif11', 'kit', 'kith', 'kpcb', 'lck', 'lkha4', 'mapk2', 'mcr', 'met', 'mk01', 'mk10', 'mk14', 'mmp13', 'mp2k1', 'nos1', 'nram', 'pa2ga', 'parp1', 'pde5a', 'pgh1', 'pgh2', 'plk1', 'pnph', 'ppara', 'ppard', 'pparg', 'prgr', 'ptn1', 'pur2', 'pygm', 'pyrd', 'reni', 'rock1', 'rxra', 'sahh', 'src', 'tgfr1', 'thb', 'thrb', 'try1', 'tryb1', 'tysy', 'urok', 'vgfr2', 'wee1', 'xiap']\n",
    "#dude_ids = ['lck', 'src', 'ada17', 'hivpr', 'mk14', 'mmp13', 'aa2ar', 'bace1', 'pparg', 'parp1', 'ace', 'thrb', 'cdk2', 'esr1', 'esr2', 'vgfr2', 'fnta', 'drd3', 'csf1r', 'dhi1', 'casp3', 'gria2', 'kit', 'dyr', 'braf', 'tryb1', 'hdac8', 'aldr', 'akt1', 'ital', 'kpcb', 'tysy', 'ppard', 'hivint', 'ppara', 'urok', 'wee1', 'reni', 'grik1', 'aces', 'fa10', 'dpp4', 'adrb2', 'jak2', 'hivrt', 'fkb1a', 'cah2', 'kif11', 'try1', 'adrb1', 'akt2', 'rock1', 'pa2ga', 'pygm', 'mapk2', 'fa7', 'tgfr1', 'mk10', 'fak1', 'gcr', 'hdac2', 'prgr', 'ptn1', 'nram', 'abl1', 'hs90a', 'egfr', 'hxk4', 'mk01', 'cxcr4', 'lkha4', 'ada', 'pur2', 'pnph', 'andr', 'rxra', 'fpps', 'cp3a4', 'met', 'ampc', 'mp2k1', 'pyrd', 'pgh1', 'kith', 'thb', 'comt', 'cp2c9', 'aofb', 'fabp4', 'mcr', 'inha', 'pgh2', 'def', 'xiap', 'glcm', 'pde5a', 'nos1', 'sahh', 'hmdh', 'igf1r', 'plk1', 'fgfr1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.10.4'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 136 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 386 tasks      | elapsed:  3.8min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  4.8min finished\n",
      "[Parallel(n_jobs=-1)]: Done 136 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 386 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  4.9min finished\n",
      "[Parallel(n_jobs=-1)]: Done 136 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 386 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  4.9min finished\n",
      "[Parallel(n_jobs=-1)]: Done 136 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 386 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  4.9min finished\n",
      "[Parallel(n_jobs=-1)]: Done 136 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 386 tasks      | elapsed:  3.8min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  4.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'rs_inactive': 0.0091554495961674275, 'ef10': 6.3915929203539825, 'rk_inactive': 0.0074772079991085073, 'ef1_perc': 0.4772808586762075, 'a_100perc': 4520, 'rk_active': 0.17389351956967933, 'n_100perc': 279460, 'rp': 0.45843816511978136, 'rs': 0.15940183370490155, 'rp_active': 0.28115104446562134, 'mse': 0.053148846793100751, 'n_1perc': 2794, 'rk': 0.12979719052056218, 'engine': 'dude', 'rs_active': 0.25547424280087039, 'a_1perc': 1334, 'precision': 0.078103377115249215, 'roc_auc': 0.78680776529843033, 'roc_log_auc': 0.24528300026119443, 'ef1': 29.509050611870126, 'ef2': 19.610025013060618, 'ef5': 10.504424778761063, 'hitrate_100perc': 0.016174049953481713, 'recall': 0.71172566371681412, 'rp_inactive': 0.014782723134440179, 'hitrate_1perc': 0.47745168217609163, 'v': 1, 'ef0.1': 58.073625158027816, 'accuracy': 0.85946110355685967}\n",
      "{'rs_inactive': 0.008091784104414225, 'ef10': 6.684217305447214, 'rk_inactive': 0.006607323254506592, 'ef1_perc': 0.4930232558139535, 'a_100perc': 4503, 'rk_active': 0.19582857345073798, 'n_100perc': 279421, 'rp': 0.47121943692776724, 'rs': 0.16399579172061976, 'rp_active': 0.29678485449793418, 'mse': 0.053741455190362102, 'n_1perc': 2794, 'rk': 0.1335318702129272, 'engine': 'dude', 'rs_active': 0.28840245866403158, 'a_1perc': 1378, 'precision': 0.079532613660952559, 'roc_auc': 0.7976808285023731, 'roc_log_auc': 0.24752310348149417, 'ef1': 30.59317147741299, 'ef2': 20.284370448328776, 'ef5': 11.049659485208798, 'hitrate_100perc': 0.016115467341395243, 'recall': 0.73462136353542085, 'rp_inactive': 0.019506352170603914, 'hitrate_1perc': 0.4931997136721546, 'v': 1, 'ef0.1': 59.17119301418102, 'accuracy': 0.85870782797284384}\n",
      "{'rs_inactive': 0.0083473530523914853, 'ef10': 6.535690892077355, 'rk_inactive': 0.0068167926545313453, 'ef1_perc': 0.4881889763779528, 'a_100perc': 4480, 'rk_active': 0.18029013091864046, 'n_100perc': 279379, 'rp': 0.46536346128783762, 'rs': 0.16321202735726811, 'rp_active': 0.27645500639296294, 'mse': 0.052239994278727331, 'n_1perc': 2793, 'rk': 0.13289941842611985, 'engine': 'dude', 'rs_active': 0.26573478523418748, 'a_1perc': 1364, 'precision': 0.077881394732403186, 'roc_auc': 0.79111235918854361, 'roc_log_auc': 0.24569053239994323, 'ef1': 30.444140185601803, 'ef2': 20.299813415546073, 'ef5': 10.986567817897795, 'hitrate_100perc': 0.016035564591468935, 'recall': 0.72142857142857142, 'rp_inactive': 0.014356641763415137, 'hitrate_1perc': 0.4883637665592553, 'v': 1, 'ef0.1': 57.9069993622449, 'accuracy': 0.85856130918930917}\n",
      "{'rs_inactive': 0.0061839168389127286, 'ef10': 6.569477967342084, 'rk_inactive': 0.0050493517353423891, 'ef1_perc': 0.48067287043664997, 'a_100perc': 4463, 'rk_active': 0.18091236381477116, 'n_100perc': 279346, 'rp': 0.45661701693980467, 'rs': 0.15955023687134687, 'rp_active': 0.28229798004723311, 'mse': 0.053967570245725766, 'n_1perc': 2793, 'rk': 0.12989713311875586, 'engine': 'dude', 'rs_active': 0.26641174365656384, 'a_1perc': 1343, 'precision': 0.078359485750598812, 'roc_auc': 0.79058626094837614, 'roc_log_auc': 0.24707433741821286, 'ef1': 30.086050563521493, 'ef2': 20.468001784334902, 'ef5': 10.92932005654742, 'hitrate_100perc': 0.015976602492965713, 'recall': 0.71835088505489586, 'rp_inactive': 0.0097304723592359331, 'hitrate_1perc': 0.4808449695667741, 'v': 1, 'ef0.1': 56.10883614480971, 'accuracy': 0.86051348506869618}\n",
      "{'rs_inactive': 0.0069991081413006374, 'ef10': 6.628898600069993, 'rk_inactive': 0.0057150504939682345, 'ef1_perc': 0.48943788041532404, 'a_100perc': 4438, 'rk_active': 0.1598399993813247, 'n_100perc': 279291, 'rp': 0.45488090922732333, 'rs': 0.16269457437699106, 'rp_active': 0.2623774101958895, 'mse': 0.053896226094035438, 'n_1perc': 2792, 'rk': 0.13246708152618139, 'engine': 'dude', 'rs_active': 0.23514782899749059, 'a_1perc': 1366, 'precision': 0.078627312388798165, 'roc_auc': 0.7946825177180582, 'roc_log_auc': 0.24797599467926335, 'ef1': 30.801170585641337, 'ef2': 20.256219719452496, 'ef5': 10.977564235058127, 'hitrate_100perc': 0.015890236348468086, 'recall': 0.72690401081568279, 'rp_inactive': 0.024119168059340102, 'hitrate_1perc': 0.4892550143266476, 'v': 1, 'ef0.1': 58.43660271679649, 'accuracy': 0.86030699163238344}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 136 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 386 tasks      | elapsed: 11.0min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed: 14.0min finished\n",
      "[Parallel(n_jobs=-1)]: Done 136 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 386 tasks      | elapsed: 11.1min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed: 14.0min finished\n",
      "[Parallel(n_jobs=-1)]: Done 136 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 386 tasks      | elapsed: 11.1min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed: 13.9min finished\n",
      "[Parallel(n_jobs=-1)]: Done 136 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 386 tasks      | elapsed: 10.9min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed: 13.8min finished\n",
      "[Parallel(n_jobs=-1)]: Done 136 tasks      | elapsed:  4.3min\n",
      "[Parallel(n_jobs=-1)]: Done 386 tasks      | elapsed: 11.0min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed: 13.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'rs_inactive': 0.0065242341164906473, 'ef10': 7.303097345132745, 'rk_inactive': 0.0053273513764506494, 'ef1_perc': 0.6110912343470483, 'a_100perc': 4520, 'rk_active': 0.1892252300622734, 'n_100perc': 279460, 'rp': 0.53647519268890853, 'rs': 0.17469654462459136, 'rp_active': 0.31446850193051895, 'mse': 0.050484789980424369, 'n_1perc': 2794, 'rk': 0.14226074932094074, 'engine': 'dude', 'rs_active': 0.27783650314260588, 'a_1perc': 1708, 'precision': 0.084347620175230884, 'roc_auc': 0.82370575140770863, 'roc_log_auc': 0.25386963190265771, 'ef1': 37.78220273243941, 'ef2': 24.808395997910303, 'ef5': 12.650442477876107, 'hitrate_100perc': 0.016174049953481713, 'recall': 0.78805309734513274, 'rp_inactive': 0.018395105697108491, 'hitrate_1perc': 0.6113099498926271, 'v': 2, 'ef0.1': 61.60662136536031, 'accuracy': 0.85820510985471987}\n",
      "{'rs_inactive': 0.0087286989840496872, 'ef10': 7.479217237457214, 'rk_inactive': 0.0071270619407165512, 'ef1_perc': 0.6268336314847943, 'a_100perc': 4503, 'rk_active': 0.20143541628158526, 'n_100perc': 279421, 'rp': 0.54143270814473254, 'rs': 0.17879990686785394, 'rp_active': 0.31201764671283888, 'mse': 0.05130765041638613, 'n_1perc': 2794, 'rk': 0.14560010851658736, 'engine': 'dude', 'rs_active': 0.29574239665079682, 'a_1perc': 1752, 'precision': 0.085231291554675348, 'roc_auc': 0.83083438982101132, 'roc_log_auc': 0.25551137389877271, 'ef1': 38.89639798869924, 'ef2': 25.147290128880503, 'ef5': 13.012661692790102, 'hitrate_100perc': 0.016115467341395243, 'recall': 0.80279813457694871, 'rp_inactive': 0.010274317405825857, 'hitrate_1perc': 0.6270579813886901, 'v': 2, 'ef0.1': 61.830572475492524, 'accuracy': 0.85796701035355249}\n",
      "{'rs_inactive': 0.0082418093312586673, 'ef10': 7.419616299612408, 'rk_inactive': 0.0067295302377981831, 'ef1_perc': 0.6252684323550465, 'a_100perc': 4480, 'rk_active': 0.19207653635020253, 'n_100perc': 279379, 'rp': 0.54186663008725156, 'rs': 0.17817588641537521, 'rp_active': 0.30572188476168244, 'mse': 0.049599737528038354, 'n_1perc': 2793, 'rk': 0.14508981679167665, 'engine': 'dude', 'rs_active': 0.28183648633512859, 'a_1perc': 1747, 'precision': 0.08585437840668364, 'roc_auc': 0.83271221763330749, 'roc_log_auc': 0.25716361624419259, 'ef1': 38.992604768509054, 'ef2': 25.031600599818493, 'ef5': 12.705311665882618, 'hitrate_100perc': 0.016035564591468935, 'recall': 0.80513392857142863, 'rp_inactive': 0.021100194247800921, 'hitrate_1perc': 0.6254923021840315, 'v': 2, 'ef0.1': 61.2477877869898, 'accuracy': 0.85940603982403829}\n",
      "{'rs_inactive': 0.0073145021149196472, 'ef10': 7.4522795768689525, 'rk_inactive': 0.0059723835194589884, 'ef1_perc': 0.6109520400858983, 'a_100perc': 4463, 'rk_active': 0.19284178724113094, 'n_100perc': 279346, 'rp': 0.53653764015231142, 'rs': 0.17596086401642091, 'rp_active': 0.30670927658170227, 'mse': 0.051056762511216304, 'n_1perc': 2793, 'rk': 0.14328446153053595, 'engine': 'dude', 'rs_active': 0.28314399153374342, 'a_1perc': 1707, 'precision': 0.084651692322288377, 'roc_auc': 0.82964601640969959, 'roc_log_auc': 0.25575060070530359, 'ef1': 38.240423165994926, 'ef2': 25.083665021962695, 'ef5': 12.918913375574501, 'hitrate_100perc': 0.015976602492965713, 'recall': 0.79968630965718124, 'rp_inactive': 0.0096024484826549468, 'hitrate_1perc': 0.611170784103115, 'v': 2, 'ef0.1': 61.9209068211645, 'accuracy': 0.85864841451103646}\n",
      "{'rs_inactive': 0.0060609132059528887, 'ef10': 7.487365753919983, 'rk_inactive': 0.0049488331868518367, 'ef1_perc': 0.623344074471894, 'a_100perc': 4438, 'rk_active': 0.18504616437914753, 'n_100perc': 279291, 'rp': 0.53318081859904209, 'rs': 0.17825789258482855, 'rp_active': 0.29085379230909647, 'mse': 0.051218411920104741, 'n_1perc': 2792, 'rk': 0.14516545655078225, 'engine': 'dude', 'rs_active': 0.27133532570737851, 'a_1perc': 1741, 'precision': 0.085412231816116152, 'roc_auc': 0.83276004547095295, 'roc_log_auc': 0.25764428232923137, 'ef1': 39.228118500074295, 'ef2': 25.39350347477526, 'ef5': 12.9558691197833, 'hitrate_100perc': 0.015890236348468086, 'recall': 0.80464173050923837, 'rp_inactive': 0.012274894051094404, 'hitrate_1perc': 0.623567335243553, 'v': 2, 'ef0.1': 62.257457509817804, 'accuracy': 0.859984747091743}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 136 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 386 tasks      | elapsed:  4.5min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  5.6min finished\n",
      "[Parallel(n_jobs=-1)]: Done 136 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 386 tasks      | elapsed:  4.4min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  5.5min finished\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "out = []\n",
    "\n",
    "n_folds = 5\n",
    "for engine, v in [(e, v) for e in ['dude', 'dock', 'vina'] for v in [1,2,3]]:\n",
    "# for engine, v in [(e, v) for e in ['vina'] for v in [3]]:\n",
    "    \n",
    "    if v == 1:\n",
    "        col_range = range(1,37)\n",
    "        np_type = np.uint16\n",
    "    elif v == 2:\n",
    "        col_range = range(1,217)\n",
    "        np_type = np.uint16\n",
    "    elif v == 3:\n",
    "        col_range = range(1,43)   \n",
    "        np_type = np.float16\n",
    "    \n",
    "    act_cutoff = 6.\n",
    "    inactive_cutoff = 5.95\n",
    "    \n",
    "    train = []\n",
    "    test = []\n",
    "    for n in range(n_folds):\n",
    "        train.append([])\n",
    "        test.append([])\n",
    "        \n",
    "    for dude_id in dude_ids:\n",
    "        # trap errors on reading\n",
    "        try:\n",
    "            actives_full = pd.read_csv('head1_full/%s/%s/%s_desc_v%i.csv.gz' % (dude_id, engine, 'actives', v), dtype={i: np_type for i in col_range})\n",
    "            decoys_full = pd.read_csv('head1_full/%s/%s/%s_desc_v%i.csv.gz' % (dude_id, engine, 'decoys', v), dtype={i: np_type for i in col_range})\n",
    "        except:\n",
    "            continue\n",
    "\n",
    "        #trap empty ones\n",
    "        if len(actives_full) == 0 or len(decoys_full) == 0:\n",
    "            continue\n",
    "\n",
    "        decoys_full['act'] = inactive_cutoff if act_cutoff > 10 else 10**(9-inactive_cutoff)\n",
    "\n",
    "        # generate one big table for dude_id\n",
    "        data = pd.concat((actives_full, decoys_full))\n",
    "\n",
    "        # normalize\n",
    "        if act_cutoff >= 10:\n",
    "            data['act'] = np.clip(data['act'], 1e-9, inactive_cutoff)\n",
    "        else:\n",
    "            data['act'] = np.clip(-np.log10(np.clip(data['act'], 1e-9, 1e9) * 1e-9), 0 , 15)\n",
    "        # binary\n",
    "        data['act_bin'] = data['act'] < act_cutoff if act_cutoff > 10 else data['act'] > act_cutoff# 1e-6 # close to zero\n",
    "        \n",
    "        # manual CV, use Stratified for even distribution\n",
    "        oddt.random_seed(0)\n",
    "        cv_split = StratifiedKFold(data['act_bin'], n_folds=n_folds, shuffle=True)\n",
    "        for n, (cv_train, cv_test) in enumerate(cv_split):\n",
    "            if len(train[n]) == 0:\n",
    "                train[n] = data.iloc[cv_train]\n",
    "            else:\n",
    "                train[n] = pd.concat((train[n], data.iloc[cv_train]))\n",
    "\n",
    "            if len(test[n]) == 0:\n",
    "                test[n] = data.iloc[cv_test]\n",
    "            else:\n",
    "                test[n] = pd.concat((test[n], data.iloc[cv_test]))\n",
    "    \n",
    "    if v == 2:\n",
    "        mtry = 100\n",
    "    else:\n",
    "        mtry = 15\n",
    "    \n",
    "    rfs = []\n",
    "    for n in range(n_folds):\n",
    "        oddt.random_seed(0)\n",
    "        # Random Forest\n",
    "        rfs.append(randomforest(n_estimators=500, n_jobs=-1, verbose=1,\n",
    "                                max_features=mtry, \n",
    "                                oob_score=True,\n",
    "                                bootstrap = True,\n",
    "                                random_state = 0,\n",
    "                               ))\n",
    "        # non-parallel\n",
    "        rfs[n].fit(train[n][col_range], \n",
    "                   train[n]['act'],\n",
    "                   sample_weight=compute_sample_weight('balanced', train[n]['act_bin']))\n",
    "    \n",
    "\n",
    "\n",
    "    test_dfs = []\n",
    "    #print 'engine: %s\\tv: %i\\t head: %i' % (engine, v, head)\n",
    "    for n in range(n_folds):\n",
    "        rfs[n].verbose = 0\n",
    "        test[n]['pred'] = rfs[n].predict(test[n][col_range])\n",
    "        test[n]['pred_bin'] = test[n]['pred']  < act_cutoff if act_cutoff > 10 else test[n]['pred'] > act_cutoff\n",
    "        \n",
    "        test_df = test[n].sort_values('pred', ascending=False)# log is descending                                        \n",
    "        test_dfs.append(test_df)\n",
    "        \n",
    "        d = {'engine': engine,\n",
    "             'v': v,\n",
    "             'roc_auc': roc_auc(test_df['act_bin'], test_df['pred_bin'], ascending_score=False),# Binary is descending\n",
    "             \n",
    "             \n",
    "             'n_1perc': len(test_df['act_bin'])/100,\n",
    "             'n_100perc': len(test_df['act_bin']),\n",
    "             'a_1perc': test_df['act_bin'][:len(test_df['act_bin'])/100].sum(),\n",
    "             'a_100perc': test_df['act_bin'].sum(),\n",
    "             'hitrate_1perc': float(test_df['act_bin'][:len(test_df['act_bin'])/100].sum())/float(len(test_df['act_bin'])/100),\n",
    "             'hitrate_100perc': float(test_df['act_bin'].sum())/float(len(test_df['act_bin'])),\n",
    "             \n",
    "             'ef1_perc': enrichment_factor(test_df['act_bin'], test_df['pred_bin'], kind='percentage'),\n",
    "             'ef0.1': enrichment_factor(test_df['act_bin'], test_df['pred_bin'], percentage=0.1),\n",
    "             'ef1': enrichment_factor(test_df['act_bin'], test_df['pred_bin']),\n",
    "             'ef2': enrichment_factor(test_df['act_bin'], test_df['pred_bin'], percentage=2),\n",
    "             'ef5': enrichment_factor(test_df['act_bin'], test_df['pred_bin'], percentage=5),\n",
    "             'ef10': enrichment_factor(test_df['act_bin'], test_df['pred_bin'], percentage=10),\n",
    "             \n",
    "             'rp': pearsonr(test_df['act'], test_df['pred'])[0],\n",
    "             'rs': spearmanr(test_df['act'], test_df['pred'])[0],\n",
    "             'rk': kendalltau(test_df['act'], test_df['pred'])[0],\n",
    "             \n",
    "             'rp_active': pearsonr(test_df['act'][test_df['act_bin']], test_df['pred'][test_df['act_bin']])[0],\n",
    "             'rs_active': spearmanr(test_df['act'][test_df['act_bin']], test_df['pred'][test_df['act_bin']])[0],\n",
    "             'rk_active': kendalltau(test_df['act'][test_df['act_bin']], test_df['pred'][test_df['act_bin']])[0],\n",
    "             \n",
    "             'rp_inactive': pearsonr(test_df['act'][~test_df['act_bin']], test_df['pred'][~test_df['act_bin']])[0],\n",
    "             'rs_inactive': spearmanr(test_df['act'][~test_df['act_bin']], test_df['pred'][~test_df['act_bin']])[0],\n",
    "             'rk_inactive': kendalltau(test_df['act'][~test_df['act_bin']], test_df['pred'][~test_df['act_bin']])[0],\n",
    "             \n",
    "             'mse': mean_squared_error(test_df['act'], test_df['pred']),\n",
    "             'roc_log_auc': roc_log_auc(test_df['act_bin'], test_df['pred_bin'], ascending_score=False),# Binary is descending\n",
    "             'precision': precision_score(test_df['act_bin'], test_df['pred_bin']),\n",
    "             'accuracy': accuracy_score(test_df['act_bin'], test_df['pred_bin']),\n",
    "             'recall': recall_score(test_df['act_bin'], test_df['pred_bin']),\n",
    "            }\n",
    "        print d\n",
    "        out.append(d)\n",
    "    pd.concat(test_dfs)[['act', 'act_bin', 'pred', 'pred_bin', 'dude_id']].to_csv('dude_pred_horizontal_v%i_e%s.csv' % (v,engine), index=False)\n",
    "    \n",
    "pd.DataFrame(out).to_csv('dude_horizontal.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(out).groupby(['engine', 'v']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}