TalkingGridSearch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hdadmin\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import gc\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import xgboost as xgb\n",
    "from xgboost import plot_importance\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = 'D:\\\\PythonSamples\\\\Talking\\\\'\n",
    "\n",
    "def timeFeatures(df):\n",
    "    # Make some new features with click_time column\n",
    "    df['datetime'] = pd.to_datetime(df['click_time'])\n",
    "    #df['dow']      = df['datetime'].dt.dayofweek\n",
    "    #df[\"doy\"]      = df[\"datetime\"].dt.dayofyear\n",
    "    df[\"hod\"]      = df[\"datetime\"].dt.hour\n",
    "    #df[\"dteom\"]    = df[\"datetime\"].dt.daysinmonth - df[\"datetime\"].dt.day\n",
    "    df.drop(['click_time', 'datetime'], axis=1, inplace=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start_time = time.time()\n",
    "\n",
    "train_columns = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']\n",
    "test_columns  = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'click_id']\n",
    "dtypes = {\n",
    "        'ip'            : 'uint32',\n",
    "        'app'           : 'uint16',\n",
    "        'device'        : 'uint16',\n",
    "        'os'            : 'uint16',\n",
    "        'channel'       : 'uint16',\n",
    "        'is_attributed' : 'uint8',\n",
    "        'click_id'      : 'uint32'\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_ref = pd.read_csv(path+\"train.csv\", usecols=train_columns, dtype=dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ref = pd.read_csv(path+\"test.csv\", usecols=test_columns, dtype=dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop the IP and the columns from target\n",
    "train_ref.drop(['is_attributed'], axis=1, inplace=True)\n",
    "gc.collect()\n",
    "#nrow_train = train.shape[0]\n",
    "merge_ref = pd.concat([train_ref, test_ref])\n",
    "del train_ref, test_ref\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#ip_count = merge_ref.groupby(['ip'])['channel'].count().reset_index()\n",
    "#ip_count.columns = ['ip', 'clicks_by_ip']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "app_count = merge_ref.groupby(['app'])['channel'].count().reset_index()\n",
    "app_count.columns = ['app', 'clicks_by_app']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del merge_ref\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_check = pd.read_csv(path+\"train.csv\", usecols=train_columns, dtype=dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#train_check.groupby(['hod'])['is_attributed'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#train_check[\"is_attributed\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "apprank = train_check.groupby(['app'])['is_attributed'].sum().reset_index().astype('int')\n",
    "apprank.columns = ['app', 'app_sum']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rnk = apprank.loc[apprank['app_sum'] != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#rnk.sort_values(by='app_sum', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hdadmin\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "rnk['app_rank'] = rnk['app_sum'].rank(ascending=False).astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#%matplotlib inline\n",
    "#plt.hist(rnk['app_sum'], normed=True, bins=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hdadmin\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "rnk.drop(['app_sum'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#apprank.loc[apprank['app_sum'] != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del apprank\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_1 = train_check.loc[train_check[\"is_attributed\"] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_0 = train_check.loc[train_check[\"is_attributed\"] == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0027108051527743367\n"
     ]
    }
   ],
   "source": [
    "fraction_val = 500000/184447045\n",
    "print(fraction_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_0f = train_0.sample(frac=fraction_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.concat([train_1, train_0f])\n",
    "del  train_0f, train_1\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ip               956846\n",
       "app              956846\n",
       "device           956846\n",
       "os               956846\n",
       "channel          956846\n",
       "click_time       956846\n",
       "is_attributed    956846\n",
       "dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test = pd.read_csv(path+\"test.csv\", usecols=test_columns, dtype=dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "37"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop the IP and the columns from target\n",
    "y = train['is_attributed']\n",
    "train.drop(['is_attributed'], axis=1, inplace=True)\n",
    "\n",
    "# Drop IP and ID from test rows\n",
    "sub = pd.DataFrame()\n",
    "sub['click_id'] = test['click_id'].astype('int')\n",
    "#test.drop(['click_id'], axis=1, inplace=True)\n",
    "#gc.collect()\n",
    "\n",
    "#nrow_train = train.shape[0]\n",
    "#merge = pd.concat([train, test])\n",
    "del test\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Count the number of clicks by ip\n",
    "#train = pd.merge(train, ip_count, on='ip', how='left', sort=False)\n",
    "#train['clicks_by_ip'] = train['clicks_by_ip'].astype('uint16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.merge(train, app_count, on=['app'], how='left', sort=False)\n",
    "train['clicks_by_app'] = train['clicks_by_app'].astype('uint16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.merge(train, rnk, on=['app'], how='left', sort=False)\n",
    "train['app_rank'].fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['app_rank'] = train['app_rank'].astype('uint16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.drop('ip', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "35"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[667.477912902832] Start to generate time features\n",
      "[668.154007434845] Start XGBoost Training\n"
     ]
    }
   ],
   "source": [
    "print('[{}] Start to generate time features'.format(time.time() - start_time))\n",
    "\n",
    "train = timeFeatures(train)\n",
    "gc.collect()\n",
    "\n",
    "print('[{}] Start XGBoost Training'.format(time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def active_time(row):\n",
    "    if row['hod'] in [17,18,19,20,21,22]:\n",
    "        return 0\n",
    "    else:\n",
    "        return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['active_time'] = train.apply(active_time, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.drop('hod', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>clicks_by_app</th>\n",
       "      <th>app_rank</th>\n",
       "      <th>active_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>app</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.093929</td>\n",
       "      <td>0.117683</td>\n",
       "      <td>-0.129948</td>\n",
       "      <td>-0.096558</td>\n",
       "      <td>0.310322</td>\n",
       "      <td>0.002489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>device</th>\n",
       "      <td>0.093929</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.532388</td>\n",
       "      <td>-0.011545</td>\n",
       "      <td>-0.040378</td>\n",
       "      <td>-0.027602</td>\n",
       "      <td>-0.004470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>os</th>\n",
       "      <td>0.117683</td>\n",
       "      <td>0.532388</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.018491</td>\n",
       "      <td>-0.020147</td>\n",
       "      <td>0.000589</td>\n",
       "      <td>-0.000779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>channel</th>\n",
       "      <td>-0.129948</td>\n",
       "      <td>-0.011545</td>\n",
       "      <td>-0.018491</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.085558</td>\n",
       "      <td>0.045764</td>\n",
       "      <td>-0.026573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>clicks_by_app</th>\n",
       "      <td>-0.096558</td>\n",
       "      <td>-0.040378</td>\n",
       "      <td>-0.020147</td>\n",
       "      <td>0.085558</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.116712</td>\n",
       "      <td>-0.024487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>app_rank</th>\n",
       "      <td>0.310322</td>\n",
       "      <td>-0.027602</td>\n",
       "      <td>0.000589</td>\n",
       "      <td>0.045764</td>\n",
       "      <td>0.116712</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.019893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>active_time</th>\n",
       "      <td>0.002489</td>\n",
       "      <td>-0.004470</td>\n",
       "      <td>-0.000779</td>\n",
       "      <td>-0.026573</td>\n",
       "      <td>-0.024487</td>\n",
       "      <td>-0.019893</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    app    device        os   channel  clicks_by_app  \\\n",
       "app            1.000000  0.093929  0.117683 -0.129948      -0.096558   \n",
       "device         0.093929  1.000000  0.532388 -0.011545      -0.040378   \n",
       "os             0.117683  0.532388  1.000000 -0.018491      -0.020147   \n",
       "channel       -0.129948 -0.011545 -0.018491  1.000000       0.085558   \n",
       "clicks_by_app -0.096558 -0.040378 -0.020147  0.085558       1.000000   \n",
       "app_rank       0.310322 -0.027602  0.000589  0.045764       0.116712   \n",
       "active_time    0.002489 -0.004470 -0.000779 -0.026573      -0.024487   \n",
       "\n",
       "               app_rank  active_time  \n",
       "app            0.310322     0.002489  \n",
       "device        -0.027602    -0.004470  \n",
       "os             0.000589    -0.000779  \n",
       "channel        0.045764    -0.026573  \n",
       "clicks_by_app  0.116712    -0.024487  \n",
       "app_rank       1.000000    -0.019893  \n",
       "active_time   -0.019893     1.000000  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#due high correlation between device and os, droping device coz it has high number of unique values.\n",
    "#train.drop('device', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#train.drop('', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>clicks_by_app</th>\n",
       "      <th>app_rank</th>\n",
       "      <th>active_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>21</td>\n",
       "      <td>28753</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>215</td>\n",
       "      <td>62860</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>21</td>\n",
       "      <td>28753</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>46</td>\n",
       "      <td>274</td>\n",
       "      <td>28753</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>274</td>\n",
       "      <td>28753</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   app  device  os  channel  clicks_by_app  app_rank  active_time\n",
       "0   35       1  13       21          28753         2            1\n",
       "1    9       1  22      215          62860         6            1\n",
       "2   35       1  13       21          28753         2            1\n",
       "3   35       1  46      274          28753         2            1\n",
       "4   35       1  13      274          28753         2            1"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7\n"
     ]
    }
   ],
   "source": [
    "print(xgb.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test = pd.read_csv(path+\"test.csv\", usecols=test_columns, dtype=dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#test = pd.merge(test, ip_count, on='ip', how='left', sort=False)\n",
    "test = pd.merge(test, app_count, on=['app'], how='left', sort=False)\n",
    "test = pd.merge(test, rnk, on=['app'], how='left', sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['app_rank'].fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "204"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#del ip_count\n",
    "del app_count\n",
    "del rnk\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#test['clicks_by_ip'] = test['clicks_by_ip'].astype('uint16')\n",
    "test['clicks_by_app'] = test['clicks_by_app'].astype('uint16')\n",
    "test['app_rank'] = test['app_rank'].astype('uint16')\n",
    "test = timeFeatures(test)\n",
    "test['active_time'] = test.apply(active_time, axis = 1)\n",
    "test.drop(['click_id', 'ip'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test.drop('hod', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#test.drop('device', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#test.drop('active_time', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>clicks_by_app</th>\n",
       "      <th>app_rank</th>\n",
       "      <th>active_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>107</td>\n",
       "      <td>62860</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>466</td>\n",
       "      <td>62860</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>128</td>\n",
       "      <td>60834</td>\n",
       "      <td>58</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>111</td>\n",
       "      <td>64259</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>328</td>\n",
       "      <td>8542</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   app  device  os  channel  clicks_by_app  app_rank  active_time\n",
       "0    9       1   3      107          62860         6            1\n",
       "1    9       1   3      466          62860         6            1\n",
       "2   21       1  19      128          60834        58            1\n",
       "3   15       1  13      111          64259        17            1\n",
       "4   12       1  13      328           8542        19            1"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18790989"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hdadmin\\Anaconda3\\lib\\site-packages\\sklearn\\grid_search.py:43: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn import preprocessing\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.cross_validation import *\n",
    "from sklearn.grid_search import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['app', 'device', 'os', 'channel', 'clicks_by_app', 'app_rank', 'active_time']\n"
     ]
    }
   ],
   "source": [
    "features = list(train.columns) \n",
    "print(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for f in train.columns:\n",
    "    if train[f].dtype=='object':\n",
    "        #print(f)\n",
    "        lbl = preprocessing.LabelEncoder()\n",
    "        lbl.fit(list(train[f].values) + list(test[f].values))\n",
    "        train[f] = lbl.transform(list(train[f].values))\n",
    "        test[f] = lbl.transform(list(test[f].values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xgb_model = xgb.XGBClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set the params(this params from Pranav kernel) for xgboost model\n",
    "parameters = {'learning_rate': [0.5,0.6],\n",
    "          'booster': [\"gbtree\"],\n",
    "          'max_depth': [4,5],\n",
    "          'subsample': [1.0],\n",
    "          'colsample_bytree': [0.9,1.0],\n",
    "          'min_child_weight':[0,1],\n",
    "          'reg_alpha':[5.0,6.0],\n",
    "          'gamma':[0.9],\n",
    "          'objective': ['binary:logistic'], \n",
    "          'scale_pos_weight':[1,2],\n",
    "          'nthread':[5],\n",
    "          'random_state': [99], \n",
    "          'silent': [1]}\n",
    "\n",
    "Raw AUC score: 0.9656840221349073\n",
    "booster: 'gbtree'\n",
    "colsample_bytree: 0.9\n",
    "gamma: 0.8\n",
    "learning_rate: 0.5\n",
    "max_depth: 5\n",
    "min_child_weight: 1\n",
    "nthread: 5\n",
    "objective: 'binary:logistic'\n",
    "random_state: 99\n",
    "reg_alpha: 5.0\n",
    "scale_pos_weight: 2\n",
    "silent: 1\n",
    "subsample: 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime.datetime(2018, 4, 8, 11, 39, 39, 900380)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import datetime\n",
    "datetime.datetime.now()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clf = GridSearchCV(xgb_model, parameters, n_jobs=5, \n",
    "                   cv=StratifiedKFold(y, n_folds=5, shuffle=True), \n",
    "                   scoring='roc_auc',\n",
    "                   verbose=2, refit=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 64 candidates, totalling 320 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Done  31 tasks      | elapsed:  8.5min\n",
      "[Parallel(n_jobs=5)]: Done 152 tasks      | elapsed: 43.3min\n",
      "[Parallel(n_jobs=5)]: Done 320 out of 320 | elapsed: 93.6min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=sklearn.cross_validation.StratifiedKFold(labels=[1 1 ..., 0 0], n_folds=5, shuffle=True, random_state=None),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=3, min_child_weight=1, missing=None, n_estimators=100,\n",
       "       n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "       silent=True, subsample=1),\n",
       "       fit_params={}, iid=True, n_jobs=5,\n",
       "       param_grid={'learning_rate': [0.4, 0.5], 'booster': ['gbtree'], 'max_depth': [4, 5], 'subsample': [1.0], 'colsample_bytree': [0.8, 0.9], 'min_child_weight': [0, 1], 'reg_alpha': [5.0, 6.0], 'gamma': [0.8], 'objective': ['binary:logistic'], 'scale_pos_weight': [1, 2], 'nthread': [5], 'random_state': [99], 'silent': [1]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, scoring='roc_auc', verbose=2)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(train[features], y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime.datetime(2018, 4, 8, 13, 13, 59, 89378)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datetime.datetime.now()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Raw AUC score: 0.9656840221349073\n",
      "booster: 'gbtree'\n",
      "colsample_bytree: 0.9\n",
      "gamma: 0.8\n",
      "learning_rate: 0.5\n",
      "max_depth: 5\n",
      "min_child_weight: 1\n",
      "nthread: 5\n",
      "objective: 'binary:logistic'\n",
      "random_state: 99\n",
      "reg_alpha: 5.0\n",
      "scale_pos_weight: 2\n",
      "silent: 1\n",
      "subsample: 1.0\n"
     ]
    }
   ],
   "source": [
    "#trust your CV!\n",
    "best_parameters, score, _ = max(clf.grid_scores_, key=lambda x: x[1])\n",
    "print('Raw AUC score:', score)\n",
    "for param_name in sorted(best_parameters.keys()):\n",
    "    print(\"%s: %r\" % (param_name, best_parameters[param_name]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_probs = clf.predict_proba(test[features])[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub['is_attributed'] = np.round(test_probs).astype(np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>click_id</th>\n",
       "      <th>is_attributed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   click_id  is_attributed\n",
       "0         0              0\n",
       "1         1              0\n",
       "2         2              0\n",
       "3         3              0\n",
       "4         4              0"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub.to_csv(path+'xgb_sub.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}