05a_multiclass_balanced.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature selection for all-APGAR model (one hot encoded features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import shap\n",
    "\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('./data/processed_data_for_model_one_hot.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Choose to drop hopsitals from data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "drop_hospitals = True\n",
    "\n",
    "if drop_hospitals:\n",
    "    selected_cols = []\n",
    "    for col in list(data):\n",
    "        if col[0:8] == 'hospital':\n",
    "            pass\n",
    "        else:\n",
    "            selected_cols.append(col)\n",
    "    \n",
    "    data = data[selected_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>parity</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prevcaes</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>noscans</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pyrexia</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>meconium</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>apgar5</th>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bmi</th>\n",
       "      <td>18</td>\n",
       "      <td>20</td>\n",
       "      <td>27</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>age</th>\n",
       "      <td>27</td>\n",
       "      <td>25</td>\n",
       "      <td>34</td>\n",
       "      <td>29</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gest</th>\n",
       "      <td>38</td>\n",
       "      <td>40</td>\n",
       "      <td>36</td>\n",
       "      <td>26</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ethnic_white</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_I</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_P</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_E</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_G</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_S</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_C</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_T</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_L</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_B</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_O</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>induction_A</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>induction_O</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>induction_P</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ctg_A</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ctg_N</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ctg_Z</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_A</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_B</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_C</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_E</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_L</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_O</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_R</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_S</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_V</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>onset_E</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>onset_I</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>onset_N</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>onset_S</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0      1      2      3      4\n",
       "parity          1.0    2.0    0.0    2.0    1.0\n",
       "prevcaes        0.0    0.0    0.0    0.0    0.0\n",
       "noscans         1.0    2.0    5.0    1.0    2.0\n",
       "pyrexia       False  False  False  False  False\n",
       "meconium      False  False   True  False  False\n",
       "apgar5         10.0    9.0   10.0    9.0   10.0\n",
       "bmi              18     20     27     33     25\n",
       "age              27     25     34     29     37\n",
       "gest             38     40     36     26     41\n",
       "ethnic_white  False  False  False  False  False\n",
       "analgesia_I   False   True  False  False   True\n",
       "analgesia_P   False   True  False   True  False\n",
       "analgesia_E   False  False   True  False  False\n",
       "analgesia_G   False  False  False  False  False\n",
       "analgesia_S   False  False  False  False  False\n",
       "analgesia_C   False  False  False  False  False\n",
       "analgesia_T   False  False  False  False   True\n",
       "analgesia_L   False  False  False  False   True\n",
       "analgesia_B   False  False  False  False  False\n",
       "analgesia_O   False  False  False  False  False\n",
       "induction_A   False  False   True  False  False\n",
       "induction_O   False  False   True  False  False\n",
       "induction_P   False  False  False  False  False\n",
       "ctg_A             0      0      0      0      0\n",
       "ctg_N             1      1      1      1      1\n",
       "ctg_Z             0      0      0      0      0\n",
       "delivery_A        0      0      0      0      0\n",
       "delivery_B        0      0      0      0      0\n",
       "delivery_C        0      0      0      0      0\n",
       "delivery_E        0      0      0      0      0\n",
       "delivery_L        0      0      0      0      0\n",
       "delivery_O        0      0      0      0      0\n",
       "delivery_R        0      0      0      0      0\n",
       "delivery_S        1      1      0      1      1\n",
       "delivery_V        0      0      1      0      0\n",
       "onset_E           0      0      0      0      0\n",
       "onset_I           0      0      1      0      0\n",
       "onset_N           0      0      0      0      0\n",
       "onset_S           1      1      0      1      1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "y = data['apgar5']\n",
    "X = data.drop('apgar5', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Set up splits\n",
    "number_of_splits = 3\n",
    "skf = StratifiedKFold(n_splits = number_of_splits)\n",
    "skf.get_n_splits(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/home/michael/MK_Sync/intrapartum_risk_pilot/05a_multiclass_balanced.ipynb Cell 10\u001b[0m line \u001b[0;36m5\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/michael/MK_Sync/intrapartum_risk_pilot/05a_multiclass_balanced.ipynb#X12sZmlsZQ%3D%3D?line=46'>47</a>\u001b[0m model \u001b[39m=\u001b[39m XGBClassifier(verbosity \u001b[39m=\u001b[39m \u001b[39m0\u001b[39m, seed\u001b[39m=\u001b[39m\u001b[39m42\u001b[39m, learning_rate\u001b[39m=\u001b[39m\u001b[39m0.5\u001b[39m)\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/michael/MK_Sync/intrapartum_risk_pilot/05a_multiclass_balanced.ipynb#X12sZmlsZQ%3D%3D?line=48'>49</a>\u001b[0m \u001b[39m# Fit model\u001b[39;00m\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/michael/MK_Sync/intrapartum_risk_pilot/05a_multiclass_balanced.ipynb#X12sZmlsZQ%3D%3D?line=49'>50</a>\u001b[0m model\u001b[39m.\u001b[39;49mfit(X_train, y_train)\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/michael/MK_Sync/intrapartum_risk_pilot/05a_multiclass_balanced.ipynb#X12sZmlsZQ%3D%3D?line=51'>52</a>\u001b[0m \u001b[39m# Get target categories from model\u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/michael/MK_Sync/intrapartum_risk_pilot/05a_multiclass_balanced.ipynb#X12sZmlsZQ%3D%3D?line=52'>53</a>\u001b[0m classes \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39mclasses_\n",
      "File \u001b[0;32m~/miniconda3/envs/sam10/lib/python3.10/site-packages/xgboost/core.py:620\u001b[0m, in \u001b[0;36mrequire_keyword_args.<locals>.throw_if.<locals>.inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    618\u001b[0m \u001b[39mfor\u001b[39;00m k, arg \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(sig\u001b[39m.\u001b[39mparameters, args):\n\u001b[1;32m    619\u001b[0m     kwargs[k] \u001b[39m=\u001b[39m arg\n\u001b[0;32m--> 620\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/sam10/lib/python3.10/site-packages/xgboost/sklearn.py:1490\u001b[0m, in \u001b[0;36mXGBClassifier.fit\u001b[0;34m(self, X, y, sample_weight, base_margin, eval_set, eval_metric, early_stopping_rounds, verbose, xgb_model, sample_weight_eval_set, base_margin_eval_set, feature_weights, callbacks)\u001b[0m\n\u001b[1;32m   1462\u001b[0m (\n\u001b[1;32m   1463\u001b[0m     model,\n\u001b[1;32m   1464\u001b[0m     metric,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1469\u001b[0m     xgb_model, eval_metric, params, early_stopping_rounds, callbacks\n\u001b[1;32m   1470\u001b[0m )\n\u001b[1;32m   1471\u001b[0m train_dmatrix, evals \u001b[39m=\u001b[39m _wrap_evaluation_matrices(\n\u001b[1;32m   1472\u001b[0m     missing\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmissing,\n\u001b[1;32m   1473\u001b[0m     X\u001b[39m=\u001b[39mX,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1487\u001b[0m     feature_types\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfeature_types,\n\u001b[1;32m   1488\u001b[0m )\n\u001b[0;32m-> 1490\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_Booster \u001b[39m=\u001b[39m train(\n\u001b[1;32m   1491\u001b[0m     params,\n\u001b[1;32m   1492\u001b[0m     train_dmatrix,\n\u001b[1;32m   1493\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mget_num_boosting_rounds(),\n\u001b[1;32m   1494\u001b[0m     evals\u001b[39m=\u001b[39;49mevals,\n\u001b[1;32m   1495\u001b[0m     early_stopping_rounds\u001b[39m=\u001b[39;49mearly_stopping_rounds,\n\u001b[1;32m   1496\u001b[0m     evals_result\u001b[39m=\u001b[39;49mevals_result,\n\u001b[1;32m   1497\u001b[0m     obj\u001b[39m=\u001b[39;49mobj,\n\u001b[1;32m   1498\u001b[0m     custom_metric\u001b[39m=\u001b[39;49mmetric,\n\u001b[1;32m   1499\u001b[0m     verbose_eval\u001b[39m=\u001b[39;49mverbose,\n\u001b[1;32m   1500\u001b[0m     xgb_model\u001b[39m=\u001b[39;49mmodel,\n\u001b[1;32m   1501\u001b[0m     callbacks\u001b[39m=\u001b[39;49mcallbacks,\n\u001b[1;32m   1502\u001b[0m )\n\u001b[1;32m   1504\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mcallable\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mobjective):\n\u001b[1;32m   1505\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mobjective \u001b[39m=\u001b[39m params[\u001b[39m\"\u001b[39m\u001b[39mobjective\u001b[39m\u001b[39m\"\u001b[39m]\n",
      "File \u001b[0;32m~/miniconda3/envs/sam10/lib/python3.10/site-packages/xgboost/core.py:620\u001b[0m, in \u001b[0;36mrequire_keyword_args.<locals>.throw_if.<locals>.inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    618\u001b[0m \u001b[39mfor\u001b[39;00m k, arg \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(sig\u001b[39m.\u001b[39mparameters, args):\n\u001b[1;32m    619\u001b[0m     kwargs[k] \u001b[39m=\u001b[39m arg\n\u001b[0;32m--> 620\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/sam10/lib/python3.10/site-packages/xgboost/training.py:185\u001b[0m, in \u001b[0;36mtrain\u001b[0;34m(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, xgb_model, callbacks, custom_metric)\u001b[0m\n\u001b[1;32m    183\u001b[0m \u001b[39mif\u001b[39;00m cb_container\u001b[39m.\u001b[39mbefore_iteration(bst, i, dtrain, evals):\n\u001b[1;32m    184\u001b[0m     \u001b[39mbreak\u001b[39;00m\n\u001b[0;32m--> 185\u001b[0m bst\u001b[39m.\u001b[39;49mupdate(dtrain, i, obj)\n\u001b[1;32m    186\u001b[0m \u001b[39mif\u001b[39;00m cb_container\u001b[39m.\u001b[39mafter_iteration(bst, i, dtrain, evals):\n\u001b[1;32m    187\u001b[0m     \u001b[39mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/sam10/lib/python3.10/site-packages/xgboost/core.py:1918\u001b[0m, in \u001b[0;36mBooster.update\u001b[0;34m(self, dtrain, iteration, fobj)\u001b[0m\n\u001b[1;32m   1915\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_validate_dmatrix_features(dtrain)\n\u001b[1;32m   1917\u001b[0m \u001b[39mif\u001b[39;00m fobj \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m-> 1918\u001b[0m     _check_call(_LIB\u001b[39m.\u001b[39;49mXGBoosterUpdateOneIter(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mhandle,\n\u001b[1;32m   1919\u001b[0m                                             ctypes\u001b[39m.\u001b[39;49mc_int(iteration),\n\u001b[1;32m   1920\u001b[0m                                             dtrain\u001b[39m.\u001b[39;49mhandle))\n\u001b[1;32m   1921\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m   1922\u001b[0m     pred \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpredict(dtrain, output_margin\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m, training\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Create list to store accuracies and chosen features\n",
    "roc_auc_by_feature_number = []\n",
    "roc_auc_by_feature_number_kfold = []\n",
    "chosen_features = []\n",
    "best_auc = 0\n",
    "\n",
    "# Initialise chosen features list and run tracker\n",
    "available_features = list(X)\n",
    "number_of_features = len(available_features)\n",
    "\n",
    "# Loop through number of features\n",
    "for i in range (20):\n",
    "    \n",
    "    # Reset best feature and accuracy\n",
    "    best_result = 0\n",
    "    best_feature = ''\n",
    "    \n",
    "    # Loop through available features\n",
    "    for feature in available_features:\n",
    "\n",
    "        # Create copy of already chosen features to avoid original being changed\n",
    "        features_to_use = chosen_features.copy()\n",
    "        # Create a list of features from features already chosen + 1 new feature\n",
    "        features_to_use.append(feature)\n",
    "        \n",
    "        # Set up a list to hold AUC results for this feature for each kfold\n",
    "        feature_roc_auc_ovr_kfold = []\n",
    "        \n",
    "        # Loop through the k-fold splits\n",
    "        counter = 0\n",
    "        for train_index, test_index in skf.split(X, y):\n",
    "            counter += 1\n",
    "    \n",
    "            # Get X and Y train/test\n",
    "            X_train, X_test = X.loc[train_index], X.loc[test_index]\n",
    "            y_train, y_test = y.loc[train_index], y.loc[test_index]\n",
    "\n",
    "            # Balance training data with random oversampling\n",
    "            ros = RandomOverSampler(random_state=42)\n",
    "            X_train, y_train = ros.fit_resample(X_train, y_train)        \n",
    "            \n",
    "            # Restrict features\n",
    "            X_train = X_train[features_to_use]\n",
    "            X_test = X_test[features_to_use]\n",
    "\n",
    "            # Define model\n",
    "            model = XGBClassifier(verbosity = 0, seed=42, learning_rate=0.5)\n",
    "\n",
    "            # Fit model\n",
    "            model.fit(X_train, y_train)\n",
    "            \n",
    "            # Get target categories from model\n",
    "            classes = model.classes_\n",
    "\n",
    "            # Get predicted probabilities\n",
    "            y_probs = model.predict_proba(X_test)\n",
    "            \n",
    "            # Calculate ROC AUC for multiclass models, using One vs Rest\n",
    "            feature_roc_auc_ovr = roc_auc_score(y_test, y_probs, labels = classes, \n",
    "                                multi_class = 'ovr', average = 'macro')\n",
    "            feature_roc_auc_ovr_kfold.append(feature_roc_auc_ovr)\n",
    "\n",
    "            # Remove model\n",
    "            del model\n",
    "        \n",
    "        # Get average result from all k-fold splits``\n",
    "        feature_auc_mean = np.mean(feature_roc_auc_ovr_kfold)\n",
    "    \n",
    "        # Update chosen feature and result if this feature is a new best\n",
    "        if feature_auc_mean > best_result:\n",
    "            best_result = feature_auc_mean\n",
    "            best_result_kfold = feature_roc_auc_ovr_kfold\n",
    "            best_feature = feature\n",
    "            \n",
    "    # k-fold splits are complete    \n",
    "    # Add mean accuracy and AUC to record of accuracy by feature number\n",
    "    roc_auc_by_feature_number.append(best_result)\n",
    "    roc_auc_by_feature_number_kfold.append(best_result_kfold)\n",
    "    chosen_features.append(best_feature)\n",
    "    available_features.remove(best_feature)\n",
    "            \n",
    "    print (f'Feature {i+1:2.0f}: {best_feature}, AUC: {best_result:0.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(chosen_features)\n",
    "s.to_csv('./output/feature_selection_full_model_rebalanced.csv', header=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}