05_multiclass.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature selection for all-APGAR model (one hot encoded features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import shap\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('./data/processed_data_for_model_one_hot.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Choose to drop hopsitals from data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "drop_hospitals = True\n",
    "\n",
    "if drop_hospitals:\n",
    "    selected_cols = []\n",
    "    for col in list(data):\n",
    "        if col[0:8] == 'hospital':\n",
    "            pass\n",
    "        else:\n",
    "            selected_cols.append(col)\n",
    "    \n",
    "    data = data[selected_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>parity</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prevcaes</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>noscans</th>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pyrexia</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>meconium</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>apgar5</th>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bmi</th>\n",
       "      <td>20</td>\n",
       "      <td>25</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>age</th>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>26</td>\n",
       "      <td>19</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gest</th>\n",
       "      <td>40</td>\n",
       "      <td>42</td>\n",
       "      <td>38</td>\n",
       "      <td>39</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ethnic_white</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_I</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_P</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_E</th>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_G</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_S</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_C</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_T</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_L</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_B</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analgesia_O</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>induction_A</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>induction_O</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>induction_P</th>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ctg_A</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ctg_N</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ctg_Z</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_A</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_B</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_C</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_E</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_L</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_O</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_R</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_S</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>delivery_V</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>onset_E</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>onset_I</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>onset_N</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>onset_S</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0      1      2      3      4\n",
       "parity          0.0    1.0    1.0    0.0    0.0\n",
       "prevcaes        0.0    0.0    0.0    0.0    0.0\n",
       "noscans         1.0    3.0    1.0    2.0    1.0\n",
       "pyrexia       False  False  False  False  False\n",
       "meconium      False  False  False  False  False\n",
       "apgar5          9.0    9.0   10.0   10.0    9.0\n",
       "bmi              20     25     21     21      0\n",
       "age              21     22     26     19     29\n",
       "gest             40     42     38     39     40\n",
       "ethnic_white  False  False  False  False  False\n",
       "analgesia_I    True   True   True   True   True\n",
       "analgesia_P   False   True  False  False  False\n",
       "analgesia_E    True  False  False   True  False\n",
       "analgesia_G   False  False  False  False  False\n",
       "analgesia_S   False  False  False  False  False\n",
       "analgesia_C   False  False  False  False  False\n",
       "analgesia_T   False  False  False  False  False\n",
       "analgesia_L   False  False  False  False  False\n",
       "analgesia_B   False  False  False  False  False\n",
       "analgesia_O   False  False  False  False   True\n",
       "induction_A   False  False  False   True  False\n",
       "induction_O   False  False  False  False  False\n",
       "induction_P    True  False  False  False  False\n",
       "ctg_A             0      0      0      0      0\n",
       "ctg_N             1      1      1      1      1\n",
       "ctg_Z             0      0      0      0      0\n",
       "delivery_A        0      0      0      0      0\n",
       "delivery_B        0      0      0      0      0\n",
       "delivery_C        0      0      0      0      0\n",
       "delivery_E        0      0      0      0      0\n",
       "delivery_L        0      0      0      0      0\n",
       "delivery_O        0      0      0      0      0\n",
       "delivery_R        0      0      0      0      0\n",
       "delivery_S        0      1      1      1      1\n",
       "delivery_V        1      0      0      0      0\n",
       "onset_E           0      0      0      0      0\n",
       "onset_I           1      0      0      0      0\n",
       "onset_N           0      0      0      0      0\n",
       "onset_S           0      1      1      1      1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "y = data['apgar5']\n",
    "X = data.drop('apgar5', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Set up splits\n",
    "number_of_splits = 3\n",
    "skf = StratifiedKFold(n_splits = number_of_splits)\n",
    "skf.get_n_splits(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature  1: gest, AUC: 0.625\n",
      "Feature  2: ctg_A, AUC: 0.689\n",
      "Feature  3: analgesia_G, AUC: 0.716\n",
      "Feature  4: meconium, AUC: 0.719\n",
      "Feature  5: delivery_B, AUC: 0.721\n",
      "Feature  6: induction_O, AUC: 0.724\n",
      "Feature  7: delivery_A, AUC: 0.726\n",
      "Feature  8: analgesia_T, AUC: 0.726\n",
      "Feature  9: delivery_R, AUC: 0.727\n",
      "Feature 10: analgesia_P, AUC: 0.728\n",
      "Feature 11: ethnic_white, AUC: 0.728\n",
      "Feature 12: analgesia_O, AUC: 0.728\n",
      "Feature 13: delivery_O, AUC: 0.730\n",
      "Feature 14: analgesia_S, AUC: 0.731\n",
      "Feature 15: onset_E, AUC: 0.731\n",
      "Feature 16: analgesia_C, AUC: 0.730\n",
      "Feature 17: onset_N, AUC: 0.731\n",
      "Feature 18: delivery_E, AUC: 0.731\n",
      "Feature 19: pyrexia, AUC: 0.732\n",
      "Feature 20: ctg_N, AUC: 0.732\n"
     ]
    }
   ],
   "source": [
    "# Create list to store accuracies and chosen features\n",
    "roc_auc_by_feature_number = []\n",
    "roc_auc_by_feature_number_kfold = []\n",
    "chosen_features = []\n",
    "best_auc = 0\n",
    "\n",
    "# Initialise chosen features list and run tracker\n",
    "available_features = list(X)\n",
    "number_of_features = len(available_features)\n",
    "\n",
    "# Loop through number of features\n",
    "for i in range (20):\n",
    "    \n",
    "    # Reset best feature and accuracy\n",
    "    best_result = 0\n",
    "    best_feature = ''\n",
    "    \n",
    "    # Loop through available features\n",
    "    for feature in available_features:\n",
    "\n",
    "        # Create copy of already chosen features to avoid original being changed\n",
    "        features_to_use = chosen_features.copy()\n",
    "        # Create a list of features from features already chosen + 1 new feature\n",
    "        features_to_use.append(feature)\n",
    "        \n",
    "        # Set up a list to hold AUC results for this feature for each kfold\n",
    "        feature_roc_auc_ovr_kfold = []\n",
    "        \n",
    "        # Loop through the k-fold splits\n",
    "        counter = 0\n",
    "        for train_index, test_index in skf.split(X, y):\n",
    "            counter += 1\n",
    "    \n",
    "            # Get X and Y train/test\n",
    "            X_train, X_test = X.loc[train_index], X.loc[test_index]\n",
    "            y_train, y_test = y.loc[train_index], y.loc[test_index]            \n",
    "            \n",
    "            # Restrict features\n",
    "            X_train = X_train[features_to_use]\n",
    "            X_test = X_test[features_to_use]\n",
    "\n",
    "            # Define model\n",
    "            model = XGBClassifier(verbosity = 0, seed=42, learning_rate=0.5)\n",
    "\n",
    "            # Fit model\n",
    "            model.fit(X_train, y_train)\n",
    "            \n",
    "            # Get target categories from model\n",
    "            classes = model.classes_\n",
    "\n",
    "            # Get predicted probabilities\n",
    "            y_probs = model.predict_proba(X_test)\n",
    "            \n",
    "            # Calculate ROC AUC for multiclass models, using One vs Rest\n",
    "            feature_roc_auc_ovr = roc_auc_score(y_test, y_probs, labels = classes, \n",
    "                                multi_class = 'ovr', average = 'macro')\n",
    "            feature_roc_auc_ovr_kfold.append(feature_roc_auc_ovr)\n",
    "\n",
    "            # Remove model\n",
    "            del model\n",
    "        \n",
    "        # Get average result from all k-fold splits``\n",
    "        feature_auc_mean = np.mean(feature_roc_auc_ovr_kfold)\n",
    "    \n",
    "        # Update chosen feature and result if this feature is a new best\n",
    "        if feature_auc_mean > best_result:\n",
    "            best_result = feature_auc_mean\n",
    "            best_result_kfold = feature_roc_auc_ovr_kfold\n",
    "            best_feature = feature\n",
    "            \n",
    "    # k-fold splits are complete    \n",
    "    # Add mean accuracy and AUC to record of accuracy by feature number\n",
    "    roc_auc_by_feature_number.append(best_result)\n",
    "    roc_auc_by_feature_number_kfold.append(best_result_kfold)\n",
    "    chosen_features.append(best_feature)\n",
    "    available_features.remove(best_feature)\n",
    "            \n",
    "    print (f'Feature {i+1:2.0f}: {best_feature}, AUC: {best_result:0.3f}')\n",
    "    \n",
    "    # Check if AUC is still increasinbg. Stop if not\n",
    "    #if best_result > best_auc:\n",
    "    #    best_auc = best_result\n",
    "    #else:\n",
    "    #    print ('AUC falling; selection stopped')\n",
    "    #    break            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0             gest\n",
       "1            ctg_A\n",
       "2      analgesia_G\n",
       "3         meconium\n",
       "4       delivery_B\n",
       "5      induction_O\n",
       "6       delivery_A\n",
       "7      analgesia_T\n",
       "8       delivery_R\n",
       "9      analgesia_P\n",
       "10    ethnic_white\n",
       "11     analgesia_O\n",
       "12      delivery_O\n",
       "13     analgesia_S\n",
       "14         onset_E\n",
       "15     analgesia_C\n",
       "16         onset_N\n",
       "17      delivery_E\n",
       "18         pyrexia\n",
       "19           ctg_N\n",
       "dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = pd.Series(chosen_features)\n",
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "s.to_csv('./output/feature_selection_full_model.csv', header=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}