isic-2024-parallel-image-lines-1-new-line.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1738f89d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:34:54.225891Z",
     "iopub.status.busy": "2024-09-02T19:34:54.225426Z",
     "iopub.status.idle": "2024-09-02T19:35:00.803154Z",
     "shell.execute_reply": "2024-09-02T19:35:00.802306Z"
    },
    "papermill": {
     "duration": 6.593014,
     "end_time": "2024-09-02T19:35:00.805639",
     "exception": false,
     "start_time": "2024-09-02T19:34:54.212625",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import polars as pl\n",
    "\n",
    "from sklearn.model_selection import StratifiedGroupKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from imblearn.pipeline import Pipeline\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "\n",
    "\n",
    "import lightgbm as lgb\n",
    "import catboost as cb\n",
    "import xgboost as xgb\n",
    "\n",
    "import optuna\n",
    "from optuna.samplers import TPESampler\n",
    "\n",
    "#downsampling techniques\n",
    "# they took long time, so we use RandomUnderSampler\n",
    "from imblearn.under_sampling import NearMiss\n",
    "from imblearn.under_sampling import ClusterCentroids\n",
    "from imblearn.under_sampling import TomekLinks\n",
    "from imblearn.under_sampling import EditedNearestNeighbours\n",
    "from imblearn.pipeline import Pipeline as ImbPipeline\n",
    "from imblearn.under_sampling import NearMiss, TomekLinks\n",
    "from sklearn.impute import SimpleImputer\n",
    "\n",
    "import time\n",
    "from sklearn.feature_selection import SelectKBest, chi2, mutual_info_classif, VarianceThreshold\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3f30c2c0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:35:00.828952Z",
     "iopub.status.busy": "2024-09-02T19:35:00.828282Z",
     "iopub.status.idle": "2024-09-02T19:35:00.838140Z",
     "shell.execute_reply": "2024-09-02T19:35:00.837287Z"
    },
    "papermill": {
     "duration": 0.023598,
     "end_time": "2024-09-02T19:35:00.840147",
     "exception": false,
     "start_time": "2024-09-02T19:35:00.816549",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import gc\n",
    "import time\n",
    "import copy\n",
    "import optuna\n",
    "\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np, pandas as pd, polars as pl\n",
    "\n",
    "from sklearn.model_selection import GroupKFold\n",
    "from sklearn.model_selection import StratifiedGroupKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "from sklearn.utils import resample\n",
    "\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from imblearn.pipeline import Pipeline\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "\n",
    "import lightgbm as lgb, catboost as cb, xgboost as xgb\n",
    "\n",
    "from optuna.samplers import TPESampler\n",
    "\n",
    "from imblearn.under_sampling import NearMiss\n",
    "from imblearn.under_sampling import ClusterCentroids\n",
    "from imblearn.under_sampling import TomekLinks\n",
    "from imblearn.under_sampling import EditedNearestNeighbours\n",
    "# from imblearn.pipeline import Pipeline as ImbPipeline\n",
    "from imblearn.under_sampling import NearMiss, TomekLinks\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.feature_selection import SelectKBest, chi2, mutual_info_classif, VarianceThreshold\n",
    "from sklearn.base import BaseEstimator, TransformerMixin"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39a64638",
   "metadata": {
    "papermill": {
     "duration": 0.010271,
     "end_time": "2024-09-02T19:35:00.861243",
     "exception": false,
     "start_time": "2024-09-02T19:35:00.850972",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# IMAGE PREDICTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ac9ef34b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:35:00.884627Z",
     "iopub.status.busy": "2024-09-02T19:35:00.884211Z",
     "iopub.status.idle": "2024-09-02T19:35:15.342700Z",
     "shell.execute_reply": "2024-09-02T19:35:15.341089Z"
    },
    "papermill": {
     "duration": 14.473494,
     "end_time": "2024-09-02T19:35:15.345323",
     "exception": false,
     "start_time": "2024-09-02T19:35:00.871829",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BEST_WEIGHT = /kaggle/input/isic-pytorch-training-baseline-image-only/AUROC0.5171_Loss0.3476_epoch35.bin\r\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:01<00:00,  1.01s/it]\r\n"
     ]
    }
   ],
   "source": [
    "#EFFNET V1B0\n",
    "!python /kaggle/input/isic-script-inference-effnetv1b0-f313ae/main.py /kaggle/input/isic-pytorch-training-baseline-image-only/AUROC0.5171_Loss0.3476_epoch35.bin\n",
    "!mv submission.csv submission_effnetv1b0.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ef0f0953",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:35:15.370462Z",
     "iopub.status.busy": "2024-09-02T19:35:15.369699Z",
     "iopub.status.idle": "2024-09-02T19:35:36.719256Z",
     "shell.execute_reply": "2024-09-02T19:35:36.717966Z"
    },
    "papermill": {
     "duration": 21.365213,
     "end_time": "2024-09-02T19:35:36.721843",
     "exception": false,
     "start_time": "2024-09-02T19:35:15.356630",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/isic-2024-pl-submission-script-and-preds/pl_submission.py:25: DtypeWarning: Columns (51,52) have mixed types. Specify dtype option on import or set low_memory=False.\r\n",
      "  df_train_meta = pd.read_csv(BASE_DATA_DIR + \"train-metadata.csv\")\r\n"
     ]
    }
   ],
   "source": [
    "#TARGET 3\n",
    "!python /kaggle/input/isic-2024-pl-submission-script-and-preds/pl_submission.py\n",
    "!mv submission.csv submission_image3.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cda77b27",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:35:36.746131Z",
     "iopub.status.busy": "2024-09-02T19:35:36.745250Z",
     "iopub.status.idle": "2024-09-02T19:35:47.538533Z",
     "shell.execute_reply": "2024-09-02T19:35:47.537361Z"
    },
    "papermill": {
     "duration": 10.807927,
     "end_time": "2024-09-02T19:35:47.540957",
     "exception": false,
     "start_time": "2024-09-02T19:35:36.733030",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BEST_WEIGHT = /kaggle/input/isic-pytorch-training-baseline-eva02/AUROC0.5177_Loss0.2829_epoch7.bin\r\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  2.64it/s]\r\n"
     ]
    }
   ],
   "source": [
    "#EVA02\n",
    "!python /kaggle/input/isic-script-inference-eva02/main.py /kaggle/input/isic-pytorch-training-baseline-eva02/AUROC0.5177_Loss0.2829_epoch7.bin\n",
    "!mv submission.csv submission_eva02.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b18a107d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:35:47.566927Z",
     "iopub.status.busy": "2024-09-02T19:35:47.566099Z",
     "iopub.status.idle": "2024-09-02T19:35:58.482646Z",
     "shell.execute_reply": "2024-09-02T19:35:58.481329Z"
    },
    "papermill": {
     "duration": 10.931927,
     "end_time": "2024-09-02T19:35:58.485169",
     "exception": false,
     "start_time": "2024-09-02T19:35:47.553242",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BEST_WEIGHT = /kaggle/input/isic-pytorch-training-edgenext/Final_model.bin\r\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  1.53it/s]\r\n"
     ]
    }
   ],
   "source": [
    "#EDGENEXT\n",
    "!python /kaggle/input/isic-script-inference-edgenext/main.py /kaggle/input/isic-pytorch-training-edgenext/Final_model.bin\n",
    "!mv submission.csv submission_edgenext.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5a63b48",
   "metadata": {
    "papermill": {
     "duration": 0.011179,
     "end_time": "2024-09-02T19:35:58.508814",
     "exception": false,
     "start_time": "2024-09-02T19:35:58.497635",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# DATA PREPROCESS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b1724041",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:35:58.534233Z",
     "iopub.status.busy": "2024-09-02T19:35:58.533589Z",
     "iopub.status.idle": "2024-09-02T19:35:58.550068Z",
     "shell.execute_reply": "2024-09-02T19:35:58.549261Z"
    },
    "papermill": {
     "duration": 0.031717,
     "end_time": "2024-09-02T19:35:58.552048",
     "exception": false,
     "start_time": "2024-09-02T19:35:58.520331",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "root = Path('/kaggle/input/isic-2024-challenge')\n",
    "\n",
    "train_path = root / 'train-metadata.csv'\n",
    "test_path = root / 'test-metadata.csv'\n",
    "subm_path = root / 'sample_submission.csv'\n",
    "\n",
    "id_col = 'isic_id'\n",
    "target_col = 'target'\n",
    "group_col = 'patient_id'\n",
    "\n",
    "err = 1e-5\n",
    "sampling_ratio = 0.01\n",
    "seed = 42\n",
    "\n",
    "num_cols = [\n",
    "    'age_approx',                        # Approximate age of patient at time of imaging.\n",
    "    'clin_size_long_diam_mm',            # Maximum diameter of the lesion (mm).+\n",
    "    'tbp_lv_A',                          # A inside  lesion.+\n",
    "    'tbp_lv_Aext',                       # A outside lesion.+\n",
    "    'tbp_lv_B',                          # B inside  lesion.+\n",
    "    'tbp_lv_Bext',                       # B outside lesion.+ \n",
    "    'tbp_lv_C',                          # Chroma inside  lesion.+\n",
    "    'tbp_lv_Cext',                       # Chroma outside lesion.+\n",
    "    'tbp_lv_H',                          # Hue inside the lesion; calculated as the angle of A* and B* in LAB* color space. Typical values range from 25 (red) to 75 (brown).+\n",
    "    'tbp_lv_Hext',                       # Hue outside lesion.+\n",
    "    'tbp_lv_L',                          # L inside lesion.+\n",
    "    'tbp_lv_Lext',                       # L outside lesion.+\n",
    "    'tbp_lv_areaMM2',                    # Area of lesion (mm^2).+\n",
    "    'tbp_lv_area_perim_ratio',           # Border jaggedness, the ratio between lesions perimeter and area. Circular lesions will have low values; irregular shaped lesions will have higher values. Values range 0-10.+\n",
    "    'tbp_lv_color_std_mean',             # Color irregularity, calculated as the variance of colors within the lesion's boundary.\n",
    "    'tbp_lv_deltaA',                     # Average A contrast (inside vs. outside lesion).+\n",
    "    'tbp_lv_deltaB',                     # Average B contrast (inside vs. outside lesion).+\n",
    "    'tbp_lv_deltaL',                     # Average L contrast (inside vs. outside lesion).+\n",
    "    'tbp_lv_deltaLB',                    #\n",
    "    'tbp_lv_deltaLBnorm',                # Contrast between the lesion and its immediate surrounding skin. Low contrast lesions tend to be faintly visible such as freckles; high contrast lesions tend to be those with darker pigment. Calculated as the average delta LB of the lesion relative to its immediate background in LAB* color space. Typical values range from 5.5 to 25.+\n",
    "    'tbp_lv_eccentricity',               # Eccentricity.+\n",
    "    'tbp_lv_minorAxisMM',                # Smallest lesion diameter (mm).+\n",
    "    'tbp_lv_nevi_confidence',            # Nevus confidence score (0-100 scale) is a convolutional neural network classifier estimated probability that the lesion is a nevus. The neural network was trained on approximately 57,000 lesions that were classified and labeled by a dermatologist.+,++\n",
    "    'tbp_lv_norm_border',                # Border irregularity (0-10 scale); the normalized average of border jaggedness and asymmetry.+\n",
    "    'tbp_lv_norm_color',                 # Color variation (0-10 scale); the normalized average of color asymmetry and color irregularity.+\n",
    "    'tbp_lv_perimeterMM',                # Perimeter of lesion (mm).+\n",
    "    'tbp_lv_radial_color_std_max',       # Color asymmetry, a measure of asymmetry of the spatial distribution of color within the lesion. This score is calculated by looking at the average standard deviation in LAB* color space within concentric rings originating from the lesion center. Values range 0-10.+\n",
    "    'tbp_lv_stdL',                       # Standard deviation of L inside  lesion.+\n",
    "    'tbp_lv_stdLExt',                    # Standard deviation of L outside lesion.+\n",
    "    'tbp_lv_symm_2axis',                 # Border asymmetry; a measure of asymmetry of the lesion's contour about an axis perpendicular to the lesion's most symmetric axis. Lesions with two axes of symmetry will therefore have low scores (more symmetric), while lesions with only one or zero axes of symmetry will have higher scores (less symmetric). This score is calculated by comparing opposite halves of the lesion contour over many degrees of rotation. The angle where the halves are most similar identifies the principal axis of symmetry, while the second axis of symmetry is perpendicular to the principal axis. Border asymmetry is reported as the asymmetry value about this second axis. Values range 0-10.+\n",
    "    'tbp_lv_symm_2axis_angle',           # Lesion border asymmetry angle.+\n",
    "    'tbp_lv_x',                          # X-coordinate of the lesion on 3D TBP.+\n",
    "    'tbp_lv_y',                          # Y-coordinate of the lesion on 3D TBP.+\n",
    "    'tbp_lv_z',                          # Z-coordinate of the lesion on 3D TBP.+\n",
    "]\n",
    "\n",
    "new_num_cols = [\n",
    "    'lesion_size_ratio',                 # tbp_lv_minorAxisMM      / clin_size_long_diam_mm\n",
    "    'lesion_shape_index',                # tbp_lv_areaMM2          / tbp_lv_perimeterMM **2\n",
    "    'hue_contrast',                      # tbp_lv_H                - tbp_lv_Hext              abs\n",
    "    'luminance_contrast',                # tbp_lv_L                - tbp_lv_Lext              abs\n",
    "    'lesion_color_difference',           # tbp_lv_deltaA **2       + tbp_lv_deltaB **2 + tbp_lv_deltaL **2  sqrt  \n",
    "    'border_complexity',                 # tbp_lv_norm_border      + tbp_lv_symm_2axis\n",
    "    'color_uniformity',                  # tbp_lv_color_std_mean   / tbp_lv_radial_color_std_max\n",
    "\n",
    "    'position_distance_3d',              # tbp_lv_x **2 + tbp_lv_y **2 + tbp_lv_z **2  sqrt\n",
    "    'perimeter_to_area_ratio',           # tbp_lv_perimeterMM      / tbp_lv_areaMM2\n",
    "    'area_to_perimeter_ratio',           # tbp_lv_areaMM2          / tbp_lv_perimeterMM\n",
    "    'lesion_visibility_score',           # tbp_lv_deltaLBnorm      + tbp_lv_norm_color\n",
    "    'symmetry_border_consistency',       # tbp_lv_symm_2axis       * tbp_lv_norm_border\n",
    "    'consistency_symmetry_border',       # tbp_lv_symm_2axis       * tbp_lv_norm_border / (tbp_lv_symm_2axis + tbp_lv_norm_border)\n",
    "\n",
    "    'color_consistency',                 # tbp_lv_stdL             / tbp_lv_Lext\n",
    "    'consistency_color',                 # tbp_lv_stdL*tbp_lv_Lext / tbp_lv_stdL + tbp_lv_Lext\n",
    "    'size_age_interaction',              # clin_size_long_diam_mm  * age_approx\n",
    "    'hue_color_std_interaction',         # tbp_lv_H                * tbp_lv_color_std_mean\n",
    "    'lesion_severity_index',             # tbp_lv_norm_border      + tbp_lv_norm_color + tbp_lv_eccentricity / 3\n",
    "    'shape_complexity_index',            # border_complexity       + lesion_shape_index\n",
    "    'color_contrast_index',              # tbp_lv_deltaA + tbp_lv_deltaB + tbp_lv_deltaL + tbp_lv_deltaLBnorm\n",
    "\n",
    "    'log_lesion_area',                   # tbp_lv_areaMM2          + 1  np.log\n",
    "    'normalized_lesion_size',            # clin_size_long_diam_mm  / age_approx\n",
    "    'mean_hue_difference',               # tbp_lv_H                + tbp_lv_Hext    / 2\n",
    "    'std_dev_contrast',                  # tbp_lv_deltaA **2 + tbp_lv_deltaB **2 + tbp_lv_deltaL **2   / 3  np.sqrt\n",
    "    'color_shape_composite_index',       # tbp_lv_color_std_mean   + bp_lv_area_perim_ratio + tbp_lv_symm_2axis   / 3\n",
    "    'lesion_orientation_3d',             # tbp_lv_y                , tbp_lv_x  np.arctan2\n",
    "    'overall_color_difference',          # tbp_lv_deltaA           + tbp_lv_deltaB + tbp_lv_deltaL   / 3\n",
    "\n",
    "    'symmetry_perimeter_interaction',    # tbp_lv_symm_2axis       * tbp_lv_perimeterMM\n",
    "    'comprehensive_lesion_index',        # tbp_lv_area_perim_ratio + tbp_lv_eccentricity + bp_lv_norm_color + tbp_lv_symm_2axis   / 4\n",
    "    'color_variance_ratio',              # tbp_lv_color_std_mean   / tbp_lv_stdLExt\n",
    "    'border_color_interaction',          # tbp_lv_norm_border      * tbp_lv_norm_color\n",
    "    'border_color_interaction_2',\n",
    "    'size_color_contrast_ratio',         # clin_size_long_diam_mm  / tbp_lv_deltaLBnorm\n",
    "    'age_normalized_nevi_confidence',    # tbp_lv_nevi_confidence  / age_approx\n",
    "    'age_normalized_nevi_confidence_2',\n",
    "    'color_asymmetry_index',             # tbp_lv_symm_2axis       * tbp_lv_radial_color_std_max\n",
    "\n",
    "    'volume_approximation_3d',           # tbp_lv_areaMM2          * sqrt(tbp_lv_x**2 + tbp_lv_y**2 + tbp_lv_z**2)\n",
    "    'color_range',                       # abs(tbp_lv_L - tbp_lv_Lext) + abs(tbp_lv_A - tbp_lv_Aext) + abs(tbp_lv_B - tbp_lv_Bext)\n",
    "    'shape_color_consistency',           # tbp_lv_eccentricity     * tbp_lv_color_std_mean\n",
    "    'border_length_ratio',               # tbp_lv_perimeterMM      / pi * sqrt(tbp_lv_areaMM2 / pi)\n",
    "    'age_size_symmetry_index',           # age_approx              * clin_size_long_diam_mm * tbp_lv_symm_2axis\n",
    "    'index_age_size_symmetry',           # age_approx              * tbp_lv_areaMM2 * tbp_lv_symm_2axis\n",
    "]\n",
    "\n",
    "cat_cols = ['sex', 'anatom_site_general', 'tbp_tile_type', 'tbp_lv_location', 'tbp_lv_location_simple', 'attribution']\n",
    "norm_cols = [f'{col}_patient_norm' for col in num_cols + new_num_cols]\n",
    "special_cols = ['count_per_patient']\n",
    "image_cols = [\"target_3\",\"target_effnetv1b0\",\"target_eva02\",\"target_edgenext\"]\n",
    "#image_cols = [\"target_3\",\"target_effnetv1b0\"]\n",
    "\n",
    "#norm_cols += image_cols\n",
    "feature_cols = num_cols + new_num_cols + cat_cols + norm_cols + special_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "35a7bcff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:35:58.577587Z",
     "iopub.status.busy": "2024-09-02T19:35:58.577271Z",
     "iopub.status.idle": "2024-09-02T19:35:58.606959Z",
     "shell.execute_reply": "2024-09-02T19:35:58.606012Z"
    },
    "papermill": {
     "duration": 0.044731,
     "end_time": "2024-09-02T19:35:58.609170",
     "exception": false,
     "start_time": "2024-09-02T19:35:58.564439",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def read_data(path):\n",
    "    return (\n",
    "        pl.read_csv(path)\n",
    "        .with_columns(\n",
    "            pl.col('age_approx').cast(pl.String).replace('NA', np.nan).cast(pl.Float64),\n",
    "        )\n",
    "        .with_columns(\n",
    "            pl.col(pl.Float64).fill_nan(pl.col(pl.Float64).median()), # You may want to impute test data with train\n",
    "        )\n",
    "        .with_columns(\n",
    "            lesion_size_ratio              = pl.col('tbp_lv_minorAxisMM') / pl.col('clin_size_long_diam_mm'),\n",
    "            lesion_shape_index             = pl.col('tbp_lv_areaMM2') / (pl.col('tbp_lv_perimeterMM') ** 2),\n",
    "            hue_contrast                   = (pl.col('tbp_lv_H') - pl.col('tbp_lv_Hext')).abs(),\n",
    "            luminance_contrast             = (pl.col('tbp_lv_L') - pl.col('tbp_lv_Lext')).abs(),\n",
    "            lesion_color_difference        = (pl.col('tbp_lv_deltaA') ** 2 + pl.col('tbp_lv_deltaB') ** 2 + pl.col('tbp_lv_deltaL') ** 2).sqrt(),\n",
    "            border_complexity              = pl.col('tbp_lv_norm_border') + pl.col('tbp_lv_symm_2axis'),\n",
    "            color_uniformity               = pl.col('tbp_lv_color_std_mean') / (pl.col('tbp_lv_radial_color_std_max') + err),\n",
    "        )\n",
    "        .with_columns(\n",
    "            position_distance_3d           = (pl.col('tbp_lv_x') ** 2 + pl.col('tbp_lv_y') ** 2 + pl.col('tbp_lv_z') ** 2).sqrt(),\n",
    "            perimeter_to_area_ratio        = pl.col('tbp_lv_perimeterMM') / pl.col('tbp_lv_areaMM2'),\n",
    "            area_to_perimeter_ratio        = pl.col('tbp_lv_areaMM2') / pl.col('tbp_lv_perimeterMM'),\n",
    "            lesion_visibility_score        = pl.col('tbp_lv_deltaLBnorm') + pl.col('tbp_lv_norm_color'),\n",
    "            combined_anatomical_site       = pl.col('anatom_site_general') + '_' + pl.col('tbp_lv_location'),\n",
    "            symmetry_border_consistency    = pl.col('tbp_lv_symm_2axis') * pl.col('tbp_lv_norm_border'),\n",
    "            consistency_symmetry_border    = pl.col('tbp_lv_symm_2axis') * pl.col('tbp_lv_norm_border') / (pl.col('tbp_lv_symm_2axis') + pl.col('tbp_lv_norm_border')),\n",
    "        )\n",
    "        .with_columns(\n",
    "            color_consistency              = pl.col('tbp_lv_stdL') / pl.col('tbp_lv_Lext'),\n",
    "            consistency_color              = pl.col('tbp_lv_stdL') * pl.col('tbp_lv_Lext') / (pl.col('tbp_lv_stdL') + pl.col('tbp_lv_Lext')),\n",
    "            size_age_interaction           = pl.col('clin_size_long_diam_mm') * pl.col('age_approx'),\n",
    "            hue_color_std_interaction      = pl.col('tbp_lv_H') * pl.col('tbp_lv_color_std_mean'),\n",
    "            lesion_severity_index          = (pl.col('tbp_lv_norm_border') + pl.col('tbp_lv_norm_color') + pl.col('tbp_lv_eccentricity')) / 3,\n",
    "            shape_complexity_index         = pl.col('border_complexity') + pl.col('lesion_shape_index'),\n",
    "            color_contrast_index           = pl.col('tbp_lv_deltaA') + pl.col('tbp_lv_deltaB') + pl.col('tbp_lv_deltaL') + pl.col('tbp_lv_deltaLBnorm'),\n",
    "        )\n",
    "        .with_columns(\n",
    "            log_lesion_area                = (pl.col('tbp_lv_areaMM2') + 1).log(),\n",
    "            normalized_lesion_size         = pl.col('clin_size_long_diam_mm') / pl.col('age_approx'),\n",
    "            mean_hue_difference            = (pl.col('tbp_lv_H') + pl.col('tbp_lv_Hext')) / 2,\n",
    "            std_dev_contrast               = ((pl.col('tbp_lv_deltaA') ** 2 + pl.col('tbp_lv_deltaB') ** 2 + pl.col('tbp_lv_deltaL') ** 2) / 3).sqrt(),\n",
    "            color_shape_composite_index    = (pl.col('tbp_lv_color_std_mean') + pl.col('tbp_lv_area_perim_ratio') + pl.col('tbp_lv_symm_2axis')) / 3,\n",
    "            lesion_orientation_3d          = pl.arctan2(pl.col('tbp_lv_y'), pl.col('tbp_lv_x')),\n",
    "            overall_color_difference       = (pl.col('tbp_lv_deltaA') + pl.col('tbp_lv_deltaB') + pl.col('tbp_lv_deltaL')) / 3,\n",
    "        )\n",
    "        .with_columns(\n",
    "            symmetry_perimeter_interaction = pl.col('tbp_lv_symm_2axis') * pl.col('tbp_lv_perimeterMM'),\n",
    "            comprehensive_lesion_index     = (pl.col('tbp_lv_area_perim_ratio') + pl.col('tbp_lv_eccentricity') + pl.col('tbp_lv_norm_color') + pl.col('tbp_lv_symm_2axis')) / 4,\n",
    "            color_variance_ratio           = pl.col('tbp_lv_color_std_mean') / pl.col('tbp_lv_stdLExt'),\n",
    "            border_color_interaction       = pl.col('tbp_lv_norm_border') * pl.col('tbp_lv_norm_color'),\n",
    "            border_color_interaction_2     = pl.col('tbp_lv_norm_border') * pl.col('tbp_lv_norm_color') / (pl.col('tbp_lv_norm_border') + pl.col('tbp_lv_norm_color')),\n",
    "            size_color_contrast_ratio      = pl.col('clin_size_long_diam_mm') / pl.col('tbp_lv_deltaLBnorm'),\n",
    "            age_normalized_nevi_confidence = pl.col('tbp_lv_nevi_confidence') / pl.col('age_approx'),\n",
    "            age_normalized_nevi_confidence_2 = (pl.col('clin_size_long_diam_mm')**2 + pl.col('age_approx')**2).sqrt(),\n",
    "            color_asymmetry_index          = pl.col('tbp_lv_radial_color_std_max') * pl.col('tbp_lv_symm_2axis'),\n",
    "        )\n",
    "        .with_columns(\n",
    "            volume_approximation_3d        = pl.col('tbp_lv_areaMM2') * (pl.col('tbp_lv_x')**2 + pl.col('tbp_lv_y')**2 + pl.col('tbp_lv_z')**2).sqrt(),\n",
    "            color_range                    = (pl.col('tbp_lv_L') - pl.col('tbp_lv_Lext')).abs() + (pl.col('tbp_lv_A') - pl.col('tbp_lv_Aext')).abs() + (pl.col('tbp_lv_B') - pl.col('tbp_lv_Bext')).abs(),\n",
    "            shape_color_consistency        = pl.col('tbp_lv_eccentricity') * pl.col('tbp_lv_color_std_mean'),\n",
    "            border_length_ratio            = pl.col('tbp_lv_perimeterMM') / (2 * np.pi * (pl.col('tbp_lv_areaMM2') / np.pi).sqrt()),\n",
    "            age_size_symmetry_index        = pl.col('age_approx') * pl.col('clin_size_long_diam_mm') * pl.col('tbp_lv_symm_2axis'),\n",
    "            index_age_size_symmetry        = pl.col('age_approx') * pl.col('tbp_lv_areaMM2') * pl.col('tbp_lv_symm_2axis'),\n",
    "        )\n",
    "        .with_columns(\n",
    "            ((pl.col(col) - pl.col(col).mean().over('patient_id')) / (pl.col(col).std().over('patient_id') + err)).alias(f'{col}_patient_norm') for col in (num_cols + new_num_cols)\n",
    "        )\n",
    "        .with_columns(\n",
    "            count_per_patient = pl.col('isic_id').count().over('patient_id'),\n",
    "        )\n",
    "        .with_columns(\n",
    "            pl.col(cat_cols).cast(pl.Categorical),\n",
    "        )\n",
    "        .to_pandas()\n",
    "        .set_index(id_col)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f9d1855c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:35:58.636070Z",
     "iopub.status.busy": "2024-09-02T19:35:58.635133Z",
     "iopub.status.idle": "2024-09-02T19:35:58.650291Z",
     "shell.execute_reply": "2024-09-02T19:35:58.649387Z"
    },
    "papermill": {
     "duration": 0.031027,
     "end_time": "2024-09-02T19:35:58.652530",
     "exception": false,
     "start_time": "2024-09-02T19:35:58.621503",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def preprocess(df_train, df_test):\n",
    "    global cat_cols\n",
    "    \n",
    "    encoder = OneHotEncoder(sparse_output=False, dtype=np.int32, handle_unknown='ignore')\n",
    "    encoder.fit(df_train[cat_cols])\n",
    "    \n",
    "    new_cat_cols = [f'onehot_{i}' for i in range(len(encoder.get_feature_names_out()))]\n",
    "\n",
    "    df_train[new_cat_cols] = encoder.transform(df_train[cat_cols])\n",
    "    df_train[new_cat_cols] = df_train[new_cat_cols].astype('category')\n",
    "\n",
    "    df_test[new_cat_cols] = encoder.transform(df_test[cat_cols])\n",
    "    df_test[new_cat_cols] = df_test[new_cat_cols].astype('category')\n",
    "    \n",
    "    \n",
    "    # effnetv1b0\n",
    "    df_eff = pd.read_csv(\"/kaggle/input/isic-inference-effnetv1b0-for-training-data/train_effnetv1b0.csv\")\n",
    "  \n",
    "    df_train = df_train.reset_index(drop=True)\n",
    "    df_eff = df_eff.reset_index(drop=True)\n",
    "    \n",
    "    #upload effnetv1b0 train predictions values\n",
    "    df_train[\"target_effnetv1b0\"] = df_eff[\"target_effnetv1b0\"]\n",
    "    df_eff = pd.read_csv(\"submission_effnetv1b0.csv\")\n",
    "    \n",
    "    df_test = df_test.reset_index(drop=True)\n",
    "    df_eff = df_eff.reset_index(drop=True)\n",
    "    \n",
    "    #upload effnetv1b0 test predictions values\n",
    "    df_test[\"target_effnetv1b0\"] = df_eff[\"target\"]\n",
    "        \n",
    "    # target 3\n",
    "    df_image_3 = pd.read_csv(\"/kaggle/input/isic-2024-pl-submission-script-and-preds/train_preds.csv\")\n",
    "    \n",
    "    df_train = df_train.reset_index(drop=True)\n",
    "    df_image_3 = df_image_3.reset_index(drop=True)\n",
    "    \n",
    "    df_train[\"target_3\"] = df_image_3[\"pred\"]\n",
    "    \n",
    "    df_3 = pd.read_csv(\"submission_image3.csv\")\n",
    "    \n",
    "    df_test = df_test.reset_index(drop=True)\n",
    "    df_image_3 = df_image_3.reset_index(drop=True)\n",
    "    \n",
    "    df_test[\"target_3\"] = df_3[\"target\"]\n",
    "    \n",
    "    #eva02\n",
    "    df_eva = pd.read_csv(\"/kaggle/input/isic-inference-eva02-for-training-data/train_eva02.csv\")\n",
    "    \n",
    "    df_train = df_train.reset_index(drop=True)\n",
    "    df_eva = df_eva.reset_index(drop=True)\n",
    "    \n",
    "    df_eva = df_eva[[\"target_eva02\"]]\n",
    "    df_train[\"target_eva02\"] = df_eva[\"target_eva02\"]\n",
    "    \n",
    "    df_eva = pd.read_csv(\"submission_eva02.csv\")\n",
    "    \n",
    "    df_test = df_test.reset_index(drop=True)\n",
    "    df_eva = df_eva.reset_index(drop=True)\n",
    "    \n",
    "    df_test[\"target_eva02\"] = df_eva[\"target\"]\n",
    "    \n",
    "    #edgenext\n",
    "    df_ednx = pd.read_csv(\"/kaggle/input/isic-inference-edgenext-for-training-data/train_edgenext.csv\")\n",
    "    df_train = df_train.reset_index(drop=True)\n",
    "    df_ednx = df_ednx.reset_index(drop=True)\n",
    "    df_ednx = df_ednx[['target_edgenext']]\n",
    "    df_train['target_edgenext'] = df_ednx['target_edgenext']\n",
    "    df_ednx = pd.read_csv(\"submission_edgenext.csv\")\n",
    "    df_test = df_test.reset_index(drop=True)\n",
    "    df_ednx = df_ednx.reset_index(drop=True)\n",
    "    df_test [\"target_edgenext\"] = df_ednx['target']\n",
    "\n",
    "\n",
    "    for col in cat_cols:\n",
    "        feature_cols.remove(col)\n",
    "\n",
    "    feature_cols.extend(new_cat_cols)\n",
    "    cat_cols = new_cat_cols\n",
    "    \n",
    "    return df_train, df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5692e712",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:35:58.679008Z",
     "iopub.status.busy": "2024-09-02T19:35:58.678227Z",
     "iopub.status.idle": "2024-09-02T19:35:58.684808Z",
     "shell.execute_reply": "2024-09-02T19:35:58.683858Z"
    },
    "papermill": {
     "duration": 0.021881,
     "end_time": "2024-09-02T19:35:58.686943",
     "exception": false,
     "start_time": "2024-09-02T19:35:58.665062",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def custom_metric(estimator, X, y_true):\n",
    "    y_hat = estimator.predict_proba(X)[:, 1]\n",
    "    min_tpr = 0.80\n",
    "    max_fpr = abs(1 - min_tpr)\n",
    "    \n",
    "    v_gt = abs(y_true - 1)\n",
    "    v_pred = np.array([1.0 - x for x in y_hat])\n",
    "    \n",
    "    partial_auc_scaled = roc_auc_score(v_gt, v_pred, max_fpr=max_fpr)\n",
    "    partial_auc = 0.5 * max_fpr**2 + (max_fpr - 0.5 * max_fpr**2) / (1.0 - 0.5) * (partial_auc_scaled - 0.5)\n",
    "    \n",
    "    return partial_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "806c0680",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:35:58.712739Z",
     "iopub.status.busy": "2024-09-02T19:35:58.712395Z",
     "iopub.status.idle": "2024-09-02T19:36:30.728965Z",
     "shell.execute_reply": "2024-09-02T19:36:30.728109Z"
    },
    "papermill": {
     "duration": 32.032351,
     "end_time": "2024-09-02T19:36:30.731463",
     "exception": false,
     "start_time": "2024-09-02T19:35:58.699112",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_25/2140420280.py:17: DtypeWarning: Columns (51,52) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df_eff = pd.read_csv(\"/kaggle/input/isic-inference-effnetv1b0-for-training-data/train_effnetv1b0.csv\")\n",
      "/tmp/ipykernel_25/2140420280.py:48: DtypeWarning: Columns (51,52) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df_eva = pd.read_csv(\"/kaggle/input/isic-inference-eva02-for-training-data/train_eva02.csv\")\n",
      "/tmp/ipykernel_25/2140420280.py:64: DtypeWarning: Columns (51,52) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df_ednx = pd.read_csv(\"/kaggle/input/isic-inference-edgenext-for-training-data/train_edgenext.csv\")\n"
     ]
    }
   ],
   "source": [
    "df_train = read_data(train_path)\n",
    "df_test = read_data(test_path)\n",
    "df_subm = pd.read_csv(subm_path, index_col=id_col)\n",
    "\n",
    "df_train, df_test = preprocess(df_train, df_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1d9e60d",
   "metadata": {
    "papermill": {
     "duration": 0.012258,
     "end_time": "2024-09-02T19:36:30.756546",
     "exception": false,
     "start_time": "2024-09-02T19:36:30.744288",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "***drop least important features with feature importance(code at the last of the notebook before test predictions)***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7c5a18c7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:36:30.783167Z",
     "iopub.status.busy": "2024-09-02T19:36:30.782801Z",
     "iopub.status.idle": "2024-09-02T19:36:30.995476Z",
     "shell.execute_reply": "2024-09-02T19:36:30.994365Z"
    },
    "papermill": {
     "duration": 0.22885,
     "end_time": "2024-09-02T19:36:30.997883",
     "exception": false,
     "start_time": "2024-09-02T19:36:30.769033",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#they are detected at the first run\n",
    "least_important_features = ['onehot_32', 'onehot_6', 'onehot_33', 'onehot_30', 'onehot_26', 'onehot_22', 'onehot_36', 'onehot_4']\n",
    "#they are detected after the least_important_features are removed and it has increased cv score also so I add it\n",
    "#least_important_features_2 = ['onehot_17', 'onehot_42', 'onehot_29', 'onehot_13', 'onehot_25']\n",
    "#least_important_features += least_important_features_2\n",
    "df_train.drop(columns =least_important_features,inplace = True)\n",
    "for feature in least_important_features:\n",
    "    cat_cols.remove(feature)\n",
    "    feature_cols.remove(feature)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c0db85a",
   "metadata": {
    "papermill": {
     "duration": 0.012285,
     "end_time": "2024-09-02T19:36:31.022895",
     "exception": false,
     "start_time": "2024-09-02T19:36:31.010610",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# MODEL INITIALIZATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "11a82ce6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:36:31.049714Z",
     "iopub.status.busy": "2024-09-02T19:36:31.048968Z",
     "iopub.status.idle": "2024-09-02T19:36:31.055232Z",
     "shell.execute_reply": "2024-09-02T19:36:31.054342Z"
    },
    "papermill": {
     "duration": 0.021962,
     "end_time": "2024-09-02T19:36:31.057244",
     "exception": false,
     "start_time": "2024-09-02T19:36:31.035282",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "import copy\n",
    "\n",
    "feature_cols_without_image_cols = copy.copy(feature_cols)\n",
    "feature_cols += image_cols\n",
    "\n",
    "class SelectColumns(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, columns):\n",
    "        self.columns = columns\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X):\n",
    "        return X[self.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d7580597",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:36:31.083858Z",
     "iopub.status.busy": "2024-09-02T19:36:31.083518Z",
     "iopub.status.idle": "2024-09-02T19:36:31.091333Z",
     "shell.execute_reply": "2024-09-02T19:36:31.090352Z"
    },
    "papermill": {
     "duration": 0.023649,
     "end_time": "2024-09-02T19:36:31.093454",
     "exception": false,
     "start_time": "2024-09-02T19:36:31.069805",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "lgb_params = {\n",
    "    'objective':        'binary',\n",
    "    'verbosity':        -1,\n",
    "    'n_iter':           200,\n",
    "    'boosting_type':    'gbdt',\n",
    "    'random_state':     seed,\n",
    "    'lambda_l1':        0.08758718919397321, \n",
    "    'lambda_l2':        0.0039689175176025465, \n",
    "    'learning_rate':    0.03231007103195577, \n",
    "    'max_depth':        4, \n",
    "    'num_leaves':       103, \n",
    "    'colsample_bytree': 0.8329551585827726, \n",
    "    'colsample_bynode': 0.4025961355653304, \n",
    "    'bagging_fraction': 0.7738954452473223, \n",
    "    'bagging_freq':     4, \n",
    "    'min_data_in_leaf': 85, \n",
    "    'scale_pos_weight': 2.7984184778875543,\n",
    "}\n",
    "\n",
    "\n",
    "sampling_ratio = 0.01\n",
    "seed =42\n",
    "\n",
    "lgb_model = Pipeline([\n",
    "    ('sampler_1', RandomOverSampler(sampling_strategy= 0.003 , random_state=seed)),\n",
    "    ('sampler_2', RandomUnderSampler(sampling_strategy=sampling_ratio, random_state=seed)),\n",
    "    ('filter', SelectColumns(feature_cols_without_image_cols)),\n",
    "    ('classifier', lgb.LGBMClassifier(**lgb_params)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9104c076",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:36:31.120861Z",
     "iopub.status.busy": "2024-09-02T19:36:31.120115Z",
     "iopub.status.idle": "2024-09-02T19:36:31.129417Z",
     "shell.execute_reply": "2024-09-02T19:36:31.128454Z"
    },
    "papermill": {
     "duration": 0.025122,
     "end_time": "2024-09-02T19:36:31.131476",
     "exception": false,
     "start_time": "2024-09-02T19:36:31.106354",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cb_params = {\n",
    "    'loss_function':     'Logloss',\n",
    "    'iterations':        250,\n",
    "    'verbose':           False,\n",
    "    'random_state':      seed,\n",
    "    'max_depth':         7, \n",
    "    'learning_rate':     0.06936242010150652, \n",
    "    'scale_pos_weight':  2.6149345838209532, \n",
    "    'l2_leaf_reg':       6.216113851699493, \n",
    "    'subsample':         0.6249261779711819, \n",
    "    'min_data_in_leaf':  24,\n",
    "    'cat_features':      cat_cols,\n",
    "}\n",
    "cb_model = Pipeline([\n",
    "    ('sampler_1', RandomOverSampler(sampling_strategy= 0.003 , random_state=seed)),\n",
    "    ('sampler_2', RandomUnderSampler(sampling_strategy=sampling_ratio, random_state=seed)),\n",
    "    ('classifier', cb.CatBoostClassifier(**cb_params)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f9c19356",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:36:31.158769Z",
     "iopub.status.busy": "2024-09-02T19:36:31.157954Z",
     "iopub.status.idle": "2024-09-02T19:36:31.165057Z",
     "shell.execute_reply": "2024-09-02T19:36:31.163941Z"
    },
    "papermill": {
     "duration": 0.023053,
     "end_time": "2024-09-02T19:36:31.167115",
     "exception": false,
     "start_time": "2024-09-02T19:36:31.144062",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "xgb_params = {\n",
    "    'enable_categorical': True,\n",
    "    'tree_method':        'hist',\n",
    "    'random_state':       seed,\n",
    "    'learning_rate':      0.08501257473292347, \n",
    "    'lambda':             8.879624125465703, \n",
    "    'alpha':              0.6779926606782505, \n",
    "    'max_depth':          6, \n",
    "    'subsample':          0.6012681388711075, \n",
    "    'colsample_bytree':   0.8437772277074493, \n",
    "    'colsample_bylevel':  0.5476090898823716, \n",
    "    'colsample_bynode':   0.9928601203635129, \n",
    "    'scale_pos_weight':   3.29440313334688,\n",
    "}\n",
    "\n",
    "xgb_model = Pipeline([\n",
    "    ('sampler_1', RandomOverSampler(sampling_strategy= 0.003 , random_state=seed)),\n",
    "    ('sampler_2', RandomUnderSampler(sampling_strategy=sampling_ratio, random_state=seed)),\n",
    "    ('classifier', xgb.XGBClassifier(**xgb_params)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "dc613f90",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:36:31.195134Z",
     "iopub.status.busy": "2024-09-02T19:36:31.194788Z",
     "iopub.status.idle": "2024-09-02T19:36:31.199761Z",
     "shell.execute_reply": "2024-09-02T19:36:31.198789Z"
    },
    "papermill": {
     "duration": 0.021526,
     "end_time": "2024-09-02T19:36:31.201724",
     "exception": false,
     "start_time": "2024-09-02T19:36:31.180198",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "estimator = VotingClassifier([\n",
    "    ('lgb', lgb_model), ('cb', cb_model), ('xgb', xgb_model),\n",
    "], voting='soft',weights=[0.30,0.47,0.28])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f090a277",
   "metadata": {
    "papermill": {
     "duration": 0.012703,
     "end_time": "2024-09-02T19:36:31.227290",
     "exception": false,
     "start_time": "2024-09-02T19:36:31.214587",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# CROSS VALIDATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a18d9dec",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:36:31.254674Z",
     "iopub.status.busy": "2024-09-02T19:36:31.254192Z",
     "iopub.status.idle": "2024-09-02T19:40:16.335674Z",
     "shell.execute_reply": "2024-09-02T19:40:16.334619Z"
    },
    "papermill": {
     "duration": 225.112466,
     "end_time": "2024-09-02T19:40:16.352512",
     "exception": false,
     "start_time": "2024-09-02T19:36:31.240046",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n",
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n",
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n",
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n",
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.1906621698400448,\n",
       " array([0.19070906, 0.1896957 , 0.19616528, 0.18614882, 0.190592  ]))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = df_train[feature_cols]\n",
    "y = df_train[target_col]\n",
    "groups = df_train[group_col]\n",
    "cv = StratifiedGroupKFold(5, shuffle=True, random_state=seed)\n",
    "\n",
    "val_score = cross_val_score(\n",
    "    estimator=estimator, \n",
    "    X=X, y=y, \n",
    "    cv=cv, \n",
    "    groups=groups,\n",
    "    scoring=custom_metric,\n",
    ")\n",
    "\n",
    "np.mean(val_score), val_score\n",
    "\n",
    "\n",
    "# -------------------CV SCORES------------------------\n",
    "# RUS ==RandomUnderSampler(0.001) , ROS ==RandomOverSampler(0.003)\n",
    "# DROP NOTHING ----------------------> (0.18775) (RUS + image features(effnet+ target3))\n",
    "# DROP least_important_features -----> (0.18747) (RUS + image features(effnet+target3))\n",
    "# DROP NOTHING ----------------------> (0.17000) (ROS+RUS + no image features)\n",
    "# DROP least_important_features -----> (0.17124) (ROS+RUS + no image features)\n",
    "# DROP least_important_features+2 ---> (0.17160) (ROS+RUS + no image features)\n",
    "\n",
    "# edgenext (0.19038159621700457, array([0.19015561, 0.18941902, 0.19590514, 0.18606745, 0.19036077]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1bfcdb1f",
   "metadata": {
    "papermill": {
     "duration": 0.013035,
     "end_time": "2024-09-02T19:40:16.378968",
     "exception": false,
     "start_time": "2024-09-02T19:40:16.365933",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# HYPERPARAMETER TUNING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3d22d542",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:40:16.408559Z",
     "iopub.status.busy": "2024-09-02T19:40:16.408153Z",
     "iopub.status.idle": "2024-09-02T19:40:16.412645Z",
     "shell.execute_reply": "2024-09-02T19:40:16.411758Z"
    },
    "papermill": {
     "duration": 0.021834,
     "end_time": "2024-09-02T19:40:16.414853",
     "exception": false,
     "start_time": "2024-09-02T19:40:16.393019",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "DO_TUNING = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b09b24a7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:40:16.443363Z",
     "iopub.status.busy": "2024-09-02T19:40:16.442635Z",
     "iopub.status.idle": "2024-09-02T19:40:16.452718Z",
     "shell.execute_reply": "2024-09-02T19:40:16.451908Z"
    },
    "papermill": {
     "duration": 0.026328,
     "end_time": "2024-09-02T19:40:16.454658",
     "exception": false,
     "start_time": "2024-09-02T19:40:16.428330",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def lgb_objective(trial):\n",
    "    params = {\n",
    "        'objective':         'binary',\n",
    "        'verbosity':         -1,\n",
    "        'n_iter': 200,\n",
    "        'boosting_type':  'gbdt',\n",
    "        'lambda_l1':         trial.suggest_float('lambda_l1', 1e-3, 10.0, log=True),\n",
    "        'lambda_l2':         trial.suggest_float('lambda_l2', 1e-3, 10.0, log=True),\n",
    "        'learning_rate':     trial.suggest_float('learning_rate', 1e-2, 1e-1, log=True),\n",
    "        'max_depth':         trial.suggest_int('max_depth', 4, 8),\n",
    "        'num_leaves':        trial.suggest_int('num_leaves', 16, 256),\n",
    "        'colsample_bytree':  trial.suggest_float('colsample_bytree', 0.4, 1.0),\n",
    "        'colsample_bynode':  trial.suggest_float('colsample_bynode', 0.4, 1.0),\n",
    "        'bagging_fraction':  trial.suggest_float('bagging_fraction', 0.4, 1.0),\n",
    "        'bagging_freq':      trial.suggest_int('bagging_freq', 1, 7),\n",
    "        'min_data_in_leaf':  trial.suggest_int('min_data_in_leaf', 5, 100),\n",
    "        'scale_pos_weight' : trial.suggest_float('scale_pos_weight', 0.8, 4.0),\n",
    "    }\n",
    "\n",
    "    estimator = Pipeline([\n",
    "        ('sampler', RandomUnderSampler(sampling_strategy=sampling_ratio)),\n",
    "        ('classifier', lgb.LGBMClassifier(**params)),\n",
    "    ])\n",
    "\n",
    "    X = df_train[feature_cols]\n",
    "    y = df_train[target_col]\n",
    "    groups = df_train[group_col]\n",
    "    cv = StratifiedGroupKFold(5, shuffle=True)\n",
    "\n",
    "    val_score = cross_val_score(\n",
    "        estimator=estimator, \n",
    "        X=X, y=y, \n",
    "        cv=cv, \n",
    "        groups=groups,\n",
    "        scoring=custom_metric,\n",
    "    )\n",
    "\n",
    "    return np.mean(val_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "659b43ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:40:16.483329Z",
     "iopub.status.busy": "2024-09-02T19:40:16.482589Z",
     "iopub.status.idle": "2024-09-02T19:40:16.491444Z",
     "shell.execute_reply": "2024-09-02T19:40:16.490566Z"
    },
    "papermill": {
     "duration": 0.025069,
     "end_time": "2024-09-02T19:40:16.493456",
     "exception": false,
     "start_time": "2024-09-02T19:40:16.468387",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def cb_objective(trial):\n",
    "    params = {\n",
    "        'loss_function':     'Logloss',\n",
    "        'iterations':        200,\n",
    "        'verbose':           False,\n",
    "        'random_state':      seed,\n",
    "        'learning_rate':     trial.suggest_float('learning_rate', 1e-2, 1e-1, log=True),\n",
    "        'max_depth':         trial.suggest_int('max_depth', 4, 8),\n",
    "        'l2_leaf_reg':       trial.suggest_float('l2_leaf_reg', 1e-3, 10.0, log=True),\n",
    "        'subsample':         trial.suggest_float('subsample', 0.4, 1.0),\n",
    "        'colsample_bylevel': trial.suggest_float('colsample_bylevel', 0.4, 1.0),\n",
    "        'min_data_in_leaf':  trial.suggest_int('min_data_in_leaf', 5, 100),\n",
    "        'scale_pos_weight':  trial.suggest_float('scale_pos_weight', 0.8, 4.0),\n",
    "        'bootstrap_type':    'Bayesian',  # Optional: depending on your use case, you may want to tune this as well\n",
    "    }\n",
    "\n",
    "    estimator = Pipeline([\n",
    "        ('sampler', RandomUnderSampler(sampling_strategy=sampling_ratio)),\n",
    "        ('classifier', cb.CatBoostClassifier(**params)),\n",
    "    ])\n",
    "\n",
    "    X = df_train[feature_cols]\n",
    "    y = df_train[target_col]\n",
    "    groups = df_train[group_col]\n",
    "    cv = StratifiedGroupKFold(5, shuffle=True)\n",
    "\n",
    "    val_score = cross_val_score(\n",
    "        estimator=estimator, \n",
    "        X=X, y=y, \n",
    "        cv=cv, \n",
    "        groups=groups,\n",
    "        scoring=custom_metric,\n",
    "    )\n",
    "\n",
    "    return np.mean(val_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "10dffdc1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:40:16.582860Z",
     "iopub.status.busy": "2024-09-02T19:40:16.582118Z",
     "iopub.status.idle": "2024-09-02T19:40:16.591690Z",
     "shell.execute_reply": "2024-09-02T19:40:16.590810Z"
    },
    "papermill": {
     "duration": 0.086024,
     "end_time": "2024-09-02T19:40:16.593617",
     "exception": false,
     "start_time": "2024-09-02T19:40:16.507593",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def xgb_objective(trial):\n",
    "    params = {\n",
    "        'objective':          'binary:logistic',\n",
    "        'n_estimators':       200,\n",
    "        'tree_method':        'hist',\n",
    "        'random_state':       seed,\n",
    "        'learning_rate':      trial.suggest_float('learning_rate', 1e-2, 1e-1, log=True),\n",
    "        'max_depth':          trial.suggest_int('max_depth', 4, 8),\n",
    "        'lambda':             trial.suggest_float('lambda', 1e-3, 10.0, log=True),\n",
    "        'alpha':              trial.suggest_float('alpha', 1e-3, 10.0, log=True),\n",
    "        'subsample':          trial.suggest_float('subsample', 0.4, 1.0),\n",
    "        'colsample_bytree':   trial.suggest_float('colsample_bytree', 0.4, 1.0),\n",
    "        'colsample_bynode':   trial.suggest_float('colsample_bynode', 0.4, 1.0),\n",
    "        'scale_pos_weight':   trial.suggest_float('scale_pos_weight', 0.8, 4.0),\n",
    "    }\n",
    "\n",
    "    estimator = Pipeline([\n",
    "        ('sampler', RandomUnderSampler(sampling_strategy=sampling_ratio)),\n",
    "        ('classifier', xgb.XGBClassifier(**params)),\n",
    "    ])\n",
    "\n",
    "    X = df_train[feature_cols]\n",
    "    y = df_train[target_col]\n",
    "    groups = df_train[group_col]\n",
    "    cv = StratifiedGroupKFold(5, shuffle=True)\n",
    "\n",
    "    val_score = cross_val_score(\n",
    "        estimator=estimator, \n",
    "        X=X, y=y, \n",
    "        cv=cv, \n",
    "        groups=groups,\n",
    "        scoring=custom_metric,\n",
    "    )\n",
    "\n",
    "    return np.mean(val_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c0927792",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:40:16.622951Z",
     "iopub.status.busy": "2024-09-02T19:40:16.622265Z",
     "iopub.status.idle": "2024-09-02T19:40:16.627581Z",
     "shell.execute_reply": "2024-09-02T19:40:16.626749Z"
    },
    "papermill": {
     "duration": 0.021742,
     "end_time": "2024-09-02T19:40:16.629518",
     "exception": false,
     "start_time": "2024-09-02T19:40:16.607776",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_TUNING:\n",
    "    # LightGBM\n",
    "    start_time = time.time()\n",
    "    study_lgb = optuna.create_study(direction='maximize', sampler=TPESampler(seed=seed))\n",
    "    study_lgb.optimize(objective_lgb, n_trials=100)\n",
    "    end_time = time.time()\n",
    "    elapsed_time_lgb = end_time - start_time\n",
    "    print(f\"LightGBM tuning took {elapsed_time_lgb:.2f} seconds.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a0efce6c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:40:16.662026Z",
     "iopub.status.busy": "2024-09-02T19:40:16.661352Z",
     "iopub.status.idle": "2024-09-02T19:40:16.666894Z",
     "shell.execute_reply": "2024-09-02T19:40:16.666039Z"
    },
    "papermill": {
     "duration": 0.023922,
     "end_time": "2024-09-02T19:40:16.668991",
     "exception": false,
     "start_time": "2024-09-02T19:40:16.645069",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_TUNING:\n",
    "    # CatBoost\n",
    "    start_time = time.time()\n",
    "    study_cb = optuna.create_study(direction='maximize', sampler=TPESampler(seed=seed))\n",
    "    study_cb.optimize(objective_cb, n_trials=100)\n",
    "    end_time = time.time()\n",
    "    elapsed_time_cb = end_time - start_time\n",
    "    print(f\"CatBoost tuning took {elapsed_time_cb:.2f} seconds.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "4907f201",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:40:16.698010Z",
     "iopub.status.busy": "2024-09-02T19:40:16.697313Z",
     "iopub.status.idle": "2024-09-02T19:40:16.703080Z",
     "shell.execute_reply": "2024-09-02T19:40:16.702133Z"
    },
    "papermill": {
     "duration": 0.022437,
     "end_time": "2024-09-02T19:40:16.705072",
     "exception": false,
     "start_time": "2024-09-02T19:40:16.682635",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_TUNING:\n",
    "    # XGBoost\n",
    "    start_time = time.time()\n",
    "    study_xgb = optuna.create_study(direction='maximize', sampler=TPESampler(seed=seed))\n",
    "    study_xgb.optimize(objective_xgb, n_trials=100)\n",
    "    end_time = time.time()\n",
    "    elapsed_time_xgb = end_time - start_time\n",
    "    print(f\"XGBoost tuning took {elapsed_time_xgb:.2f} seconds.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "408ade24",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:40:16.734382Z",
     "iopub.status.busy": "2024-09-02T19:40:16.733573Z",
     "iopub.status.idle": "2024-09-02T19:40:16.738359Z",
     "shell.execute_reply": "2024-09-02T19:40:16.737515Z"
    },
    "papermill": {
     "duration": 0.02165,
     "end_time": "2024-09-02T19:40:16.740380",
     "exception": false,
     "start_time": "2024-09-02T19:40:16.718730",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_TUNING:\n",
    "    # Print best parameters for each study\n",
    "    print(\"Best LGBM trial:\", study_lgb.best_trial)\n",
    "    print(\"Best CatBoost trial:\", study_cb.best_trial)\n",
    "    print(\"Best XGBoost trial:\", study_xgb.best_trial)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cee8a135",
   "metadata": {
    "papermill": {
     "duration": 0.013052,
     "end_time": "2024-09-02T19:40:16.767238",
     "exception": false,
     "start_time": "2024-09-02T19:40:16.754186",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# TRAINING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6c283886",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:40:16.795887Z",
     "iopub.status.busy": "2024-09-02T19:40:16.795247Z",
     "iopub.status.idle": "2024-09-02T19:41:07.929663Z",
     "shell.execute_reply": "2024-09-02T19:41:07.928639Z"
    },
    "papermill": {
     "duration": 51.165294,
     "end_time": "2024-09-02T19:41:07.946011",
     "exception": false,
     "start_time": "2024-09-02T19:40:16.780717",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:172: UserWarning: Found `n_iter` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>VotingClassifier(estimators=[(&#x27;lgb&#x27;,\n",
       "                              Pipeline(steps=[(&#x27;sampler_1&#x27;,\n",
       "                                               RandomOverSampler(random_state=42,\n",
       "                                                                 sampling_strategy=0.003)),\n",
       "                                              (&#x27;sampler_2&#x27;,\n",
       "                                               RandomUnderSampler(random_state=42,\n",
       "                                                                  sampling_strategy=0.01)),\n",
       "                                              (&#x27;filter&#x27;,\n",
       "                                               SelectColumns(columns=[&#x27;age_approx&#x27;,\n",
       "                                                                      &#x27;clin_size_long_diam_mm&#x27;,\n",
       "                                                                      &#x27;tbp_lv_A&#x27;,\n",
       "                                                                      &#x27;tbp_lv_Aext&#x27;,\n",
       "                                                                      &#x27;tbp_lv_B&#x27;,\n",
       "                                                                      &#x27;tbp_lv_Bext&#x27;,\n",
       "                                                                      &#x27;tbp_lv_C&#x27;,\n",
       "                                                                      &#x27;tbp_lv_Cext&#x27;,\n",
       "                                                                      &#x27;tbp_l...\n",
       "                                                             interaction_constraints=None,\n",
       "                                                             lambda=8.879624125465703,\n",
       "                                                             learning_rate=0.08501257473292347,\n",
       "                                                             max_bin=None,\n",
       "                                                             max_cat_threshold=None,\n",
       "                                                             max_cat_to_onehot=None,\n",
       "                                                             max_delta_step=None,\n",
       "                                                             max_depth=6,\n",
       "                                                             max_leaves=None,\n",
       "                                                             min_child_weight=None,\n",
       "                                                             missing=nan,\n",
       "                                                             monotone_constraints=None,\n",
       "                                                             multi_strategy=None,\n",
       "                                                             n_estimators=None,\n",
       "                                                             n_jobs=None, ...))]))],\n",
       "                 voting=&#x27;soft&#x27;, weights=[0.3, 0.47, 0.28])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">VotingClassifier</label><div class=\"sk-toggleable__content\"><pre>VotingClassifier(estimators=[(&#x27;lgb&#x27;,\n",
       "                              Pipeline(steps=[(&#x27;sampler_1&#x27;,\n",
       "                                               RandomOverSampler(random_state=42,\n",
       "                                                                 sampling_strategy=0.003)),\n",
       "                                              (&#x27;sampler_2&#x27;,\n",
       "                                               RandomUnderSampler(random_state=42,\n",
       "                                                                  sampling_strategy=0.01)),\n",
       "                                              (&#x27;filter&#x27;,\n",
       "                                               SelectColumns(columns=[&#x27;age_approx&#x27;,\n",
       "                                                                      &#x27;clin_size_long_diam_mm&#x27;,\n",
       "                                                                      &#x27;tbp_lv_A&#x27;,\n",
       "                                                                      &#x27;tbp_lv_Aext&#x27;,\n",
       "                                                                      &#x27;tbp_lv_B&#x27;,\n",
       "                                                                      &#x27;tbp_lv_Bext&#x27;,\n",
       "                                                                      &#x27;tbp_lv_C&#x27;,\n",
       "                                                                      &#x27;tbp_lv_Cext&#x27;,\n",
       "                                                                      &#x27;tbp_l...\n",
       "                                                             interaction_constraints=None,\n",
       "                                                             lambda=8.879624125465703,\n",
       "                                                             learning_rate=0.08501257473292347,\n",
       "                                                             max_bin=None,\n",
       "                                                             max_cat_threshold=None,\n",
       "                                                             max_cat_to_onehot=None,\n",
       "                                                             max_delta_step=None,\n",
       "                                                             max_depth=6,\n",
       "                                                             max_leaves=None,\n",
       "                                                             min_child_weight=None,\n",
       "                                                             missing=nan,\n",
       "                                                             monotone_constraints=None,\n",
       "                                                             multi_strategy=None,\n",
       "                                                             n_estimators=None,\n",
       "                                                             n_jobs=None, ...))]))],\n",
       "                 voting=&#x27;soft&#x27;, weights=[0.3, 0.47, 0.28])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><label>lgb</label></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomOverSampler</label><div class=\"sk-toggleable__content\"><pre>RandomOverSampler(random_state=42, sampling_strategy=0.003)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomUnderSampler</label><div class=\"sk-toggleable__content\"><pre>RandomUnderSampler(random_state=42, sampling_strategy=0.01)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SelectColumns</label><div class=\"sk-toggleable__content\"><pre>SelectColumns(columns=[&#x27;age_approx&#x27;, &#x27;clin_size_long_diam_mm&#x27;, &#x27;tbp_lv_A&#x27;,\n",
       "                       &#x27;tbp_lv_Aext&#x27;, &#x27;tbp_lv_B&#x27;, &#x27;tbp_lv_Bext&#x27;, &#x27;tbp_lv_C&#x27;,\n",
       "                       &#x27;tbp_lv_Cext&#x27;, &#x27;tbp_lv_H&#x27;, &#x27;tbp_lv_Hext&#x27;, &#x27;tbp_lv_L&#x27;,\n",
       "                       &#x27;tbp_lv_Lext&#x27;, &#x27;tbp_lv_areaMM2&#x27;,\n",
       "                       &#x27;tbp_lv_area_perim_ratio&#x27;, &#x27;tbp_lv_color_std_mean&#x27;,\n",
       "                       &#x27;tbp_lv_deltaA&#x27;, &#x27;tbp_lv_deltaB&#x27;, &#x27;tbp_lv_deltaL&#x27;,\n",
       "                       &#x27;tbp_lv_deltaLB&#x27;, &#x27;tbp_lv_deltaLBnorm&#x27;,\n",
       "                       &#x27;tbp_lv_eccentricity&#x27;, &#x27;tbp_lv_minorAxisMM&#x27;,\n",
       "                       &#x27;tbp_lv_nevi_confidence&#x27;, &#x27;tbp_lv_norm_border&#x27;,\n",
       "                       &#x27;tbp_lv_norm_color&#x27;, &#x27;tbp_lv_perimeterMM&#x27;,\n",
       "                       &#x27;tbp_lv_radial_color_std_max&#x27;, &#x27;tbp_lv_stdL&#x27;,\n",
       "                       &#x27;tbp_lv_stdLExt&#x27;, &#x27;tbp_lv_symm_2axis&#x27;, ...])</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LGBMClassifier</label><div class=\"sk-toggleable__content\"><pre>LGBMClassifier(bagging_fraction=0.7738954452473223, bagging_freq=4,\n",
       "               colsample_bynode=0.4025961355653304,\n",
       "               colsample_bytree=0.8329551585827726,\n",
       "               lambda_l1=0.08758718919397321, lambda_l2=0.0039689175176025465,\n",
       "               learning_rate=0.03231007103195577, max_depth=4,\n",
       "               min_data_in_leaf=85, n_iter=200, num_leaves=103,\n",
       "               objective=&#x27;binary&#x27;, random_state=42,\n",
       "               scale_pos_weight=2.7984184778875543, verbosity=-1)</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><label>cb</label></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomOverSampler</label><div class=\"sk-toggleable__content\"><pre>RandomOverSampler(random_state=42, sampling_strategy=0.003)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomUnderSampler</label><div class=\"sk-toggleable__content\"><pre>RandomUnderSampler(random_state=42, sampling_strategy=0.01)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" ><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">CatBoostClassifier</label><div class=\"sk-toggleable__content\"><pre>&lt;catboost.core.CatBoostClassifier object at 0x7af9ee7c55d0&gt;</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><label>xgb</label></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-9\" type=\"checkbox\" ><label for=\"sk-estimator-id-9\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomOverSampler</label><div class=\"sk-toggleable__content\"><pre>RandomOverSampler(random_state=42, sampling_strategy=0.003)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-10\" type=\"checkbox\" ><label for=\"sk-estimator-id-10\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomUnderSampler</label><div class=\"sk-toggleable__content\"><pre>RandomUnderSampler(random_state=42, sampling_strategy=0.01)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-11\" type=\"checkbox\" ><label for=\"sk-estimator-id-11\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(alpha=0.6779926606782505, base_score=None, booster=None,\n",
       "              callbacks=None, colsample_bylevel=0.5476090898823716,\n",
       "              colsample_bynode=0.9928601203635129,\n",
       "              colsample_bytree=0.8437772277074493, device=None,\n",
       "              early_stopping_rounds=None, enable_categorical=True,\n",
       "              eval_metric=None, feature_types=None, gamma=None,\n",
       "              grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, lambda=8.879624125465703,\n",
       "              learning_rate=0.08501257473292347, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=6, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              multi_strategy=None, n_estimators=None, n_jobs=None, ...)</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "VotingClassifier(estimators=[('lgb',\n",
       "                              Pipeline(steps=[('sampler_1',\n",
       "                                               RandomOverSampler(random_state=42,\n",
       "                                                                 sampling_strategy=0.003)),\n",
       "                                              ('sampler_2',\n",
       "                                               RandomUnderSampler(random_state=42,\n",
       "                                                                  sampling_strategy=0.01)),\n",
       "                                              ('filter',\n",
       "                                               SelectColumns(columns=['age_approx',\n",
       "                                                                      'clin_size_long_diam_mm',\n",
       "                                                                      'tbp_lv_A',\n",
       "                                                                      'tbp_lv_Aext',\n",
       "                                                                      'tbp_lv_B',\n",
       "                                                                      'tbp_lv_Bext',\n",
       "                                                                      'tbp_lv_C',\n",
       "                                                                      'tbp_lv_Cext',\n",
       "                                                                      'tbp_l...\n",
       "                                                             interaction_constraints=None,\n",
       "                                                             lambda=8.879624125465703,\n",
       "                                                             learning_rate=0.08501257473292347,\n",
       "                                                             max_bin=None,\n",
       "                                                             max_cat_threshold=None,\n",
       "                                                             max_cat_to_onehot=None,\n",
       "                                                             max_delta_step=None,\n",
       "                                                             max_depth=6,\n",
       "                                                             max_leaves=None,\n",
       "                                                             min_child_weight=None,\n",
       "                                                             missing=nan,\n",
       "                                                             monotone_constraints=None,\n",
       "                                                             multi_strategy=None,\n",
       "                                                             n_estimators=None,\n",
       "                                                             n_jobs=None, ...))]))],\n",
       "                 voting='soft', weights=[0.3, 0.47, 0.28])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = df_train[feature_cols], df_train[target_col]\n",
    "\n",
    "estimator.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c009b5cd",
   "metadata": {
    "papermill": {
     "duration": 0.015915,
     "end_time": "2024-09-02T19:41:07.976545",
     "exception": false,
     "start_time": "2024-09-02T19:41:07.960630",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# LOOKING FOR FEATURE IMPORTANCE(lgb + xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "bd4e7c20",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:41:08.009395Z",
     "iopub.status.busy": "2024-09-02T19:41:08.008215Z",
     "iopub.status.idle": "2024-09-02T19:41:08.013206Z",
     "shell.execute_reply": "2024-09-02T19:41:08.012250Z"
    },
    "papermill": {
     "duration": 0.023318,
     "end_time": "2024-09-02T19:41:08.015426",
     "exception": false,
     "start_time": "2024-09-02T19:41:07.992108",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "DO_FEATURE_IMPORTANCE_MODELS = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4de8d3e6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:41:08.046550Z",
     "iopub.status.busy": "2024-09-02T19:41:08.045895Z",
     "iopub.status.idle": "2024-09-02T19:41:08.052960Z",
     "shell.execute_reply": "2024-09-02T19:41:08.051835Z"
    },
    "papermill": {
     "duration": 0.024986,
     "end_time": "2024-09-02T19:41:08.055099",
     "exception": false,
     "start_time": "2024-09-02T19:41:08.030113",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_FEATURE_IMPORTANCE_MODELS:\n",
    "    lgb_model = estimator.named_estimators_['lgb'].named_steps['classifier']\n",
    "    lgb_feature_importance = lgb_model.booster_.feature_importance(importance_type='gain')\n",
    "    lgb_feature_importance_df = pd.DataFrame({\n",
    "        'feature': X.columns,\n",
    "        'importance': lgb_feature_importance\n",
    "    }).sort_values(by='importance', ascending=False)\n",
    "\n",
    "\n",
    "    xgb_model = estimator.named_estimators_['xgb'].named_steps['classifier']\n",
    "    xgb_feature_importance = xgb_model.get_booster().get_score(importance_type='weight')\n",
    "    xgb_feature_importance_df = pd.DataFrame({\n",
    "        'feature': xgb_feature_importance.keys(),\n",
    "        'importance': xgb_feature_importance.values()\n",
    "    }).sort_values(by='importance', ascending=False)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "03a52b2c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:41:08.085103Z",
     "iopub.status.busy": "2024-09-02T19:41:08.084786Z",
     "iopub.status.idle": "2024-09-02T19:41:08.089204Z",
     "shell.execute_reply": "2024-09-02T19:41:08.088354Z"
    },
    "papermill": {
     "duration": 0.021962,
     "end_time": "2024-09-02T19:41:08.091244",
     "exception": false,
     "start_time": "2024-09-02T19:41:08.069282",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_FEATURE_IMPORTANCE_MODELS:\n",
    "\n",
    "    print(lgb_feature_importance_df)\n",
    "    print(xgb_feature_importance_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfb875c3",
   "metadata": {
    "papermill": {
     "duration": 0.014454,
     "end_time": "2024-09-02T19:41:08.120516",
     "exception": false,
     "start_time": "2024-09-02T19:41:08.106062",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# LEAST IMPORTANT FEATURES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f1a780ce",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:41:08.162797Z",
     "iopub.status.busy": "2024-09-02T19:41:08.162319Z",
     "iopub.status.idle": "2024-09-02T19:41:08.169589Z",
     "shell.execute_reply": "2024-09-02T19:41:08.168378Z"
    },
    "papermill": {
     "duration": 0.031684,
     "end_time": "2024-09-02T19:41:08.171700",
     "exception": false,
     "start_time": "2024-09-02T19:41:08.140016",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_FEATURE_IMPORTANCE_MODELS:\n",
    "\n",
    "    # Assuming lgb_feature_importance_df is already created and contains the feature importances\n",
    "    least_important_lgb = lgb_feature_importance_df.sort_values(by='importance').head(24)\n",
    "\n",
    "    print(\"Least Important Features in LightGBM:\")\n",
    "    print(least_important_lgb)\n",
    "\n",
    "    # Convert the xgb_feature_importance to a DataFrame for easier manipulation\n",
    "    least_important_xgb = xgb_feature_importance_df.sort_values(by = \"importance\").head(6)\n",
    "\n",
    "\n",
    "    print(\"\\nLeast Important Features in XGBoost:\")\n",
    "    print(least_important_xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2968113c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:41:08.210562Z",
     "iopub.status.busy": "2024-09-02T19:41:08.209844Z",
     "iopub.status.idle": "2024-09-02T19:41:08.215685Z",
     "shell.execute_reply": "2024-09-02T19:41:08.214701Z"
    },
    "papermill": {
     "duration": 0.026204,
     "end_time": "2024-09-02T19:41:08.217698",
     "exception": false,
     "start_time": "2024-09-02T19:41:08.191494",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_FEATURE_IMPORTANCE_MODELS:\n",
    "\n",
    "    # Extract the least important feature names from both LightGBM and XGBoost\n",
    "    least_important_lgb_features = least_important_lgb['feature'].tolist()\n",
    "    least_important_xgb_features = least_important_xgb['feature'].tolist()\n",
    "\n",
    "    # Find the intersection of the two lists\n",
    "    common_least_important_features = list(set(least_important_lgb_features) & set(least_important_xgb_features))\n",
    "\n",
    "    print(\"Common Least Important Features in Both LightGBM and XGBoost:\")\n",
    "    print(common_least_important_features)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93cd4c87",
   "metadata": {
    "papermill": {
     "duration": 0.014769,
     "end_time": "2024-09-02T19:41:08.247487",
     "exception": false,
     "start_time": "2024-09-02T19:41:08.232718",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# LOOKING FOR FEATURE IMPORTANCE(Tests)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "7aed376b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:41:08.278698Z",
     "iopub.status.busy": "2024-09-02T19:41:08.278277Z",
     "iopub.status.idle": "2024-09-02T19:41:08.282682Z",
     "shell.execute_reply": "2024-09-02T19:41:08.281821Z"
    },
    "papermill": {
     "duration": 0.022482,
     "end_time": "2024-09-02T19:41:08.284714",
     "exception": false,
     "start_time": "2024-09-02T19:41:08.262232",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "DO_FEATURE_IMPORTANCE_TEST = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "2d18f632",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:41:08.316455Z",
     "iopub.status.busy": "2024-09-02T19:41:08.315547Z",
     "iopub.status.idle": "2024-09-02T19:41:08.327964Z",
     "shell.execute_reply": "2024-09-02T19:41:08.326973Z"
    },
    "papermill": {
     "duration": 0.030823,
     "end_time": "2024-09-02T19:41:08.330368",
     "exception": false,
     "start_time": "2024-09-02T19:41:08.299545",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if DO_FEATURE_IMPORTANCE_TEST:\n",
    "    X = df_train[feature_cols]\n",
    "    y = df_train[target_col]\n",
    "\n",
    "    # Separate continuous and categorical features\n",
    "    continuous_features = num_cols + norm_cols + new_num_cols\n",
    "    # Fill null values of continuous features with their median values\n",
    "    X[continuous_features] = X[continuous_features].apply(lambda x: x.fillna(x.median()))\n",
    "\n",
    "\n",
    "    # Correlation Matrix for continuous features\n",
    "    corr_matrix = df_train[continuous_features + ['target']].corr()\n",
    "    threshold = 0.01\n",
    "    relevant_features_corr = corr_matrix[abs(corr_matrix['target']) > threshold].index\n",
    "    selected_features_corr = relevant_features_corr.drop('target')\n",
    "    print(\"Selected continuous features based on correlation threshold:\")\n",
    "    print(selected_features_corr)\n",
    "    print(len(selected_features_corr))\n",
    "\n",
    "    # Chi-Square Test for categorical features\n",
    "    chi2_selector = SelectKBest(chi2, k=15)\n",
    "    chi2_selector.fit_transform(X[cat_cols], y)\n",
    "    selected_features_chi2 = X[cat_cols].columns[chi2_selector.get_support()]\n",
    "    print(\"Selected categorical features based on Chi-Square Test:\")\n",
    "    print(selected_features_chi2)\n",
    "\n",
    "    # Mutual Information for all features\n",
    "    mi_selector = SelectKBest(mutual_info_classif, k=15)\n",
    "    mi_selector.fit_transform(X, y)\n",
    "    selected_features_mi = X.columns[mi_selector.get_support()]\n",
    "    print(\"Selected features based on Mutual Information:\")\n",
    "    print(selected_features_mi)\n",
    "\n",
    "    # Variance Threshold for continuous features\n",
    "    threshold = 0.05\n",
    "    var_threshold = VarianceThreshold(threshold=threshold)\n",
    "    var_threshold.fit_transform(X[continuous_features])\n",
    "    selected_features_var = X[continuous_features].columns[var_threshold.get_support()]\n",
    "    print(\"Selected continuous features based on Variance Threshold:\")\n",
    "    print(selected_features_var)\n",
    "    print(len(selected_features_var))\n",
    "\n",
    "\n",
    "    # Combine all selected features\n",
    "    selected_features_all = set(selected_features_corr) | set(selected_features_chi2) | set(selected_features_mi) | set(selected_features_var)\n",
    "\n",
    "    # Original features\n",
    "    original_features = set(X.columns)\n",
    "\n",
    "    # Find features not selected by any method\n",
    "    least_selected_features = original_features - selected_features_all\n",
    "\n",
    "    boosting_selected_features = set()\n",
    "\n",
    "    least_selected_features_list = list(least_selected_features | boosting_selected_features)\n",
    "\n",
    "    print(least_selected_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "820a0e17",
   "metadata": {
    "papermill": {
     "duration": 0.014708,
     "end_time": "2024-09-02T19:41:08.360232",
     "exception": false,
     "start_time": "2024-09-02T19:41:08.345524",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# TEST PREDICTION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e13aa455",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-02T19:41:08.392167Z",
     "iopub.status.busy": "2024-09-02T19:41:08.391791Z",
     "iopub.status.idle": "2024-09-02T19:41:08.509103Z",
     "shell.execute_reply": "2024-09-02T19:41:08.508073Z"
    },
    "papermill": {
     "duration": 0.13602,
     "end_time": "2024-09-02T19:41:08.511816",
     "exception": false,
     "start_time": "2024-09-02T19:41:08.375796",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>isic_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ISIC_0015657</th>\n",
       "      <td>0.230463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ISIC_0015729</th>\n",
       "      <td>0.211698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ISIC_0015740</th>\n",
       "      <td>0.234048</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                target\n",
       "isic_id               \n",
       "ISIC_0015657  0.230463\n",
       "ISIC_0015729  0.211698\n",
       "ISIC_0015740  0.234048"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_subm['target'] = estimator.predict_proba(df_test[feature_cols])[:, 1]\n",
    "\n",
    "df_subm.to_csv('submission.csv')\n",
    "df_subm.head()"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "databundleVersionId": 9094797,
     "sourceId": 63056,
     "sourceType": "competition"
    },
    {
     "datasetId": 5406640,
     "sourceId": 8982084,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 5415918,
     "sourceId": 8991790,
     "sourceType": "datasetVersion"
    },
    {
     "sourceId": 186147615,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 186149019,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 187730674,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 188543089,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 188543756,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 188602899,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 188603204,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 188603902,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 190801726,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 190815443,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 190817588,
     "sourceType": "kernelVersion"
    }
   ],
   "dockerImageVersionId": 30747,
   "isGpuEnabled": true,
   "isInternetEnabled": false,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 378.213658,
   "end_time": "2024-09-02T19:41:09.453078",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-09-02T19:34:51.239420",
   "version": "2.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}