amex-ANN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b99ddfe9",
   "metadata": {
    "_cell_guid": "dcb2670b-869a-474d-b788-5c77bdbda89a",
    "_uuid": "ec0a3025-123f-4845-8547-11373afbdc3d",
    "execution": {
     "iopub.execute_input": "2022-08-01T17:49:12.003756Z",
     "iopub.status.busy": "2022-08-01T17:49:12.003255Z",
     "iopub.status.idle": "2022-08-01T17:49:19.104023Z",
     "shell.execute_reply": "2022-08-01T17:49:19.103234Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 7.112274,
     "end_time": "2022-08-01T17:49:19.106732",
     "exception": false,
     "start_time": "2022-08-01T17:49:11.994458",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "import gc\n",
    "import os\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import time\n",
    "from tensorflow import keras\n",
    "import tensorflow as tf\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler, MaxAbsScaler, PowerTransformer\n",
    "import joblib "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02a5873f",
   "metadata": {
    "papermill": {
     "duration": 0.00575,
     "end_time": "2022-08-01T17:49:19.119000",
     "exception": false,
     "start_time": "2022-08-01T17:49:19.113250",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Constants:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb7d937b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-08-01T17:49:19.132724Z",
     "iopub.status.busy": "2022-08-01T17:49:19.131822Z",
     "iopub.status.idle": "2022-08-01T17:49:19.136993Z",
     "shell.execute_reply": "2022-08-01T17:49:19.136294Z"
    },
    "papermill": {
     "duration": 0.013693,
     "end_time": "2022-08-01T17:49:19.138569",
     "exception": false,
     "start_time": "2022-08-01T17:49:19.124876",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "RANDOM_STATE = 84 \n",
    "\n",
    "NAN_VALUE = 0\n",
    "\n",
    "#k-folds:\n",
    "FOLDS = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e4a1138",
   "metadata": {},
   "outputs": [],
   "source": [
    "def seed_everything(seed):\n",
    "    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    tf.random.set_seed(seed)\n",
    "    keras.utils.set_random_seed(seed)\n",
    "    \n",
    "seed_everything(RANDOM_STATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9b444fd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-08-01T17:49:19.151750Z",
     "iopub.status.busy": "2022-08-01T17:49:19.151086Z",
     "iopub.status.idle": "2022-08-01T17:49:47.607369Z",
     "shell.execute_reply": "2022-08-01T17:49:47.606460Z"
    },
    "papermill": {
     "duration": 28.46623,
     "end_time": "2022-08-01T17:49:47.610719",
     "exception": false,
     "start_time": "2022-08-01T17:49:19.144489",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "train = pd.read_parquet(\"data_v29/train_processed.parquet\")\n",
    "train = train.sort_values(\"customer_ID\").reset_index(drop=True)\n",
    "print(train.shape)\n",
    "features = [col for col in train.columns if \"target\" not in col and \"customer_ID\" not in col]\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "434a7ab3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-08-01T17:49:49.171810Z",
     "iopub.status.busy": "2022-08-01T17:49:49.171575Z",
     "iopub.status.idle": "2022-08-01T17:49:49.174842Z",
     "shell.execute_reply": "2022-08-01T17:49:49.174029Z"
    },
    "papermill": {
     "duration": 0.012593,
     "end_time": "2022-08-01T17:49:49.176625",
     "exception": false,
     "start_time": "2022-08-01T17:49:49.164032",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/code/carlmcbrideellis/tabular-classification-with-neural-networks-keras/notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b81022a4",
   "metadata": {
    "papermill": {
     "duration": 0.006977,
     "end_time": "2022-08-01T17:49:49.190337",
     "exception": false,
     "start_time": "2022-08-01T17:49:49.183360",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### AmEx metric:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08c65266",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-08-01T17:49:49.205874Z",
     "iopub.status.busy": "2022-08-01T17:49:49.205335Z",
     "iopub.status.idle": "2022-08-01T17:49:49.219178Z",
     "shell.execute_reply": "2022-08-01T17:49:49.218528Z"
    },
    "papermill": {
     "duration": 0.023396,
     "end_time": "2022-08-01T17:49:49.221060",
     "exception": false,
     "start_time": "2022-08-01T17:49:49.197664",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/competitions/amex-default-prediction/discussion/328020\n",
    "# https://www.kaggle.com/code/rohanrao/amex-competition-metric-implementations\n",
    "\n",
    "def amex_metric_numpy(y_true: np.array, y_pred: np.array) -> float:\n",
    "\n",
    "    # count of positives and negatives\n",
    "    n_pos = y_true.sum()\n",
    "    n_neg = y_true.shape[0] - n_pos\n",
    "\n",
    "    # sorting by descring prediction values\n",
    "    indices = np.argsort(y_pred)[::-1]\n",
    "    preds, target = y_pred[indices], y_true[indices]\n",
    "\n",
    "    # filter the top 4% by cumulative row weights\n",
    "    weight = 20.0 - target * 19.0\n",
    "    cum_norm_weight = (weight / weight.sum()).cumsum()\n",
    "    four_pct_filter = cum_norm_weight <= 0.04\n",
    "\n",
    "    # default rate captured at 4%\n",
    "    d = target[four_pct_filter].sum() / n_pos\n",
    "\n",
    "    # weighted gini coefficient\n",
    "    lorentz = (target / n_pos).cumsum()\n",
    "    gini = ((lorentz - cum_norm_weight) * weight).sum()\n",
    "\n",
    "    # max weighted gini coefficient\n",
    "    gini_max = 10 * n_neg * (1 - 19 / (n_pos + 20 * n_neg))\n",
    "\n",
    "    # normalized weighted gini coefficient\n",
    "    g = gini / gini_max\n",
    "\n",
    "    return 0.5 * (g + d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa940b46",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-08-01T17:49:49.236192Z",
     "iopub.status.busy": "2022-08-01T17:49:49.235946Z",
     "iopub.status.idle": "2022-08-01T17:49:49.240205Z",
     "shell.execute_reply": "2022-08-01T17:49:49.239326Z"
    },
    "papermill": {
     "duration": 0.014106,
     "end_time": "2022-08-01T17:49:49.242132",
     "exception": false,
     "start_time": "2022-08-01T17:49:49.228026",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "original_cat_col = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n",
    "cat_col = [col + \"_last\" for col in original_cat_col]\n",
    "cat_col += [col + \"_2ndlast\" for col in original_cat_col]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7558e1b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "train.fillna(NAN_VALUE, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32c541cc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-08-01T17:49:49.257191Z",
     "iopub.status.busy": "2022-08-01T17:49:49.256941Z",
     "iopub.status.idle": "2022-08-01T18:01:34.663164Z",
     "shell.execute_reply": "2022-08-01T18:01:34.662341Z"
    },
    "papermill": {
     "duration": 705.416268,
     "end_time": "2022-08-01T18:01:34.665314",
     "exception": false,
     "start_time": "2022-08-01T17:49:49.249046",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#%%time\n",
    "skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=RANDOM_STATE)\n",
    "acc_list = []\n",
    "y_val_full = []\n",
    "y_pred_full = []\n",
    "customer_id_list = []\n",
    "\n",
    "auc = keras.metrics.AUC()\n",
    "\n",
    "for fold, (train_index, val_index) in enumerate(skf.split(train[features], train[\"target\"])):\n",
    "    \n",
    "    print(f\"starting fold {fold}\")\n",
    "    x_train_fold, x_val_fold = train[features].iloc[train_index], train[features].iloc[val_index]\n",
    "    y_train_fold, y_val_fold = train[\"target\"].iloc[train_index], train[\"target\"].iloc[val_index]\n",
    "    print(\"starting transforming\")\n",
    "    scaler  = MaxAbsScaler() #StandardScaler() #MinMaxScaler()\n",
    "    x_train_fold = scaler.fit_transform(x_train_fold)\n",
    "    x_val_fold   = scaler.transform(x_val_fold)\n",
    "    joblib.dump(scaler, f\"ANN/ANN_scalers/scaler_fit_fold{fold}.pkl\")\n",
    "    print(f\"Transformed and saved scaler fold {fold}\")\n",
    "    del scaler\n",
    "    gc.collect()\n",
    "    \n",
    "    model = keras.Sequential(\n",
    "    [\n",
    "        keras.layers.Dense(units=128, activation=\"swish\", input_shape=(x_train_fold.shape[-1],) ),\n",
    "        keras.layers.Dropout(0.3), \n",
    "        keras.layers.Dense(units=64, activation=\"swish\"),\n",
    "        keras.layers.Dense(units=32, activation=\"swish\"),\n",
    "\n",
    "        \n",
    "        # the output layer, with a single neuron\n",
    "        keras.layers.Dense(units=1, activation=\"sigmoid\"),\n",
    "    ]\n",
    "    )\n",
    "    # initial_weights = model.get_weights()\n",
    "    learning_rate = 1e-4\n",
    "\n",
    "    model.compile(optimizer=keras.optimizers.Adam(learning_rate=learning_rate), \n",
    "              loss=\"binary_crossentropy\", \n",
    "              metrics=[auc],\n",
    "             )\n",
    "    earlystop = keras.callbacks.EarlyStopping(monitor=\"val_auc\",\n",
    "                                              patience = 20, \n",
    "                                              #min_delta=0.0001,\n",
    "                                              restore_best_weights=True)\n",
    "    print(f\"ANN fold #: {fold}\")\n",
    "    model.fit(x_train_fold, y_train_fold, \n",
    "          epochs=200, \n",
    "          batch_size=2048, \n",
    "          validation_data=(x_val_fold, y_val_fold),\n",
    "          verbose=1,\n",
    "          callbacks= [earlystop],\n",
    "                          )\n",
    "    del x_train_fold, y_train_fold\n",
    "    gc.collect()\n",
    "#     tf.keras.backend.clear_session()\n",
    "    print(\"Start predicting\")\n",
    "    global y_pred\n",
    "    y_pred = model.predict(x_val_fold) #,batch_size=1000)\n",
    "    model.save(f\"ANN/models/ANN_model_fold{fold}\")\n",
    "    del x_val_fold, model\n",
    "    gc.collect()\n",
    "    print(\"metric\")\n",
    "\n",
    "    acc = amex_metric_numpy(y_val_fold.values, y_pred.ravel()) \n",
    "    print(\"appending\")\n",
    "    acc_list.append(acc)\n",
    "    y_val_full.append(y_val_fold.to_numpy().ravel())\n",
    "    y_pred_full.append(y_pred.ravel())\n",
    "    customer_id_list.append(train[\"customer_ID\"].iloc[val_index])\n",
    "    print(f\"fold {fold}, acc: {acc}\")\n",
    "    del y_val_fold, acc,y_pred\n",
    "    \n",
    "print(f\"ANN mean: {np.mean(acc_list)}\")\n",
    "\n",
    "\n",
    "y_val_full_array = np.concatenate(y_val_full, axis=None)\n",
    "y_pred_full_array = np.concatenate(y_pred_full, axis=None)\n",
    "acc_full = amex_metric_numpy(y_val_full_array,y_pred_full_array)\n",
    "print(f\"Acc over full dataset: {acc_full}\")\n",
    "\n",
    "customer_id_list_conc = np.concatenate(customer_id_list, axis=None)\n",
    "df_OOF = pd.DataFrame()\n",
    "df_OOF[\"customer_ID\"] = pd.Series(customer_id_list_conc)\n",
    "df_OOF[\"prediction\"] = pd.Series(y_pred_full_array)\n",
    "\n",
    "del y_val_full_array, y_pred_full_array,acc_full\n",
    "gc.collect()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7c78aff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 0.7840882673961607 standardscaler\n",
    "# 0.7853181450312552 maxabs\n",
    "# 0.7835603347923932 minmax\n",
    "# 0.7858811083977235 maxabs "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c678eae9",
   "metadata": {
    "papermill": {
     "duration": 0.567869,
     "end_time": "2022-08-01T18:01:35.797954",
     "exception": false,
     "start_time": "2022-08-01T18:01:35.230085",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_OOF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "901ddc97",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_OOF.to_csv(f\"ANN/ANN_V2_data29_seed{RANDOM_STATE}_OOF.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9ce5452",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-08-01T18:01:36.987569Z",
     "iopub.status.busy": "2022-08-01T18:01:36.987115Z",
     "iopub.status.idle": "2022-08-01T18:01:37.162306Z",
     "shell.execute_reply": "2022-08-01T18:01:37.160593Z"
    },
    "papermill": {
     "duration": 0.746967,
     "end_time": "2022-08-01T18:01:37.164138",
     "exception": false,
     "start_time": "2022-08-01T18:01:36.417171",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "del train\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22d22bf5",
   "metadata": {
    "papermill": {
     "duration": 0.606504,
     "end_time": "2022-08-01T18:01:38.336159",
     "exception": false,
     "start_time": "2022-08-01T18:01:37.729655",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# test:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "659290e5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-08-01T18:01:40.645932Z",
     "iopub.status.busy": "2022-08-01T18:01:40.645247Z",
     "iopub.status.idle": "2022-08-01T18:23:12.238646Z",
     "shell.execute_reply": "2022-08-01T18:23:12.237765Z"
    },
    "papermill": {
     "duration": 1292.162654,
     "end_time": "2022-08-01T18:23:12.241100",
     "exception": false,
     "start_time": "2022-08-01T18:01:40.078446",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time \n",
    "chunks = 8\n",
    "test_rows = 924621\n",
    "\n",
    "test_submission = pd.DataFrame()\n",
    "\n",
    "for chunk in range(chunks):\n",
    "\n",
    "    print(f\"chunk:{chunk+1}\")\n",
    "    test = pd.read_parquet(f\"data_v29/test_processed_chunk{chunk}.parquet\")\n",
    "    \n",
    "    print(f\"Loaded chunk#{chunk+1}\")\n",
    "    gc.collect()\n",
    "    test_submission_chunk = pd.DataFrame()\n",
    "    test_submission_chunk[\"customer_ID\"] = test[\"customer_ID\"]\n",
    "    test = test.drop(columns=[\"customer_ID\"])\n",
    "\n",
    "    test.fillna(NAN_VALUE, inplace=True)\n",
    "    \n",
    "    predictions = []\n",
    "    print(\"Starting predictions:\")\n",
    "    for fold in range(FOLDS):\n",
    "\n",
    "        model = keras.models.load_model(f\"ANN/models/ANN_model_fold{fold}\")\n",
    "        scaler = joblib.load(f\"ANN/ANN_scalers/scaler_fit_fold{fold}.pkl\")\n",
    "        test_scaled   = scaler.transform(test)\n",
    "        test_pred = model.predict(test_scaled)\n",
    "        predictions.append(test_pred.ravel())\n",
    "        \n",
    "        print(f\"prediction added for fold:{fold}\")\n",
    "        del model, test_pred,scaler\n",
    "        gc.collect()\n",
    "    mean_predictions = np.mean(np.column_stack(predictions), axis=1)\n",
    "    test_submission_chunk[\"prediction\"] = mean_predictions\n",
    "    test_submission = test_submission.append(test_submission_chunk)\n",
    "    del test_submission_chunk\n",
    "    gc.collect()\n",
    "    \n",
    "    \n",
    "print(\"Finished predicting!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4868030",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-08-01T18:23:15.818250Z",
     "iopub.status.busy": "2022-08-01T18:23:15.817887Z",
     "iopub.status.idle": "2022-08-01T18:23:15.823266Z",
     "shell.execute_reply": "2022-08-01T18:23:15.822457Z"
    },
    "papermill": {
     "duration": 0.578592,
     "end_time": "2022-08-01T18:23:15.825312",
     "exception": false,
     "start_time": "2022-08-01T18:23:15.246720",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "len(test_submission)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d03071b8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-08-01T18:23:17.022310Z",
     "iopub.status.busy": "2022-08-01T18:23:17.021654Z",
     "iopub.status.idle": "2022-08-01T18:23:17.035939Z",
     "shell.execute_reply": "2022-08-01T18:23:17.035044Z"
    },
    "papermill": {
     "duration": 0.587377,
     "end_time": "2022-08-01T18:23:17.037842",
     "exception": false,
     "start_time": "2022-08-01T18:23:16.450465",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32d1aaa8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-08-01T18:23:18.247307Z",
     "iopub.status.busy": "2022-08-01T18:23:18.246580Z",
     "iopub.status.idle": "2022-08-01T18:23:18.250610Z",
     "shell.execute_reply": "2022-08-01T18:23:18.249747Z"
    },
    "papermill": {
     "duration": 0.630771,
     "end_time": "2022-08-01T18:23:18.253205",
     "exception": false,
     "start_time": "2022-08-01T18:23:17.622434",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# test_submission.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9be0c60",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-08-01T18:23:22.543277Z",
     "iopub.status.busy": "2022-08-01T18:23:22.542920Z",
     "iopub.status.idle": "2022-08-01T18:23:26.623603Z",
     "shell.execute_reply": "2022-08-01T18:23:26.622728Z"
    },
    "papermill": {
     "duration": 4.662668,
     "end_time": "2022-08-01T18:23:26.625681",
     "exception": false,
     "start_time": "2022-08-01T18:23:21.963013",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_submission.to_csv(\"ANN/ANN_V2_dataV29_submission.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 2068.024719,
   "end_time": "2022-08-01T18:23:31.890438",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-08-01T17:49:03.865719",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}