amex-data-preparation_V23.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a7da0d88",
   "metadata": {
    "_cell_guid": "dcb2670b-869a-474d-b788-5c77bdbda89a",
    "_uuid": "ec0a3025-123f-4845-8547-11373afbdc3d",
    "execution": {
     "iopub.execute_input": "2022-07-31T19:31:01.093721Z",
     "iopub.status.busy": "2022-07-31T19:31:01.093062Z",
     "iopub.status.idle": "2022-07-31T19:31:05.669968Z",
     "shell.execute_reply": "2022-07-31T19:31:05.669106Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 4.590899,
     "end_time": "2022-07-31T19:31:05.672272",
     "exception": false,
     "start_time": "2022-07-31T19:31:01.081373",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd \n",
    "\n",
    "import gc\n",
    "import time\n",
    "import joblib\n",
    "import cudf "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "61244c26",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T19:31:05.710252Z",
     "iopub.status.busy": "2022-07-31T19:31:05.709692Z",
     "iopub.status.idle": "2022-07-31T19:31:05.725346Z",
     "shell.execute_reply": "2022-07-31T19:31:05.724648Z"
    },
    "papermill": {
     "duration": 0.027402,
     "end_time": "2022-07-31T19:31:05.727053",
     "exception": false,
     "start_time": "2022-07-31T19:31:05.699651",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "permutationimportance = pd.read_csv(\"../input/permutationimportance-amex/permutationimportance_repeats3_meantransposed_v1.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c5fdd147",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T19:31:05.746975Z",
     "iopub.status.busy": "2022-07-31T19:31:05.746444Z",
     "iopub.status.idle": "2022-07-31T19:31:05.750141Z",
     "shell.execute_reply": "2022-07-31T19:31:05.749361Z"
    },
    "papermill": {
     "duration": 0.015745,
     "end_time": "2022-07-31T19:31:05.751903",
     "exception": false,
     "start_time": "2022-07-31T19:31:05.736158",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#permutationimportance_neg = permutationimportance[permutationimportance[\"importance\"] < 0].column.values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "937d7ce1",
   "metadata": {
    "papermill": {
     "duration": 0.008811,
     "end_time": "2022-07-31T19:31:05.769736",
     "exception": false,
     "start_time": "2022-07-31T19:31:05.760925",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Constants:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "55fd0223",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T19:31:05.789038Z",
     "iopub.status.busy": "2022-07-31T19:31:05.788412Z",
     "iopub.status.idle": "2022-07-31T19:31:05.792248Z",
     "shell.execute_reply": "2022-07-31T19:31:05.791556Z"
    },
    "papermill": {
     "duration": 0.015419,
     "end_time": "2022-07-31T19:31:05.794060",
     "exception": false,
     "start_time": "2022-07-31T19:31:05.778641",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "RANDOM_STATE = 42\n",
    "\n",
    "NAN_VALUE = -127\n",
    "\n",
    "#k-folds:\n",
    "FOLDS = 5"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4355fdb",
   "metadata": {
    "papermill": {
     "duration": 0.008891,
     "end_time": "2022-07-31T19:31:05.838025",
     "exception": false,
     "start_time": "2022-07-31T19:31:05.829134",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Processing and feature engineering:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3036d5d6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T19:31:05.858200Z",
     "iopub.status.busy": "2022-07-31T19:31:05.857674Z",
     "iopub.status.idle": "2022-07-31T19:31:05.903745Z",
     "shell.execute_reply": "2022-07-31T19:31:05.903071Z"
    },
    "papermill": {
     "duration": 0.058223,
     "end_time": "2022-07-31T19:31:05.905434",
     "exception": false,
     "start_time": "2022-07-31T19:31:05.847211",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def process_and_feature(df):\n",
    "    df = df.sort_values(['customer_ID','S_2']) \n",
    "    df = df.reset_index(drop=True)\n",
    "    \n",
    "    df[\"S_2\"] = cudf.to_datetime(df[\"S_2\"])\n",
    "    \n",
    "    # setting up permutation importance variables:\n",
    "    permutationimportance_neg = permutationimportance[:50].column.values\n",
    "    top3 = permutationimportance[-3:].column.values\n",
    "    top5 = permutationimportance[-5:].column.values\n",
    "    top7 = permutationimportance[-7:].column.values\n",
    "    top10 = permutationimportance[-10:].column.values\n",
    "    top15 = permutationimportance[-15:].column.values\n",
    "    top25 = permutationimportance[-25:].column.values\n",
    "\n",
    "    # set np seed:\n",
    "    np.random.seed(RANDOM_STATE)\n",
    "    \n",
    "    # sorting on customer ID and then time, to use last properly\n",
    "    df = df.sort_values(['customer_ID','S_2'])\n",
    "    \n",
    "    \n",
    "    cat_col = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n",
    "    num_col = [col for col in df.columns if col not in (cat_col + [\"customer_ID\", \"S_2\",\"target\"])]\n",
    "    all_col = num_col + cat_col\n",
    "    \n",
    "    def convert64to32(df):\n",
    "    #float64 and int64 to 32 : https://www.kaggle.com/code/ragnar123/amex-lgbm-dart-cv-0-7977\n",
    "        float64_cols = list(df.dtypes[df.dtypes == \"float64\"].index)\n",
    "        for col in float64_cols:\n",
    "            df[col] = df[col].astype(\"float32\")\n",
    "        int64_cols = list(df.dtypes[df.dtypes == \"int64\"].index)\n",
    "        for col in int64_cols:\n",
    "            df[col] = df[col].astype(\"int32\")\n",
    "        return df\n",
    "\n",
    "    # mean and sum of the different sort of variables\n",
    "    D_col_num = []\n",
    "    S_col_num = []\n",
    "    P_col_num = []\n",
    "    B_col_num = []\n",
    "    R_col_num = []\n",
    "    for col in df.columns:\n",
    "        if col not in (cat_col + [\"customer_ID\", \"S_2\"]):\n",
    "            if \"D_\" in col:\n",
    "                D_col_num.append(col)\n",
    "            elif \"S_\" in col:\n",
    "                S_col_num.append(col)\n",
    "            elif \"P_\" in col:\n",
    "                P_col_num.append(col)\n",
    "            elif \"B_\" in col:\n",
    "                B_col_num.append(col)\n",
    "            elif \"R_\" in col:\n",
    "                R_col_num.append(col)\n",
    "        else:\n",
    "            pass\n",
    "    df[\"D_mean\"] = df[D_col_num].mean(axis=1)\n",
    "    df[\"D_sum\"] = df[D_col_num].sum(axis=1)\n",
    "    df[\"S_mean\"] = df[S_col_num].mean(axis=1)\n",
    "    df[\"S_sum\"] = df[S_col_num].sum(axis=1)\n",
    "    df[\"P_mean\"] = df[P_col_num].mean(axis=1)\n",
    "    df[\"P_sum\"] = df[P_col_num].sum(axis=1)\n",
    "    df[\"B_mean\"] = df[B_col_num].mean(axis=1)\n",
    "    df[\"B_sum\"] = df[B_col_num].sum(axis=1)\n",
    "    df[\"R_mean\"] = df[R_col_num].mean(axis=1)\n",
    "    df[\"R_sum\"] = df[R_col_num].sum(axis=1)\n",
    "\n",
    "    var_groups_cols = [\"D_mean\",\"S_mean\",\"P_mean\",\"B_mean\",\"R_mean\",\n",
    "                       \"D_sum\",\"S_sum\",\"P_sum\",\"B_sum\",\"R_sum\",\n",
    "                      ]\n",
    "    df_var_groups = df.groupby(\"customer_ID\")[var_groups_cols].agg([\"last\",\"max\",\"mean\"])\n",
    "    df_var_groups.columns = ['_'.join(x) for x in df_var_groups.columns]\n",
    "    df_var_groups = convert64to32(df_var_groups)\n",
    "\n",
    "    ######################\n",
    "    # P_2 ratios:   \n",
    "    #P_2÷B_9\n",
    "    df[\"P2divB9\"] = df[\"P_2\"]/df[\"B_9\"]\n",
    "    \n",
    "#     P2_ratio_col = [\"D_42\", \"B_3\",\"B_1\",\"B_4\",\"D_39\"]\n",
    "#     P2_ratio_col_finished = []\n",
    "#     for col in P2_ratio_col:\n",
    "#         df[f\"P2-{col}-ratio\"] = df[col]/df[\"P_2\"]\n",
    "#         P2_ratio_col_finished.append(f\"P2-{col}-ratio\")\n",
    "    P2_ratio_col_finished = [\"P2divB9\"]\n",
    "    df_P2_ratios = df.groupby(\"customer_ID\")[P2_ratio_col_finished].agg([\"last\",\"min\",\"max\",\"mean\"])\n",
    "    df_P2_ratios.columns = ['_'.join(x) for x in df_P2_ratios.columns]\n",
    "    #################################\n",
    "\n",
    "    \n",
    "    # count:\n",
    "    df_count = df.groupby(\"customer_ID\")[\"S_2\"].agg([\"count\"])\n",
    "    for col in df_count.columns:\n",
    "        df_count[col] = df_count[col].astype(\"int8\")\n",
    "    \n",
    "    # count non NaN values:\n",
    "#     df_nonNANcount = df.groupby(\"customer_ID\")[all_col][df[all_col] != -127].count().sort_index()\n",
    "#     df_nonNANcount.columns = [f\"{col}_nonNAN\" for col in df_nonNANcount.columns]\n",
    "#     for col in df_nonNANcount.columns:\n",
    "#         df_nonNANcount[col] = df_nonNANcount[col].astype(\"int8\")\n",
    "        \n",
    "    \n",
    "    # time difference:\n",
    "    df_date = df.groupby(\"customer_ID\")[[\"S_2\"]].agg([\"first\",\"last\"])\n",
    "    df_date.columns = ['_'.join(x) for x in df_date.columns]\n",
    "    df_date[\"S_2_diff\"] = ((df_date[\"S_2_last\"] - df_date[\"S_2_first\"])/np.timedelta64(1, 'D'))\n",
    "    df_date[\"S_2_diff\"] = df_date[\"S_2_diff\"].astype(\"int16\")\n",
    "    df_date = df_date.drop(columns=[\"S_2_first\",\"S_2_last\"])\n",
    "    \n",
    "    # time variables\n",
    "    df[\"weekday\"] = df.S_2.dt.weekday\n",
    "    df[\"month\"] = df.S_2.dt.month\n",
    "    df[\"percofyear\"] = np.round(((df.S_2.dt.dayofyear / 365) * 100), 1)\n",
    "    df[\"percofmonth\"] = np.round(((df.S_2.dt.day / df.S_2.dt.days_in_month) * 100), 1)\n",
    "    df[\"quarter\"] = df.S_2.dt.quarter\n",
    "    time_col = [\"weekday\",\"month\",\"percofyear\",\"percofmonth\",\"quarter\"]\n",
    "    df_time = df.groupby(\"customer_ID\")[time_col].agg([\"last\"])\n",
    "    df_time.columns = ['_'.join(x) for x in df_time.columns]\n",
    "    df_time = convert64to32(df_time)\n",
    "    \n",
    "    # 2nd last variables\n",
    "    all_2nd_col = all_col + var_groups_cols + P2_ratio_col_finished\n",
    "    df_2ndlast = df.groupby(\"customer_ID\")[all_2nd_col].nth(-2) \n",
    "    df_2ndlast.columns = [str(x) + \"_2ndlast\" for x in df_2ndlast.columns]\n",
    "    # https://www.kaggle.com/competitions/amex-default-prediction/discussion/332880#1830894 @Carl McBride Ellis\n",
    "    df_2ndlast = convert64to32(df_2ndlast)\n",
    "    \n",
    "    # categorical columns:\n",
    "    df_cat = df.groupby(\"customer_ID\")[cat_col].agg([\"first\",\"last\",\"nunique\"]) # ['count', 'last', 'nunique']\n",
    "    df_cat.columns = ['_'.join(x) for x in df_cat.columns]\n",
    "    df_cat = convert64to32(df_cat)\n",
    "    \n",
    "    # Numerical columns:\n",
    "    df_num = df.groupby(\"customer_ID\")[num_col].agg([\"first\",\"last\",\"min\",\"max\",\"mean\"]) #'mean', 'std', 'min', 'max', 'last'\n",
    "    df_num.columns = ['_'.join(x) for x in df_num.columns]\n",
    "    df_num = convert64to32(df_num)\n",
    "    \n",
    "    ######\n",
    "    global last_col\n",
    "    last_col = all_col #+ var_groups_cols\n",
    "    last_col = [f\"{col}_last\" for col in last_col]\n",
    "    last_col = [col for col in last_col if col not in permutationimportance_neg]\n",
    "    \n",
    "    \n",
    "    ############# Concat ######################################################\n",
    "    print(\"concat\")\n",
    "    df = cudf.concat([df_date,df_time, df_count,df_cat, df_num, df_var_groups, df_2ndlast,df_P2_ratios], axis=1) \n",
    "    \n",
    "    \n",
    "    del df_date,df_time, df_count,df_cat, df_num, df_var_groups, df_2ndlast, df_P2_ratios\n",
    "    gc.collect()\n",
    "    \n",
    "    # fill in NaN for 2ndlast with only 1 customer:\n",
    "    \n",
    "    for col in all_2nd_col:\n",
    "        df[f\"{col}_2ndlast\"].fillna(df[f\"{col}_last\"], inplace=True) \n",
    "    \n",
    "    # second stage:\n",
    "    \n",
    "    for col in num_col:\n",
    "        df[f\"{col}_last-mean\"] = df[f\"{col}_last\"] - df[f\"{col}_mean\"]\n",
    "#         df[f\"{col}_MaxdivMin\"] = df[f\"{col}_max\"] - df[f\"{col}_min\"]\n",
    "        # new ones V22\n",
    "        df[f\"{col}_last-first\"] = df[f\"{col}_last\"] - df[f\"{col}_first\"]\n",
    "        df[f\"{col}_lastDIVfirst\"] = (df[f\"{col}_last\"]+0.0001) / (df[f\"{col}_first\"]+0.0001) # +0.0001 to prevent inf\n",
    "\n",
    "        \n",
    "#     for col in all_2nd_col:\n",
    "#         df[f\"{col}_last-2ndlast\"] = df[f\"{col}_last\"] - df[f\"{col}_2ndlast\"]\n",
    "#         df = df.drop(columns=f\"{col}_2ndlast\")\n",
    "\n",
    "    df = df.reset_index()\n",
    "    \n",
    "    # permutation importance features:\n",
    "    \n",
    "    top_dic = {\"top3\":top3, \"top5\":top5,\"top7\":top7,\"top10\":top10,\"top15\":top15,\"top25\":top25}\n",
    "    for top in top_dic:\n",
    "        df[f\"{top}_mean\"] = df[top_dic[top]].mean(axis=1)\n",
    "        df[f\"{top}_sum\"] = df[top_dic[top]].sum(axis=1)\n",
    "        df[f\"{top}_min\"] = df[top_dic[top]].min(axis=1)\n",
    "        df[f\"{top}_max\"] = df[top_dic[top]].max(axis=1)\n",
    "    ####    \n",
    "    for i in top5:\n",
    "        for x in top5:\n",
    "            if i == x:\n",
    "                pass\n",
    "            else:\n",
    "                df[f\"{i}DIV{x}\"] = (df[i]+0.0001)/(df[x]+0.0001)\n",
    "                df[f\"{i}MIN{x}\"] = df[i]-df[x]\n",
    "                \n",
    "        df[f\"{i}_top5perc\"] = df[i]/ df[\"top5_sum\"] * 100\n",
    "    float64_cols = list(df.dtypes[df.dtypes == \"float64\"].index)\n",
    "    for col in float64_cols:\n",
    "        df[col] = df[col].astype(\"float32\") \n",
    "        \n",
    "    # dropping columns:\n",
    "    df = df.drop(columns=permutationimportance_neg)\n",
    "    \n",
    "    return df\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cd712d22",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T19:31:05.953480Z",
     "iopub.status.busy": "2022-07-31T19:31:05.953112Z",
     "iopub.status.idle": "2022-07-31T19:31:05.962932Z",
     "shell.execute_reply": "2022-07-31T19:31:05.962272Z"
    },
    "papermill": {
     "duration": 0.021704,
     "end_time": "2022-07-31T19:31:05.964689",
     "exception": false,
     "start_time": "2022-07-31T19:31:05.942985",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_process():\n",
    "    # read in data:\n",
    "    train = cudf.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/train.parquet\")\n",
    "    # Categorical columns are label encoded and missing values are replaced by -1 @Raddar\n",
    "    \n",
    "    train[\"customer_ID\"] = train[\"customer_ID\"].str[-16:].str.hex_to_int().astype(\"int64\") # not needed for test\n",
    "\n",
    "    # feature engineering:\n",
    "    train = process_and_feature(train) \n",
    "    \n",
    "    #del train\n",
    "    gc.collect()\n",
    "    train_labels = cudf.read_csv(\"../input/amex-default-prediction/train_labels.csv\")\n",
    "    train_labels[\"customer_ID\"] = train_labels[\"customer_ID\"].str[-16:].str.hex_to_int().astype(\"int64\") # not needed for test\n",
    "    \n",
    "    # add target labels:\n",
    "    train_labels.target = train_labels.target.astype('int8')\n",
    "    train = train.merge(train_labels, on=\"customer_ID\")\n",
    "    train = train.sort_values(\"customer_ID\").reset_index(drop=True)\n",
    "\n",
    "    del train_labels\n",
    "    gc.collect()\n",
    "    print(f\"shape:{train.shape}\")\n",
    "    train = train.to_pandas() # free up GPU\n",
    "    \n",
    "    \n",
    "    gc.collect()\n",
    "    return train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7b28ff55",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T19:31:05.985671Z",
     "iopub.status.busy": "2022-07-31T19:31:05.984924Z",
     "iopub.status.idle": "2022-07-31T19:31:50.201866Z",
     "shell.execute_reply": "2022-07-31T19:31:50.201009Z"
    },
    "papermill": {
     "duration": 44.230696,
     "end_time": "2022-07-31T19:31:50.204173",
     "exception": false,
     "start_time": "2022-07-31T19:31:05.973477",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "concat\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/cupy/_statistics/order.py:111: RuntimeWarning: All-NaN slice encountered\n",
      "  warnings.warn('All-NaN slice encountered', RuntimeWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/cupy/_statistics/order.py:142: RuntimeWarning: All-NaN slice encountered\n",
      "  warnings.warn('All-NaN slice encountered', RuntimeWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape:(458913, 1710)\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_final = train_process() #1710"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7333e094",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T19:31:50.375560Z",
     "iopub.status.busy": "2022-07-31T19:31:50.375247Z",
     "iopub.status.idle": "2022-07-31T19:32:20.768927Z",
     "shell.execute_reply": "2022-07-31T19:32:20.767989Z"
    },
    "papermill": {
     "duration": 30.406574,
     "end_time": "2022-07-31T19:32:20.771381",
     "exception": false,
     "start_time": "2022-07-31T19:31:50.364807",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_final.to_parquet(\"train_processed.parquet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "41cac8e2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T19:32:22.590819Z",
     "iopub.status.busy": "2022-07-31T19:32:22.590205Z",
     "iopub.status.idle": "2022-07-31T19:32:22.900545Z",
     "shell.execute_reply": "2022-07-31T19:32:22.899793Z"
    },
    "papermill": {
     "duration": 2.120338,
     "end_time": "2022-07-31T19:32:22.902302",
     "exception": false,
     "start_time": "2022-07-31T19:32:20.781964",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del train_final\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8e64bb5",
   "metadata": {
    "papermill": {
     "duration": 0.009254,
     "end_time": "2022-07-31T19:32:22.921122",
     "exception": false,
     "start_time": "2022-07-31T19:32:22.911868",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# test:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "66b43c79",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T19:32:22.941069Z",
     "iopub.status.busy": "2022-07-31T19:32:22.940382Z",
     "iopub.status.idle": "2022-07-31T19:32:23.062401Z",
     "shell.execute_reply": "2022-07-31T19:32:23.061620Z"
    },
    "papermill": {
     "duration": 0.133837,
     "end_time": "2022-07-31T19:32:23.064241",
     "exception": false,
     "start_time": "2022-07-31T19:32:22.930404",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clear GPU memory\n",
    "# torch.cuda.empty_cache()\n",
    "# cuda.select_device(0)\n",
    "# cuda.close()\n",
    "# cuda.select_device(0)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "933f01dd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T19:32:23.085070Z",
     "iopub.status.busy": "2022-07-31T19:32:23.084415Z",
     "iopub.status.idle": "2022-07-31T19:36:00.621931Z",
     "shell.execute_reply": "2022-07-31T19:36:00.621076Z"
    },
    "papermill": {
     "duration": 217.550296,
     "end_time": "2022-07-31T19:36:00.624265",
     "exception": false,
     "start_time": "2022-07-31T19:32:23.073969",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chunk:0\n",
      "chunk#0, start chunk id 0, end_chunk id 1420901\n",
      "loaded chunk\n",
      "concat\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/cupy/_statistics/order.py:111: RuntimeWarning: All-NaN slice encountered\n",
      "  warnings.warn('All-NaN slice encountered', RuntimeWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/cupy/_statistics/order.py:142: RuntimeWarning: All-NaN slice encountered\n",
      "  warnings.warn('All-NaN slice encountered', RuntimeWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feature engineered\n",
      "chunk:1\n",
      "chunk#1, start chunk id 1420901, end_chunk id 2841209\n",
      "loaded chunk\n",
      "concat\n",
      "feature engineered\n",
      "chunk:2\n",
      "chunk#2, start chunk id 2841209, end_chunk id 4261572\n",
      "loaded chunk\n",
      "concat\n",
      "feature engineered\n",
      "chunk:3\n",
      "chunk#3, start chunk id 4261572, end_chunk id 5681053\n",
      "loaded chunk\n",
      "concat\n",
      "feature engineered\n",
      "chunk:4\n",
      "chunk#4, start chunk id 5681053, end_chunk id 7102344\n",
      "loaded chunk\n",
      "concat\n",
      "feature engineered\n",
      "chunk:5\n",
      "chunk#5, start chunk id 7102344, end_chunk id 8523150\n",
      "loaded chunk\n",
      "concat\n",
      "feature engineered\n",
      "chunk:6\n",
      "chunk#6, start chunk id 8523150, end_chunk id 9944645\n",
      "loaded chunk\n",
      "concat\n",
      "feature engineered\n",
      "chunk:7\n",
      "chunk#7, start chunk id 9944645, end_chunk id 11363763\n",
      "loaded chunk\n",
      "concat\n",
      "feature engineered\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "chunks = 8\n",
    "test_rows = 11363762\n",
    "groupby_test_rows = 924621\n",
    "start_chunk = 0\n",
    "length_chunk = int(groupby_test_rows/chunks)\n",
    "\n",
    "test_full_id = cudf.DataFrame()\n",
    "\n",
    "for chunk in range(chunks):\n",
    "\n",
    "    print(f\"chunk:{chunk}\")\n",
    "    test = cudf.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/test.parquet\")\n",
    "    test = test.sort_values([\"customer_ID\",\"S_2\"])\n",
    "    test = test.reset_index(drop=True)\n",
    "    if (chunk+1) == chunks:\n",
    "        end_chunk = test_rows +1\n",
    "    else:\n",
    "        end_chunk = test.loc[test[\"customer_ID\"] == test.groupby(\"customer_ID\").agg([\"last\"]).iloc[(length_chunk*(chunk+1))].index[0]].tail(1).index[0]\n",
    "        end_chunk += 1\n",
    "    print(f\"chunk#{chunk}, start chunk id {start_chunk}, end_chunk id {end_chunk}\")\n",
    "    test = test.iloc[start_chunk:end_chunk]\n",
    "    gc.collect()\n",
    "    print(\"loaded chunk\")\n",
    "    test = process_and_feature(test)\n",
    "    gc.collect()\n",
    "    print(\"feature engineered\")\n",
    "    test = test.sort_values(\"customer_ID\").reset_index(drop=True)\n",
    "    test_full_id = test_full_id.append(test[[\"customer_ID\"]])\n",
    "    test.to_parquet(f\"test_processed_chunk{chunk}.parquet\")\n",
    "    \n",
    "    del test\n",
    "    gc.collect()\n",
    "    start_chunk = end_chunk"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 310.402032,
   "end_time": "2022-07-31T19:36:02.595688",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-07-31T19:30:52.193656",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}