m5-final-XX.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright 2020 Konstantin Yakovlev, Matthias Anderer\n",
    "\n",
    "   Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "   you may not use this file except in compliance with the License.\n",
    "   You may obtain a copy of the License at\n",
    "\n",
    "       http://www.apache.org/licenses/LICENSE-2.0\n",
    "\n",
    "   Unless required by applicable law or agreed to in writing, software\n",
    "   distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "   See the License for the specific language governing permissions and\n",
    "   limitations under the License."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "# General imports\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os, sys, gc, time, warnings, pickle, psutil, random\n",
    "\n",
    "# custom imports\n",
    "from multiprocessing import Pool        # Multiprocess Runs\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Helpers\n",
    "#################################################################################\n",
    "## Seeder\n",
    "# :seed to make all processes deterministic     # type: int\n",
    "def seed_everything(seed=0):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LOSS_MULTIPLIER = 1.0 # Set multiplier according to desired under-/overshooting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define custom loss function\n",
    "def custom_asymmetric_train(y_pred, y_true):\n",
    "    y_true = y_true.get_label()\n",
    "    residual = (y_true - y_pred).astype(\"float\")\n",
    "    grad = np.where(residual < 0, -2 * residual, -2 * residual * LOSS_MULTIPLIER)\n",
    "    hess = np.where(residual < 0, 2, 2 * LOSS_MULTIPLIER)\n",
    "    return grad, hess\n",
    "\n",
    "# define custom evaluation metric\n",
    "def custom_asymmetric_valid(y_pred, y_true):\n",
    "    y_true = y_true.get_label()\n",
    "    residual = (y_true - y_pred).astype(\"float\")\n",
    "    loss = np.where(residual < 0, (residual ** 2) , (residual ** 2) * LOSS_MULTIPLIER) \n",
    "    return \"custom_asymmetric_eval\", np.mean(loss), False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Helper to load data by store ID\n",
    "#################################################################################\n",
    "# Read data\n",
    "def get_data_by_store(store):\n",
    "    \n",
    "    # Read and contact basic feature\n",
    "    df = pd.concat([pd.read_pickle(BASE),\n",
    "                    pd.read_pickle(PRICE).iloc[:,2:],\n",
    "                    pd.read_pickle(CALENDAR).iloc[:,2:]],\n",
    "                    axis=1)\n",
    "    \n",
    "    # Leave only relevant store\n",
    "    df = df[df['store_id']==store]\n",
    "    \n",
    "    ############\n",
    "    \n",
    "    # Create features list\n",
    "    features = [col for col in list(df) if col not in remove_features]\n",
    "    df = df[['id','d',TARGET]+features]\n",
    "    \n",
    "    # Skipping first n rows\n",
    "    df = df[df['d']>=START_TRAIN].reset_index(drop=True)\n",
    "    \n",
    "    return df, features\n",
    "\n",
    "# Recombine Test set after training\n",
    "def get_base_test():\n",
    "    base_test = pd.DataFrame()\n",
    "\n",
    "    for store_id in STORES_IDS:\n",
    "        temp_df = pd.read_pickle('test_'+store_id+'.pkl')\n",
    "        temp_df['store_id'] = store_id\n",
    "        base_test = pd.concat([base_test, temp_df]).reset_index(drop=True)\n",
    "    \n",
    "    return base_test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Model params\n",
    "#################################################################################\n",
    "import lightgbm as lgb\n",
    "lgb_params = {\n",
    "        'boosting_type': 'gbdt',\n",
    "        'objective': 'tweedie',\n",
    "        'tweedie_variance_power': 1.1,\n",
    "        'metric':'rmse',\n",
    "        'n_jobs': -1,\n",
    "        'seed': 42,\n",
    "        'learning_rate': 0.2,\n",
    "        'bagging_fraction': 0.85,\n",
    "        'bagging_freq': 1, \n",
    "        'colsample_bytree': 0.85,\n",
    "        'colsample_bynode': 0.85,\n",
    "        #'min_data_per_leaf': 25,\n",
    "        #'num_leaves': 200,\n",
    "        'lambda_l1': 0.5,\n",
    "        'lambda_l2': 0.5\n",
    "}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Vars\n",
    "#################################################################################\n",
    "VER = 1                          # Our model version\n",
    "SEED = 42                        # We want all things\n",
    "seed_everything(SEED)            # to be as deterministic \n",
    "lgb_params['seed'] = SEED        # as possible\n",
    "N_CORES = psutil.cpu_count()     # Available CPU cores\n",
    "\n",
    "\n",
    "#LIMITS and const\n",
    "TARGET      = 'sales'            # Our target\n",
    "START_TRAIN = 0                  # We can skip some rows (Nans/faster training)\n",
    "END_TRAIN   = 1913+28            # End day of our train set\n",
    "P_HORIZON   = 28                 # Prediction horizon\n",
    "USE_AUX     = False               # Use or not pretrained models\n",
    "\n",
    "#FEATURES to remove\n",
    "## These features lead to overfit\n",
    "## or values not present in test set\n",
    "remove_features = ['id','state_id','store_id',\n",
    "                   'date','wm_yr_wk','d',TARGET]\n",
    "\n",
    "#PATHS for Features\n",
    "ORIGINAL = '../input/m5-forecasting-accuracy/'\n",
    "BASE     = '../input/m5-simple-fe-evaluation/grid_part_1.pkl'\n",
    "PRICE    = '../input/m5-simple-fe-evaluation/grid_part_2.pkl'\n",
    "CALENDAR = '../input/m5-simple-fe-evaluation/grid_part_3.pkl'\n",
    "\n",
    "#STORES ids\n",
    "STORES_IDS = pd.read_csv(ORIGINAL+'sales_train_validation.csv')['store_id']\n",
    "STORES_IDS = list(STORES_IDS.unique())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Train Models\n",
    "#################################################################################\n",
    "for store_id in STORES_IDS:\n",
    "    print('Train', store_id)\n",
    "    \n",
    "    # Get grid for current store\n",
    "    grid_df, features_columns = get_data_by_store(store_id)\n",
    "    \n",
    "    # Masks for \n",
    "    # Train (All data less than 1913)\n",
    "    # \"Validation\" (Last 28 days - not real validatio set)\n",
    "    # Test (All data greater than 1913 day, \n",
    "    #       with some gap for recursive features)\n",
    "    train_mask = grid_df['d']<=END_TRAIN\n",
    "    valid_mask = train_mask&(grid_df['d']>(END_TRAIN-P_HORIZON))\n",
    "    preds_mask = grid_df['d']>(END_TRAIN-100)\n",
    "    \n",
    "    # Apply masks and save lgb dataset as bin\n",
    "    # to reduce memory spikes during dtype convertations\n",
    "    # https://github.com/Microsoft/LightGBM/issues/1032\n",
    "    # \"To avoid any conversions, you should always use np.float32\"\n",
    "    # or save to bin before start training\n",
    "    # https://www.kaggle.com/c/talkingdata-adtracking-fraud-detection/discussion/53773\n",
    "    train_data = lgb.Dataset(grid_df[train_mask][features_columns], \n",
    "                       label=grid_df[train_mask][TARGET])\n",
    "    train_data.save_binary('train_data.bin')\n",
    "    train_data = lgb.Dataset('train_data.bin')\n",
    "    \n",
    "    valid_data = lgb.Dataset(grid_df[valid_mask][features_columns], \n",
    "                       label=grid_df[valid_mask][TARGET])\n",
    "    \n",
    "    # Saving part of the dataset for later predictions\n",
    "    # Removing features that we need to calculate recursively \n",
    "    grid_df = grid_df[preds_mask].reset_index(drop=True)\n",
    "    keep_cols = [col for col in list(grid_df) if '_tmp_' not in col]\n",
    "    grid_df = grid_df[keep_cols]\n",
    "    grid_df.to_pickle('test_'+store_id+'.pkl')\n",
    "    del grid_df\n",
    "    \n",
    "    # Launch seeder again to make lgb training 100% deterministic\n",
    "    # with each \"code line\" np.random \"evolves\" \n",
    "    # so we need (may want) to \"reset\" it\n",
    "    seed_everything(SEED)\n",
    "    estimator = lgb.train(lgb_params,\n",
    "                          train_data,\n",
    "                          num_boost_round = 3600, \n",
    "                          early_stopping_rounds = 50, \n",
    "                          valid_sets = [train_data, valid_data],\n",
    "                          verbose_eval = 100,\n",
    "                          fobj = custom_asymmetric_train\n",
    "\n",
    "                          )\n",
    "    \n",
    "    # Save model - it's not real '.bin' but a pickle file\n",
    "    # estimator = lgb.Booster(model_file='model.txt')\n",
    "    # can only predict with the best iteration (or the saving iteration)\n",
    "    # pickle.dump gives us more flexibility\n",
    "    # like estimator.predict(TEST, num_iteration=100)\n",
    "    # num_iteration - number of iteration want to predict with, \n",
    "    # NULL or <= 0 means use best iteration\n",
    "    model_name = 'lgb_model_'+store_id+'_v'+str(VER)+'.bin'\n",
    "    pickle.dump(estimator, open(model_name, 'wb'))\n",
    "\n",
    "    # Remove temporary files and objects \n",
    "    # to free some hdd space and ram memory\n",
    "    !rm train_data.bin\n",
    "    del train_data, valid_data, estimator\n",
    "    gc.collect()\n",
    "    \n",
    "    # \"Keep\" models features for predictions\n",
    "    MODEL_FEATURES = features_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Predict\n",
    "#################################################################################\n",
    "\n",
    "# Create Dummy DataFrame to store predictions\n",
    "all_preds = pd.DataFrame()\n",
    "\n",
    "# Join back the Test dataset with \n",
    "# a small part of the training data \n",
    "# to make recursive features\n",
    "base_test = get_base_test()\n",
    "\n",
    "# Timer to measure predictions time \n",
    "main_time = time.time()\n",
    "\n",
    "# Loop over each prediction day\n",
    "# As rolling lags are the most timeconsuming\n",
    "# we will calculate it for whole day\n",
    "for PREDICT_DAY in range(1,29):    \n",
    "    print('Predict | Day:', PREDICT_DAY)\n",
    "    start_time = time.time()\n",
    "\n",
    "    # Make temporary grid to calculate rolling lags\n",
    "    grid_df = base_test.copy()\n",
    "        \n",
    "    for store_id in STORES_IDS:\n",
    "        \n",
    "        # Read all our models and make predictions\n",
    "        # for each day/store pairs\n",
    "        model_path = 'lgb_model_'+store_id+'_v'+str(VER)+'.bin' \n",
    "        if USE_AUX:\n",
    "            model_path = AUX_MODELS + model_path\n",
    "        \n",
    "        estimator = pickle.load(open(model_path, 'rb'))\n",
    "        \n",
    "        day_mask = base_test['d']==(END_TRAIN+PREDICT_DAY)\n",
    "        store_mask = base_test['store_id']==store_id\n",
    "        \n",
    "        mask = (day_mask)&(store_mask)\n",
    "        base_test[TARGET][mask] = estimator.predict(grid_df[mask][MODEL_FEATURES])\n",
    "    \n",
    "    # Make good column naming and add \n",
    "    # to all_preds DataFrame\n",
    "    temp_df = base_test[day_mask][['id',TARGET]]\n",
    "    temp_df.columns = ['id','F'+str(PREDICT_DAY)]\n",
    "    if 'id' in list(all_preds):\n",
    "        all_preds = all_preds.merge(temp_df, on=['id'], how='left')\n",
    "    else:\n",
    "        all_preds = temp_df.copy()\n",
    "        \n",
    "    print('#'*10, ' %0.2f min round |' % ((time.time() - start_time) / 60),\n",
    "                  ' %0.2f min total |' % ((time.time() - main_time) / 60),\n",
    "                  ' %0.2f day sales |' % (temp_df['F'+str(PREDICT_DAY)].sum()))\n",
    "    del temp_df\n",
    "    \n",
    "all_preds = all_preds.reset_index(drop=True)\n",
    "all_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Export\n",
    "#################################################################################\n",
    "# Reading competition sample submission and\n",
    "# merging our predictions\n",
    "# As we have predictions only for \"_validation\" data\n",
    "# we need to do fillna() for \"_evaluation\" items\n",
    "submission = pd.read_csv(ORIGINAL+'sample_submission.csv')[['id']]\n",
    "submission = submission.merge(all_preds, on=['id'], how='left').fillna(0)\n",
    "submission.to_csv('submission_v'+str(VER)+'.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}