m5_nbeats_toplvl.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright 2020 Stephan Rabanser, Matthias Anderer\n",
    "\n",
    "   Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "   you may not use this file except in compliance with the License.\n",
    "   You may obtain a copy of the License at\n",
    "\n",
    "       http://www.apache.org/licenses/LICENSE-2.0\n",
    "\n",
    "   Unless required by applicable law or agreed to in writing, software\n",
    "   distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "   See the License for the specific language governing permissions and\n",
    "   limitations under the License."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eEnKLNgpj_V4"
   },
   "outputs": [],
   "source": [
    "## Only needed when running in colab with data on your gdrive - as you can see not production but competition hack :-)\n",
    "\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Dqcy1FMZkCsq"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "package_path = '/content/drive/My Drive/m5data/gluonts' \n",
    "sys.path.append(package_path)\n",
    "\n",
    "###### IF NOT RUN ON COLAB YOU HAVE TO MAKE SURE THAT GLUONTS PACKAGE IS IN YOUR PATH\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Cb5pBjYlkI8q"
   },
   "source": [
    "# Imports\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "x0Vbmz_3kHPd"
   },
   "outputs": [],
   "source": [
    "# Make sure to select the correct MXNet Version (here we have CUDA 10.1 and mkl)\n",
    "!pip install --upgrade pydantic ujson mxnet-cu101mkl==1.4.1 --no-deps\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SH8Zj2DFkLez"
   },
   "outputs": [],
   "source": [
    "from gluonts.dataset.common import load_datasets, ListDataset\n",
    "from gluonts.dataset.field_names import FieldName"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_b7WfCpCkM4y"
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import mxnet as mx\n",
    "from mxnet import gluon\n",
    "import numpy as np\n",
    "import random\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "import os\n",
    "from tqdm.autonotebook import tqdm\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VSEjo9lqkN4B"
   },
   "outputs": [],
   "source": [
    "from gluonts.evaluation.backtest import make_evaluation_predictions\n",
    "from gluonts.trainer import Trainer\n",
    "from gluonts.model.n_beats import NBEATSEnsembleEstimator\n",
    "from gluonts.evaluation import Evaluator\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3SNnguDQfN2M"
   },
   "outputs": [],
   "source": [
    "class M5Evaluator(Evaluator):\n",
    "          \n",
    "        def get_metrics_per_ts(self, time_series, forecast):\n",
    "              successive_diff = np.diff(time_series.values.reshape(len(time_series)))\n",
    "              successive_diff = successive_diff ** 2\n",
    "              successive_diff = successive_diff[:-prediction_length]\n",
    "              denom = np.mean(successive_diff)\n",
    "              pred_values = forecast.samples.mean(axis=0)\n",
    "              true_values = time_series.values.reshape(len(time_series))[-prediction_length:]\n",
    "              num = np.mean((pred_values - true_values)**2)\n",
    "              rmsse = num / denom\n",
    "              metrics = super().get_metrics_per_ts(time_series, forecast)\n",
    "              metrics[\"RMSSE\"] = rmsse\n",
    "              return metrics\n",
    "          \n",
    "        def get_aggregate_metrics(self, metric_per_ts):\n",
    "              wrmsse = metric_per_ts[\"RMSSE\"].mean()\n",
    "              agg_metric , _ = super().get_aggregate_metrics(metric_per_ts)\n",
    "              agg_metric[\"MRMSSE\"] = wrmsse\n",
    "              return agg_metric, metric_per_ts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AgaIZ3lkkPcH"
   },
   "source": [
    "# Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uN9GUvjbkRJ7"
   },
   "outputs": [],
   "source": [
    "single_prediction_length = 28\n",
    "submission_prediction_length = single_prediction_length * 2\n",
    "m5_input_path=\"/content/drive/My Drive/m5data/\"\n",
    "\n",
    "SUBMISSION=True\n",
    "VISUALIZE=True\n",
    "\n",
    "VERSION=2\n",
    "\n",
    "CALC_RESIDUALS = False\n",
    "\n",
    "prediction_length = single_prediction_length\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XWxCS1XOkSj4"
   },
   "source": [
    "# Set Seeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_zBAa9G4kT0C"
   },
   "outputs": [],
   "source": [
    "# Seed value\n",
    "seed_value= 247\n",
    "\n",
    "# 1. Set `PYTHONHASHSEED` environment variable at a fixed value\n",
    "os.environ['PYTHONHASHSEED']=str(seed_value)\n",
    "\n",
    "# 2. Set `python` built-in pseudo-random generator at a fixed value\n",
    "random.seed(seed_value)\n",
    "\n",
    "# 3. Set `numpy` pseudo-random generator at a fixed value\n",
    "np.random.seed(seed_value)\n",
    "\n",
    "# 4. Set gluon seed...\n",
    "mx.random.seed(seed_value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SqQo1w5wkVe5"
   },
   "source": [
    "# Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gx5kwAJHkWtA"
   },
   "outputs": [],
   "source": [
    "# Load data\n",
    "print('Loading data...')\n",
    "sell_price = pd.read_csv('%s/sell_prices.csv' % m5_input_path)\n",
    "calendar = pd.read_csv('%s/calendar.csv' % m5_input_path)\n",
    "train = pd.read_csv('%s/sales_train_evaluation.csv' % m5_input_path).set_index('id')\n",
    "sample_sub = pd.read_csv('%s/sample_submission.csv' % m5_input_path)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "2hQ0Tgs7kYAV"
   },
   "source": [
    "# Build aggregate dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aVCSWu5_ka01"
   },
   "outputs": [],
   "source": [
    "# Get column groups\n",
    "cat_cols = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']\n",
    "ts_cols = [col for col in train.columns if col not in cat_cols]\n",
    "ts_dict = {t: int(t[2:]) for t in ts_cols}\n",
    "\n",
    "# Describe data\n",
    "print('  unique forecasts: %i' % train.shape[0])\n",
    "for col in cat_cols:\n",
    "    print('   N_unique %s: %i' % (col, train[col].nunique()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wJYz8VyokkBD"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# 1. All products, all stores, all states (1 series)\n",
    "all_sales = pd.DataFrame(train[ts_cols].sum()).transpose()\n",
    "all_sales['id_str'] = 'all'\n",
    "all_sales = all_sales[ ['id_str'] +  [c for c in all_sales if c not in ['id_str']] ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LyNIJtFb39SY"
   },
   "outputs": [],
   "source": [
    "all_sales"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yBBy2ycskpdI"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# 2. All products by state (3 series)\n",
    "state_sales = train.groupby('state_id',as_index=False)[ts_cols].sum()\n",
    "state_sales['id_str'] = state_sales['state_id'] \n",
    "state_sales = state_sales[ ['id_str'] +  [c for c in state_sales if c not in ['id_str']] ]\n",
    "state_sales = state_sales.drop(['state_id'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1x3XKwY-3uGa"
   },
   "outputs": [],
   "source": [
    "state_sales"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9SDGDCMXkucp"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# 3. All products by store (10 series)\n",
    "store_sales = train.groupby('store_id',as_index=False)[ts_cols].sum()\n",
    "store_sales['id_str'] = store_sales['store_id'] \n",
    "store_sales = store_sales[ ['id_str'] +  [c for c in store_sales if c not in ['id_str']] ]\n",
    "store_sales = store_sales.drop(['store_id'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MAZIgK1clAc-"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# 4. All products by category (3 series)\n",
    "cat_sales = train.groupby('cat_id',as_index=False)[ts_cols].sum()\n",
    "cat_sales['id_str'] = cat_sales['cat_id'] \n",
    "cat_sales = cat_sales[ ['id_str'] +  [c for c in cat_sales if c not in ['id_str']] ]\n",
    "cat_sales = cat_sales.drop(['cat_id'],axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1Xjgx12glFSA"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# 5. All products by department (7 series)\n",
    "dept_sales = train.groupby('dept_id',as_index=False)[ts_cols].sum()\n",
    "dept_sales['id_str'] = dept_sales['dept_id'] \n",
    "dept_sales = dept_sales[ ['id_str'] +  [c for c in dept_sales if c not in ['id_str']] ]\n",
    "dept_sales = dept_sales.drop(['dept_id'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "e1jTKElZ5eYm"
   },
   "outputs": [],
   "source": [
    "## TOP LEVEL aggregates + TOTAL\n",
    "all_aggregates = pd.concat([all_sales,state_sales,store_sales,cat_sales,dept_sales],ignore_index=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "eBgnV1QE56mm"
   },
   "source": [
    "# Prepare dataframe for gluon-ts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yXGDFMzG5842"
   },
   "outputs": [],
   "source": [
    "train_df = all_aggregates.drop([\"id_str\"], axis=1)\n",
    "train_target_values = train_df.values\n",
    "\n",
    "if SUBMISSION == True:\n",
    "    test_target_values = [np.append(ts, np.ones(prediction_length) * np.nan) for ts in train_df.values]\n",
    "else:\n",
    "    test_target_values = train_target_values.copy()\n",
    "    train_target_values = [ts[:-prediction_length] for ts in train_df.values]\n",
    "\n",
    "m5_dates = [pd.Timestamp(\"2011-01-29\", freq='1D') for _ in range(len(all_aggregates))]\n",
    "\n",
    "train_ds = ListDataset([\n",
    "      {\n",
    "          FieldName.TARGET: target,\n",
    "          FieldName.START: start\n",
    "      }\n",
    "      for (target, start) in zip(train_target_values,\n",
    "                                          m5_dates\n",
    "                                          )\n",
    "  ], freq=\"D\")\n",
    "\n",
    "test_ds = ListDataset([\n",
    "      {\n",
    "          FieldName.TARGET: target,\n",
    "          FieldName.START: start\n",
    "      }\n",
    "      for (target, start) in zip(test_target_values,\n",
    "                                          m5_dates)\n",
    "  ], freq=\"D\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Kc0PRY488wiw"
   },
   "outputs": [],
   "source": [
    "num_signals = len(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QEkEkzMT8avg"
   },
   "outputs": [],
   "source": [
    "next(iter(train_ds))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "h2WppDVa8mak"
   },
   "source": [
    "# Define Estimators and train on aggregates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "30Yd69ItX-zx"
   },
   "outputs": [],
   "source": [
    "if True:\n",
    "\n",
    "  estimator = NBEATSEnsembleEstimator(\n",
    "      prediction_length=prediction_length,\n",
    "      #context_length=7*prediction_length,\n",
    "      meta_bagging_size = 3,  # 3, ## Change back to 10 after testing??\n",
    "      meta_context_length = [prediction_length * mlp for mlp in [3,5,7] ], ## Change back to (2,7) // 3,5,7\n",
    "      meta_loss_function = ['sMAPE'], ## Change back to all three MAPE, MASE ...\n",
    "      num_stacks = 30,\n",
    "      widths= [512],\n",
    "      freq=\"D\",\n",
    "      trainer=Trainer(\n",
    "                    learning_rate=6e-4,\n",
    "                    #clip_gradient=1.0,\n",
    "                    epochs=12, #10\n",
    "                    num_batches_per_epoch=1000,\n",
    "                    batch_size=16\n",
    "                    #ctx=mx.context.gpu()\n",
    "                )\n",
    "\n",
    "  )\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YghYzbpdmA8K"
   },
   "outputs": [],
   "source": [
    "if SUBMISSION:\n",
    "  predictor = estimator.train(train_ds)\n",
    "else:\n",
    "  predictor = estimator.train(train_ds,test_ds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8lF-Egtx-Zup"
   },
   "source": [
    "# Analyze forcasts - Errors and Visual inspection\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gdAM9oNhYx1B"
   },
   "outputs": [],
   "source": [
    "  forecast_it, ts_it = make_evaluation_predictions(\n",
    "      dataset=test_ds,\n",
    "      predictor=predictor,\n",
    "      num_samples=100\n",
    "  )\n",
    "\n",
    "  print(\"Obtaining time series conditioning values ...\")\n",
    "  tss = list(tqdm(ts_it, total=len(test_ds)))\n",
    "  print(\"Obtaining time series predictions ...\")\n",
    "  forecasts = list(tqdm(forecast_it, total=len(test_ds)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zrIFAu-CY11w"
   },
   "outputs": [],
   "source": [
    "if not SUBMISSION:\n",
    "      evaluator = M5Evaluator(quantiles=[0.5, 0.67, 0.95, 0.99])\n",
    "      agg_metrics, item_metrics = evaluator(iter(tss), iter(forecasts), num_series=len(test_ds))\n",
    "      print(json.dumps(agg_metrics, indent=4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "L7Uu4Wt9rZMY"
   },
   "source": [
    "# Visualize forecasts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Rkzz0m7UjYcP"
   },
   "outputs": [],
   "source": [
    "num_series = len(all_aggregates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZX34DYggrb7I"
   },
   "outputs": [],
   "source": [
    "if VISUALIZE:\n",
    "  \n",
    "  plot_log_path = \"./plots/\"\n",
    "  directory = os.path.dirname(plot_log_path)\n",
    "  if not os.path.exists(directory):\n",
    "      os.makedirs(directory)\n",
    "      \n",
    "  def plot_prob_forecasts(ts_entry, forecast_entry, path, sample_id, inline=True):\n",
    "      plot_length = 150\n",
    "      prediction_intervals = (50, 99)\n",
    "      legend = [\"observations\", \"median prediction\"] + [f\"{k}% prediction interval\" for k in prediction_intervals][::-1]\n",
    "\n",
    "      _, ax = plt.subplots(1, 1, figsize=(10, 7))\n",
    "      ts_entry[-plot_length:].plot(ax=ax)\n",
    "      forecast_entry.plot(prediction_intervals=prediction_intervals, color='g')\n",
    "      ax.axvline(ts_entry.index[-prediction_length], color='r')\n",
    "      plt.legend(legend, loc=\"upper left\")\n",
    "      if inline:\n",
    "          plt.show()\n",
    "          plt.clf()\n",
    "      else:\n",
    "          plt.savefig('{}forecast_{}.pdf'.format(path, sample_id))\n",
    "          plt.close()\n",
    "\n",
    "  print(\"Plotting time series predictions ...\")\n",
    "  for i in tqdm(range(num_series)):\n",
    "      ts_entry = tss[i]\n",
    "      forecast_entry = forecasts[i]\n",
    "      plot_prob_forecasts(ts_entry, forecast_entry, plot_log_path, i)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aXoFAaSK44Cp"
   },
   "source": [
    "# Predict and save forecast\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "J3qH0_rB48no"
   },
   "outputs": [],
   "source": [
    "forecasts_acc = np.zeros((len(forecasts), prediction_length))\n",
    "\n",
    "for i in range(len(forecasts)):\n",
    "    forecasts_acc[i] = forecasts[i].samples\n",
    "\n",
    "columns = []\n",
    "for i in range(1,(forecasts_acc.shape[1]+1)):\n",
    "    columns.append(\"F\"+str(i))\n",
    "forecasts_acc_df = pd.DataFrame(data=forecasts_acc, columns=columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Q74vQ5fX-LeV"
   },
   "outputs": [],
   "source": [
    "forecasts_acc_df = pd.concat([all_aggregates['id_str'],forecasts_acc_df],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6AYKs5yx6wxF"
   },
   "outputs": [],
   "source": [
    "forecasts_acc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mA7a8M7l6xS6"
   },
   "outputs": [],
   "source": [
    "forecasts_acc_df.to_csv('{}nbeats_level_predictions/nbeats_toplvl_forecasts{}.csv'.format(m5_input_path, VERSION), index=False)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "M5 NBEATS TopLevel",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}