stock_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from ESRNN import ESRNN\n",
    "plt.style.use('ggplot')\n",
    "pd.options.display.max_rows = 999\n",
    "np.set_printoptions(threshold=np.inf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Useful funtions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot\n",
    "def plot_prediction(y, y_hat):\n",
    "    n_y = len(y)\n",
    "    n_yhat = len(y_hat)\n",
    "    ds_y = np.array(range(n_y))\n",
    "    ds_yhat = np.array(range(n_y, n_y+n_yhat))\n",
    "\n",
    "    plt.plot(ds_y, y, label = 'y')\n",
    "    plt.plot(ds_yhat, y_hat, label='y_hat')\n",
    "    plt.legend(loc='upper left')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ffill_missing_dates_particular_serie(serie, min_date, max_date, freq):\n",
    "    date_range = pd.date_range(start=min_date, end=max_date, freq=freq)\n",
    "    unique_id = serie['unique_id'].unique()\n",
    "    df_balanced = pd.DataFrame({'ds':date_range, 'key':[1]*len(date_range), 'unique_id': unique_id[0]})\n",
    "\n",
    "    # Check balance\n",
    "    check_balance = df_balanced.groupby(['unique_id']).size().reset_index(name='count')\n",
    "    assert len(set(check_balance['count'].values)) <= 1\n",
    "    df_balanced = df_balanced.merge(serie, how=\"left\", on=['unique_id', 'ds'])\n",
    "\n",
    "    df_balanced['y'] = df_balanced['y'].fillna(method='ffill')\n",
    "    df_balanced['x'] = df_balanced['x'].fillna(method='ffill')\n",
    "\n",
    "\n",
    "    return df_balanced\n",
    "\n",
    "def ffill_missing_dates_per_serie(df, freq, fixed_max_date=None):\n",
    "    \"\"\"Receives a DataFrame with a date column and forward fills the missing gaps in dates, not filling dates before\n",
    "    the first appearance of a unique key\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    df: DataFrame\n",
    "        Input DataFrame\n",
    "    key: str or list\n",
    "        Name(s) of the column(s) which make a unique time series\n",
    "    date_col: str\n",
    "        Name of the column that contains the time column\n",
    "    freq: str\n",
    "        Pandas time frequency standard strings, like \"W-THU\" or \"D\" or \"M\"\n",
    "    numeric_to_fill: str or list\n",
    "        Name(s) of the columns with numeric values to fill \"fill_value\" with\n",
    "    \"\"\"\n",
    "    if fixed_max_date is None:\n",
    "        df_max_min_dates = df[['unique_id', 'ds']].groupby('unique_id').agg(['min', 'max']).reset_index()\n",
    "    else:\n",
    "        df_max_min_dates = df[['unique_id', 'ds']].groupby('unique_id').agg(['min']).reset_index()\n",
    "        df_max_min_dates['max'] = fixed_max_date\n",
    "\n",
    "    df_max_min_dates.columns = df_max_min_dates.columns.droplevel()\n",
    "    df_max_min_dates.columns = ['unique_id', 'min_date', 'max_date']\n",
    "\n",
    "    df_list = []\n",
    "    for index, row in df_max_min_dates.iterrows():\n",
    "        df_id = df[df['unique_id'] == row['unique_id']]\n",
    "        df_id = ffill_missing_dates_particular_serie(df_id, row['min_date'], row['max_date'], freq)\n",
    "        df_list.append(df_id)\n",
    "\n",
    "    df_dates = pd.concat(df_list).reset_index(drop=True).drop('key', axis=1)[['unique_id', 'ds', 'y','x']]\n",
    "\n",
    "    return df_dates"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Original stock data\n",
    "data = pd.read_csv('data/train.csv')\n",
    "data['Date'] = data['Year'].astype(str)+'-'+data['Date'].astype(str)\n",
    "data['Date'] = pd.to_datetime(data['Date'])\n",
    "data = data[['Company','Year','Date','Close']]\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean data (model assumes this name columns)\n",
    "data['unique_id'] = data['Company']+\"_\"+data['Year'].astype(str)\n",
    "data = data.rename(columns={'Date':'ds', 'Close':'y'})\n",
    "data['x'] = data['Year'].astype(str)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Series must be complete in the frequency\n",
    "data = ffill_missing_dates_per_serie(data,'D')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = data[['unique_id','ds','x']]\n",
    "y_train = data[['unique_id','ds','y']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_test = pd.read_csv('data/test.csv')\n",
    "data_test['Date'] = data_test['Year'].astype(str)+'-'+data_test['Date'].astype(str)\n",
    "data_test['Date'] = pd.to_datetime(data_test['Date'])\n",
    "data_test = data_test[['Company','Year','Date','Close']]\n",
    "data_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean data (model assumes this name columns)\n",
    "data_test['unique_id'] = data_test['Company']+\"_\"+data_test['Year'].astype(str)\n",
    "data_test = data_test.rename(columns={'Date':'ds', 'Close':'y'})\n",
    "data_test['x'] = data_test['Year'].astype(str)\n",
    "X_test = data_test[['unique_id','ds']]\n",
    "X_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model train example to view initial vs trained predictions\n",
    "Note: to reduce wigglines of prediction train with more epochs and/or increase level_variability_penalty hyperpar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model with no train to see initial prediction\n",
    "esrnn = ESRNN(max_epochs=0, batch_size=8, learning_rate=1e-3, \n",
    "              seasonality=30, input_size=30, output_size=60)\n",
    "esrnn.fit(X_train, y_train, random_seed=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat = esrnn.predict(y_train[['unique_id']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniques = X_train['unique_id'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_id = 0\n",
    "y_test_plot = y_train.loc[y_train['unique_id']==uniques[plot_id]]\n",
    "y_hat_test_plot = y_hat.loc[y_hat['unique_id']==uniques[plot_id]]\n",
    "plot_prediction(y_test_plot['y'], y_hat_test_plot['y_hat'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train model\n",
    "esrnn = ESRNN(max_epochs=50, batch_size=8, learning_rate=1e-3, \n",
    "              seasonality=30, input_size=30, output_size=60)\n",
    "esrnn.fit(X_train, y_train, random_seed=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat = esrnn.predict(y_train[['unique_id']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_id = 0\n",
    "y_test_plot = y_train.loc[y_train['unique_id']==uniques[plot_id]]\n",
    "y_hat_test_plot = y_hat.loc[y_hat['unique_id']==uniques[plot_id]]\n",
    "plot_prediction(y_test_plot['y'], y_hat_test_plot['y_hat'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predictions in stock test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat = esrnn.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}