ch11.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=1b8eaf74bf6e455ea8440642e23a1745\n",
    "%pushd book-materials\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "np.random.seed(12345)\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rc(\"figure\", figsize=(10, 6))\n",
    "PREVIOUS_MAX_ROWS = pd.options.display.max_rows\n",
    "pd.options.display.max_columns = 20\n",
    "pd.options.display.max_rows = 20\n",
    "pd.options.display.max_colwidth = 80\n",
    "np.set_printoptions(precision=4, suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b8f6434f6f9540249d053d94cee92435\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=77aa02db8c0740baa9d429ef9f9c92c3\n",
    "from datetime import datetime\n",
    "now = datetime.now()\n",
    "now\n",
    "now.year, now.month, now.day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=7939b9e6813c4810af2eb7d69016ac97\n",
    "delta = datetime(2011, 1, 7) - datetime(2008, 6, 24, 8, 15)\n",
    "delta\n",
    "delta.days\n",
    "delta.seconds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=62309078c64449508cf0353453566a3b\n",
    "from datetime import timedelta\n",
    "start = datetime(2011, 1, 7)\n",
    "start + timedelta(12)\n",
    "start - 2 * timedelta(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=2e863437108e49619f828b3399a23675\n",
    "stamp = datetime(2011, 1, 3)\n",
    "str(stamp)\n",
    "stamp.strftime(\"%Y-%m-%d\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=504bb23dbed740c697961017072ba0a5\n",
    "value = \"2011-01-03\"\n",
    "datetime.strptime(value, \"%Y-%m-%d\")\n",
    "datestrs = [\"7/6/2011\", \"8/6/2011\"]\n",
    "[datetime.strptime(x, \"%m/%d/%Y\") for x in datestrs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=66f136014e5e477aba0e773ab3137d23\n",
    "datestrs = [\"2011-07-06 12:00:00\", \"2011-08-06 00:00:00\"]\n",
    "pd.to_datetime(datestrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ab30c892b1cd40ee981e793e194fe34e\n",
    "idx = pd.to_datetime(datestrs + [None])\n",
    "idx\n",
    "idx[2]\n",
    "pd.isna(idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=c65a386b5ee44b77ae927a17a586fdce\n",
    "dates = [datetime(2011, 1, 2), datetime(2011, 1, 5),\n",
    "         datetime(2011, 1, 7), datetime(2011, 1, 8),\n",
    "         datetime(2011, 1, 10), datetime(2011, 1, 12)]\n",
    "ts = pd.Series(np.random.standard_normal(6), index=dates)\n",
    "ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b29f5a3e0b3744ac8d9d10b87b10a23d\n",
    "ts.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b345022b87e2494592f0fc50fdc3461a\n",
    "ts + ts[::2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=713bb1b1f7424b228fc940b8ff21adcf\n",
    "ts.index.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=1de1ab6f38e640c7ae177e117612248f\n",
    "stamp = ts.index[0]\n",
    "stamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=962135198c7243b9b891d420cf335e69\n",
    "stamp = ts.index[2]\n",
    "ts[stamp]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=01e57a024aa849279a3ebacc160772ca\n",
    "ts[\"2011-01-10\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d499c6cab3f64469acf7942aaa7bc5f1\n",
    "longer_ts = pd.Series(np.random.standard_normal(1000),\n",
    "                      index=pd.date_range(\"2000-01-01\", periods=1000))\n",
    "longer_ts\n",
    "longer_ts[\"2001\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=4fc49d1ae9fd4f0b8e1a69999896298c\n",
    "longer_ts[\"2001-05\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=59f52116836440a88e46e01adf808d35\n",
    "ts[datetime(2011, 1, 7):]\n",
    "ts[datetime(2011, 1, 7):datetime(2011, 1, 10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=484c72fd4b744f5bbcb8a6493b45018c\n",
    "ts\n",
    "ts[\"2011-01-06\":\"2011-01-11\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=a6db533c922e4684a8c267288880f20b\n",
    "ts.truncate(after=\"2011-01-09\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=7324548996ac413f81d5e7014af682a0\n",
    "dates = pd.date_range(\"2000-01-01\", periods=100, freq=\"W-WED\")\n",
    "long_df = pd.DataFrame(np.random.standard_normal((100, 4)),\n",
    "                       index=dates,\n",
    "                       columns=[\"Colorado\", \"Texas\",\n",
    "                                \"New York\", \"Ohio\"])\n",
    "long_df.loc[\"2001-05\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b64ec3dd408148cda95729e8c7bd9b47\n",
    "dates = pd.DatetimeIndex([\"2000-01-01\", \"2000-01-02\", \"2000-01-02\",\n",
    "                          \"2000-01-02\", \"2000-01-03\"])\n",
    "dup_ts = pd.Series(np.arange(5), index=dates)\n",
    "dup_ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=aafe9e9742b74098aac057bb029e620d\n",
    "dup_ts.index.is_unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=1963a191266e4095a6ed630a8e606956\n",
    "dup_ts[\"2000-01-03\"]  # not duplicated\n",
    "dup_ts[\"2000-01-02\"]  # duplicated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=7895e016504f421ab8bd9cafeaeffea6\n",
    "grouped = dup_ts.groupby(level=0)\n",
    "grouped.mean()\n",
    "grouped.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=215d6274cd3e4bc3bcacdd8e5ce9f838\n",
    "ts\n",
    "resampler = ts.resample(\"D\")\n",
    "resampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=c202d9e9284c4fd28200376310af6070\n",
    "index = pd.date_range(\"2012-04-01\", \"2012-06-01\")\n",
    "index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=19e8a1e8306f4ed2b5329228064c50a6\n",
    "pd.date_range(start=\"2012-04-01\", periods=20)\n",
    "pd.date_range(end=\"2012-06-01\", periods=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=8c50f185b572469eb8caa8ca321df712\n",
    "pd.date_range(\"2000-01-01\", \"2000-12-01\", freq=\"BM\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=aa62083a0254482486435a4e2d777694\n",
    "pd.date_range(\"2012-05-02 12:56:31\", periods=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=2fa04c5942ae482d87ef4eba22d7dc24\n",
    "pd.date_range(\"2012-05-02 12:56:31\", periods=5, normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=4a7a44f7b0394f4a8f9fd7bc31aae08f\n",
    "from pandas.tseries.offsets import Hour, Minute\n",
    "hour = Hour()\n",
    "hour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=502bfa3bd2b340c7bc8bd5a6e0855c63\n",
    "four_hours = Hour(4)\n",
    "four_hours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e2b7e93362014fc0a72d5c0675bba1a7\n",
    "pd.date_range(\"2000-01-01\", \"2000-01-03 23:59\", freq=\"4H\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=beef8c7807e34b1ea65756437085bab5\n",
    "Hour(2) + Minute(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=5bb25a584faa49718052115bd3b3ae39\n",
    "pd.date_range(\"2000-01-01\", periods=10, freq=\"1h30min\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=db9f1fc3d9a5419994e1adda66c83212\n",
    "monthly_dates = pd.date_range(\"2012-01-01\", \"2012-09-01\", freq=\"WOM-3FRI\")\n",
    "list(monthly_dates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b2397e270f56489e89f63a2c1732d8d5\n",
    "ts = pd.Series(np.random.standard_normal(4),\n",
    "               index=pd.date_range(\"2000-01-01\", periods=4, freq=\"M\"))\n",
    "ts\n",
    "ts.shift(2)\n",
    "ts.shift(-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=30cdaeba864a452b894ce82253efcec7\n",
    "ts.shift(2, freq=\"M\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=43fd4a3b864347008c9e58579d1318ec\n",
    "ts.shift(3, freq=\"D\")\n",
    "ts.shift(1, freq=\"90T\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d3a295c2d66d4ee49501d8e52df4006c\n",
    "from pandas.tseries.offsets import Day, MonthEnd\n",
    "now = datetime(2011, 11, 17)\n",
    "now + 3 * Day()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=840a4a58d2444b42b7f71aa5f69af613\n",
    "now + MonthEnd()\n",
    "now + MonthEnd(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=947ac8d4eae84bcb9abbd6250957070f\n",
    "offset = MonthEnd()\n",
    "offset.rollforward(now)\n",
    "offset.rollback(now)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=efd27228c8f54689b2e1e476204ba317\n",
    "ts = pd.Series(np.random.standard_normal(20),\n",
    "               index=pd.date_range(\"2000-01-15\", periods=20, freq=\"4D\"))\n",
    "ts\n",
    "ts.groupby(MonthEnd().rollforward).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b6d6087bdfb4474f91a0cf774283f1d1\n",
    "ts.resample(\"M\").mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=19c326a03d9a4cd783fbcc8cb5ff421c\n",
    "import pytz\n",
    "pytz.common_timezones[-5:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=81652670d9d84ed1bc65dff261cd0937\n",
    "tz = pytz.timezone(\"America/New_York\")\n",
    "tz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=27a1920372f0414cb335c9ac80c1a883\n",
    "dates = pd.date_range(\"2012-03-09 09:30\", periods=6)\n",
    "ts = pd.Series(np.random.standard_normal(len(dates)), index=dates)\n",
    "ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=fd6da4e7623d40babb80385bd6b80a36\n",
    "print(ts.index.tz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e03db4a0b6d34d66a88f0be927424a1b\n",
    "pd.date_range(\"2012-03-09 09:30\", periods=10, tz=\"UTC\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=0dd2997562634b6c9161adc215855962\n",
    "ts\n",
    "ts_utc = ts.tz_localize(\"UTC\")\n",
    "ts_utc\n",
    "ts_utc.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=49c15434eb0c47bf950d42390118f378\n",
    "ts_utc.tz_convert(\"America/New_York\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=01e4044008b14d7f9e4990a1e66210e3\n",
    "ts_eastern = ts.tz_localize(\"America/New_York\")\n",
    "ts_eastern.tz_convert(\"UTC\")\n",
    "ts_eastern.tz_convert(\"Europe/Berlin\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=1dcc4db3bd4b43138c22293079cafe68\n",
    "ts.index.tz_localize(\"Asia/Shanghai\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e3618c9788f040a38fac2443dac25892\n",
    "stamp = pd.Timestamp(\"2011-03-12 04:00\")\n",
    "stamp_utc = stamp.tz_localize(\"utc\")\n",
    "stamp_utc.tz_convert(\"America/New_York\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=8715a838b21f4789a6b58e4ceb0a56e4\n",
    "stamp_moscow = pd.Timestamp(\"2011-03-12 04:00\", tz=\"Europe/Moscow\")\n",
    "stamp_moscow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=0aed2bac3c41471899b081a582412f4e\n",
    "stamp_utc.value\n",
    "stamp_utc.tz_convert(\"America/New_York\").value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b778152898374964a084de3b3b63f935\n",
    "stamp = pd.Timestamp(\"2012-03-11 01:30\", tz=\"US/Eastern\")\n",
    "stamp\n",
    "stamp + Hour()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=c0d18d84985d4ede88ec95727c7fb8e9\n",
    "stamp = pd.Timestamp(\"2012-11-04 00:30\", tz=\"US/Eastern\")\n",
    "stamp\n",
    "stamp + 2 * Hour()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=8cde92a79e444f44b5f952eabb4b7491\n",
    "dates = pd.date_range(\"2012-03-07 09:30\", periods=10, freq=\"B\")\n",
    "ts = pd.Series(np.random.standard_normal(len(dates)), index=dates)\n",
    "ts\n",
    "ts1 = ts[:7].tz_localize(\"Europe/London\")\n",
    "ts2 = ts1[2:].tz_convert(\"Europe/Moscow\")\n",
    "result = ts1 + ts2\n",
    "result.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=bae9a7cce5cd496b8b3756412b670013\n",
    "p = pd.Period(\"2011\", freq=\"A-DEC\")\n",
    "p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=518401c31bc8419bbf28956e2c9be595\n",
    "p + 5\n",
    "p - 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e5ff985e90fe44579705c3d3cdf5e5b7\n",
    "pd.Period(\"2014\", freq=\"A-DEC\") - p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ef000f892654403b9f476bb7e1d9bacd\n",
    "periods = pd.period_range(\"2000-01-01\", \"2000-06-30\", freq=\"M\")\n",
    "periods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d3426c6da33b4175891f9d8d4caa4ab8\n",
    "pd.Series(np.random.standard_normal(6), index=periods)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=47db0721ff4a4ca89bdc74daae703a55\n",
    "values = [\"2001Q3\", \"2002Q2\", \"2003Q1\"]\n",
    "index = pd.PeriodIndex(values, freq=\"Q-DEC\")\n",
    "index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=348880d80dc14062bb80c374d953f7ce\n",
    "p = pd.Period(\"2011\", freq=\"A-DEC\")\n",
    "p\n",
    "p.asfreq(\"M\", how=\"start\")\n",
    "p.asfreq(\"M\", how=\"end\")\n",
    "p.asfreq(\"M\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=669af5f271964c7cbefe3265cde35c6c\n",
    "p = pd.Period(\"2011\", freq=\"A-JUN\")\n",
    "p\n",
    "p.asfreq(\"M\", how=\"start\")\n",
    "p.asfreq(\"M\", how=\"end\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ec04cdb713554c0fa0b4162e372b9876\n",
    "p = pd.Period(\"Aug-2011\", \"M\")\n",
    "p.asfreq(\"A-JUN\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=979cb059424a48f6a887fb50fdb66adf\n",
    "periods = pd.period_range(\"2006\", \"2009\", freq=\"A-DEC\")\n",
    "ts = pd.Series(np.random.standard_normal(len(periods)), index=periods)\n",
    "ts\n",
    "ts.asfreq(\"M\", how=\"start\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=51ab6bd5e33e433a8786b0a3fe239dd2\n",
    "ts.asfreq(\"B\", how=\"end\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=38b59581b62f4808a145d4b5bac7d04c\n",
    "p = pd.Period(\"2012Q4\", freq=\"Q-JAN\")\n",
    "p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=03f4b2f382e84260bfccec8df35b5903\n",
    "p.asfreq(\"D\", how=\"start\")\n",
    "p.asfreq(\"D\", how=\"end\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=629d1fcb8b5d42329da5126f166a028f\n",
    "p4pm = (p.asfreq(\"B\", how=\"end\") - 1).asfreq(\"T\", how=\"start\") + 16 * 60\n",
    "p4pm\n",
    "p4pm.to_timestamp()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=2b671198cc014db3bf349878fab5e36a\n",
    "periods = pd.period_range(\"2011Q3\", \"2012Q4\", freq=\"Q-JAN\")\n",
    "ts = pd.Series(np.arange(len(periods)), index=periods)\n",
    "ts\n",
    "new_periods = (periods.asfreq(\"B\", \"end\") - 1).asfreq(\"H\", \"start\") + 16\n",
    "ts.index = new_periods.to_timestamp()\n",
    "ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=debf05cadb0f4a94a7174e2f8859f352\n",
    "dates = pd.date_range(\"2000-01-01\", periods=3, freq=\"M\")\n",
    "ts = pd.Series(np.random.standard_normal(3), index=dates)\n",
    "ts\n",
    "pts = ts.to_period()\n",
    "pts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=6ac8d11fe6a1408cb83be89895617dd6\n",
    "dates = pd.date_range(\"2000-01-29\", periods=6)\n",
    "ts2 = pd.Series(np.random.standard_normal(6), index=dates)\n",
    "ts2\n",
    "ts2.to_period(\"M\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=de1111bbae4a4343945546f43c1af706\n",
    "pts = ts2.to_period()\n",
    "pts\n",
    "pts.to_timestamp(how=\"end\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d14658aeb9db49f090190ba82c255d4e\n",
    "data = pd.read_csv(\"examples/macrodata.csv\")\n",
    "data.head(5)\n",
    "data[\"year\"]\n",
    "data[\"quarter\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=4b1ab91f7f9d4f958b10ae49617ae6c6\n",
    "index = pd.PeriodIndex(year=data[\"year\"], quarter=data[\"quarter\"],\n",
    "                       freq=\"Q-DEC\")\n",
    "index\n",
    "data.index = index\n",
    "data[\"infl\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=07232ac59e2a49f18606b1b9422b2889\n",
    "dates = pd.date_range(\"2000-01-01\", periods=100)\n",
    "ts = pd.Series(np.random.standard_normal(len(dates)), index=dates)\n",
    "ts\n",
    "ts.resample(\"M\").mean()\n",
    "ts.resample(\"M\", kind=\"period\").mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=12dad7e46e2449eca9f2482399b82397\n",
    "dates = pd.date_range(\"2000-01-01\", periods=12, freq=\"T\")\n",
    "ts = pd.Series(np.arange(len(dates)), index=dates)\n",
    "ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b187a814d71c4a7bbb8c1c8750d9133f\n",
    "ts.resample(\"5min\").sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=450f99fca60945bab78ddf104cedabf9\n",
    "ts.resample(\"5min\", closed=\"right\").sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=6587ba3145214dc8b3f50ccb40bc88b8\n",
    "ts.resample(\"5min\", closed=\"right\", label=\"right\").sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=c48ee1ea64c2498e8ede907aea647016\n",
    "from pandas.tseries.frequencies import to_offset\n",
    "result = ts.resample(\"5min\", closed=\"right\", label=\"right\").sum()\n",
    "result.index = result.index + to_offset(\"-1s\")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e036e195212b453985a4bc219cd308f3\n",
    "ts = pd.Series(np.random.permutation(np.arange(len(dates))), index=dates)\n",
    "ts.resample(\"5min\").ohlc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=22d7658abb234e16a56b2637142c053c\n",
    "frame = pd.DataFrame(np.random.standard_normal((2, 4)),\n",
    "                     index=pd.date_range(\"2000-01-01\", periods=2,\n",
    "                                         freq=\"W-WED\"),\n",
    "                     columns=[\"Colorado\", \"Texas\", \"New York\", \"Ohio\"])\n",
    "frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=7a564b646cbc45d78d95f6333692fd84\n",
    "df_daily = frame.resample(\"D\").asfreq()\n",
    "df_daily"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ecb1c4fc9803419b849c214cddaddae3\n",
    "frame.resample(\"D\").ffill()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=a84044fc3dbe4f0597b997ec51c96490\n",
    "frame.resample(\"D\").ffill(limit=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=3f5e6bf2f6844db9ae9ac57ee806df7e\n",
    "frame.resample(\"W-THU\").ffill()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=42a429ef95bc45fdb9c595f3b3ffd163\n",
    "frame = pd.DataFrame(np.random.standard_normal((24, 4)),\n",
    "                     index=pd.period_range(\"1-2000\", \"12-2001\",\n",
    "                                           freq=\"M\"),\n",
    "                     columns=[\"Colorado\", \"Texas\", \"New York\", \"Ohio\"])\n",
    "frame.head()\n",
    "annual_frame = frame.resample(\"A-DEC\").mean()\n",
    "annual_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=21bc509f9fc340b6882974f3ec17e715\n",
    "# Q-DEC: Quarterly, year ending in December\n",
    "annual_frame.resample(\"Q-DEC\").ffill()\n",
    "annual_frame.resample(\"Q-DEC\", convention=\"end\").asfreq()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=9d82a3b714164b4dad4eceaeadeda604\n",
    "annual_frame.resample(\"Q-MAR\").ffill()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=f23204097cbd44b3a899d2cbaa35c2bd\n",
    "N = 15\n",
    "times = pd.date_range(\"2017-05-20 00:00\", freq=\"1min\", periods=N)\n",
    "df = pd.DataFrame({\"time\": times,\n",
    "                   \"value\": np.arange(N)})\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=806fd8e5d2aa413f8c990b6acebde10d\n",
    "df.set_index(\"time\").resample(\"5min\").count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=9a28095367094308ad46b129e69586aa\n",
    "df2 = pd.DataFrame({\"time\": times.repeat(3),\n",
    "                    \"key\": np.tile([\"a\", \"b\", \"c\"], N),\n",
    "                    \"value\": np.arange(N * 3.)})\n",
    "df2.head(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=0991852576124da587038d8939d3de61\n",
    "time_key = pd.Grouper(freq=\"5min\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=a7c8d07161384a4385a4d82fffa4e7ae\n",
    "resampled = (df2.set_index(\"time\")\n",
    "             .groupby([\"key\", time_key])\n",
    "             .sum())\n",
    "resampled\n",
    "resampled.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=3ff50b0ceeef40c4bbe32b4b8cf3824a\n",
    "close_px_all = pd.read_csv(\"examples/stock_px.csv\",\n",
    "                           parse_dates=True, index_col=0)\n",
    "close_px = close_px_all[[\"AAPL\", \"MSFT\", \"XOM\"]]\n",
    "close_px = close_px.resample(\"B\").ffill()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=3a7907583c5c464eb2d146e8ddcb479f\n",
    "close_px[\"AAPL\"].plot()\n",
    "#! figure,id=apple_daily_ma250,title=\"Apple price with 250-day moving average\"\n",
    "close_px[\"AAPL\"].rolling(250).mean().plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=cd2ea550f4ab44bebddcdef67fc7990b\n",
    "plt.figure()\n",
    "std250 = close_px[\"AAPL\"].pct_change().rolling(250, min_periods=10).std()\n",
    "std250[5:12]\n",
    "#! figure,id=apple_daily_std250,title=\"Apple 250-day daily return standard deviation\"\n",
    "std250.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=504b4010407f4edcac81dfa106681206\n",
    "expanding_mean = std250.expanding().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=828830f0853b45388516d1d716305702\n",
    "plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=757ccd62b4c64042add7a055b90d3f79\n",
    "plt.style.use('grayscale')\n",
    "#! figure,id=stocks_daily_ma60,title=\"Stock prices 60-day moving average (log y-axis)\"\n",
    "close_px.rolling(60).mean().plot(logy=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d1df05357bef4ef5be80524c35b9407a\n",
    "close_px.rolling(\"20D\").mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=f65d7f1f960c4466af77c7e80a9fdf4c\n",
    "plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=fac9c661db8a4c77bd6e450f0ca0f082\n",
    "aapl_px = close_px[\"AAPL\"][\"2006\":\"2007\"]\n",
    "\n",
    "ma30 = aapl_px.rolling(30, min_periods=20).mean()\n",
    "ewma30 = aapl_px.ewm(span=30).mean()\n",
    "\n",
    "aapl_px.plot(style=\"k-\", label=\"Price\")\n",
    "ma30.plot(style=\"k--\", label=\"Simple Moving Avg\")\n",
    "ewma30.plot(style=\"k-\", label=\"EW MA\")\n",
    "#! figure,id=timeseries_ewma,title=\"Simple moving average versus exponentially weighted\"\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=cf8d4580bade4701b8cc6c1b05c4ed97\n",
    "plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=850496d7c94b423f994cbc710002ccf7\n",
    "spx_px = close_px_all[\"SPX\"]\n",
    "spx_rets = spx_px.pct_change()\n",
    "returns = close_px.pct_change()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=f065c26f19e7491b8f573f8d0f0d67e3\n",
    "corr = returns[\"AAPL\"].rolling(125, min_periods=100).corr(spx_rets)\n",
    "#! figure,id=roll_correl_aapl,title=\"Six-month AAPL return correlation to S&P 500\"\n",
    "corr.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=f67e1433110f4a6a8e72034ead461e09\n",
    "plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e312071fe8a74319afc03f447030c468\n",
    "corr = returns.rolling(125, min_periods=100).corr(spx_rets)\n",
    "#! figure,id=roll_correl_all,title=\"Six-month return correlations to S&P 500\"\n",
    "corr.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=e0f7372085924896960706add4fa5f56\n",
    "plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e176480d4d034f1bbaa1b4c09caf9df6\n",
    "from scipy.stats import percentileofscore\n",
    "def score_at_2percent(x):\n",
    "    return percentileofscore(x, 0.02)\n",
    "\n",
    "result = returns[\"AAPL\"].rolling(250).apply(score_at_2percent)\n",
    "#! figure,id=roll_apply_ex,title=\"Percentile rank of 2% AAPL return over one-year window\"\n",
    "result.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=419d9badc37c41888f19b0fd158061dd\n",
    "%popd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=1ff3016eeceb4d3e9a95f134c7ab9512\n",
    "pd.options.display.max_rows = PREVIOUS_MAX_ROWS"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}