notebooks/other_utilities.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a3b2f3f4-4123-431d-a77f-4db15bd6a52a",
   "metadata": {},
   "source": [
    "# Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50c3a418-2a26-4ecb-a3f6-d019f5513ece",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install bettermdptools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4815068f-bb52-47fb-9d35-3a340663c758",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gymnasium as gym\n",
    "from bettermdptools.utils.test_env import TestEnv\n",
    "from bettermdptools.utils.grid_search import GridSearch\n",
    "from bettermdptools.utils.blackjack_wrapper import BlackjackWrapper\n",
    "from bettermdptools.algorithms.rl import RL\n",
    "from bettermdptools.algorithms.planner import Planner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "448dc393-ba4f-412d-ac89-6c71a22cfeaf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running q_learning with gamma: 0.99 epsilon decay: 0.9  iterations: 500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "runtime = 0.64 seconds\n",
      "Avg. episode reward:  0.0\n",
      "###################\n",
      "running q_learning with gamma: 0.99 epsilon decay: 0.9  iterations: 5000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "runtime = 6.49 seconds\n",
      "Avg. episode reward:  0.0\n",
      "###################\n",
      "running q_learning with gamma: 0.99 epsilon decay: 0.9  iterations: 50000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "runtime = 54.57 seconds\n",
      "Avg. episode reward:  0.88\n",
      "###################\n"
     ]
    }
   ],
   "source": [
    "frozen_lake = gym.make('FrozenLake8x8-v1', render_mode=None)\n",
    "gamma=[.99]\n",
    "epsilon_decay = [.9]\n",
    "iters = [500, 5000, 50000]\n",
    "GridSearch.q_learning_grid_search(frozen_lake, gamma, epsilon_decay, iters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "60161c93-6183-4de6-a937-7c12b9742fb7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running VI with gamma: 0.7  n_iters: 500  theta: 0.001\n",
      "runtime = 0.01 seconds\n",
      "Avg. episode reward:  -0.16\n",
      "###################\n",
      "running VI with gamma: 0.7  n_iters: 500  theta: 1e-05\n",
      "runtime = 0.02 seconds\n",
      "Avg. episode reward:  0.18\n",
      "###################\n",
      "running VI with gamma: 0.9  n_iters: 500  theta: 0.001\n",
      "runtime = 0.01 seconds\n",
      "Avg. episode reward:  -0.06\n",
      "###################\n",
      "running VI with gamma: 0.9  n_iters: 500  theta: 1e-05\n",
      "runtime = 0.01 seconds\n",
      "Avg. episode reward:  -0.1\n",
      "###################\n",
      "running VI with gamma: 0.99  n_iters: 500  theta: 0.001\n",
      "runtime = 0.01 seconds\n",
      "Avg. episode reward:  -0.1\n",
      "###################\n",
      "running VI with gamma: 0.99  n_iters: 500  theta: 1e-05\n",
      "runtime = 0.01 seconds\n",
      "Avg. episode reward:  -0.04\n",
      "###################\n"
     ]
    }
   ],
   "source": [
    "gamma=[.7, .9, .99]\n",
    "n_iters=[500]\n",
    "theta=[.001, .00001]\n",
    "base_env = gym.make('Blackjack-v1', render_mode=None)\n",
    "blackjack = BlackjackWrapper(base_env)\n",
    "GridSearch.vi_grid_search(blackjack, gamma, n_iters, theta)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83449621-e3f7-4202-a836-645fe1bbd5c5",
   "metadata": {},
   "source": [
    "# Callbacks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c329484d-db1b-48fe-b7a0-b035ad5356bb",
   "metadata": {},
   "source": [
    "RL algorithms SARSA and Q-learning have callback hooks for episode number, begin, end, and env. step.   To create a callback, override one of the callback functions in the child class MyCallbacks.  Or, you can use the add_to decorator to define the override outside of the class definition. For example, print the episode number every 1000 episodes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "787ce4d3-e2b6-459f-94a6-ae4201fa091f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 1661/10000 [00:00<00:01, 8314.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " episode= 0\n",
      " episode= 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 3302/10000 [00:00<00:00, 8044.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " episode= 2000\n",
      " episode= 3000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▉     | 4911/10000 [00:00<00:00, 7923.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " episode= 4000\n",
      " episode= 5000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 7295/10000 [00:00<00:00, 7878.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " episode= 6000\n",
      " episode= 7000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 8843/10000 [00:01<00:00, 7328.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " episode= 8000\n",
      " episode= 9000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                      "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "runtime = 1.31 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "from bettermdptools.utils.decorators import add_to\n",
    "from bettermdptools.utils.callbacks import MyCallbacks\n",
    "from bettermdptools.algorithms.rl import RL\n",
    "from bettermdptools.utils.blackjack_wrapper import BlackjackWrapper\n",
    "\n",
    "base_env = gym.make('Blackjack-v1', render_mode=None)\n",
    "blackjack = BlackjackWrapper(base_env)\n",
    "\n",
    "@add_to(MyCallbacks)\n",
    "def on_episode(self, caller, episode):\n",
    "    if episode % 1000 == 0:\n",
    "    \tprint(\" episode=\", episode)\n",
    "\n",
    "# Q-learning\n",
    "Q, V, pi, Q_track, pi_track = RL(blackjack).q_learning()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9abf238-b0ea-4438-b461-03043218481d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "bettermdptools",
   "language": "python",
   "name": "bettermdptools"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}