notebooks/blackjack.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3fc680b9-a919-4dda-9ecc-5a1a2a107477",
   "metadata": {},
   "source": [
    "# Blackjack\n",
    "Planner __init__ expects a reward and transition matrix P, which is a nested dictionary \n",
    "[OpenAI Gym](https://www.gymlibrary.ml/) style discrete environment where \n",
    "P[state][action] is a list of tuples (probability, next state, reward, terminal).\n",
    "\n",
    "The gym blackjack environment does not include this matrix, so we'll use the BlackjackWrapper class to create a wrapped gym environment, which modifies the observation space and includes P.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ec59973-2c7e-4331-9037-7b6f4a4a661e",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install bettermdptools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "900376a8-792f-41bc-988c-2bc23ff2d7d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gymnasium as gym\n",
    "from bettermdptools.utils.blackjack_wrapper import BlackjackWrapper\n",
    "from bettermdptools.utils.test_env import TestEnv\n",
    "from bettermdptools.algorithms.planner import Planner\n",
    "from bettermdptools.algorithms.rl import RL\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "265aae24-a1b3-4400-8bcc-8da16e9a4612",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "runtime = 0.03 seconds\n",
      "-0.03\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "runtime = 1.30 seconds\n",
      "0.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "base_env = gym.make('Blackjack-v1', render_mode=None)\n",
    "blackjack = BlackjackWrapper(base_env)\n",
    "\n",
    "# run VI\n",
    "V, V_track, pi = Planner(blackjack.P).value_iteration()\n",
    "\n",
    "#test policy\n",
    "test_scores = TestEnv.test_env(env=blackjack, n_iters=100, render=False, pi=pi, user_input=False)\n",
    "print(np.mean(test_scores))\n",
    "\n",
    "# Q-learning\n",
    "Q, V, pi, Q_track, pi_track = RL(blackjack).q_learning()\n",
    "\n",
    "#test policy\n",
    "test_scores = TestEnv.test_env(env=blackjack, n_iters=100, render=False, pi=pi, user_input=False)\n",
    "print(np.mean(test_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "518e348f-b5a2-409a-8c51-049410aa5a4e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "bettermdptools",
   "language": "python",
   "name": "bettermdptools"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}