plot_experiments_multiple.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This notebook is still somewhat experimental (and does not support bonefrroni corrections, normalising episodic rewards, bootstrapped / studentised confidence intervals, the option common_y_scale)\n",
    "# Setup to analyse an MDP Playground experiment\n",
    "from mdp_playground.analysis import MDPP_Analysis_Multiple as MDPP_Analysis\n",
    "\n",
    "# Set the following to True to save PDFs of plots that you generate below\n",
    "save_fig = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Data loading\n",
    "mdpp_analysis = MDPP_Analysis()\n",
    "\n",
    "# Also load evaluation runs, which may be run intermittently during training runs\n",
    "load_eval = False\n",
    "\n",
    "# load multiple experiments data\n",
    "\n",
    "# experiments = { <exp_name_1>: <dir_name_1>, ... }\n",
    "# For each experiment (also works in case of a single experiment):\n",
    "# Set dir_name to the location where the CSV files from running an experiment were saved\n",
    "# Set exp_name to the name that was given to the experiment when running it, i.e., with the -e option\n",
    "experiments = {\n",
    "    \"dqn_p_r_noises\" : \"/home/rajanr/spider_plots_data_discrete\",\n",
    "    \"rainbow_p_r_noises\" : \"/home/rajanr/spider_plots_data_discrete/rainbow_p_r_noises\",\n",
    "    \"a3c_p_r_noises\" : \"/home/rajanr/spider_plots_data_discrete\",\n",
    "    \"dqn_sparsity\" : \"/home/rajanr/spider_plots_data_discrete\",\n",
    "    \"rainbow_sparsity\" : \"/home/rajanr/spider_plots_data_discrete/rainbow_sparsity\",\n",
    "    \"a3c_sparsity\" : \"/home/rajanr/spider_plots_data_discrete\",\n",
    "    \"dqn_seq_del\" : \"/home/rajanr/spider_plots_data_discrete\",\n",
    "    \"rainbow_seq_del\" : \"/home/rajanr/spider_plots_data_discrete/rainbow_seq_del\",\n",
    "    \"a3c_seq_del\" : \"/home/rajanr/spider_plots_data_discrete\",\n",
    "    \"dqn_image_representations\": \"/home/rajanr/spider_plots_image_representations/mdpp_12540214_DQN_image_reps\",\n",
    "    \"rainbow_image_representations\": \"/home/rajanr/spider_plots_image_representations/mdpp_12579419_Rainbow_image_reps\",\n",
    "    \"a3c_image_representations\": \"/home/rajanr/spider_plots_image_representations/mdpp_12550729_A3C_image_reps\",\n",
    "}\n",
    "\n",
    "# Remember to set load_eval=False in case evaluation stats were not recorded and only training stats were recorded, otherwise there will be errors in loading the data in this cell.\n",
    "list_exp_data = mdpp_analysis.load_data(experiments, load_eval=load_eval, exp_type='grid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 1-D: Plots showing reward after 20k timesteps when varying a single dimension of hardness\n",
    "# Plots across runs: Training: with std dev across the runs\n",
    "mdpp_analysis.plot_1d_dimensions(list_exp_data, save_fig, plot_type = \"agent\")\n",
    "# mdpp_analysis.plot_1d_dimensions(list_exp_data, save_fig, use_aucs=True, plot_type = \"agent\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Plots across runs: Evaluation: with std dev across the runs\n",
    "if load_eval:\n",
    "    mdpp_analysis.plot_1d_dimensions(list_exp_data, save_fig, train=False, plot_type = \"agent\")\n",
    "#     mdpp_analysis.plot_1d_dimensions(list_exp_data, save_fig, train=False, use_aucs=True, plot_type = \"agent\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 1-D: Plots showing reward after 20k timesteps when varying a single dimension of hardness\n",
    "# Plots across runs: Training: with std dev across the runs\n",
    "mdpp_analysis.plot_1d_dimensions(list_exp_data, save_fig, plot_type = \"metric\")\n",
    "# mdpp_analysis.plot_1d_dimensions(list_exp_data, save_fig, use_aucs=True, plot_type = \"metric\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Plots across runs: Evaluation: with std dev across the runs\n",
    "if load_eval:\n",
    "    mdpp_analysis.plot_1d_dimensions(list_exp_data, save_fig, train=False, plot_type = \"metric\")\n",
    "#     mdpp_analysis.plot_1d_dimensions(list_exp_data, save_fig, use_aucs=True, train=False, plot_type = \"metric\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This and the next cell do the same as cells 3 and 4 but plot episode mean lengths instead of episode reward\n",
    "mdpp_analysis.plot_1d_dimensions(list_exp_data, save_fig, metric_num=-1, plot_type = \"agent\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if load_eval:\n",
    "    mdpp_analysis.plot_1d_dimensions(list_exp_data, save_fig, train=False, metric_num=-1, plot_type = \"agent\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2-D heatmap plots across 10 runs: Training runs: with std dev across the runs\n",
    "# There seems to be a bug with matplotlib - x and y axes tick labels are not correctly set even though we pass them. Please feel free to look into the code and suggest a correction if you find it.\n",
    "mdpp_analysis.plot_2d_heatmap(list_exp_data, save_fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 2-D heatmap plots across 10 runs: Evaluation runs: with std dev across the runs\n",
    "if load_eval:\n",
    "    mdpp_analysis.plot_2d_heatmap(list_exp_data, save_fig, train=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot learning curves: Training: Each curve corresponds to a different seed for the agent\n",
    "mdpp_analysis.plot_learning_curves(list_exp_data, save_fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot learning curves: Evaluation: Each curve corresponds to a different seed for the agent\n",
    "if load_eval:\n",
    "    mdpp_analysis.plot_learning_curves(list_exp_data, save_fig, train=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "weights = {}\n",
    "# for continuous_experiments\n",
    "weights['reward_noise'] = [.2, .2, .2, .2, .2, .0, .0, .0]\n",
    "weights['action_loss_weight'] = [.33, .33, .33, .0, .0, .0]\n",
    "\n",
    "# Plot radar(spider) plot: Training: across different dimensions of hardness\n",
    "mdpp_analysis.plot_radar(list_exp_data, save_fig=True, weights=weights, use_aucs=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Plot radar(spider) plot: Evaluation: across different dimensions of hardness\n",
    "if load_eval:\n",
    "    mdpp_analysis.plot_radar(list_exp_data, save_fig=True, train=False, use_aucs=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup to analyse an MDP Playground experiment using CAVE (https://github.com/automl/CAVE)\n",
    "from mdp_playground.analysis import MDPP_Analysis\n",
    "# Set dir_name to the location where the CSV files from running an experiment were saved\n",
    "dir_name = '/home/rajanr/mdpp_12744267_SAC_target_radius_can_use_as_sample/'\n",
    "# Set exp_name to the name that was given to the experiment when running it\n",
    "exp_name = 'sac_move_to_a_point_target_radius'\n",
    "# Set the following to True to save PDFs of plots that you generate below\n",
    "save_fig = True\n",
    "\n",
    "files_dir = '/home/rajanr/mdpp_12744267_SAC_target_radius_can_use_as_sample/'\n",
    "experiments = {\n",
    "    \"sac_move_to_a_point_target_radius\" : \"%s\"%files_dir,\n",
    "#     \"a3c_lstm_seq\" : \"%s/a3c_lstm_seq/\"%files_dir,\n",
    "#     \"a3c_del\" : \"%s/a3c_del/\"%files_dir,\n",
    "#     \"a3c_lstm_del\" : \"%s/a3c_lstm_del/\"%files_dir,\n",
    "}\n",
    "\n",
    "\n",
    "from cave.cavefacade import CAVE\n",
    "from mdp_playground.analysis.mdpp_to_cave import MDPPToCave\n",
    "import os\n",
    "\n",
    "#The converted mdpp csvs will be stored in output_dir\n",
    "output_dir = \"/home/rajanr/mdpp_to_cave\"\n",
    "mdpp_cave = MDPPToCave()\n",
    "for exp_name, dir_name in experiments.items():\n",
    "\n",
    "    cave = mdpp_cave.to_CAVE_object(dir_name, exp_name, output_dir, overwrite=True)\n",
    "\n",
    "    cave.performance_table()\n",
    "#     cave.local_parameter_importance() # Bug: https://github.com/automl/CAVE/pull/296/files\n",
    "    # cave.cave_fanova() # Only for experiments varying more than 1 dimension of hardness\n",
    "\n",
    "# Other analysis\n",
    "#     cave.parallel_coordinates() # crashes\n",
    "#     cave.cost_over_time()\n",
    "#     cave.algorithm_footprints()\n",
    "# cave.pimp_comparison_table()\n",
    "# cave.cave_ablation()\n",
    "# cave.pimp_forward_selection()\n",
    "# cave.feature_importance()\n",
    "# cave.configurator_footprint()\n",
    "# cave.algorithm_footprints()\n",
    "# cave.plot_ecdf()\n",
    "# cave.plot_scatter()\n",
    "# cave.compare_default_incumbent()\n",
    "# cave.overview_table()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}