Adding week9 materials

Author: nkarpachev

week1_09_approx_qlearning/README.md

@@ -0,0 +1,2 @@
+Approximate Q-learning implementation practice:
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/girafe-ai/ml-mipt/blob/advanced/week09_approx_qlearning/week09_approximate_Q_learning.ipynb)

week1_09_approx_qlearning/atari_wrappers.py

@@ -0,0 +1,115 @@
+# taken from OpenAI baselines.
+
+import numpy as np
+import gym
+
+class MaxAndSkipEnv(gym.Wrapper):
+    def __init__(self, env, skip=4):
+        """Return only every `skip`-th frame"""
+        gym.Wrapper.__init__(self, env)
+        # most recent raw observations (for max pooling across time steps)
+        self._obs_buffer = np.zeros((2,)+env.observation_space.shape, dtype=np.uint8)
+        self._skip       = skip
+
+    def step(self, action):
+        """Repeat action, sum reward, and max over last observations."""
+        total_reward = 0.0
+        done = None
+        for i in range(self._skip):
+            obs, reward, done, info = self.env.step(action)
+            if i == self._skip - 2: self._obs_buffer[0] = obs
+            if i == self._skip - 1: self._obs_buffer[1] = obs
+            total_reward += reward
+            if done:
+                break
+        # Note that the observation on the done=True frame
+        # doesn't matter
+        max_frame = self._obs_buffer.max(axis=0)
+
+        return max_frame, total_reward, done, info
+
+    def reset(self, **kwargs):
+        return self.env.reset(**kwargs)
+
+
+class ClipRewardEnv(gym.RewardWrapper):
+    def __init__(self, env):
+        gym.RewardWrapper.__init__(self, env)
+
+    def reward(self, reward):
+        """Bin reward to {+1, 0, -1} by its sign."""
+        return np.sign(reward)
+
+
+class FireResetEnv(gym.Wrapper):
+    def __init__(self, env):
+        """Take action on reset for environments that are fixed until firing."""
+        gym.Wrapper.__init__(self, env)
+        assert env.unwrapped.get_action_meanings()[1] == 'FIRE'
+        assert len(env.unwrapped.get_action_meanings()) >= 3
+
+    def reset(self, **kwargs):
+        self.env.reset(**kwargs)
+        obs, _, done, _ = self.env.step(1)
+        if done:
+            self.env.reset(**kwargs)
+        obs, _, done, _ = self.env.step(2)
+        if done:
+            self.env.reset(**kwargs)
+        return obs
+
+    def step(self, ac):
+        return self.env.step(ac)
+
+
+class EpisodicLifeEnv(gym.Wrapper):
+    def __init__(self, env):
+        """Make end-of-life == end-of-episode, but only reset on true game over.
+        Done by DeepMind for the DQN and co. since it helps value estimation.
+        """
+        gym.Wrapper.__init__(self, env)
+        self.lives = 0
+        self.was_real_done  = True
+
+    def step(self, action):
+        obs, reward, done, info = self.env.step(action)
+        self.was_real_done = done
+        # check current lives, make loss of life terminal,
+        # then update lives to handle bonus lives
+        lives = self.env.unwrapped.ale.lives()
+        if lives < self.lives and lives > 0:
+            # for Qbert sometimes we stay in lives == 0 condition for a few frames
+            # so it's important to keep lives > 0, so that we only reset once
+            # the environment advertises done.
+            done = True
+        self.lives = lives
+        return obs, reward, done, info
+
+    def reset(self, **kwargs):
+        """Reset only when lives are exhausted.
+        This way all states are still reachable even though lives are episodic,
+        and the learner need not know about any of this behind-the-scenes.
+        """
+        if self.was_real_done:
+            obs = self.env.reset(**kwargs)
+        else:
+            # no-op step to advance from terminal/lost life state
+            obs, _, _, _ = self.env.step(0)
+        self.lives = self.env.unwrapped.ale.lives()
+        return obs
+
+
+# in torch imgs have shape [c, h, w] instead of common [h, w, c]
+class AntiTorchWrapper(gym.ObservationWrapper):
+    def __init__(self, env):
+        gym.ObservationWrapper.__init__(self, env)
+
+        self.img_size = [env.observation_space.shape[i]
+                           for i in [1, 2, 0]
+                        ]
+        self.observation_space = gym.spaces.Box(0.0, 1.0, self.img_size)
+
+    def _observation(self, img):
+        """what happens to each observation"""
+        img = img.transpose(1, 2, 0)
+        return img

week1_09_approx_qlearning/framebuffer.py

@@ -0,0 +1,41 @@
+import numpy as np
+from gym.spaces.box import Box
+from gym.core import Wrapper
+class FrameBuffer(Wrapper):
+    def __init__(self, env, n_frames=4, dim_order='tensorflow'):
+        """A gym wrapper that reshapes, crops and scales image into the desired shapes"""
+        super(FrameBuffer, self).__init__(env)
+        self.dim_order = dim_order
+        if dim_order == 'tensorflow':
+            height, width, n_channels = env.observation_space.shape
+            obs_shape = [height, width, n_channels * n_frames]
+        elif dim_order == 'pytorch':
+            n_channels, height, width = env.observation_space.shape
+            obs_shape = [n_channels * n_frames, height, width]
+        else:
+            raise ValueError('dim_order should be "tensorflow" or "pytorch", got {}'.format(dim_order))
+        self.observation_space = Box(0.0, 1.0, obs_shape)
+        self.framebuffer = np.zeros(obs_shape, 'float32')
+        
+    def reset(self):
+        """resets breakout, returns initial frames"""
+        self.framebuffer = np.zeros_like(self.framebuffer)
+        self.update_buffer(self.env.reset())
+        return self.framebuffer
+    
+    def step(self, action):
+        """plays breakout for 1 step, returns frame buffer"""
+        new_img, reward, done, info = self.env.step(action)
+        self.update_buffer(new_img)
+        return self.framebuffer, reward, done, info
+    
+    def update_buffer(self, img):
+        if self.dim_order == 'tensorflow':
+            offset = self.env.observation_space.shape[-1]
+            axis = -1
+            cropped_framebuffer = self.framebuffer[:,:,:-offset]
+        elif self.dim_order == 'pytorch':
+            offset = self.env.observation_space.shape[0]
+            axis = 0
+            cropped_framebuffer = self.framebuffer[:-offset]
+        self.framebuffer = np.concatenate([img, cropped_framebuffer], axis = axis)

week1_09_approx_qlearning/replay_buffer.py

@@ -0,0 +1,63 @@
+# This code is shamelessly stolen from https://github.com/openai/baselines/blob/master/baselines/deepq/replay_buffer.py
+import numpy as np
+import random
+
+class ReplayBuffer(object):
+    def __init__(self, size):
+        """Create Replay buffer.
+        Parameters
+        ----------
+        size: int
+            Max number of transitions to store in the buffer. When the buffer
+            overflows the old memories are dropped.
+        """
+        self._storage = []
+        self._maxsize = size
+        self._next_idx = 0
+
+    def __len__(self):
+        return len(self._storage)
+
+    def add(self, obs_t, action, reward, obs_tp1, done):
+        data = (obs_t, action, reward, obs_tp1, done)
+
+        if self._next_idx >= len(self._storage):
+            self._storage.append(data)
+        else:
+            self._storage[self._next_idx] = data
+        self._next_idx = (self._next_idx + 1) % self._maxsize
+
+    def _encode_sample(self, idxes):
+        obses_t, actions, rewards, obses_tp1, dones = [], [], [], [], []
+        for i in idxes:
+            data = self._storage[i]
+            obs_t, action, reward, obs_tp1, done = data
+            obses_t.append(np.array(obs_t, copy=False))
+            actions.append(np.array(action, copy=False))
+            rewards.append(reward)
+            obses_tp1.append(np.array(obs_tp1, copy=False))
+            dones.append(done)
+        return np.array(obses_t), np.array(actions), np.array(rewards), np.array(obses_tp1), np.array(dones)
+
+    def sample(self, batch_size):
+        """Sample a batch of experiences.
+        Parameters
+        ----------
+        batch_size: int
+            How many transitions to sample.
+        Returns
+        -------
+        obs_batch: np.array
+            batch of observations
+        act_batch: np.array
+            batch of actions executed given obs_batch
+        rew_batch: np.array
+            rewards received as results of executing act_batch
+        next_obs_batch: np.array
+            next set of observations seen after executing act_batch
+        done_mask: np.array
+            done_mask[i] = 1 if executing act_batch[i] resulted in
+            the end of an episode and 0 otherwise.
+        """
+        idxes = [random.randint(0, len(self._storage) - 1) for _ in range(batch_size)]
+        return self._encode_sample(idxes)

week1_09_approx_qlearning/utils.py

@@ -0,0 +1,87 @@
+import numpy as np
+import psutil
+from scipy.signal import convolve, gaussian
+import torch
+from torch import nn
+import os
+
+def get_cum_discounted_rewards(rewards, gamma):
+    """
+    evaluates cumulative discounted rewards:
+    r_t + gamma * r_{t+1} + gamma^2 * r_{t_2} + ...
+    """
+    cum_rewards = []
+    cum_rewards.append(rewards[-1])
+    for r in reversed(rewards[:-1]):
+        cum_rewards.insert(0, r + gamma * cum_rewards[0])
+    return cum_rewards
+
+
+def play_and_log_episode(env, agent, gamma=0.99, t_max=10000):
+    """
+    always greedy
+    """
+    states = []
+    v_mc = []
+    v_agent = []
+    q_spreads = []
+    td_errors = []
+    rewards = []
+
+    s = env.reset()
+    for step in range(t_max):
+        states.append(s)
+        qvalues = agent.get_qvalues([s])
+        max_q_value, min_q_value = np.max(qvalues), np.min(qvalues)
+        v_agent.append(max_q_value)
+        q_spreads.append(max_q_value - min_q_value)
+        if step > 0:
+            td_errors.append(np.abs(rewards[-1] + gamma * v_agent[-1] - v_agent[-2]))
+
+        action = qvalues.argmax(axis=-1)[0]
+
+        s, r, done, _ = env.step(action)
+        rewards.append(r)
+        if done:
+            break
+    td_errors.append(np.abs(rewards[-1] + gamma * v_agent[-1] - v_agent[-2]))
+
+    v_mc = get_cum_discounted_rewards(rewards, gamma)
+
+    return_pack = {
+        'states': np.array(states),
+        'v_mc': np.array(v_mc),
+        'v_agent': np.array(v_agent),
+        'q_spreads': np.array(q_spreads),
+        'td_errors': np.array(td_errors),
+        'rewards': np.array(rewards),
+        'episode_finished': np.array(done)
+    }
+
+    return return_pack
+
+
+def img_by_obs(obs, state_dim):
+    """
+    Unwraps obs by channels.
+    observation is of shape [c, h=w, w=h]
+    """
+    return obs.reshape([-1, state_dim[2]])
+
+
+def is_enough_ram(min_available_gb = 0.1):
+    mem = psutil.virtual_memory()
+    return mem.available >= min_available_gb * (1024 ** 3)
+
+
+def linear_decay(init_val, final_val, cur_step, total_steps):
+    if cur_step >= total_steps:
+        return final_val
+    return (init_val * (total_steps - cur_step) + final_val * cur_step) / total_steps
+
+
+def smoothen(values):
+    kernel = gaussian(100, std=100)
+    # kernel = np.concatenate([np.arange(100), np.arange(99, -1, -1)])
+    kernel = kernel / np.sum(kernel)
+    return convolve(values, kernel, 'valid')