update

Author: bob7783

Diff for: rl2/a3c/main.py

@@ -13,6 +13,14 @@
 from worker import Worker
 
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+
+if tf.__version__.startswith('2'):
+  exit("Please install Tensorflow 1.x")
+
+
 ENV_NAME = "Breakout-v0"
 MAX_GLOBAL_STEPS = 5e6
 STEPS_PER_UPDATE = 5

Diff for: rl2/atari/dqn_tf.py

@@ -19,7 +19,12 @@
 
 
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
 
+if tf.__version__.startswith('2'):
+  exit("Please install Tensorflow 1.x")
 
 ##### testing only
 # MAX_EXPERIENCES = 10000
@@ -141,7 +146,11 @@ def get_minibatch(self):
       self.states[i] = self._get_state(idx - 1)
       self.new_states[i] = self._get_state(idx)
 
-    return np.transpose(self.states, axes=(0, 2, 3, 1)), self.actions[self.indices], self.rewards[self.indices], np.transpose(self.new_states, axes=(0, 2, 3, 1)), self.terminal_flags[self.indices]
+    return np.transpose(self.states, axes=(0, 2, 3, 1)), \
+      self.actions[self.indices], \
+      self.rewards[self.indices], \
+      np.transpose(self.new_states, axes=(0, 2, 3, 1)), \
+      self.terminal_flags[self.indices]
 
 
 class DQN:

Diff for: rl2/atari/dqn_theano.py

@@ -140,7 +140,11 @@ def get_minibatch(self):
       self.states[i] = self._get_state(idx - 1)
       self.new_states[i] = self._get_state(idx)
 
-    return self.states, self.actions[self.indices], self.rewards[self.indices], self.new_states, self.terminal_flags[self.indices]
+    return self.states, \
+      self.actions[self.indices], \
+      self.rewards[self.indices], \
+      self.new_states, \
+      self.terminal_flags[self.indices]
 
 
 def init_filter(shape):

Diff for: rl2/cartpole/dqn_tf.py

@@ -15,6 +15,13 @@
 from datetime import datetime
 from q_learning_bins import plot_running_avg
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+
+if tf.__version__.startswith('2'):
+  exit("Please install Tensorflow 1.x")
+
 
 # global counter
 global_iters = 0

Diff for: rl2/cartpole/dqn_theano.py

@@ -16,6 +16,10 @@
 from datetime import datetime
 from q_learning_bins import plot_running_avg
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+
 
 # global counter
 global_iters = 0

Diff for: rl2/cartpole/pg_tf.py

@@ -16,6 +16,13 @@
 from datetime import datetime
 from q_learning_bins import plot_running_avg
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+
+if tf.__version__.startswith('2'):
+  exit("Please install Tensorflow 1.x")
+
 
 # so you can test different architectures
 class HiddenLayer:

Diff for: rl2/cartpole/pg_theano.py

@@ -17,6 +17,10 @@
 from datetime import datetime
 from q_learning_bins import plot_running_avg
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+
 
 # so you can test different architectures
 class HiddenLayer:

Diff for: rl2/cartpole/q_learning.py

@@ -20,6 +20,10 @@
 from sklearn.kernel_approximation import RBFSampler
 from q_learning_bins import plot_running_avg
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+
 
 class SGDRegressor:
   def __init__(self, D):

Diff for: rl2/cartpole/q_learning_bins.py

@@ -15,6 +15,10 @@
 from gym import wrappers
 from datetime import datetime
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+
 
 # turns list of integers into an int
 # Ex.

Diff for: rl2/cartpole/random_search.py

@@ -9,6 +9,10 @@
 import numpy as np
 import matplotlib.pyplot as plt
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+
 
 def get_action(s, w):
   return 1 if s.dot(w) > 0 else 0

Diff for: rl2/cartpole/save_a_video.py

@@ -10,6 +10,11 @@
 import numpy as np
 import matplotlib.pyplot as plt
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+  
+
 
 def get_action(s, w):
   return 1 if s.dot(w) > 0 else 0
@@ -63,6 +68,5 @@ def random_search(env):
   plt.show()
 
   # play a final set of episodes
-  # env = wrappers.Monitor(env, 'my_awesome_dir')
   env = wrappers.RecordVideo(env, 'my_awesome_dir')
   print("***Final run with final weights***:", play_one_episode(env, params))

Diff for: rl2/cartpole/td_lambda.py

@@ -15,6 +15,11 @@
 from q_learning import FeatureTransformer
 from q_learning_bins import plot_running_avg
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+  
+
 
 class SGDRegressor:
   def __init__(self, D):

Diff for: rl2/cartpole/tf_warmup.py

@@ -7,6 +7,9 @@
 import tensorflow as tf
 import q_learning
 
+if tf.__version__.startswith('2'):
+  exit("Please install Tensorflow 1.x")
+
 
 class SGDRegressor:
   def __init__(self, D):

Diff for: rl2/gym_tutorial.py

@@ -6,6 +6,11 @@
 # Environment page:
 # https://gym.openai.com/envs/CartPole-v0
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+
+
 # get the environment
 env = gym.make('CartPole-v0')
 

Diff for: rl2/mountaincar/n_step.py

@@ -24,6 +24,10 @@
 import q_learning
 from q_learning import plot_cost_to_go, FeatureTransformer, Model, plot_running_avg
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+
 
 class SGDRegressor:
   def __init__(self, **kwargs):

Diff for: rl2/mountaincar/pg_tf.py

@@ -15,6 +15,13 @@
 from datetime import datetime
 from q_learning import plot_running_avg, FeatureTransformer, plot_cost_to_go
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+
+if tf.__version__.startswith('2'):
+  exit("Please install Tensorflow 1.x")
+
 
 # so you can test different architectures
 class HiddenLayer:
@@ -177,8 +184,12 @@ def play_one_td(env, pmodel, vmodel, gamma):
     totalreward += reward
 
     # update the models
-    V_next = vmodel.predict(observation)
-    G = reward + gamma*V_next
+    if done:
+      G = reward
+    else:
+      V_next = vmodel.predict(observation)
+      G = reward + gamma*V_next
+
     advantage = G - vmodel.predict(prev_observation)
     pmodel.partial_fit(prev_observation, action, advantage)
     vmodel.partial_fit(prev_observation, G)

Diff for: rl2/mountaincar/pg_tf_random.py

@@ -15,6 +15,13 @@
 from datetime import datetime
 from q_learning import plot_running_avg, FeatureTransformer
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+
+if tf.__version__.startswith('2'):
+  exit("Please install Tensorflow 1.x")
+
 
 # so you can test different architectures
 class HiddenLayer:

Diff for: rl2/mountaincar/pg_theano.py

@@ -208,7 +208,7 @@ def predict(self, X):
     return self.predict_op(X)
 
 
-def play_one_td(env, pmodel, vmodel, gamma, train=True):
+def play_one_td(env, pmodel, vmodel, gamma):
   observation = env.reset()
   done = False
   totalreward = 0
@@ -224,12 +224,15 @@ def play_one_td(env, pmodel, vmodel, gamma, train=True):
     totalreward += reward
 
     # update the models
-    if train:
+    if done:
+      G = reward
+    else:
       V_next = vmodel.predict(observation)
       G = reward + gamma*V_next
-      advantage = G - vmodel.predict(prev_observation)
-      pmodel.partial_fit(prev_observation, action, advantage)
-      vmodel.partial_fit(prev_observation, G)
+
+    advantage = G - vmodel.predict(prev_observation)
+    pmodel.partial_fit(prev_observation, action, advantage)
+    vmodel.partial_fit(prev_observation, G)
 
     iters += 1
 

Diff for: rl2/mountaincar/pg_theano_random.py

@@ -16,6 +16,10 @@
 from datetime import datetime
 from q_learning import plot_running_avg, FeatureTransformer
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+
 
 
 # so you can test different architectures

Diff for: rl2/mountaincar/q_learning.py

@@ -27,6 +27,10 @@
 from sklearn.kernel_approximation import RBFSampler
 from sklearn.linear_model import SGDRegressor
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+  
 
 # SGDRegressor defaults:
 # loss='squared_loss', penalty='l2', alpha=0.0001,
@@ -109,9 +113,13 @@ def play_one(model, env, eps, gamma):
     observation, reward, done, info = env.step(action)
 
     # update the model
-    next = model.predict(observation)
-    # assert(next.shape == (1, env.action_space.n))
-    G = reward + gamma*np.max(next[0])
+    if done:
+      G = reward
+    else:
+      Qnext = model.predict(observation)
+      # assert(next.shape == (1, env.action_space.n))
+      G = reward + gamma*np.max(Qnext[0])
+
     model.update(prev_observation, action, G)
 
     totalreward += reward
@@ -165,14 +173,14 @@ def main(show_plots=True):
   N = 300
   totalrewards = np.empty(N)
   for n in range(N):
-    # eps = 1.0/(0.1*n+1)
-    eps = 0.1*(0.97**n)
+    eps = 1.0/(0.1*n+1)
+    # eps = 0.1*(0.97**n)
     if n == 199:
       print("eps:", eps)
     # eps = 1.0/np.sqrt(n+1)
     totalreward = play_one(model, env, eps, gamma)
     totalrewards[n] = totalreward
-    if (n + 1) % 100 == 0:
+    if (n + 1) % 10 == 0:
       print("episode:", n, "total reward:", totalreward)
   print("avg reward for last 100 episodes:", totalrewards[-100:].mean())
   print("total steps:", -totalrewards.sum())

Diff for: rl2/mountaincar/td_lambda.py

@@ -23,6 +23,10 @@
 # code we already wrote
 from q_learning import plot_cost_to_go, FeatureTransformer, plot_running_avg
 
+gym_minor_version = int(gym.__version__.split('.')[1])
+if gym_minor_version >= 19:
+  exit("Please install OpenAI Gym 0.19.0 or earlier")
+
 
 class BaseModel:
   def __init__(self, D):
@@ -83,9 +87,9 @@ def play_one(model, env, eps, gamma, lambda_):
     observation, reward, done, info = env.step(action)
 
     # update the model
-    next = model.predict(observation)
-    assert(next.shape == (1, env.action_space.n))
-    G = reward + gamma*np.max(next[0])
+    Qnext = model.predict(observation)
+    assert(Qnext.shape == (1, env.action_space.n))
+    G = reward + gamma*np.max(Qnext[0])
     model.update(prev_observation, action, G, gamma, lambda_)
 
     totalreward += reward