Fix typos in PPO

Author: Omegastick

example/gym_client.cpp

@@ -21,15 +21,15 @@ const int batch_size = 40;
 const float clip_param = 0.2;
 const float discount_factor = 0.99;
 const float entropy_coef = 1e-3;
-const float gae = 0.95;
-const float kl_target = 0.05;
-const float learning_rate = 7e-4;
-const int log_interval = 1;
+const float gae = 0.9;
+const float kl_target = 0.5;
+const float learning_rate = 1e-3;
+const int log_interval = 10;
 const int max_frames = 10e+7;
 const int num_epoch = 3;
 const int num_mini_batch = 20;
 const int reward_average_window_size = 10;
-const float reward_clip_value = 10; // Post scaling
+const float reward_clip_value = 100; // Post scaling
 const bool use_gae = true;
 const bool use_lr_decay = false;
 const float value_loss_coef = 0.5;

src/algorithms/ppo.cpp

@@ -54,7 +54,7 @@ std::vector<UpdateDatum> PPO::update(RolloutStorage &rollouts, float decay_level
                        value_preds.narrow(0, 0, value_preds.size(0) - 1));
 
     // Normalize advantages
-    advantages = (advantages - advantages.mean() / (advantages.std() + 1e-5));
+    advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-5);
 
     float total_value_loss = 0;
     float total_action_loss = 0;
@@ -108,12 +108,11 @@ std::vector<UpdateDatum> PPO::update(RolloutStorage &rollouts, float decay_level
                                     mini_batch.action_log_probs);
 
             // PPO loss formula
-            auto surr_1 = ratio * mini_batch.advantages.mean();
+            auto surr_1 = ratio * mini_batch.advantages;
             auto surr_2 = (torch::clamp(ratio,
                                         1.0 - clip_param,
                                         1.0 + clip_param) *
-                           mini_batch.advantages)
-                              .mean();
+                           mini_batch.advantages);
             clip_fraction += (ratio - 1.0)
                                  .abs()
                                  .gt(clip_param)