Deploying to gh-pages from @ aa780e6 🚀

Author: PaParaZz1

_modules/ding/policy/ppg.html

@@ -581,7 +581,7 @@ <h1>Source code for ding.policy.ppg</h1><div class="highlight"><pre>
                 <span class="n">policy_output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_learn_model</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="n">policy_data</span><span class="p">[</span><span class="s1">&#39;obs&#39;</span><span class="p">],</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;compute_actor&#39;</span><span class="p">)</span>
                 <span class="n">policy_error_data</span> <span class="o">=</span> <span class="n">ppo_policy_data</span><span class="p">(</span>
                     <span class="n">policy_output</span><span class="p">[</span><span class="s1">&#39;logit&#39;</span><span class="p">],</span> <span class="n">policy_data</span><span class="p">[</span><span class="s1">&#39;logit&#39;</span><span class="p">],</span> <span class="n">policy_data</span><span class="p">[</span><span class="s1">&#39;action&#39;</span><span class="p">],</span> <span class="n">policy_adv</span><span class="p">,</span>
-                    <span class="n">policy_data</span><span class="p">[</span><span class="s1">&#39;weight&#39;</span><span class="p">]</span>
+                    <span class="n">policy_data</span><span class="p">[</span><span class="s1">&#39;weight&#39;</span><span class="p">],</span> <span class="kc">None</span>
                 <span class="p">)</span>
                 <span class="n">ppo_policy_loss</span><span class="p">,</span> <span class="n">ppo_info</span> <span class="o">=</span> <span class="n">ppo_policy_error</span><span class="p">(</span><span class="n">policy_error_data</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_clip_ratio</span><span class="p">)</span>
                 <span class="n">policy_loss</span> <span class="o">=</span> <span class="n">ppo_policy_loss</span><span class="o">.</span><span class="n">policy_loss</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">_entropy_weight</span> <span class="o">*</span> <span class="n">ppo_policy_loss</span><span class="o">.</span><span class="n">entropy_loss</span>
@@ -1236,7 +1236,7 @@ <h1>Source code for ding.policy.ppg</h1><div class="highlight"><pre>
         <span class="c1"># Policy Phase(Policy)</span>
         <span class="n">policy_output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_learn_model</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="n">policy_data</span><span class="p">[</span><span class="s1">&#39;obs&#39;</span><span class="p">],</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;compute_actor&#39;</span><span class="p">)</span>
         <span class="n">policy_error_data</span> <span class="o">=</span> <span class="n">ppo_policy_data</span><span class="p">(</span>
-            <span class="n">policy_output</span><span class="p">[</span><span class="s1">&#39;logit&#39;</span><span class="p">],</span> <span class="n">policy_data</span><span class="p">[</span><span class="s1">&#39;logit&#39;</span><span class="p">],</span> <span class="n">policy_data</span><span class="p">[</span><span class="s1">&#39;action&#39;</span><span class="p">],</span> <span class="n">policy_adv</span><span class="p">,</span> <span class="n">policy_data</span><span class="p">[</span><span class="s1">&#39;weight&#39;</span><span class="p">]</span>
+            <span class="n">policy_output</span><span class="p">[</span><span class="s1">&#39;logit&#39;</span><span class="p">],</span> <span class="n">policy_data</span><span class="p">[</span><span class="s1">&#39;logit&#39;</span><span class="p">],</span> <span class="n">policy_data</span><span class="p">[</span><span class="s1">&#39;action&#39;</span><span class="p">],</span> <span class="n">policy_adv</span><span class="p">,</span> <span class="n">policy_data</span><span class="p">[</span><span class="s1">&#39;weight&#39;</span><span class="p">],</span> <span class="kc">None</span>
         <span class="p">)</span>
         <span class="n">ppo_policy_loss</span><span class="p">,</span> <span class="n">ppo_info</span> <span class="o">=</span> <span class="n">ppo_policy_error</span><span class="p">(</span><span class="n">policy_error_data</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_clip_ratio</span><span class="p">)</span>
         <span class="n">policy_loss</span> <span class="o">=</span> <span class="n">ppo_policy_loss</span><span class="o">.</span><span class="n">policy_loss</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">_entropy_weight</span> <span class="o">*</span> <span class="n">ppo_policy_loss</span><span class="o">.</span><span class="n">entropy_loss</span>