fix a2c cannot run in paddle 1.6.0 (#232)

* fix a2c cannot run in paddle 1.6.0

* fix impala compatibility

* yapf

examples/A2C/atari_agent.py

@@ -71,7 +71,10 @@ class AtariAgent(parl.Agent):
             lr = layers.data(
                 name='lr', shape=[1], dtype='float32', append_batch_size=False)
             entropy_coeff = layers.data(
-                name='entropy_coeff', shape=[], dtype='float32')
+                name='entropy_coeff',
+                shape=[1],
+                dtype='float32',
+                append_batch_size=False)
 
             total_loss, pi_loss, vf_loss, entropy = self.alg.learn(
                 obs, actions, advantages, target_values, lr, entropy_coeff)

examples/IMPALA/atari_agent.py

@@ -58,7 +58,10 @@ class AtariAgent(parl.Agent):
             lr = layers.data(
                 name='lr', shape=[1], dtype='float32', append_batch_size=False)
             entropy_coeff = layers.data(
-                name='entropy_coeff', shape=[], dtype='float32')
+                name='entropy_coeff',
+                shape=[1],
+                dtype='float32',
+                append_batch_size=False)
 
             self.learn_reader = fluid.layers.create_py_reader_by_data(
                 capacity=32,

examples/IMPALA/train.py

@@ -123,7 +123,7 @@ class Learner(object):
                     obs_np, actions_np, behaviour_logits_np, rewards_np,
                     dones_np,
                     np.float32(self.lr),
-                    np.float32(self.entropy_coeff)
+                    np.array([self.entropy_coeff], dtype='float32')
                 ]
 
     def run_learn(self):

examples/LiftSim_baseline/A2C/lift_agent.py

@@ -67,7 +67,10 @@ class LiftAgent(parl.Agent):
             lr = layers.data(
                 name='lr', shape=[1], dtype='float32', append_batch_size=False)
             entropy_coeff = layers.data(
-                name='entropy_coeff', shape=[], dtype='float32')
+                name='entropy_coeff',
+                shape=[1],
+                dtype='float32',
+                append_batch_size=False)
 
             total_loss, pi_loss, vf_loss, entropy = self.alg.learn(
                 obs, actions, advantages, target_values, lr, entropy_coeff)

parl/algorithms/fluid/a3c.py

@@ -72,7 +72,6 @@ class A3C(Algorithm):
         policy_entropy = policy_distribution.entropy()
         entropy = layers.reduce_sum(policy_entropy)
 
-        entropy_coeff = layers.reshape(entropy_coeff, shape=[1])
         total_loss = (
             pi_loss + vf_loss * self.vf_loss_coeff + entropy * entropy_coeff)
 

parl/algorithms/fluid/impala/impala.py

@@ -78,7 +78,6 @@ class VTraceLoss(object):
         self.entropy = layers.reduce_sum(policy_entropy)
 
         # The summed weighted loss
-        entropy_coeff = layers.reshape(entropy_coeff, shape=[1])
         self.total_loss = (self.pi_loss + self.vf_loss * vf_loss_coeff +
                            self.entropy * entropy_coeff)