diff --git a/python/paddle/fluid/trainer.py b/python/paddle/fluid/trainer.py
index 64049a93cb0a267722de9cd94961b6256551330d..81cf801169987c32764e7291f91708ec36a270be 100644
--- a/python/paddle/fluid/trainer.py
+++ b/python/paddle/fluid/trainer.py
@@ -349,7 +349,10 @@ class Trainer(object):
         with self._prog_and_scope_guard():
             t = distribute_transpiler.DistributeTranspiler()
             t.transpile(
-                self.trainer_id, pservers=pserver_endpoints, trainers=trainers)
+                self.trainer_id,
+                pservers=pserver_endpoints,
+                trainers=trainers,
+                slice_var_up=False)
             if training_role == "PSERVER":
                 if self.checkpoint_cfg:
                     pserver_id = eplist.index(current_endpoint)
diff --git a/python/paddle/fluid/transpiler/distribute_transpiler.py b/python/paddle/fluid/transpiler/distribute_transpiler.py
index 121c36e477327d4d0e7b1cba1713e68ce4d06e03..416d03a4d3f886c074c128cb960f12d5caf896b9 100644
--- a/python/paddle/fluid/transpiler/distribute_transpiler.py
+++ b/python/paddle/fluid/transpiler/distribute_transpiler.py
@@ -196,8 +196,6 @@ class DistributeTranspiler(object):
         #       fc_b@GRAD_trainer_0, fc_b@GRAD_trainer_1 --> pserver2
         # shuffle the map will avoid the uneven distribution above
         grad_var_mapping_items = self.grad_var_mapping.items()
-        if not slice_var_up:
-            np.random.shuffle(grad_var_mapping_items)
 
         for orig_varname, splited_vars in grad_var_mapping_items:
             eplist = ps_dispatcher.dispatch(splited_vars)