From 7f17f9a0c5ed87ccada5ab5194e50f1775693021 Mon Sep 17 00:00:00 2001
From: JZ-LIANG <38102074+JZ-LIANG@users.noreply.github.com>
Date: Wed, 1 Sep 2021 14:08:49 +0800
Subject: [PATCH] bugfix for mp accuracy (#35326)

---
 .../fleet/meta_optimizers/tensor_parallel_optimizer.py          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py
index 5fbec7da0b5..9d099a2af24 100644
--- a/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py
+++ b/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py
@@ -124,7 +124,7 @@ class TensorParallelOptimizer(MetaOptimizerBase):
         collective_helper._init_communicator(
             self.startup_program, self.current_endpoint, self.mp_endpoints,
             self.mp_rank, self.mp_ring_id, True, self.global_ring_id, True)
-        #self._broadcast_params(self.mp_ring_id, mp_mode=True)
+        self._broadcast_params(self.mp_ring_id, mp_mode=True)
 
         # Create dp rings
         if self.nranks > self.mp_degree:
-- 
GitLab