From edd5e9a819ca7f313fcaeb10f14e654b2dd4de4c Mon Sep 17 00:00:00 2001 From: Yuang Liu Date: Tue, 8 Aug 2023 09:16:27 +0800 Subject: [PATCH] fix bug for fused_linear_grad_add and main_grad (#56030) --- .../fleet/utils/mix_precision_utils.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/python/paddle/distributed/fleet/utils/mix_precision_utils.py b/python/paddle/distributed/fleet/utils/mix_precision_utils.py index 1e25879b820..991a1fafc7b 100644 --- a/python/paddle/distributed/fleet/utils/mix_precision_utils.py +++ b/python/paddle/distributed/fleet/utils/mix_precision_utils.py @@ -55,16 +55,18 @@ class MixPrecisionLayer(nn.Layer): ), "In main_grad node, param.grad should be None, but find param[{}] has grad.".format( param.name ) - if param.main_grad is None: - param.main_grad = core.eager.Tensor( - value=tmp_grad.cast(paddle.float32).value(), - place=tmp_grad.place, - name="main_grad@" + param.name, - ) - else: - param.main_grad.add_(tmp_grad) + if tmp_grad._is_initialized(): + # Some previous pylayer may return None, should check grad validation. + if param.main_grad is None: + param.main_grad = core.eager.Tensor( + value=tmp_grad.cast(paddle.float32).value(), + place=tmp_grad.place, + name="main_grad@" + param.name, + ) + else: + param.main_grad.add_(tmp_grad) - tmp_grad._clear_data() + tmp_grad._clear_data() return None return param_hook -- GitLab