From 67ed7e12abad669d819aebdd7a5153dd7fd8693f Mon Sep 17 00:00:00 2001 From: Yuang Liu Date: Thu, 2 Sep 2021 15:40:07 +0800 Subject: [PATCH] [hybrid] [npu] fit npu nan/inf check (#35171) --- python/paddle/fluid/optimizer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index eb3d559ddc..fc5c30684b 100755 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -5323,7 +5323,13 @@ class PipelineOptimizer(object): "copy_data": False, "use_align": True, "dtype": grads[0].dtype, - self._op_role_key: self._op_role.Backward + self._op_role_key: self._op_role.Backward, + # On npu, the nan/inf check login is different with gpu. + # If there are some not initialized sections in the fused var, + # and the value in those sections are nan/inf, it will trigger the nan/inf check. + # To avoid these problematic triggers, set constant is needed for npu + "set_constant": core.is_compiled_with_npu(), + "constant": float(0.0), }) offset += 1 # For the gradient_merged_fused_var, given a init value during the coalesce op -- GitLab