From e8990e64f603f1ad9b9354223d5e13d5af3a2ee7 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Fri, 24 May 2019 14:46:04 +0800 Subject: [PATCH] Fix trust ratio in lamb (#17614) test=develop --- paddle/fluid/operators/optimizers/lamb_op.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/operators/optimizers/lamb_op.h b/paddle/fluid/operators/optimizers/lamb_op.h index 4fbe486cb1..72a9093859 100644 --- a/paddle/fluid/operators/optimizers/lamb_op.h +++ b/paddle/fluid/operators/optimizers/lamb_op.h @@ -174,10 +174,11 @@ struct LambParamUpateFunctor { inline HOSTDEVICE void operator()(size_t i) const { T lr = *lr_; - T p_norm = *param_norm_; - T tr_div_norm = *trust_ratio_div_norm_; + T p = *param_norm_; + T t = *trust_ratio_div_norm_; - lr *= p_norm / tr_div_norm; + T r = (p > 0 && t > 0) ? p / t : 1.0; + lr *= r; param_out_[i] = param_[i] - lr * trust_ratio_div_[i]; } }; -- GitLab