From 5502abb95b831fb0eb0f4a92f7223ca52b53d913 Mon Sep 17 00:00:00 2001 From: peterzhang2029 Date: Wed, 22 Nov 2017 10:32:04 +0800 Subject: [PATCH] refine docstrings --- paddle/gserver/layers/CudnnBatchNormLayer.cpp | 22 ++++--------------- paddle/gserver/layers/CudnnBatchNormLayer.h | 4 +--- proto/ModelConfig.proto | 2 +- python/paddle/trainer/config_parser.py | 5 +++-- .../paddle/trainer_config_helpers/layers.py | 4 +--- 5 files changed, 10 insertions(+), 27 deletions(-) diff --git a/paddle/gserver/layers/CudnnBatchNormLayer.cpp b/paddle/gserver/layers/CudnnBatchNormLayer.cpp index c25960d681a..8390b55026c 100644 --- a/paddle/gserver/layers/CudnnBatchNormLayer.cpp +++ b/paddle/gserver/layers/CudnnBatchNormLayer.cpp @@ -21,8 +21,6 @@ namespace paddle { REGISTER_LAYER(cudnn_batch_norm, CudnnBatchNormLayer); -const double CudnnBatchNormLayer::MIN_EPS = 1E-5; - bool CudnnBatchNormLayer::init(const LayerMap& layerMap, const ParameterMap& parameterMap) { /* Initialize the basic parent class */ @@ -61,14 +59,8 @@ void CudnnBatchNormLayer::forward(PassType passType) { real* movingMean = movingMean_->getW()->getData(); real* movingVar = movingVar_->getW()->getData(); - /** - * If epsilon_ equals to 1e-5 and eps_ is assigned the value of - * static_cast(epsilon_), The CUDNN_STATUS_BAD_PARAM error - * will occur due to eps_ value is less than - * CUDNN_BN_MIN_EPSILON. - * The following code is to ensure that the eps_ meets requirement. - */ - eps_ = std::max(MIN_EPS, static_cast(epsilon_)); + // cuDNN does not allow an epsilon value less than CUDNN_BN_MIN_EPSILON. + eps_ = std::max(CUDNN_BN_MIN_EPSILON, static_cast(epsilon_)); if (!useGlobalStats_) { REGISTER_TIMER_INFO("CudnnBatchFwTimer", getName().c_str()); @@ -137,14 +129,8 @@ void CudnnBatchNormLayer::backward(const UpdateCallback& callback) { real* savedMean = savedMean_->getData(); real* savedInvVar = savedInvVar_->getData(); - /** - * If epsilon_ equals to 1e-5 and eps_ is assigned the value of - * static_cast(epsilon_), The CUDNN_STATUS_BAD_PARAM error - * will occur due to eps_ value is less than - * CUDNN_BN_MIN_EPSILON. - * The following code is to ensure that the eps_ meets requirement. - */ - eps_ = std::max(MIN_EPS, static_cast(epsilon_)); + // cuDNN does not allow an epsilon value less than CUDNN_BN_MIN_EPSILON. + eps_ = std::max(CUDNN_BN_MIN_EPSILON, static_cast(epsilon_)); auto create = [](MatrixPtr& m, size_t h, size_t w, real** p) { Matrix::resizeOrCreate(m, h, w, false, true); diff --git a/paddle/gserver/layers/CudnnBatchNormLayer.h b/paddle/gserver/layers/CudnnBatchNormLayer.h index fb7dbc01d17..1a3f0c0cbf8 100644 --- a/paddle/gserver/layers/CudnnBatchNormLayer.h +++ b/paddle/gserver/layers/CudnnBatchNormLayer.h @@ -14,6 +14,7 @@ limitations under the License. */ #pragma once +#include #include "BatchNormBaseLayer.h" #include "Layer.h" #include "paddle/utils/Stat.h" @@ -46,9 +47,6 @@ public: void backward(const UpdateCallback& callback = nullptr) override; protected: - /// Minimum allowed value is CUDNN_BN_MIN_EPSILON defined in cudnn.h. - static const double MIN_EPS; - /// Epsilon value used in the batch normalization formula. /// Same epsilon value should be used in forward and backward functions. double eps_; diff --git a/proto/ModelConfig.proto b/proto/ModelConfig.proto index ad1251e3192..e2f5592248f 100644 --- a/proto/ModelConfig.proto +++ b/proto/ModelConfig.proto @@ -542,7 +542,7 @@ message LayerConfig { optional ReshapeConfig reshape_conf = 59; // for batch normalization layer - // small constant added to the variance to avoid numerical problems. + // The small constant added to the variance to improve numeric stability. optional double epsilon = 60 [ default = 0.00001 ]; } diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index fd232f94159..064933802f2 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -2483,8 +2483,9 @@ class BatchNormLayer(LayerBase): self.config.use_global_stats = use_global_stats if moving_average_fraction is not None: self.config.moving_average_fraction = moving_average_fraction - - self.config.epsilon = epsilon + if epsilon is not None: + assert epsilon >= 1e-5, "epsilon must be no less than 1e-5." + self.config.epsilon = epsilon input_layer = self.get_input_layer(0) image_conf = self.config.inputs[0].image_conf diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index fa5e8513904..4964c1245d8 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -3107,7 +3107,7 @@ def batch_norm_layer(input, will use the mean and variance of the current batch of test data. :type use_global_stats: bool | None. - :param epsilon: Small constant added to the variance to avoid numerical problems. + :param epsilon: The small constant added to the variance to improve numeric stability. :type epsilon: float. :param moving_average_fraction: Factor used in the moving average computation. :math:`runningMean = newMean*(1-factor) + runningMean*factor` @@ -3127,8 +3127,6 @@ def batch_norm_layer(input, (batch_norm_type == "mkldnn_batch_norm") or \ (batch_norm_type == "cudnn_batch_norm") - assert epsilon >= 1e-5, "epsilon must be no less than 1e-5." - l = Layer( name=name, img3D=img3D, -- GitLab