norm.py 55.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15 16 17 18 19 20 21 22 23 24 25 26 27
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

28
# TODO: define normalization api
29

30
import six
31

Z
zhiboniu 已提交
32 33
from ...fluid.dygraph import BatchNorm  # noqa: F401
from ...fluid.dygraph import SpectralNorm  # noqa: F401
C
ceci3 已提交
34

35
from ...framework import get_default_dtype, set_default_dtype, _non_static_mode
C
ceci3 已提交
36

Z
zhiboniu 已提交
37 38
from ..initializer import Constant
from ...framework import ParamAttr
C
ceci3 已提交
39
from ...fluid.data_feeder import check_variable_and_dtype, check_type
Z
zhiboniu 已提交
40
from ...fluid import dygraph_utils
41 42 43 44 45 46

from ..functional import batch_norm, layer_norm, instance_norm

import numpy as np
import numbers
import warnings
Z
zhiboniu 已提交
47
from ...framework import no_grad
48
from .. import functional as F
W
wanghuancoder 已提交
49
from paddle import _C_ops
Z
zhiboniu 已提交
50
from .. import Layer
Z
zhiboniu 已提交
51
from paddle import in_dynamic_mode
52
from paddle.fluid.framework import in_dygraph_mode, _in_legacy_dygraph
53

54 55
__all__ = []

C
ceci3 已提交
56

Z
zhiboniu 已提交
57
class _InstanceNormBase(Layer):
58
    """
C
cnn 已提交
59
    This class is based class for InstanceNorm1D, 2d, 3d. 
60

C
cnn 已提交
61
    See InstaceNorm1D, InstanceNorm2D or InstanceNorm3D for more details.
62 63 64 65 66 67 68 69 70 71 72 73 74
    """

    def __init__(self,
                 num_features,
                 epsilon=1e-5,
                 momentum=0.9,
                 weight_attr=None,
                 bias_attr=None,
                 data_format="NCHW",
                 name=None):
        super(_InstanceNormBase, self).__init__()

        if weight_attr == False or bias_attr == False:
75
            assert weight_attr == bias_attr, "weight_attr and bias_attr must be set to Fasle at the same time in InstanceNorm"
76 77 78
        self._epsilon = epsilon
        self._weight_attr = weight_attr
        self._bias_attr = bias_attr
79
        self._num_features = num_features
80 81 82 83 84 85 86

        if weight_attr != False and bias_attr != False:
            self.scale = self.create_parameter(
                attr=self._weight_attr,
                shape=[num_features],
                default_initializer=Constant(1.0),
                is_bias=False)
87 88 89 90
            self.bias = self.create_parameter(attr=self._bias_attr,
                                              shape=[num_features],
                                              default_initializer=Constant(0.0),
                                              is_bias=True)
91 92 93 94 95 96 97 98 99 100
        else:
            self.scale = None
            self.bias = None

    def _check_input_dim(self, input):
        raise NotImplementedError("InstanceNorm Base error")

    def forward(self, input):
        self._check_input_dim(input)

101 102 103 104
        return instance_norm(input,
                             weight=self.scale,
                             bias=self.bias,
                             eps=self._epsilon)
105

106
    def extra_repr(self):
107
        return 'num_features={}, epsilon={}'.format(self._num_features,
108 109
                                                    self._epsilon)

110

C
cnn 已提交
111
class InstanceNorm1D(_InstanceNormBase):
112
    r"""
113 114 115 116 117 118 119 120
    Applies Instance Normalization over a 3D input (a mini-batch of 1D inputs with additional channel dimension) as described in the paper Instance Normalization: The Missing Ingredient for Fast Stylization .

    DataLayout: NCL `[batch, in_channels, length]`

    :math:`input` is the input features over a mini-batch.

    ..  math::
        
121 122 123 124 125 126 127
        \mu_{\beta} &\gets \frac{1}{HW} \sum_{i=1}^{HW} x_i \qquad &//\
        \ mean\ of\ one\  feature\ map\ in\ mini-batch \\
        \sigma_{\beta}^{2} &\gets \frac{1}{HW} \sum_{i=1}^{HW}(x_i - \
        \mu_{\beta})^2 \qquad &//\ variance\ of\ one\ feature\ map\ in\ mini-batch \\
        \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\
        \sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
        y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift
128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168

    Note:
        `H` means height of feature map, `W` means width of feature map.

    Parameters:
        num_features(int): Indicate the number of channels of the input ``Tensor``.
        epsilon(float, optional): A value added to the denominator for
            numerical stability. Default is 1e-5.
        momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
        weight_attr(ParamAttr|bool, optional): The parameter attribute for Parameter `scale`
             of instance_norm. If it is set to None or one attribute of ParamAttr, instance_norm
	     will create ParamAttr as weight_attr, the name of scale can be set in ParamAttr.
	     If the Initializer of the weight_attr is not set, the parameter is initialized 
	     one. If it is set to False, will not create weight_attr. Default: None.
        bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of instance_norm.
             If it is set to None or one attribute of ParamAttr, instance_norm
	     will create ParamAttr as bias_attr, the name of bias can be set in ParamAttr. 
	     If the Initializer of the bias_attr is not set, the bias is initialized zero. 
             If it is set to False, will not create bias_attr. Default: None.
        data_format(str, optional): Specify the input data format, may be "NC", "NCL". Defalut "NCL".
        name(str, optional): Name for the InstanceNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..


    Shape:
        - x: 2-D or 3-D tensor with shape: (batch, num_features) or (batch, num_features, length).
        - output: 3-D tensor with same shape as input x.

    Returns:
        None.


    Examples:

        .. code-block:: python

          import paddle
          import numpy as np

          np.random.seed(123)
          x_data = np.random.random(size=(2, 2, 3)).astype('float32')
          x = paddle.to_tensor(x_data) 
C
cnn 已提交
169
          instance_norm = paddle.nn.InstanceNorm1D(2)
170 171
          instance_norm_out = instance_norm(x)

Z
zhang wenhui 已提交
172
          print(instance_norm_out)
173 174 175 176 177 178 179 180 181

    """

    def _check_input_dim(self, input):
        if len(input.shape) != 2 and len(input.shape) != 3:
            raise ValueError('expected 2D or 3D input (got {}D input)'.format(
                len(input.shape)))


C
cnn 已提交
182
class InstanceNorm2D(_InstanceNormBase):
183
    r"""
184 185 186 187 188 189 190 191 192
    Applies Instance Normalization over a 4D input (a mini-batch of 2D inputs with additional channel dimension) as described in the paper Instance Normalization: The Missing Ingredient for Fast Stylization .

    DataLayout: NCHW `[batch, in_channels, in_height, in_width]`


    :math:`input` is the input features over a mini-batch.

    ..  math::
        
193 194 195 196 197 198 199
        \mu_{\beta} &\gets \frac{1}{HW} \sum_{i=1}^{HW} x_i \qquad &//\
        \ mean\ of\ one\  feature\ map\ in\ mini-batch \\
        \sigma_{\beta}^{2} &\gets \frac{1}{HW} \sum_{i=1}^{HW}(x_i - \
        \mu_{\beta})^2 \qquad &//\ variance\ of\ one\ feature\ map\ in\ mini-batch \\
        \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\
        \sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
        y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift
200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239

    Note:
        `H` means height of feature map, `W` means width of feature map.

    Parameters:
        num_features(int): Indicate the number of channels of the input ``Tensor``.
        epsilon(float, optional): A value added to the denominator for
            numerical stability. Default is 1e-5.
        momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
        weight_attr(ParamAttr|bool, optional): The parameter attribute for Parameter `scale`
             of instance_norm. If it is set to None or one attribute of ParamAttr, instance_norm
	     will create ParamAttr as weight_attr, the name of scale can be set in ParamAttr.
	     If the Initializer of the weight_attr is not set, the parameter is initialized 
	     one. If it is set to False, will not create weight_attr. Default: None.
        bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of instance_norm.
             If it is set to None or one attribute of ParamAttr, instance_norm
	     will create ParamAttr as bias_attr, the name of bias can be set in ParamAttr. 
	     If the Initializer of the bias_attr is not set, the bias is initialized zero. 
             If it is set to False, will not create bias_attr. Default: None.
        data_format(str, optional): Specify the input data format, could be "NCHW". Default: NCHW.
        name(str, optional): Name for the InstanceNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..

    Shape:
        - x: 4-D tensor with shape: (batch, num_features, height, weight).
        - output: 4-D tensor with same shape as input x.

    Returns:
        None.


    Examples:

        .. code-block:: python

          import paddle
          import numpy as np

          np.random.seed(123)
          x_data = np.random.random(size=(2, 2, 2, 3)).astype('float32')
          x = paddle.to_tensor(x_data) 
C
cnn 已提交
240
          instance_norm = paddle.nn.InstanceNorm2D(2)
241 242
          instance_norm_out = instance_norm(x)

Z
zhang wenhui 已提交
243
          print(instance_norm_out)
244 245 246 247 248 249 250 251
    """

    def _check_input_dim(self, input):
        if len(input.shape) != 4:
            raise ValueError('expected 4D input (got {}D input)'.format(
                len(input.shape)))


C
cnn 已提交
252
class InstanceNorm3D(_InstanceNormBase):
253
    r"""
254 255 256 257 258 259 260 261 262
    Applies Instance Normalization over a 5D input (a mini-batch of 3D inputs with additional channel dimension) as described in the paper Instance Normalization: The Missing Ingredient for Fast Stylization .

    DataLayout: NCHW `[batch, in_channels, D, in_height, in_width]`


    :math:`input` is the input features over a mini-batch.

    ..  math::
        
263 264 265 266 267 268 269
        \mu_{\beta} &\gets \frac{1}{HW} \sum_{i=1}^{HW} x_i \qquad &//\
        \ mean\ of\ one\  feature\ map\ in\ mini-batch \\
        \sigma_{\beta}^{2} &\gets \frac{1}{HW} \sum_{i=1}^{HW}(x_i - \
        \mu_{\beta})^2 \qquad &//\ variance\ of\ one\ feature\ map\ in\ mini-batch \\
        \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\
        \sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
        y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309

    Note:
        `H` means height of feature map, `W` means width of feature map.

    Parameters:
        num_features(int): Indicate the number of channels of the input ``Tensor``.
        epsilon(float, optional): A value added to the denominator for
            numerical stability. Default is 1e-5.
        momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
        weight_attr(ParamAttr|bool, optional): The parameter attribute for Parameter `scale`
             of instance_norm. If it is set to None or one attribute of ParamAttr, instance_norm
	     will create ParamAttr as weight_attr, the name of scale can be set in ParamAttr.
	     If the Initializer of the weight_attr is not set, the parameter is initialized 
	     one. If it is set to False, will not create weight_attr. Default: None.
        bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of instance_norm.
             If it is set to None or one attribute of ParamAttr, instance_norm
	     will create ParamAttr as bias_attr, the name of bias can be set in ParamAttr. 
	     If the Initializer of the bias_attr is not set, the bias is initialized zero. 
             If it is set to False, will not create bias_attr. Default: None.
        data_format(str, optional): Specify the input data format, could be "NCDHW". Default: NCDHW.
        name(str, optional): Name for the InstanceNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..

    Shape:
        - x: 5-D tensor with shape: (batch, num_features, dims, height, weight).
        - output: 5-D tensor with same shape as input x.

    Returns:
        None.


    Examples:

        .. code-block:: python

          import paddle
          import numpy as np

          np.random.seed(123)
          x_data = np.random.random(size=(2, 2, 2, 2, 3)).astype('float32')
          x = paddle.to_tensor(x_data) 
C
cnn 已提交
310
          instance_norm = paddle.nn.InstanceNorm3D(2)
311 312
          instance_norm_out = instance_norm(x)

Z
zhang wenhui 已提交
313
          print(instance_norm_out.numpy)
314 315 316 317 318 319 320 321
    """

    def _check_input_dim(self, input):
        if len(input.shape) != 5:
            raise ValueError('expected 5D input (got {}D input)'.format(
                len(input.shape)))


Z
zhiboniu 已提交
322
class GroupNorm(Layer):
323 324 325 326 327 328 329 330
    """
    This interface is used to construct a callable object of the ``GroupNorm`` class.
    For more details, refer to code examples.
    It implements the function of the Group Normalization Layer.
    Refer to `Group Normalization <https://arxiv.org/abs/1803.08494>`_ .

    Parameters:
        num_groups(int): The number of groups that divided from channels.
331
        num_channels(int): The number of channels of input.
332 333 334 335 336 337 338 339 340 341 342 343
        epsilon(float, optional): The small value added to the variance to prevent
                                  division by zero. Default: 1e-05.
        weight_attr(ParamAttr|bool, optional): The parameter attribute for the learnable
                                         scale :math:`g`. If it is set to False, no scale will be added to the output units.
                                         If it is set to None, the bias is initialized one. Default: None.
        bias_attr(ParamAttr|bool, optional): The parameter attribute for the learnable
                                        bias :math:`b`. If it is set to False, no bias will be added to the output units.
                                        If it is set to None, the bias is initialized zero. Default: None.
        data_format(str, optional): Specify the input data format. Only NCHW is supported. Default: NCHW.
        name(str, optional): Name for the GroupNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..

    Shape:
344 345
        - x: Tensor with shape: (batch, num_features, *).
        - output: The same shape as input x.
346 347 348 349 350 351

    Returns:
        None

    Examples:
        .. code-block:: python
Z
zhang wenhui 已提交
352

353 354 355 356 357 358 359
          import paddle
          import numpy as np

          paddle.disable_static()
          np.random.seed(123)
          x_data = np.random.random(size=(2, 6, 2, 2)).astype('float32')
          x = paddle.to_tensor(x_data) 
360
          group_norm = paddle.nn.GroupNorm(num_channels=6, num_groups=6)
361 362
          group_norm_out = group_norm(x)

363
          print(group_norm_out.numpy())
364 365 366 367
    """

    def __init__(self,
                 num_groups,
368
                 num_channels,
369 370 371
                 epsilon=1e-05,
                 weight_attr=None,
                 bias_attr=None,
372
                 data_format='NCHW',
373 374 375 376 377 378 379
                 name=None):
        super(GroupNorm, self).__init__()
        self._weight_attr = weight_attr
        self._bias_attr = bias_attr
        self._epsilon = epsilon
        self._num_channels = num_channels
        self._num_groups = num_groups
380
        if data_format != 'NCHW':
381
            raise ValueError("unsupported data layout:" + data_format)
382 383 384

        param_shape = [self._num_channels]

385 386 387 388 389 390 391 392 393 394
        if weight_attr == False:
            self.weight = self.create_parameter(
                attr=None, shape=param_shape, default_initializer=Constant(1.0))
            self.weight.stop_gradient = True
        else:
            self.weight = self.create_parameter(
                attr=self._weight_attr,
                shape=param_shape,
                default_initializer=Constant(1.0))
            self.weight.stop_gradient = self._weight_attr != None and self._weight_attr.learning_rate == 0.
395

396
        if bias_attr == False:
397 398 399 400
            self.bias = self.create_parameter(attr=None,
                                              shape=param_shape,
                                              default_initializer=Constant(0.0),
                                              is_bias=True)
401 402
            self.bias.stop_gradient = True
        else:
403 404 405
            self.bias = self.create_parameter(attr=self._bias_attr,
                                              shape=param_shape,
                                              is_bias=True)
406
            self.bias.stop_gradient = self._bias_attr != None and self._bias_attr.learning_rate == 0.
407 408

    def forward(self, input):
409 410 411 412 413
        mean_out = self._helper.create_variable_for_type_inference(
            dtype=input.dtype, stop_gradient=True)
        variance_out = self._helper.create_variable_for_type_inference(
            dtype=input.dtype, stop_gradient=True)

414 415 416 417 418 419 420 421 422
        if in_dygraph_mode():
            pre_act = _C_ops.final_state_group_norm(input, self.weight,
                                                    self.bias, self._epsilon,
                                                    self._num_groups, "NCHW")

            return dygraph_utils._append_activation_in_dygraph(pre_act,
                                                               act=None)

        elif _in_legacy_dygraph():
423 424 425 426 427 428 429 430 431
            pre_act, _, _ = _C_ops.group_norm(
                input,
                self.weight,
                self.bias,
                mean_out,
                variance_out,
                'epsilon',
                self._epsilon,
                'groups',
432 433 434 435
                self._num_groups,
            )
            return dygraph_utils._append_activation_in_dygraph(pre_act,
                                                               act=None)
436

437 438 439 440 441 442 443 444 445 446
        inputs = {'X': input}
        if self.bias is not None:
            inputs['Bias'] = self.bias
        if self.weight is not None:
            inputs['Scale'] = self.weight

        # create output
        group_norm_out = self._helper.create_variable_for_type_inference(
            dtype=input.dtype)

447 448 449 450 451 452 453 454 455 456 457
        self._helper.append_op(type="group_norm",
                               inputs=inputs,
                               outputs={
                                   "Y": group_norm_out,
                                   "Mean": mean_out,
                                   "Variance": variance_out,
                               },
                               attrs={
                                   "epsilon": self._epsilon,
                                   "groups": self._num_groups
                               })
458 459 460

        return self._helper.append_activation(group_norm_out, None)

461 462 463 464
    def extra_repr(self):
        return 'num_groups={}, num_channels={}, epsilon={}'.format(
            self._num_groups, self._num_channels, self._epsilon)

465

Z
zhiboniu 已提交
466
class LayerNorm(Layer):
467
    r"""
468 469 470 471 472 473 474 475 476 477 478 479 480
    :alias_main: paddle.nn.LayerNorm
	:alias: paddle.nn.LayerNorm,paddle.nn.layer.LayerNorm,paddle.nn.layer.norm.LayerNorm
	:old_api: paddle.fluid.dygraph.LayerNorm

    This interface is used to construct a callable object of the ``LayerNorm`` class.
    For more details, refer to code examples.
    It implements the function of the Layer Normalization Layer and can be applied to mini-batch input data.
    Refer to `Layer Normalization <https://arxiv.org/pdf/1607.06450v1.pdf>`_

    The formula is as follows:

    ..  math::

481
        \mu & = \frac{1}{H}\sum_{i=1}^{H} x_i
482

483
        \sigma & = \sqrt{\frac{1}{H}\sum_{i=1}^{H}{(x_i - \mu)^2} + \epsilon}
484

485
        y & = f(\frac{g}{\sigma}(x - \mu) + b)
486 487 488

    - :math:`x`: the vector representation of the summed inputs to the neurons in that layer.
    - :math:`H`: the number of hidden units in a layers
489
    - :math:`\epsilon`: the small value added to the variance to prevent division by zero.
490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527
    - :math:`g`: the trainable scale parameter.
    - :math:`b`: the trainable bias parameter.

    Parameters:
        normalized_shape(int|list|tuple): Input shape from an expected input of
            size :math:`[*, normalized_shape[0], normalized_shape[1], ..., normalized_shape[-1]]`.
            If it is a single integer, this module will normalize over the last dimension
            which is expected to be of that specific size.
        epsilon(float, optional): The small value added to the variance to prevent
            division by zero. Default: 1e-05.
        weight_attr(ParamAttr|bool, optional): The parameter attribute for the learnable
            gain :math:`g`. If False, weight is None. If is None, a default :code:`ParamAttr` would be added as scale. The
            :attr:`param_attr` is initialized as 1 if it is added. Default: None.
        bias_attr(ParamAttr|bool, optional): The parameter attribute for the learnable
            bias :math:`b`. If is False, bias is None. If is None, a default :code:`ParamAttr` would be added as bias. The
            :attr:`bias_attr` is initialized as 0 if it is added. Default: None.
        name(str, optional): Name for the LayerNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..

    Shape:
        - x: 2-D, 3-D, 4-D or 5-D tensor.
        - output: same shape as input x.

    Returns:
        None

    Examples:

        .. code-block:: python

          import paddle
          import numpy as np

          np.random.seed(123)
          x_data = np.random.random(size=(2, 2, 2, 3)).astype('float32')
          x = paddle.to_tensor(x_data) 
          layer_norm = paddle.nn.LayerNorm(x_data.shape[1:])
          layer_norm_out = layer_norm(x)

Z
zhang wenhui 已提交
528
          print(layer_norm_out)
529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557
    """

    def __init__(self,
                 normalized_shape,
                 epsilon=1e-05,
                 weight_attr=None,
                 bias_attr=None,
                 name=None):
        super(LayerNorm, self).__init__()
        if isinstance(normalized_shape, numbers.Integral):
            normalized_shape = [normalized_shape]

        self._normalized_shape = list(normalized_shape)
        self._epsilon = epsilon
        self._weight_attr = weight_attr
        self._bias_attr = bias_attr
        param_shape = [np.prod(self._normalized_shape)]

        if weight_attr is False:
            self.weight = None
        else:
            self.weight = self.create_parameter(
                attr=self._weight_attr,
                shape=param_shape,
                default_initializer=Constant(1.0))

        if bias_attr is False:
            self.bias = None
        else:
558 559 560
            self.bias = self.create_parameter(attr=self._bias_attr,
                                              shape=param_shape,
                                              is_bias=True)
561 562

    def forward(self, input):
563 564 565 566 567
        return layer_norm(input,
                          normalized_shape=self._normalized_shape,
                          weight=self.weight,
                          bias=self.bias,
                          epsilon=self._epsilon)
568

569 570 571 572
    def extra_repr(self):
        return 'normalized_shape={}, epsilon={}'.format(self._normalized_shape,
                                                        self._epsilon)

573

Z
zhiboniu 已提交
574
class _BatchNormBase(Layer):
575 576 577 578 579 580 581 582 583 584 585
    """
    BatchNorm base .
    """

    def __init__(self,
                 num_features,
                 momentum=0.9,
                 epsilon=1e-05,
                 weight_attr=None,
                 bias_attr=None,
                 data_format='NCHW',
C
ceci3 已提交
586
                 use_global_stats=None,
587 588 589 590 591
                 name=None):
        super(_BatchNormBase, self).__init__()
        self._num_features = num_features
        self._weight_attr = weight_attr
        self._bias_attr = bias_attr
C
ceci3 已提交
592
        self._use_global_stats = use_global_stats
593 594

        if get_default_dtype() == 'float16':
G
Guoxia Wang 已提交
595 596 597
            self._dtype = 'float32'
        else:
            self._dtype = get_default_dtype()
598 599 600 601

        param_shape = [num_features]

        # create parameter
602 603
        if weight_attr == False:
            self.weight = self.create_parameter(
G
Guoxia Wang 已提交
604 605 606 607
                attr=None,
                shape=param_shape,
                dtype=self._dtype,
                default_initializer=Constant(1.0))
608 609 610 611 612
            self.weight.stop_gradient = True
        else:
            self.weight = self.create_parameter(
                attr=self._weight_attr,
                shape=param_shape,
G
Guoxia Wang 已提交
613
                dtype=self._dtype,
614 615
                default_initializer=Constant(1.0))
            self.weight.stop_gradient = self._weight_attr != None and self._weight_attr.learning_rate == 0.
616

617
        if bias_attr == False:
618 619 620 621 622
            self.bias = self.create_parameter(attr=None,
                                              shape=param_shape,
                                              dtype=self._dtype,
                                              default_initializer=Constant(0.0),
                                              is_bias=True)
623 624
            self.bias.stop_gradient = True
        else:
625 626 627 628
            self.bias = self.create_parameter(attr=self._bias_attr,
                                              shape=param_shape,
                                              dtype=self._dtype,
                                              is_bias=True)
629
            self.bias.stop_gradient = self._bias_attr != None and self._bias_attr.learning_rate == 0.
630 631 632 633 634 635 636 637

        moving_mean_name = None
        moving_variance_name = None

        if name is not None:
            moving_mean_name = name + "_mean"
            moving_variance_name = name + "_variance"

638 639 640 641 642 643 644
        self._mean = self.create_parameter(dtype=self._dtype,
                                           attr=ParamAttr(
                                               name=moving_mean_name,
                                               initializer=Constant(0.0),
                                               trainable=False,
                                               do_model_average=True),
                                           shape=param_shape)
645 646
        self._mean.stop_gradient = True

647 648 649 650 651 652 653
        self._variance = self.create_parameter(dtype=self._dtype,
                                               attr=ParamAttr(
                                                   name=moving_variance_name,
                                                   initializer=Constant(1.0),
                                                   trainable=False,
                                                   do_model_average=True),
                                               shape=param_shape)
654 655 656 657 658 659 660
        self._variance.stop_gradient = True

        self._data_format = data_format
        self._in_place = False
        self._momentum = momentum
        self._epsilon = epsilon
        self._fuse_with_relu = False
661
        self._name = name
662 663 664 665

    def _check_input_dim(self, input):
        raise NotImplementedError("BatchNorm Base error")

666 667 668
    def _check_data_format(self, input):
        raise NotImplementedError("BatchNorm Base data format error")

669 670
    def forward(self, input):

671 672
        self._check_data_format(self._data_format)

673 674
        self._check_input_dim(input)

675
        if self.training:
676 677 678
            warnings.warn(
                "When training, we now always track global mean and variance.")

679 680 681 682 683 684 685 686 687 688
        return batch_norm(input,
                          self._mean,
                          self._variance,
                          weight=self.weight,
                          bias=self.bias,
                          training=self.training,
                          momentum=self._momentum,
                          epsilon=self._epsilon,
                          data_format=self._data_format,
                          use_global_stats=self._use_global_stats)
689

690 691 692
    def extra_repr(self):
        main_str = 'num_features={}, momentum={}, epsilon={}'.format(
            self._num_features, self._momentum, self._epsilon)
693
        if self._data_format != 'NCHW':
694 695 696 697 698
            main_str += ', data_format={}'.format(self._data_format)
        if self._name is not None:
            main_str += ', name={}'.format(self._name)
        return main_str

699

C
cnn 已提交
700
class BatchNorm1D(_BatchNormBase):
701
    r"""
702 703
    Applies Batch Normalization over a 2D or 3D input (a mini-batch of 1D inputswith additional channel dimension) as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift .

704 705
    When use_global_stats = False, the :math:`\mu_{\beta}`
    and :math:`\sigma_{\beta}^{2}` are the statistics of one mini-batch.
706 707 708 709
    Calculated as follows:

    ..  math::

710 711 712 713
        \mu_{\beta} &\gets \frac{1}{m} \sum_{i=1}^{m} x_i \qquad &//\
        \ mini-batch\ mean \\
        \sigma_{\beta}^{2} &\gets \frac{1}{m} \sum_{i=1}^{m}(x_i - \
        \mu_{\beta})^2 \qquad &//\ mini-batch\ variance \\
714

715 716
    When use_global_stats = True, the :math:`\mu_{\beta}`
    and :math:`\sigma_{\beta}^{2}` are not the statistics of one mini-batch.
717 718 719 720
    They are global or running statistics (moving_mean and moving_variance). It usually got from the
    pre-trained model. Calculated as follows:

    .. math::
721 722
        moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global \ mean \\
        moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global \ variance \\
723 724 725 726 727

    The normalization function formula is as follows:

    ..  math::

728 729
        \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
        y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift
730

731 732 733
    - :math:`\epsilon` : add a smaller value to the variance to prevent division by zero
    - :math:`\gamma` : trainable proportional parameter
    - :math:`\beta` : trainable deviation parameter
734 735 736 737 738 739 740 741 742 743 744 745 746

    Parameters:
        num_features(int): Indicate the number of channels of the input ``Tensor``.
        epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5.
        momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
        weight_attr(ParamAttr|bool, optional): The parameter attribute for Parameter `scale`
            of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm
            will create ParamAttr as weight_attr. If it is set to Fasle, the weight is not learnable.
            If the Initializer of the weight_attr is not set, the parameter is initialized with Xavier. Default: None.
        bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of batch_norm.
            If it is set to None or one attribute of ParamAttr, batch_norm
            will create ParamAttr as bias_attr. If it is set to Fasle, the weight is not learnable.
            If the Initializer of the bias_attr is not set, the bias is initialized zero. Default: None.
F
Feiyu Chan 已提交
747
        data_format(str, optional): Specify the input data format, may be "NC", "NCL" or "NLC". Defalut "NCL".
C
ceci3 已提交
748
        use_global_stats(bool|None, optional): Whether to use global mean and variance. If set to False, use the statistics of one mini-batch, if set to True, use the global statistics, if set to None, use global statistics in the test phase and use the statistics of one mini-batch in the training phase. Default: None.
749 750 751
        name(str, optional): Name for the BatchNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..

    Shape:
F
Feiyu Chan 已提交
752 753
        - x: 2-D or 3-D tensor with shape: (batch, num_features) or (batch, num_features, length) when data_format is "NC" or "NCL",
            (batch, length, num_features) when data_format is "NLC".
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768
        - output: 3-D tensor with same shape as input x.

    Returns:
        None.
    

    Examples:
        .. code-block:: python

          import paddle
          import numpy as np

          np.random.seed(123)
          x_data = np.random.random(size=(2, 1, 3)).astype('float32')
          x = paddle.to_tensor(x_data) 
C
cnn 已提交
769
          batch_norm = paddle.nn.BatchNorm1D(1)
770 771
          batch_norm_out = batch_norm(x)

Z
zhang wenhui 已提交
772
          print(batch_norm_out)
773 774
    """

C
ceci3 已提交
775 776 777 778 779 780 781 782 783 784 785 786 787
    def __init__(self,
                 num_features,
                 momentum=0.9,
                 epsilon=1e-05,
                 weight_attr=None,
                 bias_attr=None,
                 data_format='NCL',
                 use_global_stats=None,
                 name=None):
        super(BatchNorm1D,
              self).__init__(num_features, momentum, epsilon, weight_attr,
                             bias_attr, data_format, use_global_stats, name)

788 789 790
    def _check_data_format(self, input):
        if input == 'NCHW' or input == 'NC' or input == 'NCL':
            self._data_format = 'NCHW'
F
Feiyu Chan 已提交
791 792
        elif input == "NHWC" or input == 'NLC':
            self._data_format = "NHWC"
793
        else:
F
Feiyu Chan 已提交
794 795
            raise ValueError(
                'expected NC , NCL, NLC or None for data_format input')
796

797 798 799 800 801 802
    def _check_input_dim(self, input):
        if len(input.shape) != 2 and len(input.shape) != 3:
            raise ValueError('expected 2D or 3D input (got {}D input)'.format(
                len(input.shape)))


C
cnn 已提交
803
class BatchNorm2D(_BatchNormBase):
804
    r"""
805 806
    Applies Batch Normalization over a 4D input (a mini-batch of 2D inputswith additional channel dimension) as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift .

807 808
    When use_global_stats = False, the :math:`\mu_{\beta}`
    and :math:`\sigma_{\beta}^{2}` are the statistics of one mini-batch.
809 810 811 812
    Calculated as follows:

    ..  math::

813 814 815 816
        \mu_{\beta} &\gets \frac{1}{m} \sum_{i=1}^{m} x_i \qquad &//
        \ mini-batch\ mean \\
        \sigma_{\beta}^{2} &\gets \frac{1}{m} \sum_{i=1}^{m}(x_i - 
        \mu_{\beta})^2 \qquad &//\ mini-batch\ variance \\
817

818 819
    When use_global_stats = True, the :math:`\mu_{\beta}`
    and :math:`\sigma_{\beta}^{2}` are not the statistics of one mini-batch.
820 821 822 823
    They are global or running statistics (moving_mean and moving_variance). It usually got from the
    pre-trained model. Calculated as follows:

    .. math::
824 825
        moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global \ mean \\
        moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global \ variance \\
826 827 828 829 830

    The normalization function formula is as follows:

    ..  math::

831 832
        \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
        y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift
833

834 835 836
    - :math:`\epsilon` : add a smaller value to the variance to prevent division by zero
    - :math:`\gamma` : trainable proportional parameter
    - :math:`\beta` : trainable deviation parameter
837 838 839 840 841 842 843 844 845 846 847 848 849

    Parameters:
        num_features(int): Indicate the number of channels of the input ``Tensor``.
        epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5.
        momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
        weight_attr(ParamAttr|bool, optional): The parameter attribute for Parameter `scale`
            of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm
            will create ParamAttr as weight_attr. If it is set to Fasle, the weight is not learnable.
            If the Initializer of the weight_attr is not set, the parameter is initialized with Xavier. Default: None.
        bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of batch_norm.
            If it is set to None or one attribute of ParamAttr, batch_norm
            will create ParamAttr as bias_attr. If it is set to Fasle, the weight is not learnable.
            If the Initializer of the bias_attr is not set, the bias is initialized zero. Default: None.
F
Feiyu Chan 已提交
850
        data_format(str, optional): Specify the input data format, the data format can be "NCHW" or "NHWC". Default: NCHW.
C
ceci3 已提交
851
        use_global_stats(bool|None, optional): Whether to use global mean and variance. If set to False, use the statistics of one mini-batch, if set to True, use the global statistics, if set to None, use global statistics in the test phase and use the statistics of one mini-batch in the training phase. Default: None.
852 853 854
        name(str, optional): Name for the BatchNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..

    Shape:
F
Feiyu Chan 已提交
855 856
        - x: 4-D tensor with shape: (batch, num_features, height, weight) when data_format is "NCHW",
            or (batch, height, weight, num_features) when data_format is "NHWC".
857 858 859 860 861 862 863 864 865 866 867 868 869 870
        - output: 4-D tensor with same shape as input x.

    Returns:
        None

    Examples:
        .. code-block:: python

          import paddle
          import numpy as np

          np.random.seed(123)
          x_data = np.random.random(size=(2, 1, 2, 3)).astype('float32')
          x = paddle.to_tensor(x_data) 
C
cnn 已提交
871
          batch_norm = paddle.nn.BatchNorm2D(1)
872 873
          batch_norm_out = batch_norm(x)

Z
zhang wenhui 已提交
874
          print(batch_norm_out)
875 876
    """

877
    def _check_data_format(self, input):
878
        if input == 'NCHW':
879
            self._data_format = input
F
Feiyu Chan 已提交
880 881
        elif input == "NHWC":
            self._data_format = input
882
        else:
F
Feiyu Chan 已提交
883
            raise ValueError('expected NCHW or NHWC for data_format input')
884

885 886 887 888 889 890
    def _check_input_dim(self, input):
        if len(input.shape) != 4:
            raise ValueError('expected 4D input (got {}D input)'.format(
                len(input.shape)))


C
cnn 已提交
891
class BatchNorm3D(_BatchNormBase):
892
    r"""
893 894
    Applies Batch Normalization over a 5D input (a mini-batch of 3D inputswith additional channel dimension) as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift .

895 896
    When use_global_stats = False, the :math:`\mu_{\beta}`
    and :math:`\sigma_{\beta}^{2}` are the statistics of one mini-batch.
897 898 899 900
    Calculated as follows:

    ..  math::

901 902 903 904
        \mu_{\beta} &\gets \frac{1}{m} \sum_{i=1}^{m} x_i \qquad &//\
        \ mini-batch\ mean \\
        \sigma_{\beta}^{2} &\gets \frac{1}{m} \sum_{i=1}^{m}(x_i - \
        \mu_{\beta})^2 \qquad &//\ mini-batch\ variance \\
905

C
ceci3 已提交
906
    When use_global_stats = True, the :math:`\\mu_{\\beta}`
907 908 909 910 911
    and :math:`\\sigma_{\\beta}^{2}` are not the statistics of one mini-batch.
    They are global or running statistics (moving_mean and moving_variance). It usually got from the
    pre-trained model. Calculated as follows:

    .. math::
912 913
        moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global \ mean \\
        moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global \ variance \\
914 915 916 917 918

    The normalization function formula is as follows:

    ..  math::

919 920
        \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
        y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift
921

922 923 924
    - :math:`\epsilon` : add a smaller value to the variance to prevent division by zero
    - :math:`\gamma` : trainable proportional parameter
    - :math:`\beta` : trainable deviation parameter
925 926 927 928 929 930 931 932 933 934 935 936 937

    Parameters:
        num_features(int): Indicate the number of channels of the input ``Tensor``.
        epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5.
        momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
        weight_attr(ParamAttr|bool, optional): The parameter attribute for Parameter `scale`
            of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm
            will create ParamAttr as weight_attr. If it is set to Fasle, the weight is not learnable.
            If the Initializer of the weight_attr is not set, the parameter is initialized with Xavier. Default: None.
        bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of batch_norm.
            If it is set to None or one attribute of ParamAttr, batch_norm
            will create ParamAttr as bias_attr. If it is set to Fasle, the weight is not learnable.
            If the Initializer of the bias_attr is not set, the bias is initialized zero. Default: None.
F
Feiyu Chan 已提交
938
        data_format(str, optional): Specify the input data format, the data format can be "NCDHW" or "NDHWC. Default: NCDHW.
C
ceci3 已提交
939
        use_global_stats(bool|None, optional): Whether to use global mean and variance. If set to False, use the statistics of one mini-batch, if set to True, use the global statistics, if set to None, use global statistics in the test phase and use the statistics of one mini-batch in the training phase. Default: None.
940 941 942
        name(str, optional): Name for the BatchNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..

    Shape:
F
Feiyu Chan 已提交
943 944
        - x: 5-D tensor with shape: (batch, num_features, dims, height, weight) when data_format is "NCDHW",
            or (batch, dims, height, weight, num_features) when data_format is "NDHWC".
945 946 947 948 949 950 951 952 953 954 955 956 957 958
        - output: 5-D tensor with same shape as input x.

    Returns:
        None

    Examples:
        .. code-block:: python

          import paddle
          import numpy as np

          np.random.seed(123)
          x_data = np.random.random(size=(2, 1, 2, 2, 3)).astype('float32')
          x = paddle.to_tensor(x_data) 
C
cnn 已提交
959
          batch_norm = paddle.nn.BatchNorm3D(1)
960 961
          batch_norm_out = batch_norm(x)

Z
zhang wenhui 已提交
962
          print(batch_norm_out)
963 964
    """

C
ceci3 已提交
965 966 967 968 969 970 971 972 973 974 975 976 977
    def __init__(self,
                 num_features,
                 momentum=0.9,
                 epsilon=1e-05,
                 weight_attr=None,
                 bias_attr=None,
                 data_format='NCDHW',
                 use_global_stats=None,
                 name=None):
        super(BatchNorm3D,
              self).__init__(num_features, momentum, epsilon, weight_attr,
                             bias_attr, data_format, use_global_stats, name)

978 979 980
    def _check_data_format(self, input):
        if input == 'NCHW' or input == 'NCDHW':
            self._data_format = 'NCHW'
F
Feiyu Chan 已提交
981 982
        elif input == "NHWC" or input == "NDHWC":
            self._data_format = 'NHWC'
983
        else:
F
Feiyu Chan 已提交
984 985
            raise ValueError(
                'expected NCDHW, NDHWC or None for data_format input')
986

987 988 989 990 991 992
    def _check_input_dim(self, input):
        if len(input.shape) != 5:
            raise ValueError('expected 5D input (got {}D input)'.format(
                len(input.shape)))


993
class SyncBatchNorm(_BatchNormBase):
994
    r"""
C
ceci3 已提交
995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010
    This interface is used to construct a callable object of the ``SyncBatchNorm`` class.
    It implements the function of the Cross-GPU Synchronized Batch Normalization Layer, and can 
    be used as a normalizer function for other operations, such as conv2d and fully connected 
    operations.
    The data is normalized by the mean and variance of the channel based on whole mini-batch
    , which including data in all gpus.
    Refer to `Batch Normalization: Accelerating Deep Network Training by Reducing
    Internal Covariate Shift <https://arxiv.org/pdf/1502.03167.pdf>`_
    for more details.

    When model in training mode, the :math:`\\mu_{\\beta}` 
    and :math:`\\sigma_{\\beta}^{2}` are the statistics of whole mini-batch data in all gpus.
    Calculated as follows:

    ..  math::

1011 1012 1013 1014
        \mu_{\beta} &\gets \frac{1}{m} \sum_{i=1}^{m} x_i \qquad &//\
        \ mini-batch\ mean \\
        \sigma_{\beta}^{2} &\gets \frac{1}{m} \sum_{i=1}^{m}(x_i - \
        \mu_{\beta})^2 \qquad &//\ mini-batch\ variance \\
C
ceci3 已提交
1015 1016 1017 1018 1019

    - :math:`x` : whole mini-batch data in all gpus
    - :math:`m` : the size of the whole mini-batch data

    When model in evaluation mode, the :math:`\\mu_{\\beta}`
1020
    and :math:`\sigma_{\beta}^{2}` are global statistics (moving_mean and moving_variance, 
C
ceci3 已提交
1021 1022 1023
    which usually got from the pre-trained model). Global statistics calculated as follows:

    .. math::
1024 1025
        moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global \ mean \\
        moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global \ variance \\
C
ceci3 已提交
1026 1027 1028 1029 1030

    The formula of normalization is as follows:
 
    ..  math::

1031 1032 1033
        \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\
        \sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
        y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift
C
ceci3 已提交
1034

1035 1036 1037
    - :math:`\epsilon` : add a smaller value to the variance to prevent division by zero
    - :math:`\gamma` : trainable scale parameter vector
    - :math:`\beta` : trainable shift parameter vector 
C
ceci3 已提交
1038

1039 1040 1041 1042 1043
    Note:
        If you want to use container to pack your model and has ``SyncBatchNorm`` in the 
        evaluation phase, please use ``nn.LayerList`` or ``nn.Sequential`` instead of 
        ``list`` to pack the model. 

C
ceci3 已提交
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071
    Parameters:
        num_features(int): Indicate the number of channels of the input ``Tensor``.
        epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5.
        momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
        weight_attr(ParamAttr|bool, optional): The parameter attribute for Parameter `scale`
             of this layer. If it is set to None or one attribute of ParamAttr, this layerr
             will create ParamAttr as param_attr. If the Initializer of the param_attr
             is not set, the parameter is initialized with Xavier. If it is set to False, 
             this layer will not have trainable scale parameter. Default: None.
        bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of this layer.
             If it is set to None or one attribute of ParamAttr, this layer
             will create ParamAttr as bias_attr. If the Initializer of the bias_attr
             is not set, the bias is initialized zero. If it is set to False, this layer will not 
             have trainable bias parameter. Default: None.

    Shapes:
        input: Tensor that the dimension from 2 to 5.
        output: Tensor with the same shape as input.

    Examples:
        .. code-block:: python

          import paddle
          import paddle.nn as nn
          import numpy as np

          x = np.array([[[[0.3, 0.4], [0.3, 0.07]], [[0.83, 0.37], [0.18, 0.93]]]]).astype('float32')
          x = paddle.to_tensor(x)
C
ceci3 已提交
1072 1073

          if paddle.is_compiled_with_cuda():
C
ceci3 已提交
1074 1075
              sync_batch_norm = nn.SyncBatchNorm(2)
              hidden1 = sync_batch_norm(x)
C
ceci3 已提交
1076
              print(hidden1)
C
ceci3 已提交
1077 1078 1079 1080 1081 1082
              # [[[[0.26824948, 1.0936325],[0.26824948, -1.6301316]],[[ 0.8095662, -0.665287],[-1.2744656, 1.1301866 ]]]]
    """

    def __init__(self,
                 num_features,
                 momentum=0.9,
1083
                 epsilon=1e-05,
C
ceci3 已提交
1084 1085 1086 1087
                 weight_attr=None,
                 bias_attr=None,
                 data_format='NCHW',
                 name=None):
1088 1089
        super(SyncBatchNorm,
              self).__init__(num_features, momentum, epsilon, weight_attr,
C
ceci3 已提交
1090
                             bias_attr, data_format, None, name)
C
ceci3 已提交
1091

C
ceci3 已提交
1092 1093 1094 1095 1096 1097 1098 1099 1100 1101
    def _check_data_format(self):
        if self._data_format in ['NCHW', 'NCDHW', 'NC', 'NCL']:
            self._data_format = 'NCHW'
        elif self._data_format in ["NHWC", "NDHWC", 'NLC']:
            self._data_format = 'NHWC'
        else:
            raise ValueError(
                'expected \'NCDHW\', \'NDHWC\', \'NCL\', \'NLC\', \'NC\', \'NCHW\', \'NHWC\' for data_format'
            )

C
ceci3 已提交
1102
    def forward(self, x):
C
ceci3 已提交
1103
        self._check_data_format()
C
ceci3 已提交
1104 1105 1106 1107 1108 1109 1110 1111
        # create output
        # mean and mean_out share the same memory
        mean_out = self._mean
        # variance and variance out share the same memory
        variance_out = self._variance

        ### train mode: use mini-batch stats, eval mode: use global stats
        ### use_global_stats only support False in sync_batch_norm
1112 1113 1114 1115 1116 1117 1118 1119
        if in_dygraph_mode():
            sync_batch_norm_out, _, _, _, _, _ = _C_ops.final_state_sync_batch_norm(
                x, self.weight, self.bias, self._mean, self._variance,
                self._momentum, self._epsilon, self._data_format,
                not self.training, False, False, False)
            return sync_batch_norm_out

        elif in_dynamic_mode():
C
ceci3 已提交
1120 1121
            attrs = ("momentum", self._momentum, "epsilon", self._epsilon,
                     "is_test", not self.training, "data_layout",
1122
                     self._data_format, "use_mkldnn", False, "fuse_with_relu",
C
ceci3 已提交
1123 1124
                     False, "use_global_stats", False, 'trainable_statistics',
                     False)
W
wanghuancoder 已提交
1125
            sync_batch_norm_out, _, _, _, _, _ = _C_ops.sync_batch_norm(
C
ceci3 已提交
1126 1127 1128 1129 1130
                x, self.weight, self.bias, self._mean, self._variance, mean_out,
                variance_out, *attrs)
            return sync_batch_norm_out

        check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'],
1131
                                 'SyncBatchNorm')
C
ceci3 已提交
1132 1133 1134 1135 1136

        attrs = {
            "momentum": self._momentum,
            "epsilon": self._epsilon,
            "is_test": not self.training,
1137
            "data_layout": self._data_format,
C
ceci3 已提交
1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
            "use_mkldnn": False,
            "fuse_with_relu": False,
            "use_global_stats": False,
            "trainable_statistics": False,
        }

        inputs = {
            "X": [x],
            "Scale": [self.weight],
            "Bias": [self.bias],
            "Mean": [self._mean],
            "Variance": [self._variance]
        }

        saved_mean = self._helper.create_variable_for_type_inference(
            dtype=self._dtype, stop_gradient=True)
        saved_variance = self._helper.create_variable_for_type_inference(
            dtype=self._dtype, stop_gradient=True)
        sync_batch_norm_out = self._helper.create_variable_for_type_inference(
            self._dtype)

        outputs = {
            "Y": [sync_batch_norm_out],
            "MeanOut": [mean_out],
            "VarianceOut": [variance_out],
            "SavedMean": [saved_mean],
            "SavedVariance": [saved_variance]
        }

1167 1168 1169 1170
        self._helper.append_op(type="sync_batch_norm",
                               inputs=inputs,
                               outputs=outputs,
                               attrs=attrs)
C
ceci3 已提交
1171
        return sync_batch_norm_out
1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189

    @classmethod
    def convert_sync_batchnorm(cls, layer):
        """
        Helper function to convert :class: `paddle.nn.BatchNorm*d` layers in the model to :class: `paddle.nn.SyncBatchNorm` layers.

        Parameters:
            layer(paddle.nn.Layer): model containing one or more `BatchNorm*d` layers.

        Returns:
            The original model with converted SyncBatchNorm layers. If BatchNorm*d layer in the model, use SyncBatchNorm layer instead.

        Examples:

            .. code-block:: python
                import paddle
                import paddle.nn as nn

C
cnn 已提交
1190
                model = nn.Sequential(nn.Conv2D(3, 5, 3), nn.BatchNorm2D(5))
1191 1192 1193 1194 1195
                sync_model = nn.SyncBatchNorm.convert_sync_batchnorm(model)

        """
        layer_output = layer
        if isinstance(layer, _BatchNormBase):
C
ceci3 已提交
1196 1197 1198
            if layer._weight_attr != None and not isinstance(
                    layer._weight_attr,
                    bool) and layer._weight_attr.name != None:
C
ceci3 已提交
1199
                layer._weight_attr.name = layer._weight_attr.name + '_sync'
C
ceci3 已提交
1200 1201
            if layer._bias_attr != None and not isinstance(
                    layer._bias_attr, bool) and layer._bias_attr.name != None:
C
ceci3 已提交
1202 1203
                layer._bias_attr.name = layer._bias_attr.name + '_sync'

1204 1205 1206 1207
            layer_output = SyncBatchNorm(layer._num_features, layer._momentum,
                                         layer._epsilon, layer._weight_attr,
                                         layer._bias_attr, layer._data_format,
                                         layer._name)
1208 1209 1210 1211 1212 1213 1214 1215

            if layer._weight_attr != False and layer._bias_attr != False:
                with no_grad():
                    layer_output.weight = layer.weight
                    layer_output.bias = layer.bias
            layer_output._mean = layer._mean
            layer_output._variance = layer._variance

C
ceci3 已提交
1216
        for name, sublayer in layer.named_children():
1217 1218 1219 1220
            layer_output.add_sublayer(name,
                                      cls.convert_sync_batchnorm(sublayer))
        del layer
        return layer_output
1221 1222


Z
zhiboniu 已提交
1223
class LocalResponseNorm(Layer):
1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280
    """
        Local Response Normalization performs a type of "lateral inhibition" by normalizing over local input regions.
        For more information, please refer to `ImageNet Classification with Deep Convolutional Neural Networks <https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf>`_

        See more details in :ref:`api_paddle_nn_functional_local_response_norm` .

        Parameters:
            size (int): The number of channels to sum over.
            alpha (float, optional): The scaling parameter, positive. Default:1e-4
            beta (float, optional): The exponent, positive. Default:0.75
            k (float, optional): An offset, positive. Default: 1.0
            data_format (str, optional): Specify the data format of the input, and the data format of the output
                will be consistent with that of the input. An optional string from:
                If input is 3-D Tensor, the string could be `"NCL"` or `"NLC"` . When it is `"NCL"`,
                the data is stored in the order of: `[batch_size, input_channels, feature_length]`.
                If input is 4-D Tensor, the string could be  `"NCHW"`, `"NHWC"`. When it is `"NCHW"`,
                the data is stored in the order of: `[batch_size, input_channels, input_height, input_width]`.
                If input is 5-D Tensor, the string could be  `"NCDHW"`, `"NDHWC"` . When it is `"NCDHW"`,
                the data is stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`.
            name (str, optional): Name for the operation (optional, default is None). For more information,
                please refer to :ref:`api_guide_Name`.

        Shape:
            - input: 3-D/4-D/5-D tensor.
            - output: 3-D/4-D/5-D tensor, the same shape as input.

        Examples:

        .. code-block:: python

            import paddle

            x = paddle.rand(shape=(3, 3, 112, 112), dtype="float32")
            m = paddle.nn.LocalResponseNorm(size=5)
            y = m(x)
            print(y.shape)  # [3, 3, 112, 112]
        """

    def __init__(self,
                 size,
                 alpha=0.0001,
                 beta=0.75,
                 k=1.0,
                 data_format="NCHW",
                 name=None):
        super(LocalResponseNorm, self).__init__()
        self.size = size
        self.alpha = alpha
        self.beta = beta
        self.k = k
        self.data_format = data_format
        self.name = name

    def forward(self, input):
        out = F.local_response_norm(input, self.size, self.alpha, self.beta,
                                    self.k, self.data_format, self.name)
        return out
1281 1282 1283 1284

    def extra_repr(self):
        main_str = 'size={}, alpha={}, beta={}, k={}'.format(
            self.size, self.alpha, self.beta, self.k)
1285
        if self.data_format != 'NCHW':
1286 1287 1288 1289
            main_str += ', data_format={}'.format(self.data_format)
        if self.name is not None:
            main_str += ', name={}'.format(self.name)
        return main_str