nn.py 122.6 KB
Newer Older
M
minqiyang 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

from six.moves import reduce
from .. import core
from ..layers import utils
20
from ..layers import nn as F
21
from .. import dygraph_utils
M
minqiyang 已提交
22
from . import layers
23
from ..framework import Variable, in_dygraph_mode, OpProtoHolder, Parameter, _dygraph_tracer, _varbase_creator, default_main_program
24
from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype
M
minqiyang 已提交
25
from ..param_attr import ParamAttr
26
from ..initializer import Normal, Constant, NumpyArrayInitializer
H
hong 已提交
27 28
from .. import unique_name
from .layer_object_helper import LayerObjectHelper
L
lujun 已提交
29
import numpy as np
30
import numbers
31
import logging
32

33
__all__ = [
34 35 36 37
    'Conv2D', 'Conv3D', 'Pool2D', 'Linear', 'BatchNorm', 'Dropout', 'Embedding',
    'GRUUnit', 'LayerNorm', 'NCE', 'PRelu', 'BilinearTensorProduct',
    'Conv2DTranspose', 'Conv3DTranspose', 'GroupNorm', 'SpectralNorm',
    'TreeConv'
38
]
M
minqiyang 已提交
39 40


X
Xin Pan 已提交
41
class Conv2D(layers.Layer):
42
    """
43 44
    This interface is used to construct a callable object of the ``Conv2D`` class.
    For more details, refer to code examples.
45 46 47
    The convolution2D layer calculates the output based on the input, filter
    and strides, paddings, dilations, groups parameters. Input and
    Output are in NCHW format, where N is batch size, C is the number of
48 49 50
    the feature map, H is the height of the feature map, and W is the width of the feature map.
    Filter's shape is [MCHW] , where M is the number of output feature map,
    C is the number of input feature map, H is the height of the filter,
51
    and W is the width of the filter. If the groups is greater than 1,
52
    C will equal the number of input feature map divided by the groups.
53
    Please refer to UFLDL's `convolution
54
    <http://ufldl.stanford.edu/tutorial/supervised/FeatureExtractionUsingConvolution/>`_
T
tianshuo78520a 已提交
55
    for more details.
56 57 58 59 60 61 62 63
    If bias attribution and activation type are provided, bias is added to the
    output of the convolution, and the corresponding activation function is
    applied to the final result.

    For each input :math:`X`, the equation is:

    .. math::

64
        Out = \\sigma (W \\ast X + b)
65 66 67

    Where:

68 69
    * :math:`X`: Input value, a ``Tensor`` with NCHW format.
    * :math:`W`: Filter value, a ``Tensor`` with shape [MCHW] .
70
    * :math:`\\ast`: Convolution operation.
71
    * :math:`b`: Bias value, a 2-D ``Tensor`` with shape [M, 1].
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
    * :math:`\\sigma`: Activation function.
    * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.

    Example:

        - Input:

          Input shape: :math:`(N, C_{in}, H_{in}, W_{in})`

          Filter shape: :math:`(C_{out}, C_{in}, H_f, W_f)`

        - Output:

          Output shape: :math:`(N, C_{out}, H_{out}, W_{out})`

        Where

        .. math::

            H_{out}&= \\frac{(H_{in} + 2 * paddings[0] - (dilations[0] * (H_f - 1) + 1))}{strides[0]} + 1 \\\\
            W_{out}&= \\frac{(W_{in} + 2 * paddings[1] - (dilations[1] * (W_f - 1) + 1))}{strides[1]} + 1

94
    Parameters:
95
        num_channels(int): The number of channels in the input image.
96
        num_filters(int): The number of filter. It is as same as the output
97 98
            feature map.
        filter_size (int or tuple): The filter size. If filter_size is a tuple,
99 100
            it must contain two integers, (filter_size_H, filter_size_W).
            Otherwise, the filter will be a square.
101
        stride (int or tuple, optional): The stride size. If stride is a tuple, it must
102
            contain two integers, (stride_H, stride_W). Otherwise, the
103 104
            stride_H = stride_W = stride. Default: 1.
        padding (int or tuple, optional): The padding size. If padding is a tuple, it must
105
            contain two integers, (padding_H, padding_W). Otherwise, the
106 107
            padding_H = padding_W = padding. Default: 0.
        dilation (int or tuple, optional): The dilation size. If dilation is a tuple, it must
108
            contain two integers, (dilation_H, dilation_W). Otherwise, the
109 110
            dilation_H = dilation_W = dilation. Default: 1.
        groups (int, optional): The groups number of the Conv2d Layer. According to grouped
111 112 113
            convolution in Alex Krizhevsky's Deep CNN paper: when group=2,
            the first half of the filters is only connected to the first half
            of the input channels, while the second half of the filters is only
114 115
            connected to the second half of the input channels. Default: 1.
        param_attr (ParamAttr, optional): The parameter attribute for learnable weights(Parameter)
116 117 118 119
            of conv2d. If it is set to None or one attribute of ParamAttr, conv2d
            will create ParamAttr as param_attr. If the Initializer of the param_attr
            is not set, the parameter is initialized with :math:`Normal(0.0, std)`,
            and the :math:`std` is :math:`(\\frac{2.0 }{filter\_elem\_num})^{0.5}`. Default: None.
120
        bias_attr (ParamAttr or bool, optional): The attribute for the bias of conv2d.
121 122 123 124
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, conv2d
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
            is not set, the bias is initialized zero. Default: None.
125 126 127 128 129
        use_cudnn (bool, optional): Use cudnn kernel or not, it is valid only when the cudnn
            library is installed. Default: True.
        act (str, optional): Activation type, if it is set to None, activation is not appended.
            Default: None.
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
130

131 132 133 134
    Attribute:
        **weight** (Parameter): the learnable weights of filter of this layer.

        **bias** (Parameter or None): the learnable bias of this layer.
135

136 137 138
    Returns:
        None
    
139
    Raises:
140
        ValueError: if ``use_cudnn`` is not a bool value.
141 142 143

    Examples:
        .. code-block:: python
L
lujun 已提交
144

145 146 147 148 149
          from paddle.fluid.dygraph.base import to_variable
          import paddle.fluid as fluid
          from paddle.fluid.dygraph import Conv2D
          import numpy as np

150
          data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32')
151
          with fluid.dygraph.guard():
152
              conv2d = Conv2D(3, 2, 3)
153 154
              data = to_variable(data)
              conv = conv2d(data)
155 156 157

    """

M
minqiyang 已提交
158
    def __init__(self,
159
                 num_channels,
M
minqiyang 已提交
160 161 162 163 164 165 166 167
                 num_filters,
                 filter_size,
                 stride=1,
                 padding=0,
                 dilation=1,
                 groups=None,
                 param_attr=None,
                 bias_attr=None,
168 169 170
                 use_cudnn=True,
                 act=None,
                 dtype='float32'):
M
minqiyang 已提交
171
        assert param_attr is not False, "param_attr should not be False here."
172 173
        super(Conv2D, self).__init__()
        self._num_channels = num_channels
M
minqiyang 已提交
174 175 176 177
        self._groups = groups
        self._stride = utils.convert_to_list(stride, 2, 'stride')
        self._padding = utils.convert_to_list(padding, 2, 'padding')
        self._dilation = utils.convert_to_list(dilation, 2, 'dilation')
178
        self._act = act
M
minqiyang 已提交
179 180 181
        if not isinstance(use_cudnn, bool):
            raise ValueError("use_cudnn should be True or False")
        self._use_cudnn = use_cudnn
182 183 184 185 186
        self._filter_size = filter_size
        self._num_filters = num_filters
        self._param_attr = param_attr
        self._bias_attr = bias_attr
        self._dtype = dtype
187

188 189 190 191 192
        if (self._num_channels == self._groups and
                num_filters % self._num_channels == 0 and not self._use_cudnn):
            self._l_type = 'depthwise_conv2d'
        else:
            self._l_type = 'conv2d'
M
minqiyang 已提交
193

194
        self._num_channels = num_channels
195 196
        if self._groups is None:
            num_filter_channels = self._num_channels
M
minqiyang 已提交
197
        else:
198
            if self._num_channels % self._groups != 0:
M
minqiyang 已提交
199
                raise ValueError("num_channels must be divisible by groups.")
200 201
            num_filter_channels = self._num_channels // self._groups
        filter_size = utils.convert_to_list(self._filter_size, 2, 'filter_size')
202
        filter_shape = [self._num_filters, num_filter_channels] + filter_size
M
minqiyang 已提交
203 204

        def _get_default_param_initializer():
205 206
            filter_elem_num = filter_size[0] * filter_size[
                1] * self._num_channels
M
minqiyang 已提交
207 208 209
            std = (2.0 / filter_elem_num)**0.5
            return Normal(0.0, std, 0)

210
        self.weight = self.create_parameter(
211
            attr=self._param_attr,
M
minqiyang 已提交
212 213 214 215
            shape=filter_shape,
            dtype=self._dtype,
            default_initializer=_get_default_param_initializer())

216
        self.bias = self.create_parameter(
217 218
            attr=self._bias_attr,
            shape=[self._num_filters],
M
minqiyang 已提交
219 220
            dtype=self._dtype,
            is_bias=True)
M
minqiyang 已提交
221 222

    def forward(self, input):
223 224 225 226 227 228 229 230 231 232 233
        if in_dygraph_mode() and self._l_type == 'conv2d':
            attrs = ('strides', self._stride, 'paddings', self._padding,
                     'dilations', self._dilation, 'groups', self._groups
                     if self._groups else 1, 'use_cudnn', self._use_cudnn)
            out = core.ops.conv2d(input, self.weight, *attrs)
            pre_bias = out

            pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias,
                                                            1)
            return dygraph_utils._append_activation_in_dygraph(pre_act,
                                                               self._act)
234 235
        inputs = {
            'Input': [input],
236
            'Filter': [self.weight],
237 238 239 240 241 242 243 244 245
        }
        attrs = {
            'strides': self._stride,
            'paddings': self._padding,
            'dilations': self._dilation,
            'groups': self._groups if self._groups else 1,
            'use_cudnn': self._use_cudnn,
            'use_mkldnn': False,
        }
246 247 248

        check_variable_and_dtype(input, 'input',
                                 ['float16', 'float32', 'float64'], 'Conv2D')
M
minqiyang 已提交
249 250 251
        pre_bias = self._helper.create_variable_for_type_inference(
            dtype=self._dtype)

M
minqiyang 已提交
252 253 254 255
        self._helper.append_op(
            type=self._l_type,
            inputs={
                'Input': input,
256
                'Filter': self.weight,
M
minqiyang 已提交
257
            },
M
minqiyang 已提交
258
            outputs={"Output": pre_bias},
259
            attrs=attrs)
M
minqiyang 已提交
260

261
        if self.bias is not None:
262 263 264 265 266
            pre_act = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
            self._helper.append_op(
                type='elementwise_add',
                inputs={'X': [pre_bias],
267
                        'Y': [self.bias]},
268 269 270 271
                outputs={'Out': [pre_act]},
                attrs={'axis': 1})
        else:
            pre_act = pre_bias
M
minqiyang 已提交
272

L
lujun 已提交
273
        # Currently, we don't support inplace in dygraph mode
274
        return self._helper.append_activation(pre_act, act=self._act)
M
minqiyang 已提交
275 276


L
lujun 已提交
277
class Conv3D(layers.Layer):
278 279 280 281 282
    """
    **Convlution3D Layer**

    The convolution3D layer calculates the output based on the input, filter
    and strides, paddings, dilations, groups parameters. Input(Input) and
D
DuYao 已提交
283 284
    Output(Output) are multidimensional tensors with a shape of 
    :math:`[N, C, D, H, W]` . Where N is batch size, C is the number of
285 286 287 288 289 290 291 292 293 294 295 296 297 298
    channels, D is the depth of the feature, H is the height of the feature,
    and W is the width of the feature. Convlution3D is similar with Convlution2D
    but adds one dimension(depth). If bias attribution and activation type are
    provided, bias is added to the output of the convolution, and the
    corresponding activation function is applied to the final result.

    For each input :math:`X`, the equation is:

    .. math::

        Out = \sigma (W \\ast X + b)

    In the above equation:

D
DuYao 已提交
299
    * :math:`X`: Input value, a tensor with NCDHW or NDHWC format.
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
    * :math:`W`: Filter value, a tensor with MCDHW format.
    * :math:`\\ast`: Convolution operation.
    * :math:`b`: Bias value, a 2-D tensor with shape [M, 1].
    * :math:`\\sigma`: Activation function.
    * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.

    Example:

        - Input:

          Input shape: :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})`

          Filter shape: :math:`(C_{out}, C_{in}, D_f, H_f, W_f)`

        - Output:
          Output shape: :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})`

        Where

        .. math::

            D_{out}&= \\frac{(D_{in} + 2 * paddings[0] - (dilations[0] * (D_f - 1) + 1))}{strides[0]} + 1 \\\\
            H_{out}&= \\frac{(H_{in} + 2 * paddings[1] - (dilations[1] * (H_f - 1) + 1))}{strides[1]} + 1 \\\\
            W_{out}&= \\frac{(W_{in} + 2 * paddings[2] - (dilations[2] * (W_f - 1) + 1))}{strides[2]} + 1

325
    Parameters:
326
        num_channels(int): The number of channels in the input image.
L
lujun 已提交
327
        num_filters(int): The number of filter. It is as same as the output image channel.
D
DuYao 已提交
328
        filter_size (int|tuple, optional): The filter size. If filter_size is a tuple,
329
            it must contain three integers, (filter_size_D, filter_size_H, filter_size_W).
D
DuYao 已提交
330 331 332
            Otherwise, the filter will be a square, filter_size_depth = filter_size_height
            = filter_size_width = filter_size.
        stride (int|tuple, optional): The stride size. If stride is a tuple, it must
333
            contain three integers, (stride_D, stride_H, stride_W). Otherwise, the
D
DuYao 已提交
334 335
            stride_D = stride_H = stride_W = stride. The default value is 1.
        padding (int|tuple, optional): The padding size. If padding is a tuple, it must
336
            contain three integers, (padding_D, padding_H, padding_W). Otherwise, the
D
DuYao 已提交
337 338
            padding_D = padding_H = padding_W = padding. The default value is 0.
        dilation (int|tuple, optional): The dilation size. If dilation is a tuple, it must
339
            contain three integers, (dilation_D, dilation_H, dilation_W). Otherwise, the
D
DuYao 已提交
340 341
            dilation_D = dilation_H = dilation_W = dilation. The default value is 1.
        groups (int, optional): The groups number of the Conv3d Layer. According to grouped
342 343 344
            convolution in Alex Krizhevsky's Deep CNN paper: when group=2,
            the first half of the filters is only connected to the first half
            of the input channels, while the second half of the filters is only
D
DuYao 已提交
345 346
            connected to the second half of the input channels. The default value is 1.
        param_attr (ParamAttr, optional): The parameter attribute for learnable parameters/weights
347 348 349
            of conv3d. If it is set to None or one attribute of ParamAttr, conv3d
            will create ParamAttr as param_attr. If it is set to None, the parameter
            is initialized with :math:`Normal(0.0, std)`, and the :math:`std` is
D
DuYao 已提交
350 351
            :math:`(\\frac{2.0 }{filter\_elem\_num})^{0.5}`. The default value is None.
        bias_attr (ParamAttr|bool, optional): The parameter attribute for the bias of conv3d.
352 353 354
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, conv3d
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
D
DuYao 已提交
355 356 357 358 359
            is not set, the bias is initialized zero. The default value is None.
        use_cudnn (bool, optional): Use cudnn kernel or not, it is valid only when the cudnn
            library is installed. The default value is True.
        act (str, optional): Activation type, if it is set to None, activation is not appended.
            The default value is None.
360
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
361

D
DuYao 已提交
362 363 364 365
    Attribute:
        **weight** (Parameter): the learnable weights of filters of this layer.

        **bias** (Parameter): the learnable bias of this layer.
366

367
    Returns:
D
DuYao 已提交
368
        None.
369 370 371 372 373 374 375 376

    Raises:
        ValueError: If the shapes of input, filter_size, stride, padding and
                    groups mismatch.

    Examples:
        .. code-block:: python

377 378 379 380 381 382
          import paddle.fluid as fluid
          import numpy

          with fluid.dygraph.guard():
              data = numpy.random.random((5, 3, 12, 32, 32)).astype('float32')
              conv3d = fluid.dygraph.nn.Conv3D(
383
                    num_channels=3, num_filters=2, filter_size=3, act="relu")
384 385
              ret = conv3d(fluid.dygraph.base.to_variable(data))

386 387
    """

L
lujun 已提交
388
    def __init__(self,
389
                 num_channels,
L
lujun 已提交
390 391 392 393 394 395 396 397 398
                 num_filters,
                 filter_size,
                 stride=1,
                 padding=0,
                 dilation=1,
                 groups=None,
                 param_attr=None,
                 bias_attr=None,
                 use_cudnn=True,
399 400
                 act=None,
                 dtype='float32'):
L
lujun 已提交
401
        assert param_attr is not False, "param_attr should not be False here."
402 403
        super(Conv3D, self).__init__()
        self._num_channels = num_channels
L
lujun 已提交
404 405 406
        self._groups = groups
        self._stride = utils.convert_to_list(stride, 3, 'stride')
        self._padding = utils.convert_to_list(padding, 3, 'padding')
407
        self._dilation = utils.convert_to_list(dilation, 3, 'dilation')
L
lujun 已提交
408 409
        self._act = act
        self._use_cudnn = use_cudnn
410 411 412 413
        self._filter_size = filter_size
        self._num_filters = num_filters
        self._param_attr = param_attr
        self._bias_attr = bias_attr
414
        self._dtype = dtype
415 416

        if self._groups is None:
417
            num_filter_channels = self._num_channels
L
lujun 已提交
418
        else:
419
            if self._num_channels % self._groups != 0:
L
lujun 已提交
420
                raise ValueError("num_channels must be divisible by groups.")
421
            num_filter_channels = self._num_channels // self._groups
L
lujun 已提交
422

423 424
        filter_size = utils.convert_to_list(self._filter_size, 3, 'filter_size')
        filter_shape = [self._num_filters, num_filter_channels] + filter_size
L
lujun 已提交
425 426 427

        def _get_default_param_initializer():
            filter_elem_num = filter_size[0] * filter_size[1] * filter_size[
428
                2] * self._num_channels
L
lujun 已提交
429 430 431
            std = (2.0 / filter_elem_num)**0.5
            return Normal(0.0, std, 0)

432
        self.weight = self.create_parameter(
433
            attr=self._param_attr,
L
lujun 已提交
434 435 436 437
            shape=filter_shape,
            dtype=self._dtype,
            default_initializer=_get_default_param_initializer())

438
        self.bias = self.create_parameter(
439 440
            attr=self._bias_attr,
            shape=[self._num_filters],
L
lujun 已提交
441 442 443 444 445 446 447 448
            dtype=self._dtype,
            is_bias=True)

    def forward(self, input):
        pre_bias = self._helper.create_variable_for_type_inference(
            dtype=self._dtype)

        self._helper.append_op(
449
            type='conv3d',
L
lujun 已提交
450 451
            inputs={
                'Input': input,
452
                'Filter': self.weight,
L
lujun 已提交
453 454 455 456 457 458 459 460 461 462 463
            },
            outputs={"Output": pre_bias},
            attrs={
                'strides': self._stride,
                'paddings': self._padding,
                'dilations': self._dilation,
                'groups': self._groups if self._groups else 1,
                'use_cudnn': self._use_cudnn,
                'use_mkldnn': False
            })

464
        if self.bias is not None:
465 466 467 468 469
            pre_act = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
            self._helper.append_op(
                type='elementwise_add',
                inputs={'X': [pre_bias],
470
                        'Y': [self.bias]},
471 472 473 474
                outputs={'Out': [pre_act]},
                attrs={'axis': 1})
        else:
            pre_act = pre_bias
L
lujun 已提交
475 476 477 478 479

        return self._helper.append_activation(pre_act, act=self._act)


class Conv3DTranspose(layers.Layer):
L
lujun 已提交
480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
    """
    **Convlution3D transpose layer**

    The convolution3D transpose layer calculates the output based on the input,
    filter, and dilations, strides, paddings. Input(Input) and output(Output)
    are in NCDHW format. Where N is batch size, C is the number of channels,
    D is the depth of the feature, H is the height of the feature, and W
    is the width of the feature. Parameters(dilations, strides, paddings) are
    two elements. These two elements represent height and width, respectively.
    The details of convolution transpose layer, please refer to the following
    explanation and references `therein <http://www.matthewzeiler.com/wp-content/uploads/2017/07/cvpr2010.pdf>`_.
    If bias attribution and activation type are provided, bias is added to
    the output of the convolution, and the corresponding activation function
    is applied to the final result.

    For each input :math:`X`, the equation is:

    .. math::

        Out = \sigma (W \\ast X + b)

    In the above equation:

    * :math:`X`: Input value, a tensor with NCDHW format.
    * :math:`W`: Filter value, a tensor with MCDHW format.
    * :math:`\\ast`: Convolution operation.
    * :math:`b`: Bias value, a 2-D tensor with shape [M, 1].
    * :math:`\\sigma`: Activation function.
    * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.

    Example:

        - Input:

          Input shape: :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})`

          Filter shape: :math:`(C_{in}, C_{out}, D_f, H_f, W_f)`

        - Output:

          Output shape: :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})`

        Where

        .. math::

D
DuYao 已提交
526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544
           D^\prime_{out} &= (D_{in} - 1) * strides[0] - 2 * paddings[0] + dilations[0] * (D_f - 1) + 1 \\\\
           H^\prime_{out} &= (H_{in} - 1) * strides[1] - 2 * paddings[1] + dilations[1] * (H_f - 1) + 1 \\\\
           W^\prime_{out} &= (W_{in} - 1) * strides[2] - 2 * paddings[2] + dilations[2] * (W_f - 1) + 1 \\\\
           D_{out} &\in [ D^\prime_{out}, D^\prime_{out} + strides[0] ] \\\\
           H_{out} &\in [ H^\prime_{out}, H^\prime_{out} + strides[1] ] \\\\

    **Note**:

          The conv3d_transpose can be seen as the backward of the conv3d. For conv3d, 
          when stride > 1, conv3d maps multiple input shape to the same output shape, 
          so for conv3d_transpose, when stride > 1, input shape maps multiple output shape.
          If output_size is None, :math:`H_{out} = H^\prime_{out}, :math:`H_{out} = \
          H^\prime_{out}, W_{out} = W^\prime_{out}`; else, the :math:`D_{out}` of the output 
          size must between :math:`D^\prime_{out}` and :math:`D^\prime_{out} + strides[0]`, 
          the :math:`H_{out}` of the output size must between :math:`H^\prime_{out}` 
          and :math:`H^\prime_{out} + strides[1]`, and the :math:`W_{out}` of the output size must 
          between :math:`W^\prime_{out}` and :math:`W^\prime_{out} + strides[2]`, 
          conv3d_transpose can compute the kernel size automatically.

L
lujun 已提交
545

546
    Parameters:
547
        num_channels(int): The number of channels in the input image.
L
lujun 已提交
548 549
        num_filters(int): The number of the filter. It is as same as the output
            image channel.
550
        filter_size(int|tuple): The filter size. If filter_size is a tuple,
L
lujun 已提交
551
            it must contain three integers, (filter_size_D, filter_size_H, filter_size_W).
552
            Otherwise, the filter will be a square.
D
DuYao 已提交
553 554 555 556 557 558 559 560 561 562 563 564 565 566 567
        padding(int|tuple, optional): The padding size. The padding argument effectively
             adds `dilation * (kernel - 1)` amount of zero-padding on both sides of input. If `padding` is a string,
             either 'VALID' or 'SAME' supported, which is the padding algorithm. If `padding`
             is a tuple or list, it could be in three forms: `[pad_depth, pad_height, pad_width]` or
            `[pad_depth_front, pad_depth_back, pad_height_top, pad_height_bottom, pad_width_left, pad_width_right]`,
            and when `data_format` is `'NCDHW'`, `padding` can be in the form
            `[[0,0], [0,0], [pad_depth_front, pad_depth_back], [pad_height_top, pad_height_bottom], [pad_width_left, pad_width_right]]`.
            when `data_format` is `'NDHWC'`, `padding` can be in the form
            `[[0,0], [pad_depth_front, pad_depth_back], [pad_height_top, pad_height_bottom], [pad_width_left, pad_width_right], [0,0]]`.
            The default value is 0.
        stride(int|tuple, optional): The stride size. It means the stride in transposed convolution. 
            If stride is a tuple, it must contain three integers, (stride_depth, stride_height, 
            stride_width). Otherwise, stride_depth = stride_height = stride_width = stride. 
            The default value is 1.
        dilation(int|tuple, optional): The dilation size. If dilation is a tuple, it must
L
lujun 已提交
568
            contain three integers, (dilation_D, dilation_H, dilation_W). Otherwise, the
D
DuYao 已提交
569 570
            dilation_D = dilation_H = dilation_W = dilation. The default value is 1.
        groups(int, optional): The groups number of the Conv3d transpose layer. Inspired by
L
lujun 已提交
571 572 573 574
            grouped convolution in Alex Krizhevsky's Deep CNN paper, in which
            when group=2, the first half of the filters is only connected to the
            first half of the input channels, while the second half of the
            filters is only connected to the second half of the input channels.
D
DuYao 已提交
575 576
            The default value is 1.
        param_attr (ParamAttr, optional): The parameter attribute for learnable parameters/weights
L
lujun 已提交
577 578
            of conv3d_transpose. If it is set to None or one attribute of ParamAttr, conv3d_transpose
            will create ParamAttr as param_attr. If the Initializer of the param_attr
D
DuYao 已提交
579 580
            is not set, the parameter is initialized with Xavier. The default value is None.
        bias_attr (ParamAttr|bool, optional): The parameter attribute for the bias of conv3d_transpose.
L
lujun 已提交
581 582 583
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, conv3d_transpose
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
D
DuYao 已提交
584 585 586 587 588 589 590
            is not set, the bias is initialized zero. The default value is None.
        use_cudnn(bool, optional): Use cudnn kernel or not, it is valid only when the cudnn
            library is installed. The default value is True.
        act (str, optional): Activation type, if it is set to None, activation is not appended.
            The default value is None.
        name(str, optional): The default value is None. Normally there is no need for user 
            to set this property. For more information, please refer to :ref:`api_guide_Name`.
L
lujun 已提交
591

D
DuYao 已提交
592 593 594 595
    Attribute:
        **weight** (Parameter): the learnable weights of filters of this layer.

        **bias** (Parameter): the learnable bias of this layer.
596

L
lujun 已提交
597
    Returns:
D
DuYao 已提交
598
        None.
L
lujun 已提交
599 600 601 602 603 604 605 606

    Raises:
        ValueError: If the shapes of input, filter_size, stride, padding and
                    groups mismatch.

    Examples:
       .. code-block:: python

607 608 609 610 611 612
         import paddle.fluid as fluid
         import numpy

         with fluid.dygraph.guard():
             data = numpy.random.random((5, 3, 12, 32, 32)).astype('float32')
             conv3dTranspose = fluid.dygraph.nn.Conv3DTranspose(
613
                    num_channels=3,
614 615 616 617 618
                    num_filters=12,
                    filter_size=12,
                    use_cudnn=False)
             ret = conv3dTranspose(fluid.dygraph.base.to_variable(data))

L
lujun 已提交
619 620
    """

L
lujun 已提交
621
    def __init__(self,
622
                 num_channels,
L
lujun 已提交
623
                 num_filters,
624
                 filter_size,
L
lujun 已提交
625 626 627 628 629 630 631 632
                 padding=0,
                 stride=1,
                 dilation=1,
                 groups=None,
                 param_attr=None,
                 bias_attr=None,
                 use_cudnn=True,
                 act=None,
633 634
                 dtype='float32'):
        super(Conv3DTranspose, self).__init__()
L
lujun 已提交
635 636 637 638 639 640 641
        if not isinstance(use_cudnn, bool):
            raise ValueError("use_cudnn should be True or False")
        assert param_attr is not False, "param_attr should not be False in conv3d_transpose."
        self._padding = utils.convert_to_list(padding, 3, 'padding')
        self._stride = utils.convert_to_list(stride, 3, 'stride')
        self._dilation = utils.convert_to_list(dilation, 3, 'dilation')
        self._param_attr = param_attr
642
        self._num_channels = num_channels
L
lujun 已提交
643 644 645 646 647 648
        self._filter_size = filter_size
        self._groups = 1 if groups is None else groups
        self._num_filters = num_filters
        self._use_cudnn = use_cudnn
        self._bias_attr = bias_attr
        self._act = act
649
        self._dtype = dtype
L
lujun 已提交
650

651 652
        self._filter_size = utils.convert_to_list(
            self._filter_size, 3, 'conv3d_transpose.filter_size')
L
lujun 已提交
653

654 655
        filter_shape = [self._num_channels, self._num_filters // self._groups
                        ] + self._filter_size
656
        self.weight = self.create_parameter(
L
lujun 已提交
657
            dtype=self._dtype, shape=filter_shape, attr=self._param_attr)
658 659 660 661 662
        self.bias = self.create_parameter(
            attr=self._bias_attr,
            shape=[self._num_filters],
            dtype=self._dtype,
            is_bias=True)
L
lujun 已提交
663 664 665 666 667 668 669

    def forward(self, input):
        pre_bias = self._helper.create_variable_for_type_inference(
            dtype=self._dtype)
        self._helper.append_op(
            type="conv3d_transpose",
            inputs={'Input': [input],
670
                    'Filter': [self.weight]},
L
lujun 已提交
671 672 673 674 675 676 677 678 679 680 681 682 683 684 685
            outputs={'Output': pre_bias},
            attrs={
                'strides': self._stride,
                'paddings': self._padding,
                'dilations': self._dilation,
                'groups': self._groups if self._groups else 1,
                'use_cudnn': self._use_cudnn
            })

        if self._bias_attr:
            pre_act = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
            self._helper.append_op(
                type='elementwise_add',
                inputs={'X': [pre_bias],
686
                        'Y': [self.bias]},
L
lujun 已提交
687 688 689 690 691 692 693 694 695
                outputs={'Out': [pre_act]},
                attrs={'axis': 1})
        else:
            pre_act = pre_bias

        # Currently, we don't support inplace in imperative mode
        return self._helper.append_activation(pre_act, act=self._act)


X
Xin Pan 已提交
696
class Pool2D(layers.Layer):
697
    """
698 699 700 701 702
    This interface is used to construct a callable object of the ``Pool2D`` class.
    For more details, refer to code examples.
    The pooling2d operation calculates the output based on the input, pool_type and pool_size, pool_stride,
    pool_padding parameters.Input and output are in NCHW format, where N is batch size, C is the number of feature map,
    H is the height of the feature map, and W is the width of the feature map.
L
lujun 已提交
703 704
    Parameters(ksize, strides, paddings) are two elements. These two elements represent height and width, respectively.
    The input(X) size and output(Out) size may be different.
705

706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749
    Example:

        - Input:

          Input shape: :math:`(N, C, H_{in}, W_{in})`

        - Output:

          Output shape: :math:`(N, C, H_{out}, W_{out})`

        If ``ceil_mode`` = False:

        .. math::

            H_{out} = \\frac{(H_{in} - ksize[0] + 2 * paddings[0])}{strides[0]} + 1 \\\\
            W_{out} = \\frac{(W_{in} - ksize[1] + 2 * paddings[1])}{strides[1]} + 1

        If ``ceil_mode`` = True:

        .. math::

            H_{out} = \\frac{(H_{in} - ksize[0] + 2 * paddings[0] + strides[0] - 1)}{strides[0]} + 1 \\\\
            W_{out} = \\frac{(W_{in} - ksize[1] + 2 * paddings[1] + strides[1] - 1)}{strides[1]} + 1

        If ``exclusive`` = False:

        .. math::

            hstart &= i * strides[0] - paddings[0] \\\\
            hend   &= hstart + ksize[0] \\\\
            wstart &= j * strides[1] - paddings[1] \\\\
            wend   &= wstart + ksize[1] \\\\
            Output(i ,j) &= \\frac{sum(Input[hstart:hend, wstart:wend])}{ksize[0] * ksize[1]}

        If ``exclusive`` = True:

        .. math::

            hstart &= max(0, i * strides[0] - paddings[0])\\\\
            hend &= min(H, hstart + ksize[0]) \\\\
            wstart &= max(0, j * strides[1] - paddings[1]) \\\\
            wend & = min(W, wstart + ksize[1]) \\\\
            Output(i ,j) & = \\frac{sum(Input[hstart:hend, wstart:wend])}{(hend - hstart) * (wend - wstart)}

750
    Parameters:
751
        pool_size (int or list or tuple, optional): The pool kernel size. If pool kernel size is a tuple or list,
752
            it must contain two integers, (pool_size_Height, pool_size_Width).
753 754 755 756
            Otherwise, the pool kernel size will be a square of an int. Default: -1.
        pool_type(str, optional) : The pooling type, can be "max" for max-pooling and "avg" for average-pooling. 
            Default: max.
        pool_stride (int or list or tuple, optional): The pool stride size. If pool stride size is a tuple or list,
L
lujun 已提交
757
            it must contain two integers, (pool_stride_Height, pool_stride_Width). Otherwise,
758 759 760
            the pool stride size will be a square of an int. Default: 1.
        pool_padding (int or list or tuple, optional): The padding size for pooling operation. 
            If ``pool_padding`` is a tuple,
761
            it must contain two integers, (pool_padding_on_Height, pool_padding_on_Width).
762 763 764 765 766 767 768
            Otherwise, the padding size for pooling operation will be a square of an int. Default: 0.
        global_pooling (bool, optional): Whether to use the global pooling. If global_pooling = true,
            kernel size and paddings will be ignored. Default: False.
        use_cudnn (bool, optional): Only used in cudnn kernel, need install cudnn. Default: True.
        ceil_mode (bool, optional): Whether to use the ceil function to calculate output height and width.
            False is the default. If it is set to False, the floor function will be used. Default: False.
        exclusive (bool, optional): Whether to exclude padding points in average pooling mode. Default: True.
769 770

    Returns:
771
        None
772 773 774 775 776 777 778 779 780 781

    Raises:
        ValueError: If 'pool_type' is not "max" nor "avg"
        ValueError: If 'global_pooling' is False and 'pool_size' is -1
        ValueError: If 'use_cudnn' is not a bool value.

    Examples:

        .. code-block:: python

L
lujun 已提交
782
          import paddle.fluid as fluid
783 784
          from paddle.fluid.dygraph.base import to_variable
          import numpy as np
L
lujun 已提交
785 786

          with fluid.dygraph.guard():
787
             data = numpy.random.random((3, 32, 32, 5)).astype('float32')
788
             pool2d = fluid.dygraph.Pool2D(pool_size=2,
789 790 791
                            pool_type='max',
                            pool_stride=1,
                            global_pooling=False)
792
             pool2d_res = pool2d(to_variable(data))
793 794 795

    """

M
minqiyang 已提交
796 797 798 799 800 801 802 803
    def __init__(self,
                 pool_size=-1,
                 pool_type="max",
                 pool_stride=1,
                 pool_padding=0,
                 global_pooling=False,
                 use_cudnn=True,
                 ceil_mode=False,
804
                 exclusive=True):
M
minqiyang 已提交
805 806 807 808 809 810 811 812 813 814 815 816 817
        if pool_type not in ["max", "avg"]:
            raise ValueError(
                "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.",
                str(pool_type))

        if global_pooling is False and pool_size == -1:
            raise ValueError(
                "When the global_pooling is False, pool_size must be passed "
                "and be a valid value. Received pool_size: " + str(pool_size))

        if not isinstance(use_cudnn, bool):
            raise ValueError("use_cudnn should be True or False")

818
        super(Pool2D, self).__init__()
M
minqiyang 已提交
819 820 821 822 823 824 825 826 827 828 829 830 831

        self._pool_type = pool_type
        self._pool_size = utils.convert_to_list(pool_size, 2, 'pool_size')
        self._pool_padding = utils.convert_to_list(pool_padding, 2,
                                                   'pool_padding')
        self._pool_stride = utils.convert_to_list(pool_stride, 2, 'pool_stride')
        self._global_pooling = global_pooling
        self._use_cudnn = use_cudnn
        self._ceil_mode = ceil_mode
        self._exclusive = exclusive
        self._l_type = 'pool2d'

    def forward(self, input):
832 833 834 835 836 837 838 839
        if in_dygraph_mode():
            attrs = ('pooling_type', self._pool_type, 'ksize', self._pool_size,
                     'global_pooling', self._global_pooling, 'strides',
                     self._pool_stride, 'paddings', self._pool_padding,
                     'use_cudnn', self._use_cudnn, 'ceil_mode', self._ceil_mode,
                     'use_mkldnn', False, 'exclusive', self._exclusive)
            return core.ops.pool2d(input, *attrs)

840 841 842 843 844 845 846 847 848 849 850 851 852
        attrs = {
            "pooling_type": self._pool_type,
            "ksize": self._pool_size,
            "global_pooling": self._global_pooling,
            "strides": self._pool_stride,
            "paddings": self._pool_padding,
            "use_cudnn": self._use_cudnn,
            "ceil_mode": self._ceil_mode,
            "use_mkldnn": False,
            "exclusive": self._exclusive,
        }
        inputs = {"X": [input]}

M
minqiyang 已提交
853 854
        pool_out = self._helper.create_variable_for_type_inference(self._dtype)

M
minqiyang 已提交
855 856 857
        self._helper.append_op(
            type=self._l_type,
            inputs={"X": input},
M
minqiyang 已提交
858
            outputs={"Out": pool_out},
859
            attrs=attrs)
M
minqiyang 已提交
860
        return pool_out
M
minqiyang 已提交
861 862


S
songyouwei 已提交
863 864 865 866 867 868 869 870 871 872
class Linear(layers.Layer):
    """
    Fully-connected linear transformation layer:

    .. math::

        Out = Act({XW + b})

    where :math:`X` is the input Tensor, :math:`W` and :math:`b` are weight and bias respectively.

873
    Linear layer takes only one ``Tensor`` input.
S
songyouwei 已提交
874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932
    The Linear layer multiplies input tensor with weight matrix and
    produces an output Tensor of shape [N, *, `output_dim`],
    where N is batch size and `*` means any number of additional dimensions.
    If ``bias_attr`` is not None, a bias variable will be created and added to the output.
    Finally, if ``act`` is not None, it will be applied to the output as well.

    Parameters:
        input_dim(int): The number of input units in this layer.
        output_dim(int): The number of output units in this layer.
        param_attr(ParamAttr or list of ParamAttr, optional): The parameter attribute for learnable
            weights(Parameter) of this layer. Default: None.
        bias_attr(ParamAttr or list of ParamAttr, optional): The attribute for the bias
            of this layer. If it is set to False, no bias will be added to the output units.
            If it is set to None, the bias is initialized zero. Default: None.
        act(str, optional): Activation to be applied to the output of this layer. Default: None.
        dtype(str, optional): Dtype used for weight, it can be "float32" or "float64". Default: "float32".

    Attributes:
        **weight** (Parameter): the learnable weights of this layer.

        **bias** (Parameter or None): the learnable bias of this layer.

    Returns:
        None

    Examples:
        .. code-block:: python

          from paddle.fluid.dygraph.base import to_variable
          import paddle.fluid as fluid
          from paddle.fluid.dygraph import Linear
          import numpy as np

          data = np.random.uniform(-1, 1, [30, 10, 32]).astype('float32')
          with fluid.dygraph.guard():
              linear = Linear(32, 64)
              data = to_variable(data)
              res = linear(data)  # [30, 10, 64]
    """

    def __init__(self,
                 input_dim,
                 output_dim,
                 param_attr=None,
                 bias_attr=None,
                 act=None,
                 dtype="float32"):
        super(Linear, self).__init__()
        self._act = act
        self._dtype = dtype
        self.weight = self.create_parameter(
            shape=[input_dim, output_dim],
            attr=param_attr,
            dtype=dtype,
            is_bias=False)
        self.bias = self.create_parameter(
            shape=[output_dim], attr=bias_attr, dtype=dtype, is_bias=True)

    def forward(self, input):
933
        if in_dygraph_mode():
934 935
            pre_bias = core.ops.mul(input, self.weight, 'x_num_col_dims',
                                    len(input.shape) - 1, 'y_num_col_dims', 1)
936 937 938 939 940 941

            pre_act = dygraph_utils._append_bias_in_dygraph(
                pre_bias, self.bias, axis=len(input.shape) - 1)

            return dygraph_utils._append_activation_in_dygraph(pre_act,
                                                               self._act)
942 943 944 945

        check_variable_and_dtype(input, 'input',
                                 ['float16', 'float32', 'float64'], "Linear")

946 947 948 949 950
        attrs = {
            "x_num_col_dims": len(input.shape) - 1,
            "y_num_col_dims": 1,
        }
        inputs = {"X": [input], "Y": [self.weight]}
951

S
songyouwei 已提交
952 953
        tmp = self._helper.create_variable_for_type_inference(self._dtype)
        self._helper.append_op(
S
songyouwei 已提交
954
            type="mul", inputs=inputs, outputs={"Out": tmp}, attrs=attrs)
S
songyouwei 已提交
955 956 957 958 959 960 961 962 963 964 965 966 967 968
        if self.bias:
            pre_activation = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
            self._helper.append_op(
                type='elementwise_add',
                inputs={'X': [tmp],
                        'Y': [self.bias]},
                outputs={'Out': [pre_activation]},
                attrs={'axis': len(input.shape) - 1})
        else:
            pre_activation = tmp
        return self._helper.append_activation(pre_activation, act=self._act)


M
minqiyang 已提交
969
class BatchNorm(layers.Layer):
970
    """
971 972 973 974 975
    This interface is used to construct a callable object of the ``BatchNorm`` class.
    For more details, refer to code examples.
    It implements the function of the Batch Normalization Layer and can be used 
    as a normalizer function for conv2d and fully connected operations.
    The data is normalized by the mean and variance of the channel based on the current batch data.
976 977 978 979
    Refer to `Batch Normalization: Accelerating Deep Network Training by Reducing
    Internal Covariate Shift <https://arxiv.org/pdf/1502.03167.pdf>`_
    for more details.

980 981 982
    When use_global_stats = False, the :math:`\\mu_{\\beta}` 
    and :math:`\\sigma_{\\beta}^{2}` are the statistics of one mini-batch.
    Calculated as follows:
983 984 985 986 987 988 989 990

    ..  math::

        \\mu_{\\beta} &\\gets \\frac{1}{m} \\sum_{i=1}^{m} x_i \\qquad &//\\
        \ mini-batch\ mean \\\\
        \\sigma_{\\beta}^{2} &\\gets \\frac{1}{m} \\sum_{i=1}^{m}(x_i - \\
        \\mu_{\\beta})^2 \\qquad &//\ mini-batch\ variance \\\\

991 992
    - :math:`x` : mini-batch data
    - :math:`m` : the size of the mini-batch data
993 994 995

    When use_global_stats = True, the :math:`\\mu_{\\beta}`
    and :math:`\\sigma_{\\beta}^{2}` are not the statistics of one mini-batch.
996 997 998 999 1000 1001
    They are global or running statistics (moving_mean and moving_variance). It usually got from the
    pre-trained model. Calculated as follows:

    .. math::
        moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global mean \\
        moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global variance \\
1002

1003 1004
    The normalization function formula is as follows:
 
1005 1006 1007
    ..  math::

        \\hat{x_i} &\\gets \\frac{x_i - \\mu_\\beta} {\\sqrt{\\
1008 1009 1010 1011 1012 1013
        \\sigma_{\\beta}^{2} + \\epsilon}} \\qquad &//\ normalize \\\\
        y_i &\\gets \\gamma \\hat{x_i} + \\beta \\qquad &//\ scale\ and\ shift

    - :math:`\\epsilon` : add a smaller value to the variance to prevent division by zero
    - :math:`\\gamma` : trainable proportional parameter
    - :math:`\\beta` : trainable deviation parameter
1014

1015
    Parameters:
1016
        num_channels(int): Indicate the number of channels of the input ``Tensor``.
T
tianshuo78520a 已提交
1017
        act(str, optional): Activation to be applied to the output of batch normalization. Default: None.
1018 1019 1020
        is_test (bool, optional): A flag indicating whether it is in test phrase or not.
             This flag only has effect on static graph mode. For dygraph mode, please use ``eval()``.
             Default: False.
1021 1022 1023
        momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
        epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5.
        param_attr(ParamAttr, optional): The parameter attribute for Parameter `scale`
1024 1025 1026
             of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm
             will create ParamAttr as param_attr. If the Initializer of the param_attr
             is not set, the parameter is initialized with Xavier. Default: None.
1027
        bias_attr(ParamAttr, optional): The parameter attribute for the bias of batch_norm.
1028 1029 1030
             If it is set to None or one attribute of ParamAttr, batch_norm
             will create ParamAttr as bias_attr. If the Initializer of the bias_attr
             is not set, the bias is initialized zero. Default: None.
1031 1032 1033 1034 1035 1036
        dtype(str, optional): Indicate the data type of the input ``Tensor``,
             which can be float32 or float64. Default: float32.
        data_layout(str, optional): Specify the input data format, the data format can be "NCHW" or "NHWC". Default: NCHW.
        in_place(bool, optional): Make the input and output of batch norm reuse memory. Default: False.
        moving_mean_name(str, optional): The name of moving_mean which store the global Mean. Default: None.
        moving_variance_name(str, optional): The name of the moving_variance which store the global Variance. Default: None.
1037 1038
        do_model_average_for_mean_and_var(bool, optional): Whether parameter mean and variance should do model
            average when model average is enabled. Default: True.
1039
        use_global_stats(bool, optional): Whether to use global mean and
1040 1041 1042
            variance. In inference or test mode, set use_global_stats to true
            or is_test to true, and the behavior is equivalent.
            In train mode, when setting use_global_stats True, the global mean
1043 1044 1045 1046
            and variance are also used during train period. Default: False.
        trainable_statistics(bool, optional): Whether to calculate mean and var in eval mode. In eval mode, when
            setting trainable_statistics True, mean and variance will be calculated by current batch statistics.
            Default: False.
1047 1048

    Returns:
1049
        None
1050 1051 1052

    Examples:
        .. code-block:: python
L
lujun 已提交
1053 1054

          import paddle.fluid as fluid
1055 1056
          from paddle.fluid.dygraph.base import to_variable
          import numpy as np
L
lujun 已提交
1057

1058
          x = np.random.random(size=(3, 10, 3, 7)).astype('float32')
L
lujun 已提交
1059
          with fluid.dygraph.guard():
1060
              x = to_variable(x)
1061
              batch_norm = fluid.BatchNorm(10)
1062
              hidden1 = batch_norm(x)
1063 1064
    """

M
minqiyang 已提交
1065 1066 1067 1068 1069 1070 1071 1072
    def __init__(self,
                 num_channels,
                 act=None,
                 is_test=False,
                 momentum=0.9,
                 epsilon=1e-05,
                 param_attr=None,
                 bias_attr=None,
1073
                 dtype='float32',
M
minqiyang 已提交
1074 1075 1076 1077
                 data_layout='NCHW',
                 in_place=False,
                 moving_mean_name=None,
                 moving_variance_name=None,
1078
                 do_model_average_for_mean_and_var=True,
1079 1080
                 use_global_stats=False,
                 trainable_statistics=False):
1081
        super(BatchNorm, self).__init__()
1082
        self._param_attr = param_attr
1083
        self._bias_attr = bias_attr
1084
        self._act = act
M
minqiyang 已提交
1085 1086 1087

        assert bias_attr is not False, "bias_attr should not be False in batch_norm."

1088 1089
        if dtype == "float16":
            self._dtype = "float32"
M
minqiyang 已提交
1090 1091 1092 1093 1094 1095
        else:
            self._dtype = dtype

        param_shape = [num_channels]

        # create parameter
1096
        self.weight = self.create_parameter(
1097
            attr=self._param_attr,
M
minqiyang 已提交
1098 1099 1100
            shape=param_shape,
            dtype=self._dtype,
            default_initializer=Constant(1.0))
1101
        self.weight.stop_gradient = use_global_stats and self._param_attr.learning_rate == 0.
M
minqiyang 已提交
1102

1103
        self.bias = self.create_parameter(
1104
            attr=self._bias_attr,
M
minqiyang 已提交
1105 1106 1107
            shape=param_shape,
            dtype=self._dtype,
            is_bias=True)
1108
        self.bias.stop_gradient = use_global_stats and self._param_attr.learning_rate == 0.
M
minqiyang 已提交
1109

1110
        self._mean = self.create_parameter(
M
minqiyang 已提交
1111 1112 1113 1114 1115 1116 1117
            attr=ParamAttr(
                name=moving_mean_name,
                initializer=Constant(0.0),
                trainable=False,
                do_model_average=do_model_average_for_mean_and_var),
            shape=param_shape,
            dtype=self._dtype)
1118
        self._mean.stop_gradient = True
M
minqiyang 已提交
1119

1120
        self._variance = self.create_parameter(
M
minqiyang 已提交
1121 1122 1123 1124 1125 1126 1127
            attr=ParamAttr(
                name=moving_variance_name,
                initializer=Constant(1.0),
                trainable=False,
                do_model_average=do_model_average_for_mean_and_var),
            shape=param_shape,
            dtype=self._dtype)
1128
        self._variance.stop_gradient = True
M
minqiyang 已提交
1129 1130

        self._in_place = in_place
1131
        self._data_layout = data_layout
M
minqiyang 已提交
1132 1133 1134
        self._momentum = momentum
        self._epsilon = epsilon
        self._is_test = is_test
1135
        self._fuse_with_relu = False
M
minqiyang 已提交
1136
        self._use_global_stats = use_global_stats
1137
        self._trainable_statistics = trainable_statistics
M
minqiyang 已提交
1138 1139 1140 1141 1142 1143 1144

    def forward(self, input):
        # create output
        # mean and mean_out share the same memory
        mean_out = self._mean
        # variance and variance out share the same memory
        variance_out = self._variance
1145 1146

        if in_dygraph_mode():
1147
            _is_test = not self.training and not self._trainable_statistics
1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158
            attrs = ("momentum", self._momentum, "epsilon", self._epsilon,
                     "is_test", _is_test, "data_layout", self._data_layout,
                     "use_mkldnn", False, "fuse_with_relu",
                     self._fuse_with_relu, "use_global_stats",
                     self._use_global_stats)
            batch_norm_out, _, _, _, _ = core.ops.batch_norm(
                input, self.weight, self.bias, self._mean, self._variance,
                mean_out, variance_out, *attrs)
            return dygraph_utils._append_activation_in_dygraph(
                batch_norm_out, act=self._act)

1159 1160 1161
        check_variable_and_dtype(input, 'input',
                                 ['float16', 'float32', 'float64'], 'BatchNorm')

1162 1163 1164 1165 1166 1167 1168
        attrs = {
            "momentum": self._momentum,
            "epsilon": self._epsilon,
            "is_test": self._is_test,
            "data_layout": self._data_layout,
            "use_mkldnn": False,
            "fuse_with_relu": self._fuse_with_relu,
1169
            "use_global_stats": self._use_global_stats
1170
        }
M
minqiyang 已提交
1171

1172 1173 1174 1175 1176 1177 1178 1179
        inputs = {
            "X": [input],
            "Scale": [self.weight],
            "Bias": [self.bias],
            "Mean": [self._mean],
            "Variance": [self._variance]
        }

1180 1181 1182 1183 1184 1185
        saved_mean = self._helper.create_variable_for_type_inference(
            dtype=self._dtype, stop_gradient=True)
        saved_variance = self._helper.create_variable_for_type_inference(
            dtype=self._dtype, stop_gradient=True)
        batch_norm_out = input if self._in_place else self._helper.create_variable_for_type_inference(
            self._dtype)
1186 1187 1188 1189 1190 1191 1192 1193 1194

        outputs = {
            "Y": [batch_norm_out],
            "MeanOut": [mean_out],
            "VarianceOut": [variance_out],
            "SavedMean": [saved_mean],
            "SavedVariance": [saved_variance]
        }

M
minqiyang 已提交
1195
        self._helper.append_op(
1196
            type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs)
M
minqiyang 已提交
1197

L
lujun 已提交
1198
        # Currently, we don't support inplace in dygraph mode
1199
        return self._helper.append_activation(batch_norm_out, self._act)
1200 1201


1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310
class Dropout(layers.Layer):
    """
   This interface is used to construct a callable object of the ``Dropout`` class.
   For more details, refer to code examples.

   Drop or keep each element of input independently. Dropout is a regularization
   technique for reducing overfitting by preventing neuron co-adaption during
   training. The dropout operator randomly sets (according to the given dropout
   probability) the outputs of some units to zero, while others are remain
   unchanged.

   Dropout layer can be removed for efficiency concern.

   Parameters:
       p (float, optional): Probability of setting units to zero. Default: 0.5
       seed (int, optional): A Python integer used to create random seeds. If this
                   parameter is set to None, a random seed is used.
                   NOTE: If an integer seed is given, always the same output
                   units will be dropped. DO NOT use a fixed seed in training. Default: None.
       dropout_implementation(string, optional): ['downgrade_in_infer'(default)|'upscale_in_train']

                                       1. downgrade_in_infer(default), downgrade the outcome at inference

                                          - train: out = input * mask
                                          - inference: out = input * (1.0 - p)

                                          (mask is a tensor same shape with input, value is 0 or 1
                                          ratio of 0 is dropout_prob)
                                       2. upscale_in_train, upscale the outcome at training time

                                          - train: out = input * mask / ( 1.0 - p )
                                          - inference: out = input

                                          (mask is a tensor same shape with input, value is 0 or 1
                                          ratio of 0 is p)
       is_test (bool, optional): A flag indicating whether it is in test phrase or not.
                   This flag only has effect on static graph mode. For dygraph mode, please use ``eval()``.
                   Default: False.

   Returns:
       None

   Examples:

       .. code-block:: python

           import paddle.fluid as fluid
           from paddle.fluid.dygraph.base import to_variable
           import numpy as np

           x = np.random.random(size=(3, 10, 3, 7)).astype('float32')
           with fluid.dygraph.guard():
               x = to_variable(x)
               m = fluid.dygraph.Dropout(p=0.5)
               droped_train = m(x)
               # switch to eval mode
               m.eval()
               droped_eval = m(x)
   """

    def __init__(self,
                 p=0.5,
                 seed=None,
                 dropout_implementation="downgrade_in_infer",
                 is_test=False):
        super(Dropout, self).__init__()
        assert isinstance(p, (float, int)), "p argument should be a number"
        assert 0 <= p <= 1, "p argument should between 0 and 1"
        self._dropout_prob = p
        assert seed is None or isinstance(
            seed, int), "seed argument should be None or a integer"
        self._seed = seed
        assert dropout_implementation in (
            'downgrade_in_infer', 'upscale_in_train'
        ), "dropout_implementation argument should be 'downgrade_in_infer' or 'upscale_in_train'"
        self._dropout_implementation = dropout_implementation
        self._is_test = is_test

    def forward(self, input):
        prog = default_main_program()
        if (self._seed is None or self._seed == 0) and prog.random_seed != 0:
            self._seed = prog.random_seed
        attrs = {
            'dropout_prob': self._dropout_prob,
            'is_test': not self.training
            if in_dygraph_mode() else self._is_test,
            'fix_seed': self._seed is not None,
            'seed': self._seed if self._seed is not None else 0,
            'dropout_implementation': self._dropout_implementation,
        }

        if in_dygraph_mode():
            attrs = sum(attrs.items(), ())
            out, mask = core.ops.dropout(input, *attrs)
            return out

        out = self._helper.create_variable_for_type_inference(dtype=input.dtype)
        mask = self._helper.create_variable_for_type_inference(
            dtype=core.VarDesc.VarType.UINT8, stop_gradient=True)

        self._helper.append_op(
            type='dropout',
            inputs={'X': [input]},
            outputs={'Out': [out],
                     'Mask': [mask]},
            attrs=attrs)
        return out


1311 1312 1313 1314
class Embedding(layers.Layer):
    """
    **Embedding Layer**

Z
zhongpu 已提交
1315 1316 1317 1318 1319 1320
    This interface is used to construct a callable object of the ``Embedding`` class.
    For specific usage, refer to code examples. It implements the function of the Embedding Layer.
    This layer is used to lookup embeddings vector of ids provided by :attr:`input` .
    It automatically constructs a 2D embedding matrix based on the
    input :attr:`size` (vocab_size, emb_size) and :attr:`dtype` .

1321 1322
    The shape of output Tensor is generated by appending an emb_size dimension to the
    last dimension of the input Tensor shape.
Z
zhongpu 已提交
1323

1324
    **Note:** The id in :attr:`input` must satisfy :math:`0 =< id < size[0]` ,
Z
zhongpu 已提交
1325 1326 1327 1328 1329 1330 1331
    otherwise the program will throw an exception and exit.

    .. code-block:: text

        Case 1:

        input is a Tensor. padding_idx = -1
1332 1333
            input.data = [[1, 3], [2, 4], [4, 127]
            input.shape = [3, 2]
Z
zhongpu 已提交
1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346
        Given size = [128, 16]
        output is a Tensor:
            out.shape = [3, 2, 16]
            out.data = [[[0.129435295, 0.244512452, ..., 0.436322452],
                        [0.345421456, 0.524563927, ..., 0.144534654]],

                        [[0.345249859, 0.124939536, ..., 0.194353745],
                        [0.945345345, 0.435394634, ..., 0.435345365]],
                        
                        [[0.945345345, 0.435394634, ..., 0.435345365],
                        [0.0,         0.0,         ..., 0.0        ]]]  # padding data
        The input padding_idx is less than 0, it is automatically converted to padding_idx = -1 + 128 = 127
        It will pad all-zero data when ids is 127.
1347

1348
    Parameters:
L
lujun 已提交
1349 1350
        size(tuple|list): The shape of the look up table parameter. It should have two elements which indicate the size
            of the dictionary of embeddings and the size of each embedding vector respectively.
Z
zhongpu 已提交
1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
        is_sparse(bool): The flag indicating whether to use sparse update. This parameter only
            affects the performance of the backwards gradient update. It is recommended to set 
            True because sparse update is faster. But some optimizer does not support sparse update,
            such as :ref:`api_fluid_optimizer_AdadeltaOptimizer` , :ref:`api_fluid_optimizer_AdamaxOptimizer` , 
            :ref:`api_fluid_optimizer_DecayedAdagradOptimizer` , :ref:`api_fluid_optimizer_FtrlOptimizer` ,
            :ref:`api_fluid_optimizer_LambOptimizer` and :ref:`api_fluid_optimizer_LarsMomentumOptimizer` .
            In these case, is_sparse must be False. Default: False.
        is_distributed(bool): Whether to store the embedding matrix in a distributed manner. Only used
            in multi-machine distributed CPU training. Default: False.
        padding_idx(int|long|None): padding_idx needs to be in the interval [-vocab_size, vocab_size). 
            If :math:`padding\_idx < 0`, the :math:`padding\_idx` will automatically be converted
            to :math:`vocab\_size + padding\_idx` . It will output all-zero padding data whenever lookup
            encounters :math:`padding\_idx` in id. And the padding data will not be updated while training.
            If set None, it makes no effect to output. Default: None.
        param_attr(ParamAttr): To specify the weight parameter property. Default: None, which means the
            default weight parameter property is used. See usage for details in :ref:`api_fluid_ParamAttr` . In addition,
            user-defined or pre-trained word vectors can be loaded with the :attr:`param_attr` parameter. 
            The local word vector needs to be transformed into numpy format, and the shape of local word
T
tianshuo78520a 已提交
1369
            vector should be consistent with :attr:`size` . Then :ref:`api_fluid_initializer_NumpyArrayInitializer`
Z
zhongpu 已提交
1370 1371 1372
            is used to load custom or pre-trained word vectors. See code example 2 for details.
        dtype(np.dtype|core.VarDesc.VarType|str): It refers to the data type of output Tensor.
            It must be "float32" or "float64". Default: "float32".
1373

Z
zhongpu 已提交
1374 1375
    Attribute:
        **weight** (Parameter): the learnable weights of this layer.
1376

1377
    Returns:
Z
zhongpu 已提交
1378
        Variable: Embedding Tensor or LoDTensor mapped by input. The data type is the same as :attr:`dtype` .
1379 1380

    Examples:
1381

1382 1383
        .. code-block:: python

L
lujun 已提交
1384 1385 1386 1387
          import paddle.fluid as fluid
          import paddle.fluid.dygraph.base as base
          import numpy as np

Z
zhongpu 已提交
1388
          # example 1
1389 1390
          inp_word = np.array([[2, 3, 5], [4, 2, 1]]).astype('int64')
          inp_word.shape  # [2, 3]
1391 1392
          dict_size = 20
          with fluid.dygraph.guard():
L
lujun 已提交
1393
              emb = fluid.dygraph.Embedding(
1394 1395 1396
                  size=[dict_size, 32],
                  param_attr='emb.w',
                  is_sparse=False)
L
lujun 已提交
1397
              static_rlt3 = emb(base.to_variable(inp_word))
1398
              static_rlt3.shape  # [2, 3, 32]
Z
zhongpu 已提交
1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412

          # example 2: load custom or pre-trained word vectors
          weight_data = np.random.random(size=(128, 100))  # word vectors with numpy format
          w_param_attrs = fluid.ParamAttr(
              name="emb_weight",
              learning_rate=0.5,
              initializer=fluid.initializer.NumpyArrayInitializer(weight_data),
              trainable=True)
          with fluid.dygraph.guard():
              emb = fluid.dygraph.Embedding(
                  size=[128, 100],
                  param_attr= w_param_attrs,
                  is_sparse=False)
              static_rlt3 = emb(base.to_variable(inp_word))          
1413 1414
    """

1415 1416 1417 1418 1419 1420 1421
    def __init__(self,
                 size,
                 is_sparse=False,
                 is_distributed=False,
                 padding_idx=None,
                 param_attr=None,
                 dtype='float32'):
1422
        super(Embedding, self).__init__()
1423 1424 1425 1426
        self._size = size
        self._is_sparse = is_sparse
        self._is_distributed = is_distributed
        self._padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else (
J
JiabinYang 已提交
1427
            size[0] + padding_idx)
1428 1429 1430

        self._param_attr = param_attr
        self._dtype = dtype
J
JiabinYang 已提交
1431
        self._remote_prefetch = self._is_sparse and (not self._is_distributed)
1432 1433 1434
        if self._remote_prefetch:
            assert self._is_sparse is True and self._is_distributed is False

1435
        self.weight = self.create_parameter(
1436 1437 1438 1439 1440 1441
            attr=self._param_attr,
            shape=self._size,
            dtype=self._dtype,
            is_bias=False)

    def forward(self, input):
1442 1443 1444 1445 1446 1447
        if in_dygraph_mode():
            return core.ops.lookup_table_v2(
                self.weight, input, 'is_sparse', self._is_sparse,
                'is_distributed', self._is_distributed, 'remote_prefetch',
                self._remote_prefetch, 'padding_idx', self._padding_idx)

1448 1449 1450 1451 1452 1453
        attrs = {
            'is_sparse': self._is_sparse,
            'is_distributed': self._is_distributed,
            'remote_prefetch': self._remote_prefetch,
            'padding_idx': self._padding_idx
        }
1454

1455 1456
        out = self._helper.create_variable_for_type_inference(self._dtype)
        self._helper.append_op(
1457
            type='lookup_table_v2',
1458
            inputs={'Ids': input,
1459
                    'W': self.weight},
1460
            outputs={'Out': out},
1461
            attrs=attrs)
1462 1463

        return out
M
minqiyang 已提交
1464 1465


1466
class LayerNorm(layers.Layer):
1467
    """
1468 1469 1470
    This interface is used to construct a callable object of the ``LayerNorm`` class.
    For more details, refer to code examples.
    It implements the function of the Layer Normalization Layer and can be applied to mini-batch input data.
1471
    Refer to `Layer Normalization <https://arxiv.org/pdf/1607.06450v1.pdf>`_
1472

1473
    The formula is as follows:
1474

1475
    ..  math::
1476

1477
        \\mu & = \\frac{1}{H}\\sum_{i=1}^{H} x_i
1478

1479
        \\sigma & = \\sqrt{\\frac{1}{H}\sum_{i=1}^{H}{(x_i - \\mu)^2} + \\epsilon}
1480

1481
        y & = f(\\frac{g}{\\sigma}(x - \\mu) + b)
1482

1483 1484 1485 1486 1487
    - :math:`x`: the vector representation of the summed inputs to the neurons in that layer.
    - :math:`H`: the number of hidden units in a layers
    - :math:`\\epsilon`: the small value added to the variance to prevent division by zero.
    - :math:`g`: the trainable scale parameter.
    - :math:`b`: the trainable bias parameter.
1488

1489
    Parameters:
1490 1491 1492 1493
        normalized_shape(int or list or tuple): Input shape from an expected input of
            size :math:`[*, normalized_shape[0], normalized_shape[1], ..., normalized_shape[-1]]`.
            If it is a single integer, this module will normalize over the last dimension
            which is expected to be of that specific size.
1494
        scale(bool, optional): Whether to learn the adaptive gain :math:`g` after
L
lujun 已提交
1495
            normalization. Default: True.
1496
        shift(bool, optional): Whether to learn the adaptive bias :math:`b` after
L
lujun 已提交
1497
            normalization. Default: True.
1498
        epsilon(float, optional): The small value added to the variance to prevent
L
lujun 已提交
1499
            division by zero. Default: 1e-05.
1500
        param_attr(ParamAttr, optional): The parameter attribute for the learnable
1501 1502 1503
            gain :math:`g`. If :attr:`scale` is False, :attr:`param_attr` is
            omitted. If :attr:`scale` is True and :attr:`param_attr` is None,
            a default :code:`ParamAttr` would be added as scale. The
L
lujun 已提交
1504
            :attr:`param_attr` is initialized as 1 if it is added. Default: None.
1505
        bias_attr(ParamAttr, optional): The parameter attribute for the learnable
1506 1507 1508
            bias :math:`b`. If :attr:`shift` is False, :attr:`bias_attr` is
            omitted. If :attr:`shift` is True and :attr:`param_attr` is None,
            a default :code:`ParamAttr` would be added as bias. The
L
lujun 已提交
1509
            :attr:`bias_attr` is initialized as 0 if it is added. Default: None.
T
tianshuo78520a 已提交
1510
        act(str, optional): Activation to be applied to the output of layer normalization.
L
lujun 已提交
1511
                  Default: None.
1512 1513
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".

1514
    Returns:
1515
        None
1516

1517
    Examples:
1518

1519 1520 1521
        .. code-block:: python

          import paddle.fluid as fluid
1522
          from paddle.fluid.dygraph.base import to_variable
1523 1524
          import numpy

1525
          x = numpy.random.random((3, 32, 32)).astype('float32')
1526
          with fluid.dygraph.guard():
1527
              x = to_variable(x)
1528
              layerNorm = fluid.LayerNorm([32, 32])
1529
              ret = layerNorm(x)
1530

1531
    """
1532

1533
    def __init__(self,
1534
                 normalized_shape,
1535 1536 1537 1538 1539
                 scale=True,
                 shift=True,
                 epsilon=1e-05,
                 param_attr=None,
                 bias_attr=None,
1540 1541 1542 1543 1544
                 act=None,
                 dtype='float32'):
        super(LayerNorm, self).__init__()
        if isinstance(normalized_shape, numbers.Integral):
            normalized_shape = [normalized_shape]
H
hong 已提交
1545

1546
        self._normalized_shape = list(normalized_shape)
1547 1548 1549 1550 1551 1552
        self._scale = scale
        self._shift = shift
        self._epsilon = epsilon
        self._param_attr = param_attr
        self._bias_attr = bias_attr
        self._act = act
1553 1554
        self._dtype = dtype
        param_shape = [np.prod(self._normalized_shape)]
1555
        if self._scale:
1556
            self.weight = self.create_parameter(
1557 1558 1559 1560
                attr=self._param_attr,
                shape=param_shape,
                dtype=self._dtype,
                default_initializer=Constant(1.0))
1561 1562
        else:
            if self._param_attr:
T
tianshuo78520a 已提交
1563
                logging.warn("param_attr are only available with scale is True")
1564
            self.weight = None
1565

1566 1567
        if self._shift:
            assert self._bias_attr is not False
1568
            self.bias = self.create_parameter(
1569 1570 1571 1572
                attr=self._bias_attr,
                shape=param_shape,
                dtype=self._dtype,
                is_bias=True)
1573 1574
        else:
            if self._bias_attr:
T
tianshuo78520a 已提交
1575
                logging.warn("bias_attr are only available with shift is True")
1576
            self.bias = None
1577 1578

    def forward(self, input):
1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589
        input_shape = list(input.shape)
        input_ndim = len(input_shape)
        normalized_ndim = len(self._normalized_shape)
        self._begin_norm_axis = input_ndim - normalized_ndim
        if input_ndim < normalized_ndim or input_shape[
                self._begin_norm_axis:] != self._normalized_shape:
            str_normalized_shape = str(self._normalized_shape)
            raise ValueError(
                'Given normalized_shape is ' + str_normalized_shape +
                ', expected input with shape [*, ' + str_normalized_shape[
                    1:] + ', but got input shape ' + str(input_shape))
1590 1591 1592 1593 1594 1595 1596 1597

        if in_dygraph_mode():
            pre_act, _, _ = core.ops.layer_norm(
                input, self.weight, self.bias, 'epsilon', self._epsilon,
                'begin_norm_axis', self._begin_norm_axis)
            return dygraph_utils._append_activation_in_dygraph(
                pre_act, act=self._act)

1598 1599 1600
        check_variable_and_dtype(input, 'input', ['float32', 'float64'],
                                 'LayerNorm')

1601
        inputs = dict()
1602
        inputs['X'] = [input]
1603
        if self._scale:
1604
            inputs['Scale'] = [self.weight]
1605
        if self._shift:
1606 1607 1608 1609 1610 1611
            inputs['Bias'] = [self.bias]
        attrs = {
            "epsilon": self._epsilon,
            "begin_norm_axis": self._begin_norm_axis
        }

1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632
        # create output
        mean_out = self._helper.create_variable_for_type_inference(
            dtype=self._dtype, stop_gradient=True)
        variance_out = self._helper.create_variable_for_type_inference(
            dtype=self._dtype, stop_gradient=True)
        layer_norm_out = self._helper.create_variable_for_type_inference(
            self._dtype)

        self._helper.append_op(
            type="layer_norm",
            inputs=inputs,
            outputs={
                "Y": layer_norm_out,
                "Mean": mean_out,
                "Variance": variance_out,
            },
            attrs={
                "epsilon": self._epsilon,
                "begin_norm_axis": self._begin_norm_axis
            })

1633
        return self._helper.append_activation(layer_norm_out, act=self._act)
1634 1635


M
minqiyang 已提交
1636 1637 1638
class GRUUnit(layers.Layer):
    """
    **GRU unit layer**
D
DuYao 已提交
1639 1640 1641 1642 1643
    
    It creates a callable object from GRUUnit class.
    If origin_mode is True, then the equation of a gru step is from paper
    `Learning Phrase Representations using RNN Encoder-Decoder for Statistical 
    Machine Translation <https://arxiv.org/pdf/1406.1078.pdf>`_
M
minqiyang 已提交
1644 1645 1646 1647 1648 1649 1650 1651 1652 1653

        .. math::
            u_t & = actGate(xu_{t} + W_u h_{t-1} + b_u)

            r_t & = actGate(xr_{t} + W_r h_{t-1} + b_r)

            m_t & = actNode(xm_t + W_c dot(r_t, h_{t-1}) + b_m)

            h_t & = dot(u_t, h_{t-1}) + dot((1-u_t), m_t)

D
DuYao 已提交
1654
    If origin_mode is False, then the equation of a gru step is from paper
M
minqiyang 已提交
1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679
    `Empirical Evaluation of Gated Recurrent Neural Networks on Sequence
    Modeling <https://arxiv.org/pdf/1412.3555.pdf>`_

        .. math::
            u_t & = actGate(xu_{t} + W_u h_{t-1} + b_u)

            r_t & = actGate(xr_{t} + W_r h_{t-1} + b_r)

            m_t & = actNode(xm_t + W_c dot(r_t, h_{t-1}) + b_m)

            h_t & = dot((1-u_t), h_{t-1}) + dot(u_t, m_t)


    The inputs of gru unit includes :math:`z_t`, :math:`h_{t-1}`. In terms
    of the equation above, the :math:`z_t` is split into 3 parts -
    :math:`xu_t`, :math:`xr_t` and :math:`xm_t`. This means that in order to
    implement a full GRU unit operator for an input, a fully
    connected layer has to be applied, such that :math:`z_t = W_{fc}x_t`.

    The terms :math:`u_t` and :math:`r_t` represent the update and reset gates
    of the GRU cell. Unlike LSTM, GRU has one lesser gate. However, there is
    an intermediate candidate hidden output, which is denoted by :math:`m_t`.
    This layer has three outputs :math:`h_t`, :math:`dot(r_t, h_{t-1})`
    and concatenation of :math:`u_t`, :math:`r_t` and :math:`m_t`.

1680
    Parameters:
L
lujun 已提交
1681
        size (int): The input dimension value.
D
DuYao 已提交
1682 1683 1684 1685 1686 1687 1688 1689 1690
        param_attr(ParamAttr, optional): The parameter attribute for the learnable
            hidden-hidden weight matrix. 
            
            **Note**:
    
                1. The shape of the weight matrix is :math:`[T, 3*D]`, where D is the hidden size.
                2. All elements in the weight matrix can be divided into two parts. The first 
                   part are weights of the update gate and reset gate with shape :math:`[D, 2*D]`, 
                   and the second part are weights for candidate hidden state with shape :math:`[D, D]`.
M
minqiyang 已提交
1691 1692 1693 1694


            If it is set to None or one attribute of ParamAttr, gru_unit will
            create ParamAttr as param_attr. If the Initializer of the param_attr
D
DuYao 已提交
1695 1696 1697 1698
            is not set, the parameter is initialized with Xavier. The default 
            value is None.
        bias_attr (ParamAttr|bool, optional): The parameter attribute for the bias
            of GRU.Note that the bias with :math:`[1, 3*D]` concatenates
M
minqiyang 已提交
1699 1700 1701 1702 1703
            the bias in the update gate, reset gate and candidate calculations.
            If it is set to False, no bias will be applied to the update gate,
            reset gate and candidate calculations. If it is set to None or one
            attribute of ParamAttr, gru_unit will create ParamAttr as
            bias_attr. If the Initializer of the bias_attr is not set, the bias
D
DuYao 已提交
1704
            is initialized zero. The default value is None.
L
lujun 已提交
1705
        activation (str): The activation type for cell (actNode).
D
DuYao 已提交
1706
                             The default value is 'tanh'.
L
lujun 已提交
1707
        gate_activation (str): The activation type for gates (actGate).
D
DuYao 已提交
1708 1709 1710
                                  The default value is 'sigmoid'.
        dtype(str): The dtype of the layers. The data type can be set as
            'float32', 'float64'. The default value is 'float32'.
M
minqiyang 已提交
1711

D
DuYao 已提交
1712 1713 1714 1715
    Attribute:
        **weight** (Parameter): the learnable weights of this layer.

        **bias** (Parameter): the learnable bias of this layer.
1716

M
minqiyang 已提交
1717
    Returns:
D
DuYao 已提交
1718 1719 1720 1721
        tuple: The hidden value, reset-hidden value and gate values. The hidden value
        is a 2-D tensor with shape  :math:`[T, D]` . The reset-hidden value is a
        2-D tensor with shape  :math:`[T, D]` . The gate value is a 2-D tensor with 
        shape  :math:`[T, 3*D]`.
L
lujun 已提交
1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734

    Examples:

        .. code-block:: python

          import paddle.fluid as fluid
          import paddle.fluid.dygraph.base as base
          import numpy

          lod = [[2, 4, 3]]
          D = 5
          T = sum(lod[0])

D
DuYao 已提交
1735
          input = numpy.random.rand(T, 3 * D).astype('float32')
L
lujun 已提交
1736 1737 1738
          hidden_input = numpy.random.rand(T, D).astype('float32')
          with fluid.dygraph.guard():
              x = numpy.random.random((3, 32, 32)).astype('float32')
1739
              gru = fluid.dygraph.GRUUnit(size=D * 3)
L
lujun 已提交
1740 1741 1742
              dy_ret = gru(
                base.to_variable(input), base.to_variable(hidden_input))

M
minqiyang 已提交
1743 1744 1745 1746 1747 1748 1749 1750 1751 1752
    """

    def __init__(self,
                 size,
                 param_attr=None,
                 bias_attr=None,
                 activation='tanh',
                 gate_activation='sigmoid',
                 origin_mode=False,
                 dtype='float32'):
1753
        super(GRUUnit, self).__init__()
1754
        self._bias_attr = bias_attr
M
minqiyang 已提交
1755 1756 1757 1758 1759
        activation_dict = dict(
            identity=0,
            sigmoid=1,
            tanh=2,
            relu=3, )
H
Hongyu Liu 已提交
1760 1761
        self.activation = activation_dict[activation]
        self.gate_activation = activation_dict[gate_activation]
M
minqiyang 已提交
1762

M
minqiyang 已提交
1763
        self._dtype = dtype
M
minqiyang 已提交
1764 1765
        size = size // 3
        # create weight
1766
        self.weight = self.create_parameter(
M
minqiyang 已提交
1767
            attr=param_attr, shape=[size, 3 * size], dtype=dtype)
M
minqiyang 已提交
1768 1769

        # create bias
M
minqiyang 已提交
1770
        bias_size = [1, 3 * size]
1771
        self._bias_size = bias_size
1772
        self.bias = self.create_parameter(
M
minqiyang 已提交
1773
            attr=bias_attr, shape=bias_size, dtype=dtype, is_bias=True)
M
minqiyang 已提交
1774

M
minqiyang 已提交
1775
    def forward(self, input, hidden):
1776 1777 1778 1779 1780 1781
        if in_dygraph_mode():
            gate, reset_hidden_pre, updated_hidden = core.ops.gru_unit(
                input, hidden, self.weight, self.bias, 'activation',
                self.activation, 'gate_activation', self.gate_activation)
            return updated_hidden, reset_hidden_pre, gate

1782 1783 1784 1785 1786
        inputs = {
            'Input': [input],
            'HiddenPrev': [hidden],
            'Weight': [self.weight]
        }
1787
        if self.bias is not None:
1788 1789 1790 1791 1792
            inputs['Bias'] = [self.bias]
        attrs = {
            'activation': self.activation,
            'gate_activation': self.gate_activation,
        }
M
minqiyang 已提交
1793 1794 1795 1796 1797
        gate = self._helper.create_variable_for_type_inference(self._dtype)
        reset_hidden_pre = self._helper.create_variable_for_type_inference(
            self._dtype)
        updated_hidden = self._helper.create_variable_for_type_inference(
            self._dtype)
M
minqiyang 已提交
1798 1799 1800 1801 1802 1803 1804 1805 1806
        self._helper.append_op(
            type='gru_unit',
            inputs=inputs,
            outputs={
                'Gate': gate,
                'ResetHiddenPrev': reset_hidden_pre,
                'Hidden': updated_hidden,
            },
            attrs={
H
Hongyu Liu 已提交
1807 1808
                'activation': self.activation,
                'gate_activation': self.gate_activation,
M
minqiyang 已提交
1809 1810 1811
            })

        return updated_hidden, reset_hidden_pre, gate
1812 1813 1814 1815


class NCE(layers.Layer):
    """
1816 1817 1818 1819 1820
    This interface is used to construct a callable object of the ``NCE`` class.
    For more details, refer to code examples.
    It implements the function of the ``NCE`` loss function.
    By default this function uses a uniform distribution for sampling, and it
    compute and return the noise-contrastive estimation training loss. See
1821
    `Noise-contrastive estimation: A new estimation principle for unnormalized statistical models <http://www.jmlr.org/proceedings/papers/v9/gutmann10a/gutmann10a.pdf>`_ .
1822

1823
    Parameters:
1824 1825
        num_total_classes (int): Total number of classes in all samples.
        dim (int): Dimension of input (possibly embedding dim).
1826
        param_attr (ParamAttr, optional): The parameter attribute for learnable weights(Parameter)
1827 1828 1829
             of nce. If it is set to None or one attribute of ParamAttr, nce
             will create ParamAttr as param_attr. If the Initializer of the param_attr
             is not set, the parameter is initialized with Xavier. Default: None.
1830
        bias_attr (ParamAttr or bool, optional): The attribute for the bias of nce.
1831 1832 1833 1834
             If it is set to False, no bias will be added to the output units.
             If it is set to None or one attribute of ParamAttr, nce
             will create ParamAttr as bias_attr. If the Initializer of the bias_attr
             is not set, the bias is initialized zero. Default: None.
1835
        num_neg_samples (int, optional): The number of negative classes. The default value is 10.
T
tianshuo78520a 已提交
1836
        sampler (str, optional): The sampler used to sample class from negative classes.
1837 1838
                       It can be 'uniform', 'log_uniform' or 'custom_dist'.
                       default: 'uniform'.
1839
        custom_dist (float[], optional): A float[] with size=num_total_classes.
1840
                       It is used when sampler is set to 'custom_dist'.
1841
                       custom_dist[i] is the probability of i-th class to be sampled.
L
lujun 已提交
1842
                       Default: None.
1843 1844
        seed (int, optional): The seed used in sampler. Default: 0.
        is_sparse(bool, optional): The flag indicating whether to use sparse update. If is_sparse is True, the weight@GRAD and bias@GRAD will be changed to SelectedRows. Default: False.
1845
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
1846

1847 1848
    Attribute:
        **weight** (Parameter): the learnable weights of this layer.
1849

1850 1851
        **bias** (Parameter or None): the learnable bias of this layer.
    
1852
    Returns:
1853
        None
1854 1855 1856 1857

    Examples:
        .. code-block:: python

1858 1859 1860
            import numpy as np
            import paddle.fluid as fluid

1861
            window_size = 5
1862 1863
            dict_size = 20
            label_word = int(window_size // 2) + 1
1864
            inp_word = np.array([[1], [2], [3], [4], [5]]).astype('int64')
1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885
            nid_freq_arr = np.random.dirichlet(np.ones(20) * 1000).astype('float32')

            with fluid.dygraph.guard():
                words = []
                for i in range(window_size):
                    words.append(fluid.dygraph.base.to_variable(inp_word[i]))

                emb = fluid.Embedding(
                    size=[dict_size, 32],
                    param_attr='emb.w',
                    is_sparse=False)

                embs3 = []
                for i in range(window_size):
                    if i == label_word:
                        continue

                    emb_rlt = emb(words[i])
                    embs3.append(emb_rlt)

                embs3 = fluid.layers.concat(input=embs3, axis=1)
1886
                nce = fluid.NCE(
1887
                             num_total_classes=dict_size,
1888
                             dim=embs3.shape[1],
1889 1890 1891 1892 1893 1894 1895
                             num_neg_samples=2,
                             sampler="custom_dist",
                             custom_dist=nid_freq_arr.tolist(),
                             seed=1,
                             param_attr='nce.w',
                             bias_attr='nce.b')

1896 1897
                wl = fluid.layers.unsqueeze(words[label_word], axes=[0])
                nce_loss3 = nce(embs3, wl)
1898 1899 1900 1901 1902

    """

    def __init__(self,
                 num_total_classes,
1903
                 dim,
1904
                 sample_weight=None,
1905 1906 1907 1908 1909 1910
                 param_attr=None,
                 bias_attr=None,
                 num_neg_samples=None,
                 sampler="uniform",
                 custom_dist=None,
                 seed=0,
1911 1912 1913
                 is_sparse=False,
                 dtype='float32'):
        super(NCE, self).__init__()
1914 1915 1916
        self._param_attr = param_attr
        self._bias_attr = bias_attr
        self._num_total_classes = num_total_classes
1917
        self._dtype = dtype
1918
        self._inputs = dict()
1919
        self._inputs['SampleWeight'] = sample_weight if sample_weight is not None else []
1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006
        if sampler == "uniform":
            sampler = 0
        elif sampler == "log_uniform":
            sampler = 1
        elif sampler == "custom_dist":
            assert custom_dist is not None
            # assert isinstance(custom_dist, Variable)

            custom_dist_len = len(custom_dist)
            alias_probs_ = [0] * custom_dist_len
            alias_ = [0] * custom_dist_len
            bigs = []
            littles = []
            for i in range(custom_dist_len):
                normal_prob = custom_dist[i] * custom_dist_len
                if normal_prob - 1.0 > 0:
                    bigs.append((i, normal_prob))
                elif 1.0 - normal_prob > 0:
                    littles.append((i, normal_prob))
                else:
                    alias_probs_[i] = normal_prob
                    alias_[i] = -1

            while len(bigs) and len(littles):
                big = bigs.pop(0)
                little = littles.pop(0)

                big_idx = big[0]
                big_prob = big[1]

                alias_probs_[little[0]] = little[1]
                alias_[little[0]] = big_idx
                big_left = big[1] + little[1] - 1
                if big_left - 1.0 > 0:
                    bigs.append((big_idx, big_left))
                elif 1.0 - big_left > 0:
                    littles.append((big_idx, big_left))
                else:
                    alias_probs_[big_idx] = big_left
                    alias_[big_idx] = -1

            if len(bigs):
                big = bigs.pop(0)
                alias_probs_[big[0]] = 1.0
                alias_[big[0]] = -1
            if len(littles):
                little = littles.pop(0)
                alias_probs_[little[0]] = 1.0
                alias_[little[0]] = -1

            def _init_by_numpy_array(numpy_array):
                ret = self.create_parameter(
                    attr=ParamAttr(),
                    shape=numpy_array.shape,
                    dtype=numpy_array.dtype,
                    default_initializer=NumpyArrayInitializer(numpy_array))
                ret.stop_gradient = True
                return ret

            self._inputs['CustomDistProbs'] = _init_by_numpy_array(
                np.array(custom_dist).astype('float32'))
            self._inputs['CustomDistAlias'] = _init_by_numpy_array(
                np.array(alias_).astype('int32'))
            self._inputs['CustomDistAliasProbs'] = _init_by_numpy_array(
                np.array(alias_probs_).astype('float32'))
            sampler = 2
        else:
            raise Exception("Unsupported sampler type.")

        if num_neg_samples is None:
            num_neg_samples = 10
        else:
            num_neg_samples = int(num_neg_samples)
        self._num_neg_samples = num_neg_samples
        remote_prefetch = is_sparse
        print(
            "With sparse mode, if your models has only small parameter prefetch may cause speed down"
        )
        self._attrs = {
            'num_total_classes': int(num_total_classes),
            'num_neg_samples': num_neg_samples,
            'seed': seed,
            'sampler': sampler,
            'is_sparse': is_sparse,
            'remote_prefetch': remote_prefetch
        }

2007
        self.weight = self.create_parameter(
2008 2009 2010
            attr=self._param_attr,
            shape=[self._num_total_classes, dim],
            is_bias=False,
2011
            dtype=self._dtype)
2012
        if self._bias_attr:
2013
            self.bias = self.create_parameter(
2014 2015 2016
                attr=self._bias_attr,
                shape=[self._num_total_classes, 1],
                is_bias=True,
2017
                dtype=self._dtype)
2018 2019
            self._inputs['Bias'] = self.bias
        self._inputs['Weight'] = self.weight
2020

2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049
    def forward(self, input, label, sample_weight=None):
        assert isinstance(input, Variable)
        assert isinstance(label, Variable)

        self._inputs['Input'] = input
        self._inputs['Label'] = label
        self._inputs['SampleWeight'] = sample_weight if sample_weight is not None else []

        cost = self._helper.create_variable_for_type_inference(
            dtype=input.dtype)
        sample_logits = self._helper.create_variable_for_type_inference(
            dtype=input.dtype)
        sample_labels = self._helper.create_variable_for_type_inference(
            dtype=label.dtype)

        self._helper.append_op(
            type='nce',
            inputs=self._inputs,
            outputs={
                'Cost': cost,
                'SampleLogits': sample_logits,
                'SampleLabels': sample_labels
            },
            attrs=self._attrs)
        return cost / (self._num_neg_samples + 1)


class PRelu(layers.Layer):
    """
2050 2051 2052 2053
    This interface is used to construct a callable object of the ``PRelu`` class.
    For more details, refer to code examples.
    It implements three activation methods of the ``PRelu`` activation function.

2054 2055 2056 2057 2058
    Equation:

    .. math::
        y = \max(0, x) + \\alpha * \min(0, x)

2059
    Parameters:
L
lujun 已提交
2060
        mode (str): The mode for weight sharing. It supports all, channel
2061 2062 2063
          and element. all: all elements share same weight
          channel:elements in a channel share same weight
          element:each element has a weight
S
songyouwei 已提交
2064 2065 2066
        channel (int, optional): The number of channels.
          This argument is required when mode is "channel".
          Default: None.
2067
        input_shape (list or tuple, optional): The shape of input.
S
songyouwei 已提交
2068 2069
          This argument is required when mode is "element".
          Default: None.
2070 2071
        param_attr(ParamAttr, optional): The parameter attribute for the learnable
          weight (alpha). Default: None.
2072
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
2073

2074 2075 2076
    Attribute:
        **weight** (Parameter): the learnable weights of this layer.
    
2077
    Returns:
2078
        None
2079 2080 2081 2082 2083

    Examples:

        .. code-block:: python

L
lujun 已提交
2084
          import paddle.fluid as fluid
2085
          from paddle.fluid.dygraph.base import to_variable
L
lujun 已提交
2086 2087 2088 2089
          import numpy as np

          inp_np = np.ones([5, 200, 100, 100]).astype('float32')
          with fluid.dygraph.guard():
2090
              inp_np = to_variable(inp_np)
S
songyouwei 已提交
2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101
              prelu0 = fluid.PRelu(
                 mode='all',
                 param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant(1.0)))
              dy_rlt0 = prelu0(inp_np)
              prelu1 = fluid.PRelu(
                 mode='channel',
                 channel=200,
                 param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant(1.0)))
              dy_rlt1 = prelu1(inp_np)
              prelu2 = fluid.PRelu(
                 mode='element',
2102
                 input_shape=inp_np.shape,
L
lujun 已提交
2103
                 param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant(1.0)))
S
songyouwei 已提交
2104
              dy_rlt2 = prelu2(inp_np)
L
lujun 已提交
2105

2106 2107
    """

S
songyouwei 已提交
2108 2109 2110 2111 2112
    def __init__(self,
                 mode,
                 channel=None,
                 input_shape=None,
                 param_attr=None,
2113
                 dtype='float32'):
2114 2115
        # need specify name_scope since snake-cased 'PRelu' is 'p_relu'
        super(PRelu, self).__init__(name_scope='prelu')
2116 2117
        self._mode = mode
        self._param_attr = param_attr
2118
        self._dtype = dtype
S
songyouwei 已提交
2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132
        if mode == 'all':
            self._alpha_shape = [1]
        elif mode == 'channel':
            assert isinstance(
                channel,
                int), "channel argument is required when mode is 'channel'."
            self._alpha_shape = [1, channel, 1, 1]
        elif mode == 'element':
            assert isinstance(input_shape, (
                list, tuple
            )), "input_shape argument is required when mode is 'element'."
            self._alpha_shape = [1] + list(input_shape)[1:]
        else:
            raise ValueError('mode should be one of all, channel, element.')
2133
        self.weight = self.create_parameter(
2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144
            attr=self._param_attr,
            shape=self._alpha_shape,
            dtype='float32',
            is_bias=False,
            default_initializer=Constant(1.0))

    def forward(self, input):
        out = self._helper.create_variable_for_type_inference(self._dtype)
        self._helper.append_op(
            type="prelu",
            inputs={"X": input,
2145
                    'Alpha': self.weight},
2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165
            attrs={"mode": self._mode},
            outputs={"Out": out})
        return out


class BilinearTensorProduct(layers.Layer):
    """
    **Add Bilinear Tensor Product Layer**

    This layer performs bilinear tensor product on two inputs.
    For example:

    .. math::
      out_{i} = x * W_{i} * {y^\mathrm{T}}, i=0,1,...,size-1

    In this formula:
     - :math:`x`: the first input contains M elements, shape is [batch_size, M].
     - :math:`y`: the second input contains N elements, shape is [batch_size, N].
     - :math:`W_{i}`: the i-th learned weight, shape is [M, N]
     - :math:`out_{i}`: the i-th element of out, shape is [batch_size, size].
D
DuYao 已提交
2166
     - :math:`y^\mathrm{T}`: the transpose of :math:`y`.
2167

2168
    Parameters:
2169 2170 2171 2172 2173
       input1_dim (int): The dimension of each first input.
       input2_dim (int): The dimension of each second input.
       output_dim (int): The dimension of output of this layer.
       name (str, optional): The default value is None. Normally there is no need for user
           to set this property. For more information, please refer to :ref:`api_guide_Name`. Default: None.
D
DuYao 已提交
2174 2175 2176 2177
       act (str, optional): Activation to be applied to the output of this layer. The default value is None.
       param_attr (ParamAttr, optional): The parameter attribute for the learnable w, parameters/weights of 
           this layer. The default value is None.
       bias_attr (ParamAttr, optional): The parameter attribute for the bias
2178
           of this layer. If it is set to False, no bias will be added to the output units.
D
DuYao 已提交
2179
           If it is set to None, the bias is initialized zero. The default value is None.
2180
       dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
2181

D
DuYao 已提交
2182 2183 2184 2185
    Attribute:
        **weight** (Parameter): the learnable weights of this layer.

        **bias** (Parameter): the learnable bias of this layer.
2186

2187 2188 2189 2190 2191 2192
    Returns:
       Variable: A 2-D Tensor of shape [batch_size, size].

    Examples:
       .. code-block:: python

2193 2194 2195 2196 2197 2198 2199
         import paddle.fluid as fluid
         import numpy

         with fluid.dygraph.guard():
             layer1 = numpy.random.random((5, 5)).astype('float32')
             layer2 = numpy.random.random((5, 4)).astype('float32')
             bilinearTensorProduct = fluid.dygraph.nn.BilinearTensorProduct(
2200
                    input1_dim=5, input2_dim=4, output_dim=1000)
2201 2202
             ret = bilinearTensorProduct(fluid.dygraph.base.to_variable(layer1),
                                fluid.dygraph.base.to_variable(layer2))
2203 2204 2205
    """

    def __init__(self,
2206 2207 2208
                 input1_dim,
                 input2_dim,
                 output_dim,
2209 2210 2211
                 name=None,
                 act=None,
                 param_attr=None,
2212 2213 2214
                 bias_attr=None,
                 dtype='float32'):
        super(BilinearTensorProduct, self).__init__()
2215 2216 2217 2218
        self._param_attr = param_attr
        self._bias_attr = bias_attr
        self._act = act
        self._name = name
2219 2220 2221
        self._input1_dim = input1_dim
        self._input2_dim = input2_dim
        self._output_dim = output_dim
2222
        self._inputs = dict()
2223
        self._dtype = dtype
2224

2225
        param_shape = [self._output_dim, self._input1_dim, self._input2_dim]
2226
        self.weight = self.create_parameter(
2227 2228 2229 2230
            attr=self._param_attr,
            shape=param_shape,
            dtype=self._dtype,
            is_bias=False)
2231
        bias_size = [1, self._output_dim]
2232
        self.bias = self.create_parameter(
2233 2234 2235 2236
            attr=self._bias_attr,
            shape=bias_size,
            dtype=self._dtype,
            is_bias=True)
2237 2238

    def forward(self, x, y):
2239
        self._inputs = {"X": x, "Y": y, "Weight": self.weight}
2240
        if self.bias is not None:
2241
            self._inputs["Bias"] = self.bias
2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255
        if self._name is not None:
            out = self._helper.create_variable(
                name=".".join([self.full_name(), self._name]),
                dtype=self._dtype,
                persistable=False)
        else:
            out = self._helper.create_variable(
                dtype=self._dtype, persistable=False)
        self._helper.append_op(
            type="bilinear_tensor_product",
            inputs=self._inputs,
            outputs={"Out": out})

        # add activation
2256
        return self._helper.append_activation(out, act=self._act)
2257 2258 2259 2260


class Conv2DTranspose(layers.Layer):
    """
2261 2262
    This interface is used to construct a callable object of the ``Conv2DTranspose`` class.
    For more details, refer to code examples.
2263
    The convolution2D transpose layer calculates the output based on the input,
2264 2265 2266
    filter, and dilations, strides, paddings. Input and output
    are in NCHW format. Where N is batch size, C is the number of feature map,
    H is the height of the feature map, and W is the width of the feature map.
2267 2268
    Filter's shape is [MCHW] , where M is the number of input feature map,
    C is the number of output feature map, H is the height of the filter,
2269 2270
    and W is the width of the filter. If the groups is greater than 1,
    C will equal the number of input feature map divided by the groups.
2271 2272 2273
    If bias attribution and activation type are provided, bias is added to
    the output of the convolution, and the corresponding activation function
    is applied to the final result.
2274 2275
    The details of convolution transpose layer, please refer to the following explanation and references
    `conv2dtranspose <http://www.matthewzeiler.com/wp-content/uploads/2017/07/cvpr2010.pdf>`_ .
2276 2277 2278 2279 2280 2281 2282 2283 2284

    For each input :math:`X`, the equation is:

    .. math::

        Out = \sigma (W \\ast X + b)

    Where:

2285 2286
    * :math:`X`: Input value, a ``Tensor`` with NCHW format.
    * :math:`W`: Filter value, a ``Tensor`` with shape [MCHW] .
2287
    * :math:`\\ast`: Convolution operation.
2288
    * :math:`b`: Bias value, a 2-D ``Tensor`` with shape [M, 1].
2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312
    * :math:`\\sigma`: Activation function.
    * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.

    Example:

        - Input:

          Input shape: :math:`(N, C_{in}, H_{in}, W_{in})`

          Filter shape: :math:`(C_{in}, C_{out}, H_f, W_f)`

        - Output:

          Output shape: :math:`(N, C_{out}, H_{out}, W_{out})`

        Where

        .. math::

           H^\prime_{out} &= (H_{in} - 1) * strides[0] - 2 * paddings[0] + dilations[0] * (H_f - 1) + 1 \\\\
           W^\prime_{out} &= (W_{in} - 1) * strides[1] - 2 * paddings[1] + dilations[1] * (W_f - 1) + 1 \\\\
           H_{out} &\in [ H^\prime_{out}, H^\prime_{out} + strides[0] ) \\\\
           W_{out} &\in [ W^\prime_{out}, W^\prime_{out} + strides[1] )

2313
    Parameters:
2314
        num_channels(int): The number of channels in the input image.
2315
        num_filters(int): The number of the filter. It is as same as the output
2316
            feature map.
2317 2318 2319
        filter_size(int or tuple): The filter size. If filter_size is a tuple,
            it must contain two integers, (filter_size_H, filter_size_W).
            Otherwise, the filter will be a square.
2320
        output_size(int or tuple, optional): The output image size. If output size is a
2321 2322 2323
            tuple, it must contain two integers, (image_H, image_W). None if use
            filter_size, padding, and stride to calculate output_size.
            if output_size and filter_size are specified at the same time, They
L
lujun 已提交
2324
            should follow the formula above. Default: None.
2325
        padding(int or tuple, optional): The padding size. If padding is a tuple, it must
2326
            contain two integers, (padding_H, padding_W). Otherwise, the
2327 2328
            padding_H = padding_W = padding. Default: 0.
        stride(int or tuple, optional): The stride size. If stride is a tuple, it must
2329
            contain two integers, (stride_H, stride_W). Otherwise, the
2330 2331
            stride_H = stride_W = stride. Default: 1.
        dilation(int or tuple, optional): The dilation size. If dilation is a tuple, it must
2332
            contain two integers, (dilation_H, dilation_W). Otherwise, the
2333 2334
            dilation_H = dilation_W = dilation. Default: 1.
        groups(int, optional): The groups number of the Conv2d transpose layer. Inspired by
2335 2336 2337 2338
            grouped convolution in Alex Krizhevsky's Deep CNN paper, in which
            when group=2, the first half of the filters is only connected to the
            first half of the input channels, while the second half of the
            filters is only connected to the second half of the input channels.
2339 2340
            Default: 1.
        param_attr (ParamAttr, optional): The parameter attribute for learnable weights(Parameter)
2341 2342 2343
            of conv2d_transpose. If it is set to None or one attribute of ParamAttr, conv2d_transpose
            will create ParamAttr as param_attr. If the Initializer of the param_attr
            is not set, the parameter is initialized with Xavier. Default: None.
2344
        bias_attr (ParamAttr or bool, optional): The attribute for the bias of conv2d_transpose.
2345 2346 2347 2348
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, conv2d_transpose
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
            is not set, the bias is initialized zero. Default: None.
2349
        use_cudnn(bool, optional): Use cudnn kernel or not, it is valid only when the cudnn
2350
            library is installed. Default: True.
2351
        act (str, optional): Activation type, if it is set to None, activation is not appended.
2352
            Default: None.
2353
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
2354

2355 2356
    Attribute:
        **weight** (Parameter): the learnable weights of filters of this layer.
2357

2358
        **bias** (Parameter or None): the learnable bias of this layer.
2359

2360 2361
    Returns:
        None
2362 2363 2364 2365

    Examples:
       .. code-block:: python

2366
          import paddle.fluid as fluid
2367
          import numpy as np
2368 2369

          with fluid.dygraph.guard():
2370
              data = np.random.random((3, 32, 32, 5)).astype('float32')
2371
              conv2DTranspose = fluid.dygraph.nn.Conv2DTranspose(
2372
                    num_channels=32, num_filters=2, filter_size=3)
2373 2374
              ret = conv2DTranspose(fluid.dygraph.base.to_variable(data))

2375 2376 2377
    """

    def __init__(self,
2378
                 num_channels,
2379
                 num_filters,
2380
                 filter_size,
2381 2382 2383 2384 2385 2386 2387 2388
                 output_size=None,
                 padding=0,
                 stride=1,
                 dilation=1,
                 groups=None,
                 param_attr=None,
                 bias_attr=None,
                 use_cudnn=True,
2389 2390 2391
                 act=None,
                 dtype='float32'):
        super(Conv2DTranspose, self).__init__()
2392 2393 2394
        assert param_attr is not False, "param_attr should not be False in conv2d_transpose."
        self._param_attr = param_attr
        self._bias_attr = bias_attr
2395
        self._act = act
2396
        self._groups = groups
2397
        self._num_channels = num_channels
2398 2399 2400 2401 2402 2403 2404
        self._num_filters = num_filters
        self._use_cudnn = use_cudnn
        self._padding = padding
        self._stride = stride
        self._dilation = dilation
        self._filter_size = filter_size
        self._output_size = output_size
2405
        self._dtype = dtype
2406

2407 2408 2409
        if (self._num_channels == self._groups and
                self._num_filters == self._num_channels and
                not self._use_cudnn):
2410
            self._op_type = 'depthwise_conv2d_transpose'
2411 2412
        else:
            self._op_type = 'conv2d_transpose'
2413 2414 2415 2416 2417

        self._padding = utils.convert_to_list(self._padding, 2, 'padding')
        self._stride = utils.convert_to_list(self._stride, 2, 'stride')
        self._dilation = utils.convert_to_list(self._dilation, 2, 'dilation')

2418 2419
        self._filter_size = utils.convert_to_list(
            self._filter_size, 2, 'conv2d_transpose.filter_size')
2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430

        if self._output_size is None:
            self._output_size = []
        elif isinstance(self._output_size, list) or isinstance(
                self._output_size, int):
            self._output_size = utils.convert_to_list(self._output_size, 2,
                                                      'output_size')
        else:
            raise ValueError("output_size should be list or int")
        self._padding = utils.convert_to_list(self._padding, 2, 'padding')
        self._groups = 1 if self._groups is None else self._groups
2431
        filter_shape = [self._num_channels, self._num_filters // self._groups
2432 2433
                        ] + self._filter_size

2434
        self.weight = self.create_parameter(
2435
            dtype=self._dtype, shape=filter_shape, attr=self._param_attr)
2436

2437
        self.bias = self.create_parameter(
2438 2439 2440 2441 2442
            attr=self._bias_attr,
            shape=[self._num_filters],
            dtype=self._dtype,
            is_bias=True)

2443
    def forward(self, input):
2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455
        if in_dygraph_mode():
            op = getattr(core.ops, self._op_type)
            out = op(input, self.weight, 'output_size', self._output_size,
                     'strides', self._stride, 'paddings', self._padding,
                     'dilations', self._dilation, 'groups', self._groups,
                     'use_cudnn', self._use_cudnn)
            pre_bias = out
            pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias,
                                                            1)
            return dygraph_utils._append_activation_in_dygraph(
                pre_act, act=self._act)

2456 2457 2458 2459 2460 2461 2462 2463 2464 2465
        inputs = {'Input': [input], 'Filter': [self.weight]}
        attrs = {
            'output_size': self._output_size,
            'strides': self._stride,
            'paddings': self._padding,
            'dilations': self._dilation,
            'groups': self._groups,
            'use_cudnn': self._use_cudnn
        }

2466 2467 2468 2469
        pre_bias = self._helper.create_variable_for_type_inference(
            dtype=input.dtype)
        self._helper.append_op(
            type=self._op_type,
2470
            inputs=inputs,
2471
            outputs={'Output': pre_bias},
2472
            attrs=attrs)
2473

2474
        if self.bias is not None:
2475 2476 2477 2478 2479
            pre_act = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
            self._helper.append_op(
                type='elementwise_add',
                inputs={'X': [pre_bias],
2480
                        'Y': [self.bias]},
2481 2482 2483 2484 2485 2486
                outputs={'Out': [pre_act]},
                attrs={'axis': 1})
        else:
            pre_act = pre_bias

        out = self._helper.append_activation(pre_act, act=self._act)
2487 2488 2489 2490 2491 2492 2493 2494 2495
        return out


class SequenceConv(layers.Layer):
    """
    This function creates the op for sequence_conv, using the inputs and
    other convolutional configurations for the filters and stride as given
    in the input parameters to the function.

2496
    Parameters:
L
lujun 已提交
2497
        name_scope(str): The name of this class.
2498
        num_filters (int): number of filters.
L
lujun 已提交
2499 2500 2501
        filter_size (int): the filter size (H and W). Default: 3.
        filter_stride (int): stride of the filter. Default: 1.
        padding (bool|None): if True, add paddings. Default: None
2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513
        bias_attr (ParamAttr|bool|None): The parameter attribute for the bias of sequence_conv.
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, sequence_conv
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
            is not set, the bias is initialized zero. Default: None.
        param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights
            of sequence_conv. If it is set to None or one attribute of ParamAttr, sequence_conv
            will create ParamAttr as param_attr. If the Initializer of the param_attr
            is not set, the parameter is initialized with Xavier. Default: None.
        act (str): Activation type, if it is set to None, activation is not appended.
            Default: None.

2514 2515 2516 2517
    Attributes:
        weight (Parameter): the learnable weights of filters of this layer.
        bias (Parameter|None): the learnable bias of this layer.

2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530
    Returns:
        Variable: output of sequence_conv
    """

    def __init__(self,
                 name_scope,
                 num_filters,
                 filter_size=3,
                 filter_stride=1,
                 padding=None,
                 bias_attr=None,
                 param_attr=None,
                 act=None):
L
lujun 已提交
2531
        assert not in_dygraph_mode(
2532
        ), "SequenceConv is not supported by dynamic graph mode yet!"
2533 2534 2535 2536 2537 2538 2539
        super(SequenceConv, self).__init__(name_scope)
        self._num_filters = num_filters
        self._filter_size = filter_size
        self._filter_stride = filter_stride
        self._padding = padding
        self._bias_attr = bias_attr
        self._param_attr = param_attr
2540
        self._act = act
2541

2542
    def _build_once(self, input):
2543 2544
        self._dtype = self._helper.input_dtype(input)
        filter_shape = [self._filter_size * input.shape[1], self._num_filters]
2545
        self.weight = self.create_parameter(
2546
            attr=self._param_attr, shape=filter_shape, dtype=self._dtype)
2547

2548
        self.bias = self.create_parameter(
2549 2550 2551 2552 2553
            attr=self._bias_attr,
            shape=[self._num_filters],
            dtype=self._dtype,
            is_bias=True)

2554 2555 2556 2557 2558 2559
    def forward(self, input):
        pre_bias = self._helper.create_variable_for_type_inference(self._dtype)
        self._helper.append_op(
            type='sequence_conv',
            inputs={
                'X': [input],
2560
                'Filter': [self.weight],
2561 2562 2563 2564 2565 2566 2567
            },
            outputs={"Out": pre_bias},
            attrs={
                'contextStride': self._filter_stride,
                'contextStart': -int(self._filter_size // 2),
                'contextLength': self._filter_size
            })
2568

2569
        if self.bias is not None:
2570 2571 2572 2573 2574
            pre_act = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
            self._helper.append_op(
                type='elementwise_add',
                inputs={'X': [pre_bias],
2575
                        'Y': [self.bias]},
2576 2577 2578 2579 2580 2581
                outputs={'Out': [pre_act]},
                attrs={'axis': 1})
        else:
            pre_act = pre_bias

        return self._helper.append_activation(pre_act, act=self._act)
L
lujun 已提交
2582 2583 2584


class RowConv(layers.Layer):
2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602
    """
    ***Row-convolution operator***

    The row convolution is called lookahead convolution.  This operator was introduced in the following paper for DeepSpeech2:
    http://www.cs.cmu.edu/~dyogatam/papers/wang+etal.iclrworkshop2016.pdf

    The main motivation is that a bidirectional RNN, useful in DeepSpeech like speech models, learns representation for a sequence by performing a
    forward and a backward pass through the entire sequence. However, unlike
    unidirectional RNNs, bidirectional RNNs are challenging to deploy in an online
    and low-latency setting. The lookahead convolution incorporates information
    from future subsequences in a computationally efficient manner to improve
    unidirectional recurrent neural networks. The row convolution operator is
    different from the 1D sequence convolution, and is computed as follows:

    Given an input sequence X of length t and input dimension D, and a filter (W) of size context * D.

    More details about row_conv please refer to the design document https://github.com/PaddlePaddle/Paddle/issues/2228#issuecomment-303903645 .

2603
    Parameters:
L
lujun 已提交
2604
        name_scope(str): The name of this class.
2605 2606 2607
        future_context_size (int): Future context size. Please note, the shape
            of convolution kernel is [future_context_size + 1, D].
        param_attr (ParamAttr): Attributes of parameters, including
L
lujun 已提交
2608 2609
            name, initializer etc. Default: None.
        act (str): Non-linear activation to be applied to output variable. Default: None.
2610

2611 2612 2613
    Attributes:
        weight (Parameter): the learnable weights of this layer.

2614
    Returns:
L
lujun 已提交
2615 2616
        the output(Out) is a LodTensor, which supports variable time-length input sequences.
        The underlying tensor in this LodTensor is a matrix with shape T x N, i.e., the same shape as X.
2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid
          import numpy

          with fluid.dygraph.guard():
              x = numpy.random.random((16)).astype('float32')
              rowConv = fluid.dygraph.nn.RowConv(
                    'RowConv', future_context_size=2)
              ret = rowConv(fluid.dygraph.base.to_variable(x))

    """

L
lujun 已提交
2632 2633 2634 2635 2636
    def __init__(self,
                 name_scope,
                 future_context_size,
                 param_attr=None,
                 act=None):
L
lujun 已提交
2637
        assert not in_dygraph_mode(
2638
        ), "RowConv is not supported by dynamic graph mode yet!"
L
lujun 已提交
2639 2640 2641 2642 2643
        super(RowConv, self).__init__(name_scope)
        self._act = act
        self._param_attr = param_attr
        self._future_context_size = future_context_size

2644
    def _build_once(self, input):
L
lujun 已提交
2645 2646
        self._dtype = self._helper.input_dtype(input)
        filter_shape = [self._future_context_size + 1, input.shape[1]]
2647
        self.weight = self.create_parameter(
2648 2649 2650 2651
            attr=self._param_attr,
            shape=filter_shape,
            dtype=self._dtype,
            is_bias=False)
L
lujun 已提交
2652 2653 2654 2655 2656 2657

    def forward(self, input):
        out = self._helper.create_variable_for_type_inference(self._dtype)
        self._helper.append_op(
            type='row_conv',
            inputs={'X': [input],
2658
                    'Filter': [self.weight]},
L
lujun 已提交
2659 2660 2661 2662 2663 2664
            outputs={'Out': [out]})
        return self._helper.append_activation(out, act=self._act)


class GroupNorm(layers.Layer):
    """
2665 2666 2667 2668 2669 2670
    This interface is used to construct a callable object of the ``GroupNorm`` class.
    For more details, refer to code examples.
    It implements the function of the Group Normalization Layer.
    Refer to `Group Normalization <https://arxiv.org/abs/1803.08494>`_ .

    Parameters:
2671
        channels(int): The number of channels of input.
2672 2673 2674 2675 2676 2677 2678 2679 2680
        groups(int): The number of groups that divided from channels.
        epsilon(float, optional): The small value added to the variance to prevent
                                  division by zero. Default: 1e-05.
        param_attr(ParamAttr, optional): The parameter attribute for the learnable
                                         scale :math:`g`. If it is set to False, no scale will be added to the output units.
                                         If it is set to None, the bias is initialized one. Default: None.
        bias_attr(ParamAttr, optional): The parameter attribute for the learnable
                                        bias :math:`b`. If it is set to False, no bias will be added to the output units.
                                        If it is set to None, the bias is initialized zero. Default: None.
T
tianshuo78520a 已提交
2681
        act(str, optional): Activation to be applied to the output of group normalization. Default: None.
2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694
        data_layout(str, optional): Specify the input data format. Only NCHW is supported. Default: NCHW.

    Returns:
        None

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid
          import numpy as np

          with fluid.dygraph.guard():
              x = np.random.random((8, 32, 32)).astype('float32')
2695
              groupNorm = fluid.dygraph.nn.GroupNorm(channels=32, groups=4)
2696
              ret = groupNorm(fluid.dygraph.base.to_variable(x))
L
lujun 已提交
2697 2698 2699 2700

    """

    def __init__(self,
2701
                 channels,
L
lujun 已提交
2702 2703 2704 2705 2706
                 groups,
                 epsilon=1e-05,
                 param_attr=None,
                 bias_attr=None,
                 act=None,
2707 2708 2709
                 data_layout='NCHW',
                 dtype='float32'):
        super(GroupNorm, self).__init__()
L
lujun 已提交
2710 2711 2712
        self._param_attr = param_attr
        self._bias_attr = bias_attr
        self._epsilon = epsilon
2713
        self._channels = channels
L
lujun 已提交
2714 2715
        self._groups = groups
        self._act = act
2716
        self._dtype = dtype
L
lujun 已提交
2717 2718 2719
        if data_layout != 'NCHW':
            raise ValueError("unsupported data layout:" + data_layout)

2720
        param_shape = [self._channels]
L
lujun 已提交
2721

2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732
        self.weight = self.create_parameter(
            attr=self._param_attr or False,
            shape=param_shape,
            dtype=self._dtype,
            default_initializer=Constant(1.0))

        self.bias = self.create_parameter(
            attr=self._bias_attr or False,
            shape=param_shape,
            dtype=self._dtype,
            is_bias=True)
L
lujun 已提交
2733 2734 2735

    def forward(self, input):
        inputs = {'X': input}
2736
        if self.bias is not None:
2737
            inputs['Bias'] = self.bias
2738
        if self.weight is not None:
2739
            inputs['Scale'] = self.weight
L
lujun 已提交
2740 2741

        # create output
2742
        mean_out = self._helper.create_variable_for_type_inference(
L
lujun 已提交
2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763
            dtype=self._dtype, stop_gradient=True)
        variance_out = self._helper.create_variable_for_type_inference(
            dtype=self._dtype, stop_gradient=True)
        group_norm_out = self._helper.create_variable_for_type_inference(
            dtype=self._dtype)

        self._helper.append_op(
            type="group_norm",
            inputs=inputs,
            outputs={
                "Y": group_norm_out,
                "Mean": mean_out,
                "Variance": variance_out,
            },
            attrs={"epsilon": self._epsilon,
                   "groups": self._groups})

        return self._helper.append_activation(group_norm_out, self._act)


class SpectralNorm(layers.Layer):
2764
    """
2765 2766
    This interface is used to construct a callable object of the ``SpectralNorm`` class.
    For more details, refer to code examples. It implements the function of the Spectral Normalization Layer.
2767 2768 2769 2770 2771 2772 2773 2774 2775 2776
    This layer calculates the spectral normalization value of weight parameters of
    fc, conv1d, conv2d, conv3d layers which should be 2-D, 3-D, 4-D, 5-D
    Parameters. Calculations are showed as follows.

    Step 1:
    Generate vector U in shape of [H], and V in shape of [W].
    While H is the :attr:`dim` th dimension of the input weights,
    and W is the product result of remaining dimensions.

    Step 2:
T
tianshuo78520a 已提交
2777
    :attr:`power_iters` should be a positive integer, do following
2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797
    calculations with U and V for :attr:`power_iters` rounds.

    .. math::

        \mathbf{v} := \\frac{\mathbf{W}^{T} \mathbf{u}}{\|\mathbf{W}^{T} \mathbf{u}\|_2}

        \mathbf{u} := \\frac{\mathbf{W}^{T} \mathbf{v}}{\|\mathbf{W}^{T} \mathbf{v}\|_2}

    Step 3:
    Calculate :math:`\sigma(\mathbf{W})` and normalize weight values.

    .. math::

        \sigma(\mathbf{W}) = \mathbf{u}^{T} \mathbf{W} \mathbf{v}

        \mathbf{W} = \\frac{\mathbf{W}}{\sigma(\mathbf{W})}


    Refer to `Spectral Normalization <https://arxiv.org/abs/1802.05957>`_ .

2798
    Parameters:
2799
        weight_shape(list or tuple): The shape of weight parameter.
2800 2801 2802 2803
        dim(int, optional): The index of dimension which should be permuted to the first before reshaping Input(Weight) to matrix, it should be set as 0 if Input(Weight) is the weight of fc layer, and should be set as 1 if Input(Weight) is the weight of conv layer. Default: 0.
        power_iters(int, optional): The number of power iterations to calculate spectral norm. Default: 1.
        eps(float, optional): The epsilon for numerical stability in calculating norms. Default: 1e-12.
        name (str, optional): The default value is None.  Normally there is no need for user to set this property.  For more information, please refer to :ref:`api_guide_Name` .
2804
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
2805 2806

    Returns:
2807
        None
2808 2809 2810 2811 2812

    Examples:
       .. code-block:: python

            import paddle.fluid as fluid
2813
            import numpy as np
2814 2815

            with fluid.dygraph.guard():
2816 2817 2818
                weight = np.random.random((2, 8, 32, 32)).astype('float32')
                spectralNorm = fluid.dygraph.nn.SpectralNorm(weight.shape, dim=1, power_iters=2)
                ret = spectralNorm(fluid.dygraph.base.to_variable(weight))
2819 2820 2821

    """

2822 2823 2824 2825 2826 2827 2828
    def __init__(self,
                 weight_shape,
                 dim=0,
                 power_iters=1,
                 eps=1e-12,
                 dtype='float32'):
        super(SpectralNorm, self).__init__()
L
lujun 已提交
2829 2830 2831
        self._power_iters = power_iters
        self._eps = eps
        self._dim = dim
2832
        self._dtype = dtype
L
lujun 已提交
2833

2834 2835 2836
        self._weight_shape = list(weight_shape)
        h = self._weight_shape[self._dim]
        w = np.prod(self._weight_shape) // h
L
lujun 已提交
2837

2838
        self.weight_u = self.create_parameter(
L
lujun 已提交
2839 2840 2841 2842
            attr=ParamAttr(),
            shape=[h],
            dtype=self._dtype,
            default_initializer=Normal(0., 1.))
2843
        self.weight_u.stop_gradient = True
L
lujun 已提交
2844

2845
        self.weight_v = self.create_parameter(
L
lujun 已提交
2846 2847 2848 2849
            attr=ParamAttr(),
            shape=[w],
            dtype=self._dtype,
            default_initializer=Normal(0., 1.))
2850
        self.weight_v.stop_gradient = True
L
lujun 已提交
2851 2852

    def forward(self, weight):
2853
        inputs = {'Weight': weight, 'U': self.weight_u, 'V': self.weight_v}
L
lujun 已提交
2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868
        out = self._helper.create_variable_for_type_inference(self._dtype)
        self._helper.append_op(
            type="spectral_norm",
            inputs=inputs,
            outputs={"Out": out, },
            attrs={
                "dim": self._dim,
                "power_iters": self._power_iters,
                "eps": self._eps,
            })

        return out


class TreeConv(layers.Layer):
2869
    """
2870 2871 2872 2873 2874 2875 2876 2877 2878 2879
    This interface is used to construct a callable object of the ``TreeConv`` class.
    For more details, refer to code examples.
    Tree-Based Convolution is a kind of convolution based on tree structure.
    Tree-Based Convolution is a part of Tree-Based Convolution Neural Network(TBCNN),
    which is used to classify tree structures, such as Abstract Syntax Tree.
    Tree-Based Convolution proposed a kind of data structure called continuous binary tree,
    which regards multiway tree as binary tree.
    The paper of Tree-Based Convolution Operator is here: `tree-based convolution <https://arxiv.org/abs/1409.5718v1/>`_ .
    
    Parameters:
2880
        feature_size(int): last dimension of nodes_vector.
2881 2882 2883 2884 2885 2886 2887
        output_size(int): output feature width.
        num_filters(int, optional): number of filters, Default: 1.
        max_depth(int, optional): max depth of filters, Default: 2.
        act(str, optional): activation function, Default: tanh.
        param_attr(ParamAttr, optional): the parameter attribute for the filters, Default: None.
        bias_attr(ParamAttr, optional): the parameter attribute for the bias of this layer, Default: None.
        name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` .
2888
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
2889

2890 2891
    Attribute:
        **weight** (Parameter): the learnable weights of filters of this layer.
2892

2893
        **bias** (Parameter or None): the learnable bias of this layer.
2894

2895 2896
    Returns:
        None
L
lujun 已提交
2897

2898
    Examples:
L
lujun 已提交
2899

2900
        .. code-block:: python
2901

2902 2903
          import paddle.fluid as fluid
          import numpy
2904

2905 2906 2907 2908
          with fluid.dygraph.guard():
              nodes_vector = numpy.random.random((1, 10, 5)).astype('float32')
              edge_set = numpy.random.random((1, 9, 2)).astype('int32')
              treeConv = fluid.dygraph.nn.TreeConv(
2909
                feature_size=5, output_size=6, num_filters=1, max_depth=2)
2910
              ret = treeConv(fluid.dygraph.base.to_variable(nodes_vector), fluid.dygraph.base.to_variable(edge_set))
2911 2912
    """

L
lujun 已提交
2913
    def __init__(self,
2914
                 feature_size,
L
lujun 已提交
2915 2916 2917 2918 2919 2920
                 output_size,
                 num_filters=1,
                 max_depth=2,
                 act='tanh',
                 param_attr=None,
                 bias_attr=None,
2921 2922 2923
                 name=None,
                 dtype='float32'):
        super(TreeConv, self).__init__()
L
lujun 已提交
2924
        self._name = name
2925
        self._feature_size = feature_size
L
lujun 已提交
2926 2927 2928 2929 2930 2931
        self._output_size = output_size
        self._act = act
        self._max_depth = max_depth
        self._num_filters = num_filters
        self._bias_attr = bias_attr
        self._param_attr = param_attr
2932 2933
        self._dtype = dtype
        w_shape = [self._feature_size, 3, self._output_size, self._num_filters]
L
lujun 已提交
2934
        if self._bias_attr:
2935
            self.bias = self.create_parameter(
L
lujun 已提交
2936 2937 2938 2939
                attr=self._bias_attr,
                shape=[self._num_filters],
                dtype=self._dtype,
                is_bias=True)
2940
        self.weight = self.create_parameter(
L
lujun 已提交
2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957
            attr=self._param_attr,
            shape=w_shape,
            dtype=self._dtype,
            is_bias=False)

    def forward(self, nodes_vector, edge_set):
        if self._name:
            out = self.create_variable(
                name=self._name, dtype=self._dtype, persistable=False)
        else:
            out = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
        self._helper.append_op(
            type='tree_conv',
            inputs={
                'NodesVector': nodes_vector,
                'EdgeSet': edge_set,
2958
                'Filter': self.weight
L
lujun 已提交
2959 2960 2961 2962 2963 2964 2965 2966 2967
            },
            outputs={'Out': out, },
            attrs={'max_depth': self._max_depth})
        if self._bias_attr:
            pre_activation = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
            self._helper.append_op(
                type='elementwise_add',
                inputs={'X': [out],
2968
                        'Y': [self.bias]},
L
lujun 已提交
2969 2970 2971 2972 2973
                outputs={'Out': [pre_activation]},
                attrs={'axis': 1})
        else:
            pre_activation = out
        return self._helper.append_activation(pre_activation, act=self._act)