nn.py 116.2 KB
Newer Older
M
minqiyang 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

from six.moves import reduce
from .. import core
from ..layers import utils
20
from ..dygraph import dygraph_utils
M
minqiyang 已提交
21
from . import layers
22
from ..framework import Variable, in_dygraph_mode, OpProtoHolder, Parameter, _dygraph_tracer, _varbase_creator
M
minqiyang 已提交
23
from ..param_attr import ParamAttr
24
from ..initializer import Normal, Constant, NumpyArrayInitializer
H
hong 已提交
25 26
from .. import unique_name
from .layer_object_helper import LayerObjectHelper
L
lujun 已提交
27
import numpy as np
28
import numbers
29
import logging
30

31
__all__ = [
32 33 34
    'Conv2D', 'Conv3D', 'Pool2D', 'Linear', 'BatchNorm', 'Embedding', 'GRUUnit',
    'LayerNorm', 'NCE', 'PRelu', 'BilinearTensorProduct', 'Conv2DTranspose',
    'Conv3DTranspose', 'GroupNorm', 'SpectralNorm', 'TreeConv'
35
]
M
minqiyang 已提交
36 37


X
Xin Pan 已提交
38
class Conv2D(layers.Layer):
39
    """
40 41
    This interface is used to construct a callable object of the ``Conv2D`` class.
    For more details, refer to code examples.
42 43 44
    The convolution2D layer calculates the output based on the input, filter
    and strides, paddings, dilations, groups parameters. Input and
    Output are in NCHW format, where N is batch size, C is the number of
45 46 47
    the feature map, H is the height of the feature map, and W is the width of the feature map.
    Filter's shape is [MCHW] , where M is the number of output feature map,
    C is the number of input feature map, H is the height of the filter,
48
    and W is the width of the filter. If the groups is greater than 1,
49
    C will equal the number of input feature map divided by the groups.
50
    Please refer to UFLDL's `convolution
51
    <http://ufldl.stanford.edu/tutorial/supervised/FeatureExtractionUsingConvolution/>`_
52 53 54 55 56 57 58 59 60
    for more detials.
    If bias attribution and activation type are provided, bias is added to the
    output of the convolution, and the corresponding activation function is
    applied to the final result.

    For each input :math:`X`, the equation is:

    .. math::

61
        Out = \\sigma (W \\ast X + b)
62 63 64

    Where:

65 66
    * :math:`X`: Input value, a ``Tensor`` with NCHW format.
    * :math:`W`: Filter value, a ``Tensor`` with shape [MCHW] .
67
    * :math:`\\ast`: Convolution operation.
68
    * :math:`b`: Bias value, a 2-D ``Tensor`` with shape [M, 1].
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
    * :math:`\\sigma`: Activation function.
    * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.

    Example:

        - Input:

          Input shape: :math:`(N, C_{in}, H_{in}, W_{in})`

          Filter shape: :math:`(C_{out}, C_{in}, H_f, W_f)`

        - Output:

          Output shape: :math:`(N, C_{out}, H_{out}, W_{out})`

        Where

        .. math::

            H_{out}&= \\frac{(H_{in} + 2 * paddings[0] - (dilations[0] * (H_f - 1) + 1))}{strides[0]} + 1 \\\\
            W_{out}&= \\frac{(W_{in} + 2 * paddings[1] - (dilations[1] * (W_f - 1) + 1))}{strides[1]} + 1

91
    Parameters:
92
        num_channels(int): The number of channels in the input image.
93
        num_filters(int): The number of filter. It is as same as the output
94 95
            feature map.
        filter_size (int or tuple): The filter size. If filter_size is a tuple,
96 97
            it must contain two integers, (filter_size_H, filter_size_W).
            Otherwise, the filter will be a square.
98
        stride (int or tuple, optional): The stride size. If stride is a tuple, it must
99
            contain two integers, (stride_H, stride_W). Otherwise, the
100 101
            stride_H = stride_W = stride. Default: 1.
        padding (int or tuple, optional): The padding size. If padding is a tuple, it must
102
            contain two integers, (padding_H, padding_W). Otherwise, the
103 104
            padding_H = padding_W = padding. Default: 0.
        dilation (int or tuple, optional): The dilation size. If dilation is a tuple, it must
105
            contain two integers, (dilation_H, dilation_W). Otherwise, the
106 107
            dilation_H = dilation_W = dilation. Default: 1.
        groups (int, optional): The groups number of the Conv2d Layer. According to grouped
108 109 110
            convolution in Alex Krizhevsky's Deep CNN paper: when group=2,
            the first half of the filters is only connected to the first half
            of the input channels, while the second half of the filters is only
111 112
            connected to the second half of the input channels. Default: 1.
        param_attr (ParamAttr, optional): The parameter attribute for learnable weights(Parameter)
113 114 115 116
            of conv2d. If it is set to None or one attribute of ParamAttr, conv2d
            will create ParamAttr as param_attr. If the Initializer of the param_attr
            is not set, the parameter is initialized with :math:`Normal(0.0, std)`,
            and the :math:`std` is :math:`(\\frac{2.0 }{filter\_elem\_num})^{0.5}`. Default: None.
117
        bias_attr (ParamAttr or bool, optional): The attribute for the bias of conv2d.
118 119 120 121
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, conv2d
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
            is not set, the bias is initialized zero. Default: None.
122 123 124 125 126
        use_cudnn (bool, optional): Use cudnn kernel or not, it is valid only when the cudnn
            library is installed. Default: True.
        act (str, optional): Activation type, if it is set to None, activation is not appended.
            Default: None.
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
127

128 129 130 131
    Attribute:
        **weight** (Parameter): the learnable weights of filter of this layer.

        **bias** (Parameter or None): the learnable bias of this layer.
132

133 134 135
    Returns:
        None
    
136
    Raises:
137
        ValueError: if ``use_cudnn`` is not a bool value.
138 139 140

    Examples:
        .. code-block:: python
L
lujun 已提交
141

142 143 144 145 146
          from paddle.fluid.dygraph.base import to_variable
          import paddle.fluid as fluid
          from paddle.fluid.dygraph import Conv2D
          import numpy as np

147
          data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32')
148
          with fluid.dygraph.guard():
149
              conv2d = Conv2D(3, 2, 3)
150 151
              data = to_variable(data)
              conv = conv2d(data)
152 153 154

    """

M
minqiyang 已提交
155
    def __init__(self,
156
                 num_channels,
M
minqiyang 已提交
157 158 159 160 161 162 163 164
                 num_filters,
                 filter_size,
                 stride=1,
                 padding=0,
                 dilation=1,
                 groups=None,
                 param_attr=None,
                 bias_attr=None,
165 166 167
                 use_cudnn=True,
                 act=None,
                 dtype='float32'):
M
minqiyang 已提交
168
        assert param_attr is not False, "param_attr should not be False here."
169 170
        super(Conv2D, self).__init__()
        self._num_channels = num_channels
M
minqiyang 已提交
171 172 173 174
        self._groups = groups
        self._stride = utils.convert_to_list(stride, 2, 'stride')
        self._padding = utils.convert_to_list(padding, 2, 'padding')
        self._dilation = utils.convert_to_list(dilation, 2, 'dilation')
175
        self._act = act
M
minqiyang 已提交
176 177 178
        if not isinstance(use_cudnn, bool):
            raise ValueError("use_cudnn should be True or False")
        self._use_cudnn = use_cudnn
179 180 181 182 183
        self._filter_size = filter_size
        self._num_filters = num_filters
        self._param_attr = param_attr
        self._bias_attr = bias_attr
        self._dtype = dtype
184

S
songyouwei 已提交
185 186 187 188 189
        if (self._num_channels == self._groups and
                num_filters % self._num_channels == 0 and not self._use_cudnn):
            self._l_type = 'depthwise_conv2d'
        else:
            self._l_type = 'conv2d'
M
minqiyang 已提交
190

191
        self._num_channels = num_channels
192 193
        if self._groups is None:
            num_filter_channels = self._num_channels
M
minqiyang 已提交
194
        else:
195
            if self._num_channels % self._groups != 0:
M
minqiyang 已提交
196
                raise ValueError("num_channels must be divisible by groups.")
197 198
            num_filter_channels = self._num_channels // self._groups
        filter_size = utils.convert_to_list(self._filter_size, 2, 'filter_size')
199
        filter_shape = [self._num_filters, num_filter_channels] + filter_size
M
minqiyang 已提交
200 201

        def _get_default_param_initializer():
202 203
            filter_elem_num = filter_size[0] * filter_size[
                1] * self._num_channels
M
minqiyang 已提交
204 205 206
            std = (2.0 / filter_elem_num)**0.5
            return Normal(0.0, std, 0)

207
        self.weight = self.create_parameter(
208
            attr=self._param_attr,
M
minqiyang 已提交
209 210 211 212
            shape=filter_shape,
            dtype=self._dtype,
            default_initializer=_get_default_param_initializer())

213
        self.bias = self.create_parameter(
214 215
            attr=self._bias_attr,
            shape=[self._num_filters],
M
minqiyang 已提交
216 217
            dtype=self._dtype,
            is_bias=True)
M
minqiyang 已提交
218 219

    def forward(self, input):
220 221
        inputs = {
            'Input': [input],
222
            'Filter': [self.weight],
223 224 225 226 227 228 229 230 231 232
        }
        attrs = {
            'strides': self._stride,
            'paddings': self._padding,
            'dilations': self._dilation,
            'groups': self._groups if self._groups else 1,
            'use_cudnn': self._use_cudnn,
            'use_mkldnn': False,
        }

S
songyouwei 已提交
233
        if in_dygraph_mode() and self._l_type == 'conv2d':
234 235 236
            outs = core.ops.conv2d(inputs, attrs)
            pre_bias = outs['Output'][0]

237 238
            pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias,
                                                            1)
239 240 241 242

            return dygraph_utils._append_activation_in_dygraph(pre_act,
                                                               self._act)

M
minqiyang 已提交
243 244 245
        pre_bias = self._helper.create_variable_for_type_inference(
            dtype=self._dtype)

M
minqiyang 已提交
246 247 248 249
        self._helper.append_op(
            type=self._l_type,
            inputs={
                'Input': input,
250
                'Filter': self.weight,
M
minqiyang 已提交
251
            },
M
minqiyang 已提交
252
            outputs={"Output": pre_bias},
253
            attrs=attrs)
M
minqiyang 已提交
254

255
        if self.bias is not None:
256 257 258 259 260
            pre_act = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
            self._helper.append_op(
                type='elementwise_add',
                inputs={'X': [pre_bias],
261
                        'Y': [self.bias]},
262 263 264 265
                outputs={'Out': [pre_act]},
                attrs={'axis': 1})
        else:
            pre_act = pre_bias
M
minqiyang 已提交
266

L
lujun 已提交
267
        # Currently, we don't support inplace in dygraph mode
268
        return self._helper.append_activation(pre_act, act=self._act)
M
minqiyang 已提交
269 270


L
lujun 已提交
271
class Conv3D(layers.Layer):
272 273 274 275 276
    """
    **Convlution3D Layer**

    The convolution3D layer calculates the output based on the input, filter
    and strides, paddings, dilations, groups parameters. Input(Input) and
D
DuYao 已提交
277 278
    Output(Output) are multidimensional tensors with a shape of 
    :math:`[N, C, D, H, W]` . Where N is batch size, C is the number of
279 280 281 282 283 284 285 286 287 288 289 290 291 292
    channels, D is the depth of the feature, H is the height of the feature,
    and W is the width of the feature. Convlution3D is similar with Convlution2D
    but adds one dimension(depth). If bias attribution and activation type are
    provided, bias is added to the output of the convolution, and the
    corresponding activation function is applied to the final result.

    For each input :math:`X`, the equation is:

    .. math::

        Out = \sigma (W \\ast X + b)

    In the above equation:

D
DuYao 已提交
293
    * :math:`X`: Input value, a tensor with NCDHW or NDHWC format.
294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318
    * :math:`W`: Filter value, a tensor with MCDHW format.
    * :math:`\\ast`: Convolution operation.
    * :math:`b`: Bias value, a 2-D tensor with shape [M, 1].
    * :math:`\\sigma`: Activation function.
    * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.

    Example:

        - Input:

          Input shape: :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})`

          Filter shape: :math:`(C_{out}, C_{in}, D_f, H_f, W_f)`

        - Output:
          Output shape: :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})`

        Where

        .. math::

            D_{out}&= \\frac{(D_{in} + 2 * paddings[0] - (dilations[0] * (D_f - 1) + 1))}{strides[0]} + 1 \\\\
            H_{out}&= \\frac{(H_{in} + 2 * paddings[1] - (dilations[1] * (H_f - 1) + 1))}{strides[1]} + 1 \\\\
            W_{out}&= \\frac{(W_{in} + 2 * paddings[2] - (dilations[2] * (W_f - 1) + 1))}{strides[2]} + 1

319
    Parameters:
320
        num_channels(int): The number of channels in the input image.
L
lujun 已提交
321
        num_filters(int): The number of filter. It is as same as the output image channel.
D
DuYao 已提交
322
        filter_size (int|tuple, optional): The filter size. If filter_size is a tuple,
323
            it must contain three integers, (filter_size_D, filter_size_H, filter_size_W).
D
DuYao 已提交
324 325 326
            Otherwise, the filter will be a square, filter_size_depth = filter_size_height
            = filter_size_width = filter_size.
        stride (int|tuple, optional): The stride size. If stride is a tuple, it must
327
            contain three integers, (stride_D, stride_H, stride_W). Otherwise, the
D
DuYao 已提交
328 329
            stride_D = stride_H = stride_W = stride. The default value is 1.
        padding (int|tuple, optional): The padding size. If padding is a tuple, it must
330
            contain three integers, (padding_D, padding_H, padding_W). Otherwise, the
D
DuYao 已提交
331 332
            padding_D = padding_H = padding_W = padding. The default value is 0.
        dilation (int|tuple, optional): The dilation size. If dilation is a tuple, it must
333
            contain three integers, (dilation_D, dilation_H, dilation_W). Otherwise, the
D
DuYao 已提交
334 335
            dilation_D = dilation_H = dilation_W = dilation. The default value is 1.
        groups (int, optional): The groups number of the Conv3d Layer. According to grouped
336 337 338
            convolution in Alex Krizhevsky's Deep CNN paper: when group=2,
            the first half of the filters is only connected to the first half
            of the input channels, while the second half of the filters is only
D
DuYao 已提交
339 340
            connected to the second half of the input channels. The default value is 1.
        param_attr (ParamAttr, optional): The parameter attribute for learnable parameters/weights
341 342 343
            of conv3d. If it is set to None or one attribute of ParamAttr, conv3d
            will create ParamAttr as param_attr. If it is set to None, the parameter
            is initialized with :math:`Normal(0.0, std)`, and the :math:`std` is
D
DuYao 已提交
344 345
            :math:`(\\frac{2.0 }{filter\_elem\_num})^{0.5}`. The default value is None.
        bias_attr (ParamAttr|bool, optional): The parameter attribute for the bias of conv3d.
346 347 348
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, conv3d
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
D
DuYao 已提交
349 350 351 352 353
            is not set, the bias is initialized zero. The default value is None.
        use_cudnn (bool, optional): Use cudnn kernel or not, it is valid only when the cudnn
            library is installed. The default value is True.
        act (str, optional): Activation type, if it is set to None, activation is not appended.
            The default value is None.
354
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
355

D
DuYao 已提交
356 357 358 359
    Attribute:
        **weight** (Parameter): the learnable weights of filters of this layer.

        **bias** (Parameter): the learnable bias of this layer.
360

361
    Returns:
D
DuYao 已提交
362
        None.
363 364 365 366 367 368 369 370

    Raises:
        ValueError: If the shapes of input, filter_size, stride, padding and
                    groups mismatch.

    Examples:
        .. code-block:: python

371 372 373 374 375 376
          import paddle.fluid as fluid
          import numpy

          with fluid.dygraph.guard():
              data = numpy.random.random((5, 3, 12, 32, 32)).astype('float32')
              conv3d = fluid.dygraph.nn.Conv3D(
377
                    num_channels=3, num_filters=2, filter_size=3, act="relu")
378 379
              ret = conv3d(fluid.dygraph.base.to_variable(data))

380 381
    """

L
lujun 已提交
382
    def __init__(self,
383
                 num_channels,
L
lujun 已提交
384 385 386 387 388 389 390 391 392
                 num_filters,
                 filter_size,
                 stride=1,
                 padding=0,
                 dilation=1,
                 groups=None,
                 param_attr=None,
                 bias_attr=None,
                 use_cudnn=True,
393 394
                 act=None,
                 dtype='float32'):
L
lujun 已提交
395
        assert param_attr is not False, "param_attr should not be False here."
396 397
        super(Conv3D, self).__init__()
        self._num_channels = num_channels
L
lujun 已提交
398 399 400
        self._groups = groups
        self._stride = utils.convert_to_list(stride, 3, 'stride')
        self._padding = utils.convert_to_list(padding, 3, 'padding')
401
        self._dilation = utils.convert_to_list(dilation, 3, 'dilation')
L
lujun 已提交
402 403
        self._act = act
        self._use_cudnn = use_cudnn
404 405 406 407
        self._filter_size = filter_size
        self._num_filters = num_filters
        self._param_attr = param_attr
        self._bias_attr = bias_attr
408
        self._dtype = dtype
409 410

        if self._groups is None:
411
            num_filter_channels = self._num_channels
L
lujun 已提交
412
        else:
413
            if self._num_channels % self._groups != 0:
L
lujun 已提交
414
                raise ValueError("num_channels must be divisible by groups.")
415
            num_filter_channels = self._num_channels // self._groups
L
lujun 已提交
416

417 418
        filter_size = utils.convert_to_list(self._filter_size, 3, 'filter_size')
        filter_shape = [self._num_filters, num_filter_channels] + filter_size
L
lujun 已提交
419 420 421

        def _get_default_param_initializer():
            filter_elem_num = filter_size[0] * filter_size[1] * filter_size[
422
                2] * self._num_channels
L
lujun 已提交
423 424 425
            std = (2.0 / filter_elem_num)**0.5
            return Normal(0.0, std, 0)

426
        self.weight = self.create_parameter(
427
            attr=self._param_attr,
L
lujun 已提交
428 429 430 431
            shape=filter_shape,
            dtype=self._dtype,
            default_initializer=_get_default_param_initializer())

432
        self.bias = self.create_parameter(
433 434
            attr=self._bias_attr,
            shape=[self._num_filters],
L
lujun 已提交
435 436 437 438 439 440 441 442
            dtype=self._dtype,
            is_bias=True)

    def forward(self, input):
        pre_bias = self._helper.create_variable_for_type_inference(
            dtype=self._dtype)

        self._helper.append_op(
443
            type='conv3d',
L
lujun 已提交
444 445
            inputs={
                'Input': input,
446
                'Filter': self.weight,
L
lujun 已提交
447 448 449 450 451 452 453 454 455 456 457
            },
            outputs={"Output": pre_bias},
            attrs={
                'strides': self._stride,
                'paddings': self._padding,
                'dilations': self._dilation,
                'groups': self._groups if self._groups else 1,
                'use_cudnn': self._use_cudnn,
                'use_mkldnn': False
            })

458
        if self.bias is not None:
459 460 461 462 463
            pre_act = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
            self._helper.append_op(
                type='elementwise_add',
                inputs={'X': [pre_bias],
464
                        'Y': [self.bias]},
465 466 467 468
                outputs={'Out': [pre_act]},
                attrs={'axis': 1})
        else:
            pre_act = pre_bias
L
lujun 已提交
469 470 471 472 473

        return self._helper.append_activation(pre_act, act=self._act)


class Conv3DTranspose(layers.Layer):
L
lujun 已提交
474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519
    """
    **Convlution3D transpose layer**

    The convolution3D transpose layer calculates the output based on the input,
    filter, and dilations, strides, paddings. Input(Input) and output(Output)
    are in NCDHW format. Where N is batch size, C is the number of channels,
    D is the depth of the feature, H is the height of the feature, and W
    is the width of the feature. Parameters(dilations, strides, paddings) are
    two elements. These two elements represent height and width, respectively.
    The details of convolution transpose layer, please refer to the following
    explanation and references `therein <http://www.matthewzeiler.com/wp-content/uploads/2017/07/cvpr2010.pdf>`_.
    If bias attribution and activation type are provided, bias is added to
    the output of the convolution, and the corresponding activation function
    is applied to the final result.

    For each input :math:`X`, the equation is:

    .. math::

        Out = \sigma (W \\ast X + b)

    In the above equation:

    * :math:`X`: Input value, a tensor with NCDHW format.
    * :math:`W`: Filter value, a tensor with MCDHW format.
    * :math:`\\ast`: Convolution operation.
    * :math:`b`: Bias value, a 2-D tensor with shape [M, 1].
    * :math:`\\sigma`: Activation function.
    * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.

    Example:

        - Input:

          Input shape: :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})`

          Filter shape: :math:`(C_{in}, C_{out}, D_f, H_f, W_f)`

        - Output:

          Output shape: :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})`

        Where

        .. math::

D
DuYao 已提交
520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538
           D^\prime_{out} &= (D_{in} - 1) * strides[0] - 2 * paddings[0] + dilations[0] * (D_f - 1) + 1 \\\\
           H^\prime_{out} &= (H_{in} - 1) * strides[1] - 2 * paddings[1] + dilations[1] * (H_f - 1) + 1 \\\\
           W^\prime_{out} &= (W_{in} - 1) * strides[2] - 2 * paddings[2] + dilations[2] * (W_f - 1) + 1 \\\\
           D_{out} &\in [ D^\prime_{out}, D^\prime_{out} + strides[0] ] \\\\
           H_{out} &\in [ H^\prime_{out}, H^\prime_{out} + strides[1] ] \\\\

    **Note**:

          The conv3d_transpose can be seen as the backward of the conv3d. For conv3d, 
          when stride > 1, conv3d maps multiple input shape to the same output shape, 
          so for conv3d_transpose, when stride > 1, input shape maps multiple output shape.
          If output_size is None, :math:`H_{out} = H^\prime_{out}, :math:`H_{out} = \
          H^\prime_{out}, W_{out} = W^\prime_{out}`; else, the :math:`D_{out}` of the output 
          size must between :math:`D^\prime_{out}` and :math:`D^\prime_{out} + strides[0]`, 
          the :math:`H_{out}` of the output size must between :math:`H^\prime_{out}` 
          and :math:`H^\prime_{out} + strides[1]`, and the :math:`W_{out}` of the output size must 
          between :math:`W^\prime_{out}` and :math:`W^\prime_{out} + strides[2]`, 
          conv3d_transpose can compute the kernel size automatically.

L
lujun 已提交
539

540
    Parameters:
541
        num_channels(int): The number of channels in the input image.
L
lujun 已提交
542 543
        num_filters(int): The number of the filter. It is as same as the output
            image channel.
544
        filter_size(int|tuple): The filter size. If filter_size is a tuple,
L
lujun 已提交
545
            it must contain three integers, (filter_size_D, filter_size_H, filter_size_W).
546
            Otherwise, the filter will be a square.
D
DuYao 已提交
547 548 549 550 551 552 553 554 555 556 557 558 559 560 561
        padding(int|tuple, optional): The padding size. The padding argument effectively
             adds `dilation * (kernel - 1)` amount of zero-padding on both sides of input. If `padding` is a string,
             either 'VALID' or 'SAME' supported, which is the padding algorithm. If `padding`
             is a tuple or list, it could be in three forms: `[pad_depth, pad_height, pad_width]` or
            `[pad_depth_front, pad_depth_back, pad_height_top, pad_height_bottom, pad_width_left, pad_width_right]`,
            and when `data_format` is `'NCDHW'`, `padding` can be in the form
            `[[0,0], [0,0], [pad_depth_front, pad_depth_back], [pad_height_top, pad_height_bottom], [pad_width_left, pad_width_right]]`.
            when `data_format` is `'NDHWC'`, `padding` can be in the form
            `[[0,0], [pad_depth_front, pad_depth_back], [pad_height_top, pad_height_bottom], [pad_width_left, pad_width_right], [0,0]]`.
            The default value is 0.
        stride(int|tuple, optional): The stride size. It means the stride in transposed convolution. 
            If stride is a tuple, it must contain three integers, (stride_depth, stride_height, 
            stride_width). Otherwise, stride_depth = stride_height = stride_width = stride. 
            The default value is 1.
        dilation(int|tuple, optional): The dilation size. If dilation is a tuple, it must
L
lujun 已提交
562
            contain three integers, (dilation_D, dilation_H, dilation_W). Otherwise, the
D
DuYao 已提交
563 564
            dilation_D = dilation_H = dilation_W = dilation. The default value is 1.
        groups(int, optional): The groups number of the Conv3d transpose layer. Inspired by
L
lujun 已提交
565 566 567 568
            grouped convolution in Alex Krizhevsky's Deep CNN paper, in which
            when group=2, the first half of the filters is only connected to the
            first half of the input channels, while the second half of the
            filters is only connected to the second half of the input channels.
D
DuYao 已提交
569 570
            The default value is 1.
        param_attr (ParamAttr, optional): The parameter attribute for learnable parameters/weights
L
lujun 已提交
571 572
            of conv3d_transpose. If it is set to None or one attribute of ParamAttr, conv3d_transpose
            will create ParamAttr as param_attr. If the Initializer of the param_attr
D
DuYao 已提交
573 574
            is not set, the parameter is initialized with Xavier. The default value is None.
        bias_attr (ParamAttr|bool, optional): The parameter attribute for the bias of conv3d_transpose.
L
lujun 已提交
575 576 577
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, conv3d_transpose
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
D
DuYao 已提交
578 579 580 581 582 583 584
            is not set, the bias is initialized zero. The default value is None.
        use_cudnn(bool, optional): Use cudnn kernel or not, it is valid only when the cudnn
            library is installed. The default value is True.
        act (str, optional): Activation type, if it is set to None, activation is not appended.
            The default value is None.
        name(str, optional): The default value is None. Normally there is no need for user 
            to set this property. For more information, please refer to :ref:`api_guide_Name`.
L
lujun 已提交
585

D
DuYao 已提交
586 587 588 589
    Attribute:
        **weight** (Parameter): the learnable weights of filters of this layer.

        **bias** (Parameter): the learnable bias of this layer.
590

L
lujun 已提交
591
    Returns:
D
DuYao 已提交
592
        None.
L
lujun 已提交
593 594 595 596 597 598 599 600

    Raises:
        ValueError: If the shapes of input, filter_size, stride, padding and
                    groups mismatch.

    Examples:
       .. code-block:: python

601 602 603 604 605 606
         import paddle.fluid as fluid
         import numpy

         with fluid.dygraph.guard():
             data = numpy.random.random((5, 3, 12, 32, 32)).astype('float32')
             conv3dTranspose = fluid.dygraph.nn.Conv3DTranspose(
607
                    num_channels=3,
608 609 610 611 612
                    num_filters=12,
                    filter_size=12,
                    use_cudnn=False)
             ret = conv3dTranspose(fluid.dygraph.base.to_variable(data))

L
lujun 已提交
613 614
    """

L
lujun 已提交
615
    def __init__(self,
616
                 num_channels,
L
lujun 已提交
617
                 num_filters,
618
                 filter_size,
L
lujun 已提交
619 620 621 622 623 624 625 626
                 padding=0,
                 stride=1,
                 dilation=1,
                 groups=None,
                 param_attr=None,
                 bias_attr=None,
                 use_cudnn=True,
                 act=None,
627 628
                 dtype='float32'):
        super(Conv3DTranspose, self).__init__()
L
lujun 已提交
629 630 631 632 633 634 635
        if not isinstance(use_cudnn, bool):
            raise ValueError("use_cudnn should be True or False")
        assert param_attr is not False, "param_attr should not be False in conv3d_transpose."
        self._padding = utils.convert_to_list(padding, 3, 'padding')
        self._stride = utils.convert_to_list(stride, 3, 'stride')
        self._dilation = utils.convert_to_list(dilation, 3, 'dilation')
        self._param_attr = param_attr
636
        self._num_channels = num_channels
L
lujun 已提交
637 638 639 640 641 642
        self._filter_size = filter_size
        self._groups = 1 if groups is None else groups
        self._num_filters = num_filters
        self._use_cudnn = use_cudnn
        self._bias_attr = bias_attr
        self._act = act
643
        self._dtype = dtype
L
lujun 已提交
644

645 646
        self._filter_size = utils.convert_to_list(
            self._filter_size, 3, 'conv3d_transpose.filter_size')
L
lujun 已提交
647

648 649
        filter_shape = [self._num_channels, self._num_filters // self._groups
                        ] + self._filter_size
650
        self.weight = self.create_parameter(
L
lujun 已提交
651 652
            dtype=self._dtype, shape=filter_shape, attr=self._param_attr)
        if self._bias_attr:
653
            self.bias = self.create_parameter(
L
lujun 已提交
654 655 656 657 658 659 660 661 662 663 664
                attr=self._bias_attr,
                shape=[self._num_filters],
                dtype=self._dtype,
                is_bias=True)

    def forward(self, input):
        pre_bias = self._helper.create_variable_for_type_inference(
            dtype=self._dtype)
        self._helper.append_op(
            type="conv3d_transpose",
            inputs={'Input': [input],
665
                    'Filter': [self.weight]},
L
lujun 已提交
666 667 668 669 670 671 672 673 674 675 676 677 678 679 680
            outputs={'Output': pre_bias},
            attrs={
                'strides': self._stride,
                'paddings': self._padding,
                'dilations': self._dilation,
                'groups': self._groups if self._groups else 1,
                'use_cudnn': self._use_cudnn
            })

        if self._bias_attr:
            pre_act = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
            self._helper.append_op(
                type='elementwise_add',
                inputs={'X': [pre_bias],
681
                        'Y': [self.bias]},
L
lujun 已提交
682 683 684 685 686 687 688 689 690
                outputs={'Out': [pre_act]},
                attrs={'axis': 1})
        else:
            pre_act = pre_bias

        # Currently, we don't support inplace in imperative mode
        return self._helper.append_activation(pre_act, act=self._act)


X
Xin Pan 已提交
691
class Pool2D(layers.Layer):
692
    """
693 694 695 696 697
    This interface is used to construct a callable object of the ``Pool2D`` class.
    For more details, refer to code examples.
    The pooling2d operation calculates the output based on the input, pool_type and pool_size, pool_stride,
    pool_padding parameters.Input and output are in NCHW format, where N is batch size, C is the number of feature map,
    H is the height of the feature map, and W is the width of the feature map.
L
lujun 已提交
698 699
    Parameters(ksize, strides, paddings) are two elements. These two elements represent height and width, respectively.
    The input(X) size and output(Out) size may be different.
700

701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744
    Example:

        - Input:

          Input shape: :math:`(N, C, H_{in}, W_{in})`

        - Output:

          Output shape: :math:`(N, C, H_{out}, W_{out})`

        If ``ceil_mode`` = False:

        .. math::

            H_{out} = \\frac{(H_{in} - ksize[0] + 2 * paddings[0])}{strides[0]} + 1 \\\\
            W_{out} = \\frac{(W_{in} - ksize[1] + 2 * paddings[1])}{strides[1]} + 1

        If ``ceil_mode`` = True:

        .. math::

            H_{out} = \\frac{(H_{in} - ksize[0] + 2 * paddings[0] + strides[0] - 1)}{strides[0]} + 1 \\\\
            W_{out} = \\frac{(W_{in} - ksize[1] + 2 * paddings[1] + strides[1] - 1)}{strides[1]} + 1

        If ``exclusive`` = False:

        .. math::

            hstart &= i * strides[0] - paddings[0] \\\\
            hend   &= hstart + ksize[0] \\\\
            wstart &= j * strides[1] - paddings[1] \\\\
            wend   &= wstart + ksize[1] \\\\
            Output(i ,j) &= \\frac{sum(Input[hstart:hend, wstart:wend])}{ksize[0] * ksize[1]}

        If ``exclusive`` = True:

        .. math::

            hstart &= max(0, i * strides[0] - paddings[0])\\\\
            hend &= min(H, hstart + ksize[0]) \\\\
            wstart &= max(0, j * strides[1] - paddings[1]) \\\\
            wend & = min(W, wstart + ksize[1]) \\\\
            Output(i ,j) & = \\frac{sum(Input[hstart:hend, wstart:wend])}{(hend - hstart) * (wend - wstart)}

745
    Parameters:
746
        pool_size (int or list or tuple, optional): The pool kernel size. If pool kernel size is a tuple or list,
747
            it must contain two integers, (pool_size_Height, pool_size_Width).
748 749 750 751
            Otherwise, the pool kernel size will be a square of an int. Default: -1.
        pool_type(str, optional) : The pooling type, can be "max" for max-pooling and "avg" for average-pooling. 
            Default: max.
        pool_stride (int or list or tuple, optional): The pool stride size. If pool stride size is a tuple or list,
L
lujun 已提交
752
            it must contain two integers, (pool_stride_Height, pool_stride_Width). Otherwise,
753 754 755
            the pool stride size will be a square of an int. Default: 1.
        pool_padding (int or list or tuple, optional): The padding size for pooling operation. 
            If ``pool_padding`` is a tuple,
756
            it must contain two integers, (pool_padding_on_Height, pool_padding_on_Width).
757 758 759 760 761 762 763
            Otherwise, the padding size for pooling operation will be a square of an int. Default: 0.
        global_pooling (bool, optional): Whether to use the global pooling. If global_pooling = true,
            kernel size and paddings will be ignored. Default: False.
        use_cudnn (bool, optional): Only used in cudnn kernel, need install cudnn. Default: True.
        ceil_mode (bool, optional): Whether to use the ceil function to calculate output height and width.
            False is the default. If it is set to False, the floor function will be used. Default: False.
        exclusive (bool, optional): Whether to exclude padding points in average pooling mode. Default: True.
764 765

    Returns:
766
        None
767 768 769 770 771 772 773 774 775 776

    Raises:
        ValueError: If 'pool_type' is not "max" nor "avg"
        ValueError: If 'global_pooling' is False and 'pool_size' is -1
        ValueError: If 'use_cudnn' is not a bool value.

    Examples:

        .. code-block:: python

L
lujun 已提交
777
          import paddle.fluid as fluid
778 779
          from paddle.fluid.dygraph.base import to_variable
          import numpy as np
L
lujun 已提交
780 781

          with fluid.dygraph.guard():
782
             data = numpy.random.random((3, 32, 32, 5)).astype('float32')
783
             pool2d = fluid.dygraph.Pool2D(pool_size=2,
784 785 786
                            pool_type='max',
                            pool_stride=1,
                            global_pooling=False)
787
             pool2d_res = pool2d(to_variable(data))
788 789 790

    """

M
minqiyang 已提交
791 792 793 794 795 796 797 798
    def __init__(self,
                 pool_size=-1,
                 pool_type="max",
                 pool_stride=1,
                 pool_padding=0,
                 global_pooling=False,
                 use_cudnn=True,
                 ceil_mode=False,
799
                 exclusive=True):
M
minqiyang 已提交
800 801 802 803 804 805 806 807 808 809 810 811 812
        if pool_type not in ["max", "avg"]:
            raise ValueError(
                "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.",
                str(pool_type))

        if global_pooling is False and pool_size == -1:
            raise ValueError(
                "When the global_pooling is False, pool_size must be passed "
                "and be a valid value. Received pool_size: " + str(pool_size))

        if not isinstance(use_cudnn, bool):
            raise ValueError("use_cudnn should be True or False")

813
        super(Pool2D, self).__init__()
M
minqiyang 已提交
814 815 816 817 818 819 820 821 822 823 824 825 826

        self._pool_type = pool_type
        self._pool_size = utils.convert_to_list(pool_size, 2, 'pool_size')
        self._pool_padding = utils.convert_to_list(pool_padding, 2,
                                                   'pool_padding')
        self._pool_stride = utils.convert_to_list(pool_stride, 2, 'pool_stride')
        self._global_pooling = global_pooling
        self._use_cudnn = use_cudnn
        self._ceil_mode = ceil_mode
        self._exclusive = exclusive
        self._l_type = 'pool2d'

    def forward(self, input):
827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843
        attrs = {
            "pooling_type": self._pool_type,
            "ksize": self._pool_size,
            "global_pooling": self._global_pooling,
            "strides": self._pool_stride,
            "paddings": self._pool_padding,
            "use_cudnn": self._use_cudnn,
            "ceil_mode": self._ceil_mode,
            "use_mkldnn": False,
            "exclusive": self._exclusive,
        }
        inputs = {"X": [input]}

        if in_dygraph_mode():
            outs = core.ops.pool2d(inputs, attrs)
            return outs['Out'][0]

M
minqiyang 已提交
844 845
        pool_out = self._helper.create_variable_for_type_inference(self._dtype)

M
minqiyang 已提交
846 847 848
        self._helper.append_op(
            type=self._l_type,
            inputs={"X": input},
M
minqiyang 已提交
849
            outputs={"Out": pool_out},
850
            attrs=attrs)
M
minqiyang 已提交
851
        return pool_out
M
minqiyang 已提交
852 853


S
songyouwei 已提交
854 855 856 857 858 859 860 861 862 863
class Linear(layers.Layer):
    """
    Fully-connected linear transformation layer:

    .. math::

        Out = Act({XW + b})

    where :math:`X` is the input Tensor, :math:`W` and :math:`b` are weight and bias respectively.

864
    Linear layer takes only one ``Tensor`` input.
S
songyouwei 已提交
865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923
    The Linear layer multiplies input tensor with weight matrix and
    produces an output Tensor of shape [N, *, `output_dim`],
    where N is batch size and `*` means any number of additional dimensions.
    If ``bias_attr`` is not None, a bias variable will be created and added to the output.
    Finally, if ``act`` is not None, it will be applied to the output as well.

    Parameters:
        input_dim(int): The number of input units in this layer.
        output_dim(int): The number of output units in this layer.
        param_attr(ParamAttr or list of ParamAttr, optional): The parameter attribute for learnable
            weights(Parameter) of this layer. Default: None.
        bias_attr(ParamAttr or list of ParamAttr, optional): The attribute for the bias
            of this layer. If it is set to False, no bias will be added to the output units.
            If it is set to None, the bias is initialized zero. Default: None.
        act(str, optional): Activation to be applied to the output of this layer. Default: None.
        dtype(str, optional): Dtype used for weight, it can be "float32" or "float64". Default: "float32".

    Attributes:
        **weight** (Parameter): the learnable weights of this layer.

        **bias** (Parameter or None): the learnable bias of this layer.

    Returns:
        None

    Examples:
        .. code-block:: python

          from paddle.fluid.dygraph.base import to_variable
          import paddle.fluid as fluid
          from paddle.fluid.dygraph import Linear
          import numpy as np

          data = np.random.uniform(-1, 1, [30, 10, 32]).astype('float32')
          with fluid.dygraph.guard():
              linear = Linear(32, 64)
              data = to_variable(data)
              res = linear(data)  # [30, 10, 64]
    """

    def __init__(self,
                 input_dim,
                 output_dim,
                 param_attr=None,
                 bias_attr=None,
                 act=None,
                 dtype="float32"):
        super(Linear, self).__init__()
        self._act = act
        self._dtype = dtype
        self.weight = self.create_parameter(
            shape=[input_dim, output_dim],
            attr=param_attr,
            dtype=dtype,
            is_bias=False)
        self.bias = self.create_parameter(
            shape=[output_dim], attr=bias_attr, dtype=dtype, is_bias=True)

    def forward(self, input):
924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940
        attrs = {
            "transpose_X": False,
            "transpose_Y": False,
            "alpha": 1,
        }
        inputs = {"X": [input], "Y": [self.weight]}

        if in_dygraph_mode():
            outs = core.ops.matmul(inputs, attrs)
            pre_bias = outs['Out'][0]

            pre_act = dygraph_utils._append_bias_in_dygraph(
                pre_bias, self.bias, axis=len(input.shape) - 1)

            return dygraph_utils._append_activation_in_dygraph(pre_act,
                                                               self._act)

S
songyouwei 已提交
941 942
        tmp = self._helper.create_variable_for_type_inference(self._dtype)
        self._helper.append_op(
943
            type="matmul", inputs=inputs, outputs={"Out": tmp}, attrs=attrs)
S
songyouwei 已提交
944 945 946 947 948 949 950 951 952 953 954 955 956 957
        if self.bias:
            pre_activation = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
            self._helper.append_op(
                type='elementwise_add',
                inputs={'X': [tmp],
                        'Y': [self.bias]},
                outputs={'Out': [pre_activation]},
                attrs={'axis': len(input.shape) - 1})
        else:
            pre_activation = tmp
        return self._helper.append_activation(pre_activation, act=self._act)


M
minqiyang 已提交
958
class BatchNorm(layers.Layer):
959
    """
960 961 962 963 964
    This interface is used to construct a callable object of the ``BatchNorm`` class.
    For more details, refer to code examples.
    It implements the function of the Batch Normalization Layer and can be used 
    as a normalizer function for conv2d and fully connected operations.
    The data is normalized by the mean and variance of the channel based on the current batch data.
965 966 967 968
    Refer to `Batch Normalization: Accelerating Deep Network Training by Reducing
    Internal Covariate Shift <https://arxiv.org/pdf/1502.03167.pdf>`_
    for more details.

969 970 971
    When use_global_stats = False, the :math:`\\mu_{\\beta}` 
    and :math:`\\sigma_{\\beta}^{2}` are the statistics of one mini-batch.
    Calculated as follows:
972 973 974 975 976 977 978 979

    ..  math::

        \\mu_{\\beta} &\\gets \\frac{1}{m} \\sum_{i=1}^{m} x_i \\qquad &//\\
        \ mini-batch\ mean \\\\
        \\sigma_{\\beta}^{2} &\\gets \\frac{1}{m} \\sum_{i=1}^{m}(x_i - \\
        \\mu_{\\beta})^2 \\qquad &//\ mini-batch\ variance \\\\

980 981
    - :math:`x` : mini-batch data
    - :math:`m` : the size of the mini-batch data
982 983 984

    When use_global_stats = True, the :math:`\\mu_{\\beta}`
    and :math:`\\sigma_{\\beta}^{2}` are not the statistics of one mini-batch.
985 986 987 988 989 990
    They are global or running statistics (moving_mean and moving_variance). It usually got from the
    pre-trained model. Calculated as follows:

    .. math::
        moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global mean \\
        moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global variance \\
991

992 993
    The normalization function formula is as follows:
 
994 995 996
    ..  math::

        \\hat{x_i} &\\gets \\frac{x_i - \\mu_\\beta} {\\sqrt{\\
997 998 999 1000 1001 1002
        \\sigma_{\\beta}^{2} + \\epsilon}} \\qquad &//\ normalize \\\\
        y_i &\\gets \\gamma \\hat{x_i} + \\beta \\qquad &//\ scale\ and\ shift

    - :math:`\\epsilon` : add a smaller value to the variance to prevent division by zero
    - :math:`\\gamma` : trainable proportional parameter
    - :math:`\\beta` : trainable deviation parameter
1003

1004
    Parameters:
1005 1006 1007 1008 1009 1010
        num_channels(int): Indicate the number of channels of the input ``Tensor``.
        act(str, optional): Activation to be applied to the output of batch normalizaiton. Default: None.
        is_test (bool, optional): A flag indicating whether it is in test phrase or not. Default: False.
        momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
        epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5.
        param_attr(ParamAttr, optional): The parameter attribute for Parameter `scale`
1011 1012 1013
             of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm
             will create ParamAttr as param_attr. If the Initializer of the param_attr
             is not set, the parameter is initialized with Xavier. Default: None.
1014
        bias_attr(ParamAttr, optional): The parameter attribute for the bias of batch_norm.
1015 1016 1017
             If it is set to None or one attribute of ParamAttr, batch_norm
             will create ParamAttr as bias_attr. If the Initializer of the bias_attr
             is not set, the bias is initialized zero. Default: None.
1018 1019 1020 1021 1022 1023
        dtype(str, optional): Indicate the data type of the input ``Tensor``,
             which can be float32 or float64. Default: float32.
        data_layout(str, optional): Specify the input data format, the data format can be "NCHW" or "NHWC". Default: NCHW.
        in_place(bool, optional): Make the input and output of batch norm reuse memory. Default: False.
        moving_mean_name(str, optional): The name of moving_mean which store the global Mean. Default: None.
        moving_variance_name(str, optional): The name of the moving_variance which store the global Variance. Default: None.
1024 1025
        do_model_average_for_mean_and_var(bool, optional): Whether parameter mean and variance should do model
            average when model average is enabled. Default: True.
1026
        use_global_stats(bool, optional): Whether to use global mean and
1027 1028 1029
            variance. In inference or test mode, set use_global_stats to true
            or is_test to true, and the behavior is equivalent.
            In train mode, when setting use_global_stats True, the global mean
1030 1031 1032 1033
            and variance are also used during train period. Default: False.
        trainable_statistics(bool, optional): Whether to calculate mean and var in eval mode. In eval mode, when
            setting trainable_statistics True, mean and variance will be calculated by current batch statistics.
            Default: False.
1034 1035

    Returns:
1036
        None
1037 1038 1039

    Examples:
        .. code-block:: python
L
lujun 已提交
1040 1041

          import paddle.fluid as fluid
1042 1043
          from paddle.fluid.dygraph.base import to_variable
          import numpy as np
L
lujun 已提交
1044

1045
          x = np.random.random(size=(3, 10, 3, 7)).astype('float32')
L
lujun 已提交
1046
          with fluid.dygraph.guard():
1047
              x = to_variable(x)
1048
              batch_norm = fluid.BatchNorm(10)
1049
              hidden1 = batch_norm(x)
1050 1051
    """

M
minqiyang 已提交
1052 1053 1054 1055 1056 1057 1058 1059
    def __init__(self,
                 num_channels,
                 act=None,
                 is_test=False,
                 momentum=0.9,
                 epsilon=1e-05,
                 param_attr=None,
                 bias_attr=None,
1060
                 dtype='float32',
M
minqiyang 已提交
1061 1062 1063 1064
                 data_layout='NCHW',
                 in_place=False,
                 moving_mean_name=None,
                 moving_variance_name=None,
1065
                 do_model_average_for_mean_and_var=True,
1066 1067
                 use_global_stats=False,
                 trainable_statistics=False):
1068
        super(BatchNorm, self).__init__()
1069
        self._param_attr = param_attr
1070
        self._bias_attr = bias_attr
1071
        self._act = act
M
minqiyang 已提交
1072 1073 1074

        assert bias_attr is not False, "bias_attr should not be False in batch_norm."

1075 1076
        if dtype == "float16":
            self._dtype = "float32"
M
minqiyang 已提交
1077 1078 1079 1080 1081 1082
        else:
            self._dtype = dtype

        param_shape = [num_channels]

        # create parameter
1083
        self.weight = self.create_parameter(
1084
            attr=self._param_attr,
M
minqiyang 已提交
1085 1086 1087
            shape=param_shape,
            dtype=self._dtype,
            default_initializer=Constant(1.0))
1088
        self.weight.stop_gradient = use_global_stats and self._param_attr.learning_rate == 0.
M
minqiyang 已提交
1089

1090
        self.bias = self.create_parameter(
1091
            attr=self._bias_attr,
M
minqiyang 已提交
1092 1093 1094
            shape=param_shape,
            dtype=self._dtype,
            is_bias=True)
1095
        self.bias.stop_gradient = use_global_stats and self._param_attr.learning_rate == 0.
M
minqiyang 已提交
1096

1097
        self._mean = self.create_parameter(
M
minqiyang 已提交
1098 1099 1100 1101 1102 1103 1104
            attr=ParamAttr(
                name=moving_mean_name,
                initializer=Constant(0.0),
                trainable=False,
                do_model_average=do_model_average_for_mean_and_var),
            shape=param_shape,
            dtype=self._dtype)
1105
        self._mean.stop_gradient = True
M
minqiyang 已提交
1106

1107
        self._variance = self.create_parameter(
M
minqiyang 已提交
1108 1109 1110 1111 1112 1113 1114
            attr=ParamAttr(
                name=moving_variance_name,
                initializer=Constant(1.0),
                trainable=False,
                do_model_average=do_model_average_for_mean_and_var),
            shape=param_shape,
            dtype=self._dtype)
1115
        self._variance.stop_gradient = True
M
minqiyang 已提交
1116 1117

        self._in_place = in_place
1118
        self._data_layout = data_layout
M
minqiyang 已提交
1119 1120 1121
        self._momentum = momentum
        self._epsilon = epsilon
        self._is_test = is_test
1122
        self._fuse_with_relu = False
M
minqiyang 已提交
1123
        self._use_global_stats = use_global_stats
1124
        self._trainable_statistics = trainable_statistics
M
minqiyang 已提交
1125 1126 1127 1128 1129 1130

    def forward(self, input):
        # create output
        # mean and mean_out share the same memory
        mean_out = self._mean
        # variance and variance out share the same memory
1131

M
minqiyang 已提交
1132
        variance_out = self._variance
1133 1134 1135 1136 1137 1138 1139 1140 1141 1142
        attrs = {
            "momentum": self._momentum,
            "epsilon": self._epsilon,
            "is_test": self._is_test,
            "data_layout": self._data_layout,
            "use_mkldnn": False,
            "fuse_with_relu": self._fuse_with_relu,
            "use_global_stats": self._use_global_stats,
            "trainable_statistics": self._trainable_statistics
        }
M
minqiyang 已提交
1143

1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178
        inputs = {
            "X": [input],
            "Scale": [self.weight],
            "Bias": [self.bias],
            "Mean": [self._mean],
            "Variance": [self._variance]
        }

        if in_dygraph_mode():
            attrs['is_test'] = not _dygraph_tracer()._train_mode
            saved_mean = _varbase_creator(dtype=self._dtype)
            saved_variance = _varbase_creator(dtype=self._dtype)
            batch_norm_out = _varbase_creator(dtype=self._dtype)
            batch_norm_out.stop_gradient = False
            # inplace is not supported currently
        else:
            saved_mean = self._helper.create_variable_for_type_inference(
                dtype=self._dtype, stop_gradient=True)
            saved_variance = self._helper.create_variable_for_type_inference(
                dtype=self._dtype, stop_gradient=True)
            batch_norm_out = input if self._in_place else self._helper.create_variable_for_type_inference(
                self._dtype)

        outputs = {
            "Y": [batch_norm_out],
            "MeanOut": [mean_out],
            "VarianceOut": [variance_out],
            "SavedMean": [saved_mean],
            "SavedVariance": [saved_variance]
        }

        if in_dygraph_mode():
            outs = core.ops.batch_norm(inputs, attrs, outputs)
            return dygraph_utils._append_activation_in_dygraph(
                batch_norm_out, act=self._act)
M
minqiyang 已提交
1179 1180

        self._helper.append_op(
1181
            type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs)
M
minqiyang 已提交
1182

L
lujun 已提交
1183
        # Currently, we don't support inplace in dygraph mode
1184
        return self._helper.append_activation(batch_norm_out, self._act)
1185 1186


1187 1188 1189 1190
class Embedding(layers.Layer):
    """
    **Embedding Layer**

Z
zhongpu 已提交
1191 1192 1193 1194 1195 1196
    This interface is used to construct a callable object of the ``Embedding`` class.
    For specific usage, refer to code examples. It implements the function of the Embedding Layer.
    This layer is used to lookup embeddings vector of ids provided by :attr:`input` .
    It automatically constructs a 2D embedding matrix based on the
    input :attr:`size` (vocab_size, emb_size) and :attr:`dtype` .

1197 1198
    The shape of output Tensor is generated by appending an emb_size dimension to the
    last dimension of the input Tensor shape.
Z
zhongpu 已提交
1199

1200
    **Note:** The id in :attr:`input` must satisfy :math:`0 =< id < size[0]` ,
Z
zhongpu 已提交
1201 1202 1203 1204 1205 1206 1207
    otherwise the program will throw an exception and exit.

    .. code-block:: text

        Case 1:

        input is a Tensor. padding_idx = -1
1208 1209
            input.data = [[1, 3], [2, 4], [4, 127]
            input.shape = [3, 2]
Z
zhongpu 已提交
1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222
        Given size = [128, 16]
        output is a Tensor:
            out.shape = [3, 2, 16]
            out.data = [[[0.129435295, 0.244512452, ..., 0.436322452],
                        [0.345421456, 0.524563927, ..., 0.144534654]],

                        [[0.345249859, 0.124939536, ..., 0.194353745],
                        [0.945345345, 0.435394634, ..., 0.435345365]],
                        
                        [[0.945345345, 0.435394634, ..., 0.435345365],
                        [0.0,         0.0,         ..., 0.0        ]]]  # padding data
        The input padding_idx is less than 0, it is automatically converted to padding_idx = -1 + 128 = 127
        It will pad all-zero data when ids is 127.
1223

1224
    Parameters:
L
lujun 已提交
1225 1226
        size(tuple|list): The shape of the look up table parameter. It should have two elements which indicate the size
            of the dictionary of embeddings and the size of each embedding vector respectively.
Z
zhongpu 已提交
1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
        is_sparse(bool): The flag indicating whether to use sparse update. This parameter only
            affects the performance of the backwards gradient update. It is recommended to set 
            True because sparse update is faster. But some optimizer does not support sparse update,
            such as :ref:`api_fluid_optimizer_AdadeltaOptimizer` , :ref:`api_fluid_optimizer_AdamaxOptimizer` , 
            :ref:`api_fluid_optimizer_DecayedAdagradOptimizer` , :ref:`api_fluid_optimizer_FtrlOptimizer` ,
            :ref:`api_fluid_optimizer_LambOptimizer` and :ref:`api_fluid_optimizer_LarsMomentumOptimizer` .
            In these case, is_sparse must be False. Default: False.
        is_distributed(bool): Whether to store the embedding matrix in a distributed manner. Only used
            in multi-machine distributed CPU training. Default: False.
        padding_idx(int|long|None): padding_idx needs to be in the interval [-vocab_size, vocab_size). 
            If :math:`padding\_idx < 0`, the :math:`padding\_idx` will automatically be converted
            to :math:`vocab\_size + padding\_idx` . It will output all-zero padding data whenever lookup
            encounters :math:`padding\_idx` in id. And the padding data will not be updated while training.
            If set None, it makes no effect to output. Default: None.
        param_attr(ParamAttr): To specify the weight parameter property. Default: None, which means the
            default weight parameter property is used. See usage for details in :ref:`api_fluid_ParamAttr` . In addition,
            user-defined or pre-trained word vectors can be loaded with the :attr:`param_attr` parameter. 
            The local word vector needs to be transformed into numpy format, and the shape of local word
            vector shoud be consistent with :attr:`size` . Then :ref:`api_fluid_initializer_NumpyArrayInitializer`
            is used to load custom or pre-trained word vectors. See code example 2 for details.
        dtype(np.dtype|core.VarDesc.VarType|str): It refers to the data type of output Tensor.
            It must be "float32" or "float64". Default: "float32".
1249

Z
zhongpu 已提交
1250 1251
    Attribute:
        **weight** (Parameter): the learnable weights of this layer.
1252

1253
    Returns:
Z
zhongpu 已提交
1254
        Variable: Embedding Tensor or LoDTensor mapped by input. The data type is the same as :attr:`dtype` .
1255 1256

    Examples:
1257

1258 1259
        .. code-block:: python

L
lujun 已提交
1260 1261 1262 1263
          import paddle.fluid as fluid
          import paddle.fluid.dygraph.base as base
          import numpy as np

Z
zhongpu 已提交
1264
          # example 1
1265 1266
          inp_word = np.array([[2, 3, 5], [4, 2, 1]]).astype('int64')
          inp_word.shape  # [2, 3]
1267 1268
          dict_size = 20
          with fluid.dygraph.guard():
L
lujun 已提交
1269
              emb = fluid.dygraph.Embedding(
1270 1271 1272
                  size=[dict_size, 32],
                  param_attr='emb.w',
                  is_sparse=False)
L
lujun 已提交
1273
              static_rlt3 = emb(base.to_variable(inp_word))
1274
              static_rlt3.shape  # [2, 3, 32]
Z
zhongpu 已提交
1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288

          # example 2: load custom or pre-trained word vectors
          weight_data = np.random.random(size=(128, 100))  # word vectors with numpy format
          w_param_attrs = fluid.ParamAttr(
              name="emb_weight",
              learning_rate=0.5,
              initializer=fluid.initializer.NumpyArrayInitializer(weight_data),
              trainable=True)
          with fluid.dygraph.guard():
              emb = fluid.dygraph.Embedding(
                  size=[128, 100],
                  param_attr= w_param_attrs,
                  is_sparse=False)
              static_rlt3 = emb(base.to_variable(inp_word))          
1289 1290
    """

1291 1292 1293 1294 1295 1296 1297
    def __init__(self,
                 size,
                 is_sparse=False,
                 is_distributed=False,
                 padding_idx=None,
                 param_attr=None,
                 dtype='float32'):
1298
        super(Embedding, self).__init__()
1299 1300 1301 1302
        self._size = size
        self._is_sparse = is_sparse
        self._is_distributed = is_distributed
        self._padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else (
J
JiabinYang 已提交
1303
            size[0] + padding_idx)
1304 1305 1306

        self._param_attr = param_attr
        self._dtype = dtype
J
JiabinYang 已提交
1307
        self._remote_prefetch = self._is_sparse and (not self._is_distributed)
1308 1309 1310
        if self._remote_prefetch:
            assert self._is_sparse is True and self._is_distributed is False

1311
        self.weight = self.create_parameter(
1312 1313 1314 1315 1316 1317
            attr=self._param_attr,
            shape=self._size,
            dtype=self._dtype,
            is_bias=False)

    def forward(self, input):
1318 1319 1320 1321 1322 1323
        attrs = {
            'is_sparse': self._is_sparse,
            'is_distributed': self._is_distributed,
            'remote_prefetch': self._remote_prefetch,
            'padding_idx': self._padding_idx
        }
1324

1325
        if in_dygraph_mode():
1326
            inputs = {'Ids': [input], 'W': [self.weight]}
1327 1328 1329
            outs = core.ops.lookup_table_v2(inputs, attrs)
            return outs['Out'][0]

1330 1331
        out = self._helper.create_variable_for_type_inference(self._dtype)
        self._helper.append_op(
1332
            type='lookup_table_v2',
1333
            inputs={'Ids': input,
1334
                    'W': self.weight},
1335
            outputs={'Out': out},
1336
            attrs=attrs)
1337 1338

        return out
M
minqiyang 已提交
1339 1340


1341
class LayerNorm(layers.Layer):
1342
    """
1343 1344 1345
    This interface is used to construct a callable object of the ``LayerNorm`` class.
    For more details, refer to code examples.
    It implements the function of the Layer Normalization Layer and can be applied to mini-batch input data.
1346
    Refer to `Layer Normalization <https://arxiv.org/pdf/1607.06450v1.pdf>`_
1347

1348
    The formula is as follows:
1349

1350
    ..  math::
1351

1352
        \\mu & = \\frac{1}{H}\\sum_{i=1}^{H} x_i
1353

1354
        \\sigma & = \\sqrt{\\frac{1}{H}\sum_{i=1}^{H}{(x_i - \\mu)^2} + \\epsilon}
1355

1356
        y & = f(\\frac{g}{\\sigma}(x - \\mu) + b)
1357

1358 1359 1360 1361 1362
    - :math:`x`: the vector representation of the summed inputs to the neurons in that layer.
    - :math:`H`: the number of hidden units in a layers
    - :math:`\\epsilon`: the small value added to the variance to prevent division by zero.
    - :math:`g`: the trainable scale parameter.
    - :math:`b`: the trainable bias parameter.
1363

1364
    Parameters:
1365 1366 1367 1368
        normalized_shape(int or list or tuple): Input shape from an expected input of
            size :math:`[*, normalized_shape[0], normalized_shape[1], ..., normalized_shape[-1]]`.
            If it is a single integer, this module will normalize over the last dimension
            which is expected to be of that specific size.
1369
        scale(bool, optional): Whether to learn the adaptive gain :math:`g` after
L
lujun 已提交
1370
            normalization. Default: True.
1371
        shift(bool, optional): Whether to learn the adaptive bias :math:`b` after
L
lujun 已提交
1372
            normalization. Default: True.
1373
        epsilon(float, optional): The small value added to the variance to prevent
L
lujun 已提交
1374
            division by zero. Default: 1e-05.
1375
        param_attr(ParamAttr, optional): The parameter attribute for the learnable
1376 1377 1378
            gain :math:`g`. If :attr:`scale` is False, :attr:`param_attr` is
            omitted. If :attr:`scale` is True and :attr:`param_attr` is None,
            a default :code:`ParamAttr` would be added as scale. The
L
lujun 已提交
1379
            :attr:`param_attr` is initialized as 1 if it is added. Default: None.
1380
        bias_attr(ParamAttr, optional): The parameter attribute for the learnable
1381 1382 1383
            bias :math:`b`. If :attr:`shift` is False, :attr:`bias_attr` is
            omitted. If :attr:`shift` is True and :attr:`param_attr` is None,
            a default :code:`ParamAttr` would be added as bias. The
L
lujun 已提交
1384
            :attr:`bias_attr` is initialized as 0 if it is added. Default: None.
1385
        act(str, optional): Activation to be applied to the output of layer normalizaiton.
L
lujun 已提交
1386
                  Default: None.
1387 1388
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".

1389
    Returns:
1390
        None
1391

1392
    Examples:
1393

1394 1395 1396
        .. code-block:: python

          import paddle.fluid as fluid
1397
          from paddle.fluid.dygraph.base import to_variable
1398 1399
          import numpy

1400
          x = numpy.random.random((3, 32, 32)).astype('float32')
1401
          with fluid.dygraph.guard():
1402
              x = to_variable(x)
1403
              layerNorm = fluid.LayerNorm([32, 32])
1404
              ret = layerNorm(x)
1405

1406
    """
1407

1408
    def __init__(self,
1409
                 normalized_shape,
1410 1411 1412 1413 1414
                 scale=True,
                 shift=True,
                 epsilon=1e-05,
                 param_attr=None,
                 bias_attr=None,
1415 1416 1417 1418 1419
                 act=None,
                 dtype='float32'):
        super(LayerNorm, self).__init__()
        if isinstance(normalized_shape, numbers.Integral):
            normalized_shape = [normalized_shape]
H
hong 已提交
1420

1421
        self._normalized_shape = list(normalized_shape)
1422 1423 1424 1425 1426 1427
        self._scale = scale
        self._shift = shift
        self._epsilon = epsilon
        self._param_attr = param_attr
        self._bias_attr = bias_attr
        self._act = act
1428 1429
        self._dtype = dtype
        param_shape = [np.prod(self._normalized_shape)]
1430
        if self._scale:
1431
            self.weight = self.create_parameter(
1432 1433 1434 1435
                attr=self._param_attr,
                shape=param_shape,
                dtype=self._dtype,
                default_initializer=Constant(1.0))
1436 1437 1438 1439
        else:
            if self._param_attr:
                logging.warn("param_attr are only avaliable with scale is True")

1440 1441
        if self._shift:
            assert self._bias_attr is not False
1442
            self.bias = self.create_parameter(
1443 1444 1445 1446
                attr=self._bias_attr,
                shape=param_shape,
                dtype=self._dtype,
                is_bias=True)
1447 1448 1449
        else:
            if self._bias_attr:
                logging.warn("bias_attr are only avaliable with shift is True")
1450 1451

    def forward(self, input):
1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462
        input_shape = list(input.shape)
        input_ndim = len(input_shape)
        normalized_ndim = len(self._normalized_shape)
        self._begin_norm_axis = input_ndim - normalized_ndim
        if input_ndim < normalized_ndim or input_shape[
                self._begin_norm_axis:] != self._normalized_shape:
            str_normalized_shape = str(self._normalized_shape)
            raise ValueError(
                'Given normalized_shape is ' + str_normalized_shape +
                ', expected input with shape [*, ' + str_normalized_shape[
                    1:] + ', but got input shape ' + str(input_shape))
1463
        inputs = dict()
1464
        inputs['X'] = [input]
1465
        if self._scale:
1466
            inputs['Scale'] = [self.weight]
1467
        if self._shift:
1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480
            inputs['Bias'] = [self.bias]

        attrs = {
            "epsilon": self._epsilon,
            "begin_norm_axis": self._begin_norm_axis
        }

        if in_dygraph_mode():
            outs = core.ops.layer_norm(inputs, attrs)
            pre_act = outs['Y'][0]
            return dygraph_utils._append_activation_in_dygraph(
                pre_act, act=self._act)

1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501
        # create output
        mean_out = self._helper.create_variable_for_type_inference(
            dtype=self._dtype, stop_gradient=True)
        variance_out = self._helper.create_variable_for_type_inference(
            dtype=self._dtype, stop_gradient=True)
        layer_norm_out = self._helper.create_variable_for_type_inference(
            self._dtype)

        self._helper.append_op(
            type="layer_norm",
            inputs=inputs,
            outputs={
                "Y": layer_norm_out,
                "Mean": mean_out,
                "Variance": variance_out,
            },
            attrs={
                "epsilon": self._epsilon,
                "begin_norm_axis": self._begin_norm_axis
            })

1502
        return self._helper.append_activation(layer_norm_out, act=self._act)
1503 1504


M
minqiyang 已提交
1505 1506 1507
class GRUUnit(layers.Layer):
    """
    **GRU unit layer**
D
DuYao 已提交
1508 1509 1510 1511 1512
    
    It creates a callable object from GRUUnit class.
    If origin_mode is True, then the equation of a gru step is from paper
    `Learning Phrase Representations using RNN Encoder-Decoder for Statistical 
    Machine Translation <https://arxiv.org/pdf/1406.1078.pdf>`_
M
minqiyang 已提交
1513 1514 1515 1516 1517 1518 1519 1520 1521 1522

        .. math::
            u_t & = actGate(xu_{t} + W_u h_{t-1} + b_u)

            r_t & = actGate(xr_{t} + W_r h_{t-1} + b_r)

            m_t & = actNode(xm_t + W_c dot(r_t, h_{t-1}) + b_m)

            h_t & = dot(u_t, h_{t-1}) + dot((1-u_t), m_t)

D
DuYao 已提交
1523
    If origin_mode is False, then the equation of a gru step is from paper
M
minqiyang 已提交
1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548
    `Empirical Evaluation of Gated Recurrent Neural Networks on Sequence
    Modeling <https://arxiv.org/pdf/1412.3555.pdf>`_

        .. math::
            u_t & = actGate(xu_{t} + W_u h_{t-1} + b_u)

            r_t & = actGate(xr_{t} + W_r h_{t-1} + b_r)

            m_t & = actNode(xm_t + W_c dot(r_t, h_{t-1}) + b_m)

            h_t & = dot((1-u_t), h_{t-1}) + dot(u_t, m_t)


    The inputs of gru unit includes :math:`z_t`, :math:`h_{t-1}`. In terms
    of the equation above, the :math:`z_t` is split into 3 parts -
    :math:`xu_t`, :math:`xr_t` and :math:`xm_t`. This means that in order to
    implement a full GRU unit operator for an input, a fully
    connected layer has to be applied, such that :math:`z_t = W_{fc}x_t`.

    The terms :math:`u_t` and :math:`r_t` represent the update and reset gates
    of the GRU cell. Unlike LSTM, GRU has one lesser gate. However, there is
    an intermediate candidate hidden output, which is denoted by :math:`m_t`.
    This layer has three outputs :math:`h_t`, :math:`dot(r_t, h_{t-1})`
    and concatenation of :math:`u_t`, :math:`r_t` and :math:`m_t`.

1549
    Parameters:
L
lujun 已提交
1550
        size (int): The input dimension value.
D
DuYao 已提交
1551 1552 1553 1554 1555 1556 1557 1558 1559
        param_attr(ParamAttr, optional): The parameter attribute for the learnable
            hidden-hidden weight matrix. 
            
            **Note**:
    
                1. The shape of the weight matrix is :math:`[T, 3*D]`, where D is the hidden size.
                2. All elements in the weight matrix can be divided into two parts. The first 
                   part are weights of the update gate and reset gate with shape :math:`[D, 2*D]`, 
                   and the second part are weights for candidate hidden state with shape :math:`[D, D]`.
M
minqiyang 已提交
1560 1561 1562 1563


            If it is set to None or one attribute of ParamAttr, gru_unit will
            create ParamAttr as param_attr. If the Initializer of the param_attr
D
DuYao 已提交
1564 1565 1566 1567
            is not set, the parameter is initialized with Xavier. The default 
            value is None.
        bias_attr (ParamAttr|bool, optional): The parameter attribute for the bias
            of GRU.Note that the bias with :math:`[1, 3*D]` concatenates
M
minqiyang 已提交
1568 1569 1570 1571 1572
            the bias in the update gate, reset gate and candidate calculations.
            If it is set to False, no bias will be applied to the update gate,
            reset gate and candidate calculations. If it is set to None or one
            attribute of ParamAttr, gru_unit will create ParamAttr as
            bias_attr. If the Initializer of the bias_attr is not set, the bias
D
DuYao 已提交
1573
            is initialized zero. The default value is None.
L
lujun 已提交
1574
        activation (str): The activation type for cell (actNode).
D
DuYao 已提交
1575
                             The default value is 'tanh'.
L
lujun 已提交
1576
        gate_activation (str): The activation type for gates (actGate).
D
DuYao 已提交
1577 1578 1579
                                  The default value is 'sigmoid'.
        dtype(str): The dtype of the layers. The data type can be set as
            'float32', 'float64'. The default value is 'float32'.
M
minqiyang 已提交
1580

D
DuYao 已提交
1581 1582 1583 1584
    Attribute:
        **weight** (Parameter): the learnable weights of this layer.

        **bias** (Parameter): the learnable bias of this layer.
1585

M
minqiyang 已提交
1586
    Returns:
D
DuYao 已提交
1587 1588 1589 1590
        tuple: The hidden value, reset-hidden value and gate values. The hidden value
        is a 2-D tensor with shape  :math:`[T, D]` . The reset-hidden value is a
        2-D tensor with shape  :math:`[T, D]` . The gate value is a 2-D tensor with 
        shape  :math:`[T, 3*D]`.
L
lujun 已提交
1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603

    Examples:

        .. code-block:: python

          import paddle.fluid as fluid
          import paddle.fluid.dygraph.base as base
          import numpy

          lod = [[2, 4, 3]]
          D = 5
          T = sum(lod[0])

D
DuYao 已提交
1604
          input = numpy.random.rand(T, 3 * D).astype('float32')
L
lujun 已提交
1605 1606 1607
          hidden_input = numpy.random.rand(T, D).astype('float32')
          with fluid.dygraph.guard():
              x = numpy.random.random((3, 32, 32)).astype('float32')
1608
              gru = fluid.dygraph.GRUUnit(size=D * 3)
L
lujun 已提交
1609 1610 1611
              dy_ret = gru(
                base.to_variable(input), base.to_variable(hidden_input))

M
minqiyang 已提交
1612 1613 1614 1615 1616 1617 1618 1619 1620 1621
    """

    def __init__(self,
                 size,
                 param_attr=None,
                 bias_attr=None,
                 activation='tanh',
                 gate_activation='sigmoid',
                 origin_mode=False,
                 dtype='float32'):
1622
        super(GRUUnit, self).__init__()
1623
        self._bias_attr = bias_attr
M
minqiyang 已提交
1624 1625 1626 1627 1628
        activation_dict = dict(
            identity=0,
            sigmoid=1,
            tanh=2,
            relu=3, )
H
Hongyu Liu 已提交
1629 1630
        self.activation = activation_dict[activation]
        self.gate_activation = activation_dict[gate_activation]
M
minqiyang 已提交
1631

M
minqiyang 已提交
1632
        self._dtype = dtype
M
minqiyang 已提交
1633 1634
        size = size // 3
        # create weight
1635
        self.weight = self.create_parameter(
M
minqiyang 已提交
1636
            attr=param_attr, shape=[size, 3 * size], dtype=dtype)
M
minqiyang 已提交
1637 1638

        # create bias
M
minqiyang 已提交
1639
        bias_size = [1, 3 * size]
1640
        self._bias_size = bias_size
1641
        self.bias = self.create_parameter(
M
minqiyang 已提交
1642
            attr=bias_attr, shape=bias_size, dtype=dtype, is_bias=True)
M
minqiyang 已提交
1643

M
minqiyang 已提交
1644
    def forward(self, input, hidden):
1645 1646 1647 1648 1649
        inputs = {
            'Input': [input],
            'HiddenPrev': [hidden],
            'Weight': [self.weight]
        }
1650
        if self.bias:
1651 1652 1653 1654 1655 1656 1657 1658 1659 1660
            inputs['Bias'] = [self.bias]
        attrs = {
            'activation': self.activation,
            'gate_activation': self.gate_activation,
        }

        if in_dygraph_mode():
            outs = core.ops.gru_unit(inputs, attrs)
            return outs['Hidden'][0], outs['ResetHiddenPrev'][0], outs['Gate'][
                0]
M
minqiyang 已提交
1661 1662 1663 1664 1665 1666

        gate = self._helper.create_variable_for_type_inference(self._dtype)
        reset_hidden_pre = self._helper.create_variable_for_type_inference(
            self._dtype)
        updated_hidden = self._helper.create_variable_for_type_inference(
            self._dtype)
M
minqiyang 已提交
1667 1668 1669 1670 1671 1672 1673 1674 1675
        self._helper.append_op(
            type='gru_unit',
            inputs=inputs,
            outputs={
                'Gate': gate,
                'ResetHiddenPrev': reset_hidden_pre,
                'Hidden': updated_hidden,
            },
            attrs={
H
Hongyu Liu 已提交
1676 1677
                'activation': self.activation,
                'gate_activation': self.gate_activation,
M
minqiyang 已提交
1678 1679 1680
            })

        return updated_hidden, reset_hidden_pre, gate
1681 1682 1683 1684


class NCE(layers.Layer):
    """
1685 1686 1687 1688 1689
    This interface is used to construct a callable object of the ``NCE`` class.
    For more details, refer to code examples.
    It implements the function of the ``NCE`` loss function.
    By default this function uses a uniform distribution for sampling, and it
    compute and return the noise-contrastive estimation training loss. See
1690
    `Noise-contrastive estimation: A new estimation principle for unnormalized statistical models <http://www.jmlr.org/proceedings/papers/v9/gutmann10a/gutmann10a.pdf>`_ .
1691

1692
    Parameters:
1693 1694
        num_total_classes (int): Total number of classes in all samples.
        dim (int): Dimension of input (possibly embedding dim).
1695
        param_attr (ParamAttr, optional): The parameter attribute for learnable weights(Parameter)
1696 1697 1698
             of nce. If it is set to None or one attribute of ParamAttr, nce
             will create ParamAttr as param_attr. If the Initializer of the param_attr
             is not set, the parameter is initialized with Xavier. Default: None.
1699
        bias_attr (ParamAttr or bool, optional): The attribute for the bias of nce.
1700 1701 1702 1703
             If it is set to False, no bias will be added to the output units.
             If it is set to None or one attribute of ParamAttr, nce
             will create ParamAttr as bias_attr. If the Initializer of the bias_attr
             is not set, the bias is initialized zero. Default: None.
1704 1705
        num_neg_samples (int, optional): The number of negative classes. The default value is 10.
        sampler (str, optional): The sampler used to sample class from negtive classes.
1706 1707
                       It can be 'uniform', 'log_uniform' or 'custom_dist'.
                       default: 'uniform'.
1708
        custom_dist (float[], optional): A float[] with size=num_total_classes.
1709
                       It is used when sampler is set to 'custom_dist'.
1710
                       custom_dist[i] is the probability of i-th class to be sampled.
L
lujun 已提交
1711
                       Default: None.
1712 1713
        seed (int, optional): The seed used in sampler. Default: 0.
        is_sparse(bool, optional): The flag indicating whether to use sparse update. If is_sparse is True, the weight@GRAD and bias@GRAD will be changed to SelectedRows. Default: False.
1714
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
1715

1716 1717
    Attribute:
        **weight** (Parameter): the learnable weights of this layer.
1718

1719 1720
        **bias** (Parameter or None): the learnable bias of this layer.
    
1721
    Returns:
1722
        None
1723 1724 1725 1726

    Examples:
        .. code-block:: python

1727 1728 1729
            import numpy as np
            import paddle.fluid as fluid

1730
            window_size = 5
1731 1732
            dict_size = 20
            label_word = int(window_size // 2) + 1
1733
            inp_word = np.array([[1], [2], [3], [4], [5]]).astype('int64')
1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754
            nid_freq_arr = np.random.dirichlet(np.ones(20) * 1000).astype('float32')

            with fluid.dygraph.guard():
                words = []
                for i in range(window_size):
                    words.append(fluid.dygraph.base.to_variable(inp_word[i]))

                emb = fluid.Embedding(
                    size=[dict_size, 32],
                    param_attr='emb.w',
                    is_sparse=False)

                embs3 = []
                for i in range(window_size):
                    if i == label_word:
                        continue

                    emb_rlt = emb(words[i])
                    embs3.append(emb_rlt)

                embs3 = fluid.layers.concat(input=embs3, axis=1)
1755
                nce = fluid.NCE(
1756
                             num_total_classes=dict_size,
1757
                             dim=embs3.shape[1],
1758 1759 1760 1761 1762 1763 1764
                             num_neg_samples=2,
                             sampler="custom_dist",
                             custom_dist=nid_freq_arr.tolist(),
                             seed=1,
                             param_attr='nce.w',
                             bias_attr='nce.b')

1765 1766
                wl = fluid.layers.unsqueeze(words[label_word], axes=[0])
                nce_loss3 = nce(embs3, wl)
1767 1768 1769 1770 1771

    """

    def __init__(self,
                 num_total_classes,
1772
                 dim,
1773
                 sample_weight=None,
1774 1775 1776 1777 1778 1779
                 param_attr=None,
                 bias_attr=None,
                 num_neg_samples=None,
                 sampler="uniform",
                 custom_dist=None,
                 seed=0,
1780 1781 1782
                 is_sparse=False,
                 dtype='float32'):
        super(NCE, self).__init__()
1783 1784 1785
        self._param_attr = param_attr
        self._bias_attr = bias_attr
        self._num_total_classes = num_total_classes
1786
        self._dtype = dtype
1787
        self._inputs = dict()
1788
        self._inputs['SampleWeight'] = sample_weight if sample_weight is not None else []
1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875
        if sampler == "uniform":
            sampler = 0
        elif sampler == "log_uniform":
            sampler = 1
        elif sampler == "custom_dist":
            assert custom_dist is not None
            # assert isinstance(custom_dist, Variable)

            custom_dist_len = len(custom_dist)
            alias_probs_ = [0] * custom_dist_len
            alias_ = [0] * custom_dist_len
            bigs = []
            littles = []
            for i in range(custom_dist_len):
                normal_prob = custom_dist[i] * custom_dist_len
                if normal_prob - 1.0 > 0:
                    bigs.append((i, normal_prob))
                elif 1.0 - normal_prob > 0:
                    littles.append((i, normal_prob))
                else:
                    alias_probs_[i] = normal_prob
                    alias_[i] = -1

            while len(bigs) and len(littles):
                big = bigs.pop(0)
                little = littles.pop(0)

                big_idx = big[0]
                big_prob = big[1]

                alias_probs_[little[0]] = little[1]
                alias_[little[0]] = big_idx
                big_left = big[1] + little[1] - 1
                if big_left - 1.0 > 0:
                    bigs.append((big_idx, big_left))
                elif 1.0 - big_left > 0:
                    littles.append((big_idx, big_left))
                else:
                    alias_probs_[big_idx] = big_left
                    alias_[big_idx] = -1

            if len(bigs):
                big = bigs.pop(0)
                alias_probs_[big[0]] = 1.0
                alias_[big[0]] = -1
            if len(littles):
                little = littles.pop(0)
                alias_probs_[little[0]] = 1.0
                alias_[little[0]] = -1

            def _init_by_numpy_array(numpy_array):
                ret = self.create_parameter(
                    attr=ParamAttr(),
                    shape=numpy_array.shape,
                    dtype=numpy_array.dtype,
                    default_initializer=NumpyArrayInitializer(numpy_array))
                ret.stop_gradient = True
                return ret

            self._inputs['CustomDistProbs'] = _init_by_numpy_array(
                np.array(custom_dist).astype('float32'))
            self._inputs['CustomDistAlias'] = _init_by_numpy_array(
                np.array(alias_).astype('int32'))
            self._inputs['CustomDistAliasProbs'] = _init_by_numpy_array(
                np.array(alias_probs_).astype('float32'))
            sampler = 2
        else:
            raise Exception("Unsupported sampler type.")

        if num_neg_samples is None:
            num_neg_samples = 10
        else:
            num_neg_samples = int(num_neg_samples)
        self._num_neg_samples = num_neg_samples
        remote_prefetch = is_sparse
        print(
            "With sparse mode, if your models has only small parameter prefetch may cause speed down"
        )
        self._attrs = {
            'num_total_classes': int(num_total_classes),
            'num_neg_samples': num_neg_samples,
            'seed': seed,
            'sampler': sampler,
            'is_sparse': is_sparse,
            'remote_prefetch': remote_prefetch
        }

1876
        self.weight = self.create_parameter(
1877 1878 1879
            attr=self._param_attr,
            shape=[self._num_total_classes, dim],
            is_bias=False,
1880
            dtype=self._dtype)
1881
        if self._bias_attr:
1882
            self.bias = self.create_parameter(
1883 1884 1885
                attr=self._bias_attr,
                shape=[self._num_total_classes, 1],
                is_bias=True,
1886
                dtype=self._dtype)
1887 1888
            self._inputs['Bias'] = self.bias
        self._inputs['Weight'] = self.weight
1889

1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918
    def forward(self, input, label, sample_weight=None):
        assert isinstance(input, Variable)
        assert isinstance(label, Variable)

        self._inputs['Input'] = input
        self._inputs['Label'] = label
        self._inputs['SampleWeight'] = sample_weight if sample_weight is not None else []

        cost = self._helper.create_variable_for_type_inference(
            dtype=input.dtype)
        sample_logits = self._helper.create_variable_for_type_inference(
            dtype=input.dtype)
        sample_labels = self._helper.create_variable_for_type_inference(
            dtype=label.dtype)

        self._helper.append_op(
            type='nce',
            inputs=self._inputs,
            outputs={
                'Cost': cost,
                'SampleLogits': sample_logits,
                'SampleLabels': sample_labels
            },
            attrs=self._attrs)
        return cost / (self._num_neg_samples + 1)


class PRelu(layers.Layer):
    """
1919 1920 1921 1922
    This interface is used to construct a callable object of the ``PRelu`` class.
    For more details, refer to code examples.
    It implements three activation methods of the ``PRelu`` activation function.

1923 1924 1925 1926 1927
    Equation:

    .. math::
        y = \max(0, x) + \\alpha * \min(0, x)

1928
    Parameters:
L
lujun 已提交
1929
        mode (str): The mode for weight sharing. It supports all, channel
1930 1931 1932
          and element. all: all elements share same weight
          channel:elements in a channel share same weight
          element:each element has a weight
S
songyouwei 已提交
1933 1934 1935
        channel (int, optional): The number of channels.
          This argument is required when mode is "channel".
          Default: None.
1936
        input_shape (list or tuple, optional): The shape of input.
S
songyouwei 已提交
1937 1938
          This argument is required when mode is "element".
          Default: None.
1939 1940
        param_attr(ParamAttr, optional): The parameter attribute for the learnable
          weight (alpha). Default: None.
1941
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
1942

1943 1944 1945
    Attribute:
        **weight** (Parameter): the learnable weights of this layer.
    
1946
    Returns:
1947
        None
1948 1949 1950 1951 1952

    Examples:

        .. code-block:: python

L
lujun 已提交
1953
          import paddle.fluid as fluid
1954
          from paddle.fluid.dygraph.base import to_variable
L
lujun 已提交
1955 1956 1957 1958
          import numpy as np

          inp_np = np.ones([5, 200, 100, 100]).astype('float32')
          with fluid.dygraph.guard():
1959
              inp_np = to_variable(inp_np)
S
songyouwei 已提交
1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970
              prelu0 = fluid.PRelu(
                 mode='all',
                 param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant(1.0)))
              dy_rlt0 = prelu0(inp_np)
              prelu1 = fluid.PRelu(
                 mode='channel',
                 channel=200,
                 param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant(1.0)))
              dy_rlt1 = prelu1(inp_np)
              prelu2 = fluid.PRelu(
                 mode='element',
1971
                 input_shape=inp_np.shape,
L
lujun 已提交
1972
                 param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant(1.0)))
S
songyouwei 已提交
1973
              dy_rlt2 = prelu2(inp_np)
L
lujun 已提交
1974

1975 1976
    """

S
songyouwei 已提交
1977 1978 1979 1980 1981
    def __init__(self,
                 mode,
                 channel=None,
                 input_shape=None,
                 param_attr=None,
1982
                 dtype='float32'):
1983 1984
        # need specify name_scope since snake-cased 'PRelu' is 'p_relu'
        super(PRelu, self).__init__(name_scope='prelu')
1985 1986
        self._mode = mode
        self._param_attr = param_attr
1987
        self._dtype = dtype
S
songyouwei 已提交
1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001
        if mode == 'all':
            self._alpha_shape = [1]
        elif mode == 'channel':
            assert isinstance(
                channel,
                int), "channel argument is required when mode is 'channel'."
            self._alpha_shape = [1, channel, 1, 1]
        elif mode == 'element':
            assert isinstance(input_shape, (
                list, tuple
            )), "input_shape argument is required when mode is 'element'."
            self._alpha_shape = [1] + list(input_shape)[1:]
        else:
            raise ValueError('mode should be one of all, channel, element.')
2002
        self.weight = self.create_parameter(
2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013
            attr=self._param_attr,
            shape=self._alpha_shape,
            dtype='float32',
            is_bias=False,
            default_initializer=Constant(1.0))

    def forward(self, input):
        out = self._helper.create_variable_for_type_inference(self._dtype)
        self._helper.append_op(
            type="prelu",
            inputs={"X": input,
2014
                    'Alpha': self.weight},
2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034
            attrs={"mode": self._mode},
            outputs={"Out": out})
        return out


class BilinearTensorProduct(layers.Layer):
    """
    **Add Bilinear Tensor Product Layer**

    This layer performs bilinear tensor product on two inputs.
    For example:

    .. math::
      out_{i} = x * W_{i} * {y^\mathrm{T}}, i=0,1,...,size-1

    In this formula:
     - :math:`x`: the first input contains M elements, shape is [batch_size, M].
     - :math:`y`: the second input contains N elements, shape is [batch_size, N].
     - :math:`W_{i}`: the i-th learned weight, shape is [M, N]
     - :math:`out_{i}`: the i-th element of out, shape is [batch_size, size].
D
DuYao 已提交
2035
     - :math:`y^\mathrm{T}`: the transpose of :math:`y`.
2036

2037
    Parameters:
2038 2039 2040 2041 2042
       input1_dim (int): The dimension of each first input.
       input2_dim (int): The dimension of each second input.
       output_dim (int): The dimension of output of this layer.
       name (str, optional): The default value is None. Normally there is no need for user
           to set this property. For more information, please refer to :ref:`api_guide_Name`. Default: None.
D
DuYao 已提交
2043 2044 2045 2046
       act (str, optional): Activation to be applied to the output of this layer. The default value is None.
       param_attr (ParamAttr, optional): The parameter attribute for the learnable w, parameters/weights of 
           this layer. The default value is None.
       bias_attr (ParamAttr, optional): The parameter attribute for the bias
2047
           of this layer. If it is set to False, no bias will be added to the output units.
D
DuYao 已提交
2048
           If it is set to None, the bias is initialized zero. The default value is None.
2049
       dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
2050

D
DuYao 已提交
2051 2052 2053 2054
    Attribute:
        **weight** (Parameter): the learnable weights of this layer.

        **bias** (Parameter): the learnable bias of this layer.
2055

2056 2057 2058 2059 2060 2061
    Returns:
       Variable: A 2-D Tensor of shape [batch_size, size].

    Examples:
       .. code-block:: python

2062 2063 2064 2065 2066 2067 2068
         import paddle.fluid as fluid
         import numpy

         with fluid.dygraph.guard():
             layer1 = numpy.random.random((5, 5)).astype('float32')
             layer2 = numpy.random.random((5, 4)).astype('float32')
             bilinearTensorProduct = fluid.dygraph.nn.BilinearTensorProduct(
2069
                    input1_dim=5, input2_dim=4, output_dim=1000)
2070 2071
             ret = bilinearTensorProduct(fluid.dygraph.base.to_variable(layer1),
                                fluid.dygraph.base.to_variable(layer2))
2072 2073 2074
    """

    def __init__(self,
2075 2076 2077
                 input1_dim,
                 input2_dim,
                 output_dim,
2078 2079 2080
                 name=None,
                 act=None,
                 param_attr=None,
2081 2082 2083
                 bias_attr=None,
                 dtype='float32'):
        super(BilinearTensorProduct, self).__init__()
2084 2085 2086 2087
        self._param_attr = param_attr
        self._bias_attr = bias_attr
        self._act = act
        self._name = name
2088 2089 2090
        self._input1_dim = input1_dim
        self._input2_dim = input2_dim
        self._output_dim = output_dim
2091
        self._inputs = dict()
2092
        self._dtype = dtype
2093

2094
        param_shape = [self._output_dim, self._input1_dim, self._input2_dim]
2095
        self.weight = self.create_parameter(
2096 2097 2098 2099
            attr=self._param_attr,
            shape=param_shape,
            dtype=self._dtype,
            is_bias=False)
2100
        bias_size = [1, self._output_dim]
2101
        self.bias = self.create_parameter(
2102 2103 2104 2105
            attr=self._bias_attr,
            shape=bias_size,
            dtype=self._dtype,
            is_bias=True)
2106 2107

    def forward(self, x, y):
2108 2109 2110
        self._inputs = {"X": x, "Y": y, "Weight": self.weight}
        if self.bias:
            self._inputs["Bias"] = self.bias
2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124
        if self._name is not None:
            out = self._helper.create_variable(
                name=".".join([self.full_name(), self._name]),
                dtype=self._dtype,
                persistable=False)
        else:
            out = self._helper.create_variable(
                dtype=self._dtype, persistable=False)
        self._helper.append_op(
            type="bilinear_tensor_product",
            inputs=self._inputs,
            outputs={"Out": out})

        # add activation
2125
        return self._helper.append_activation(out, act=self._act)
2126 2127 2128 2129


class Conv2DTranspose(layers.Layer):
    """
2130 2131
    This interface is used to construct a callable object of the ``Conv2DTranspose`` class.
    For more details, refer to code examples.
2132
    The convolution2D transpose layer calculates the output based on the input,
2133 2134 2135
    filter, and dilations, strides, paddings. Input and output
    are in NCHW format. Where N is batch size, C is the number of feature map,
    H is the height of the feature map, and W is the width of the feature map.
2136 2137
    Filter's shape is [MCHW] , where M is the number of input feature map,
    C is the number of output feature map, H is the height of the filter,
2138 2139
    and W is the width of the filter. If the groups is greater than 1,
    C will equal the number of input feature map divided by the groups.
2140 2141 2142
    If bias attribution and activation type are provided, bias is added to
    the output of the convolution, and the corresponding activation function
    is applied to the final result.
2143 2144
    The details of convolution transpose layer, please refer to the following explanation and references
    `conv2dtranspose <http://www.matthewzeiler.com/wp-content/uploads/2017/07/cvpr2010.pdf>`_ .
2145 2146 2147 2148 2149 2150 2151 2152 2153

    For each input :math:`X`, the equation is:

    .. math::

        Out = \sigma (W \\ast X + b)

    Where:

2154 2155
    * :math:`X`: Input value, a ``Tensor`` with NCHW format.
    * :math:`W`: Filter value, a ``Tensor`` with shape [MCHW] .
2156
    * :math:`\\ast`: Convolution operation.
2157
    * :math:`b`: Bias value, a 2-D ``Tensor`` with shape [M, 1].
2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181
    * :math:`\\sigma`: Activation function.
    * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.

    Example:

        - Input:

          Input shape: :math:`(N, C_{in}, H_{in}, W_{in})`

          Filter shape: :math:`(C_{in}, C_{out}, H_f, W_f)`

        - Output:

          Output shape: :math:`(N, C_{out}, H_{out}, W_{out})`

        Where

        .. math::

           H^\prime_{out} &= (H_{in} - 1) * strides[0] - 2 * paddings[0] + dilations[0] * (H_f - 1) + 1 \\\\
           W^\prime_{out} &= (W_{in} - 1) * strides[1] - 2 * paddings[1] + dilations[1] * (W_f - 1) + 1 \\\\
           H_{out} &\in [ H^\prime_{out}, H^\prime_{out} + strides[0] ) \\\\
           W_{out} &\in [ W^\prime_{out}, W^\prime_{out} + strides[1] )

2182
    Parameters:
2183
        num_channels(int): The number of channels in the input image.
2184
        num_filters(int): The number of the filter. It is as same as the output
2185
            feature map.
2186 2187 2188
        filter_size(int or tuple): The filter size. If filter_size is a tuple,
            it must contain two integers, (filter_size_H, filter_size_W).
            Otherwise, the filter will be a square.
2189
        output_size(int or tuple, optional): The output image size. If output size is a
2190 2191 2192
            tuple, it must contain two integers, (image_H, image_W). None if use
            filter_size, padding, and stride to calculate output_size.
            if output_size and filter_size are specified at the same time, They
L
lujun 已提交
2193
            should follow the formula above. Default: None.
2194
        padding(int or tuple, optional): The padding size. If padding is a tuple, it must
2195
            contain two integers, (padding_H, padding_W). Otherwise, the
2196 2197
            padding_H = padding_W = padding. Default: 0.
        stride(int or tuple, optional): The stride size. If stride is a tuple, it must
2198
            contain two integers, (stride_H, stride_W). Otherwise, the
2199 2200
            stride_H = stride_W = stride. Default: 1.
        dilation(int or tuple, optional): The dilation size. If dilation is a tuple, it must
2201
            contain two integers, (dilation_H, dilation_W). Otherwise, the
2202 2203
            dilation_H = dilation_W = dilation. Default: 1.
        groups(int, optional): The groups number of the Conv2d transpose layer. Inspired by
2204 2205 2206 2207
            grouped convolution in Alex Krizhevsky's Deep CNN paper, in which
            when group=2, the first half of the filters is only connected to the
            first half of the input channels, while the second half of the
            filters is only connected to the second half of the input channels.
2208 2209
            Default: 1.
        param_attr (ParamAttr, optional): The parameter attribute for learnable weights(Parameter)
2210 2211 2212
            of conv2d_transpose. If it is set to None or one attribute of ParamAttr, conv2d_transpose
            will create ParamAttr as param_attr. If the Initializer of the param_attr
            is not set, the parameter is initialized with Xavier. Default: None.
2213
        bias_attr (ParamAttr or bool, optional): The attribute for the bias of conv2d_transpose.
2214 2215 2216 2217
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, conv2d_transpose
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
            is not set, the bias is initialized zero. Default: None.
2218
        use_cudnn(bool, optional): Use cudnn kernel or not, it is valid only when the cudnn
2219
            library is installed. Default: True.
2220
        act (str, optional): Activation type, if it is set to None, activation is not appended.
2221
            Default: None.
2222
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
2223

2224 2225
    Attribute:
        **weight** (Parameter): the learnable weights of filters of this layer.
2226

2227
        **bias** (Parameter or None): the learnable bias of this layer.
2228

2229 2230
    Returns:
        None
2231 2232 2233 2234

    Examples:
       .. code-block:: python

2235
          import paddle.fluid as fluid
2236
          import numpy as np
2237 2238

          with fluid.dygraph.guard():
2239
              data = np.random.random((3, 32, 32, 5)).astype('float32')
2240
              conv2DTranspose = fluid.dygraph.nn.Conv2DTranspose(
2241
                    num_channels=32, num_filters=2, filter_size=3)
2242 2243
              ret = conv2DTranspose(fluid.dygraph.base.to_variable(data))

2244 2245 2246
    """

    def __init__(self,
2247
                 num_channels,
2248
                 num_filters,
2249
                 filter_size,
2250 2251 2252 2253 2254 2255 2256 2257
                 output_size=None,
                 padding=0,
                 stride=1,
                 dilation=1,
                 groups=None,
                 param_attr=None,
                 bias_attr=None,
                 use_cudnn=True,
2258 2259 2260
                 act=None,
                 dtype='float32'):
        super(Conv2DTranspose, self).__init__()
2261 2262 2263
        assert param_attr is not False, "param_attr should not be False in conv2d_transpose."
        self._param_attr = param_attr
        self._bias_attr = bias_attr
2264
        self._act = act
2265
        self._groups = groups
2266
        self._num_channels = num_channels
2267 2268 2269 2270 2271 2272 2273
        self._num_filters = num_filters
        self._use_cudnn = use_cudnn
        self._padding = padding
        self._stride = stride
        self._dilation = dilation
        self._filter_size = filter_size
        self._output_size = output_size
2274
        self._dtype = dtype
2275

2276 2277 2278
        if (self._num_channels == self._groups and
                self._num_filters == self._num_channels and
                not self._use_cudnn):
2279
            self._op_type = 'depthwise_conv2d_transpose'
2280 2281
        else:
            self._op_type = 'conv2d_transpose'
2282 2283 2284 2285 2286

        self._padding = utils.convert_to_list(self._padding, 2, 'padding')
        self._stride = utils.convert_to_list(self._stride, 2, 'stride')
        self._dilation = utils.convert_to_list(self._dilation, 2, 'dilation')

2287 2288
        self._filter_size = utils.convert_to_list(
            self._filter_size, 2, 'conv2d_transpose.filter_size')
2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299

        if self._output_size is None:
            self._output_size = []
        elif isinstance(self._output_size, list) or isinstance(
                self._output_size, int):
            self._output_size = utils.convert_to_list(self._output_size, 2,
                                                      'output_size')
        else:
            raise ValueError("output_size should be list or int")
        self._padding = utils.convert_to_list(self._padding, 2, 'padding')
        self._groups = 1 if self._groups is None else self._groups
2300
        filter_shape = [self._num_channels, self._num_filters // self._groups
2301 2302
                        ] + self._filter_size

2303
        self.weight = self.create_parameter(
2304
            dtype=self._dtype, shape=filter_shape, attr=self._param_attr)
2305

2306
        self.bias = self.create_parameter(
2307 2308 2309 2310 2311
            attr=self._bias_attr,
            shape=[self._num_filters],
            dtype=self._dtype,
            is_bias=True)

2312
    def forward(self, input):
2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331
        inputs = {'Input': [input], 'Filter': [self.weight]}
        attrs = {
            'output_size': self._output_size,
            'strides': self._stride,
            'paddings': self._padding,
            'dilations': self._dilation,
            'groups': self._groups,
            'use_cudnn': self._use_cudnn
        }

        if in_dygraph_mode():
            op = getattr(core.ops, self._op_type)
            outs = op(inputs, attrs)
            pre_bias = outs['Output'][0]
            pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias,
                                                            1)
            return dygraph_utils._append_activation_in_dygraph(
                pre_act, act=self._act)

2332 2333 2334 2335
        pre_bias = self._helper.create_variable_for_type_inference(
            dtype=input.dtype)
        self._helper.append_op(
            type=self._op_type,
2336
            inputs=inputs,
2337
            outputs={'Output': pre_bias},
2338
            attrs=attrs)
2339

2340
        if self.bias is not None:
2341 2342 2343 2344 2345
            pre_act = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
            self._helper.append_op(
                type='elementwise_add',
                inputs={'X': [pre_bias],
2346
                        'Y': [self.bias]},
2347 2348 2349 2350 2351 2352
                outputs={'Out': [pre_act]},
                attrs={'axis': 1})
        else:
            pre_act = pre_bias

        out = self._helper.append_activation(pre_act, act=self._act)
2353 2354 2355 2356 2357 2358 2359 2360 2361
        return out


class SequenceConv(layers.Layer):
    """
    This function creates the op for sequence_conv, using the inputs and
    other convolutional configurations for the filters and stride as given
    in the input parameters to the function.

2362
    Parameters:
L
lujun 已提交
2363
        name_scope(str): The name of this class.
2364
        num_filters (int): number of filters.
L
lujun 已提交
2365 2366 2367
        filter_size (int): the filter size (H and W). Default: 3.
        filter_stride (int): stride of the filter. Default: 1.
        padding (bool|None): if True, add paddings. Default: None
2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379
        bias_attr (ParamAttr|bool|None): The parameter attribute for the bias of sequence_conv.
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, sequence_conv
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
            is not set, the bias is initialized zero. Default: None.
        param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights
            of sequence_conv. If it is set to None or one attribute of ParamAttr, sequence_conv
            will create ParamAttr as param_attr. If the Initializer of the param_attr
            is not set, the parameter is initialized with Xavier. Default: None.
        act (str): Activation type, if it is set to None, activation is not appended.
            Default: None.

2380 2381 2382 2383
    Attributes:
        weight (Parameter): the learnable weights of filters of this layer.
        bias (Parameter|None): the learnable bias of this layer.

2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396
    Returns:
        Variable: output of sequence_conv
    """

    def __init__(self,
                 name_scope,
                 num_filters,
                 filter_size=3,
                 filter_stride=1,
                 padding=None,
                 bias_attr=None,
                 param_attr=None,
                 act=None):
L
lujun 已提交
2397
        assert not in_dygraph_mode(
2398
        ), "SequenceConv is not supported by dynamic graph mode yet!"
2399 2400 2401 2402 2403 2404 2405
        super(SequenceConv, self).__init__(name_scope)
        self._num_filters = num_filters
        self._filter_size = filter_size
        self._filter_stride = filter_stride
        self._padding = padding
        self._bias_attr = bias_attr
        self._param_attr = param_attr
2406
        self._act = act
2407

2408
    def _build_once(self, input):
2409 2410
        self._dtype = self._helper.input_dtype(input)
        filter_shape = [self._filter_size * input.shape[1], self._num_filters]
2411
        self.weight = self.create_parameter(
2412
            attr=self._param_attr, shape=filter_shape, dtype=self._dtype)
2413

2414
        self.bias = self.create_parameter(
2415 2416 2417 2418 2419
            attr=self._bias_attr,
            shape=[self._num_filters],
            dtype=self._dtype,
            is_bias=True)

2420 2421 2422 2423 2424 2425
    def forward(self, input):
        pre_bias = self._helper.create_variable_for_type_inference(self._dtype)
        self._helper.append_op(
            type='sequence_conv',
            inputs={
                'X': [input],
2426
                'Filter': [self.weight],
2427 2428 2429 2430 2431 2432 2433
            },
            outputs={"Out": pre_bias},
            attrs={
                'contextStride': self._filter_stride,
                'contextStart': -int(self._filter_size // 2),
                'contextLength': self._filter_size
            })
2434

2435
        if self.bias is not None:
2436 2437 2438 2439 2440
            pre_act = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
            self._helper.append_op(
                type='elementwise_add',
                inputs={'X': [pre_bias],
2441
                        'Y': [self.bias]},
2442 2443 2444 2445 2446 2447
                outputs={'Out': [pre_act]},
                attrs={'axis': 1})
        else:
            pre_act = pre_bias

        return self._helper.append_activation(pre_act, act=self._act)
L
lujun 已提交
2448 2449 2450


class RowConv(layers.Layer):
2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468
    """
    ***Row-convolution operator***

    The row convolution is called lookahead convolution.  This operator was introduced in the following paper for DeepSpeech2:
    http://www.cs.cmu.edu/~dyogatam/papers/wang+etal.iclrworkshop2016.pdf

    The main motivation is that a bidirectional RNN, useful in DeepSpeech like speech models, learns representation for a sequence by performing a
    forward and a backward pass through the entire sequence. However, unlike
    unidirectional RNNs, bidirectional RNNs are challenging to deploy in an online
    and low-latency setting. The lookahead convolution incorporates information
    from future subsequences in a computationally efficient manner to improve
    unidirectional recurrent neural networks. The row convolution operator is
    different from the 1D sequence convolution, and is computed as follows:

    Given an input sequence X of length t and input dimension D, and a filter (W) of size context * D.

    More details about row_conv please refer to the design document https://github.com/PaddlePaddle/Paddle/issues/2228#issuecomment-303903645 .

2469
    Parameters:
L
lujun 已提交
2470
        name_scope(str): The name of this class.
2471 2472 2473
        future_context_size (int): Future context size. Please note, the shape
            of convolution kernel is [future_context_size + 1, D].
        param_attr (ParamAttr): Attributes of parameters, including
L
lujun 已提交
2474 2475
            name, initializer etc. Default: None.
        act (str): Non-linear activation to be applied to output variable. Default: None.
2476

2477 2478 2479
    Attributes:
        weight (Parameter): the learnable weights of this layer.

2480
    Returns:
L
lujun 已提交
2481 2482
        the output(Out) is a LodTensor, which supports variable time-length input sequences.
        The underlying tensor in this LodTensor is a matrix with shape T x N, i.e., the same shape as X.
2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid
          import numpy

          with fluid.dygraph.guard():
              x = numpy.random.random((16)).astype('float32')
              rowConv = fluid.dygraph.nn.RowConv(
                    'RowConv', future_context_size=2)
              ret = rowConv(fluid.dygraph.base.to_variable(x))

    """

L
lujun 已提交
2498 2499 2500 2501 2502
    def __init__(self,
                 name_scope,
                 future_context_size,
                 param_attr=None,
                 act=None):
L
lujun 已提交
2503
        assert not in_dygraph_mode(
2504
        ), "RowConv is not supported by dynamic graph mode yet!"
L
lujun 已提交
2505 2506 2507 2508 2509
        super(RowConv, self).__init__(name_scope)
        self._act = act
        self._param_attr = param_attr
        self._future_context_size = future_context_size

2510
    def _build_once(self, input):
L
lujun 已提交
2511 2512
        self._dtype = self._helper.input_dtype(input)
        filter_shape = [self._future_context_size + 1, input.shape[1]]
2513
        self.weight = self.create_parameter(
2514 2515 2516 2517
            attr=self._param_attr,
            shape=filter_shape,
            dtype=self._dtype,
            is_bias=False)
L
lujun 已提交
2518 2519 2520 2521 2522 2523

    def forward(self, input):
        out = self._helper.create_variable_for_type_inference(self._dtype)
        self._helper.append_op(
            type='row_conv',
            inputs={'X': [input],
2524
                    'Filter': [self.weight]},
L
lujun 已提交
2525 2526 2527 2528 2529 2530
            outputs={'Out': [out]})
        return self._helper.append_activation(out, act=self._act)


class GroupNorm(layers.Layer):
    """
2531 2532 2533 2534 2535 2536
    This interface is used to construct a callable object of the ``GroupNorm`` class.
    For more details, refer to code examples.
    It implements the function of the Group Normalization Layer.
    Refer to `Group Normalization <https://arxiv.org/abs/1803.08494>`_ .

    Parameters:
2537
        channels(int): The number of channels of input.
2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560
        groups(int): The number of groups that divided from channels.
        epsilon(float, optional): The small value added to the variance to prevent
                                  division by zero. Default: 1e-05.
        param_attr(ParamAttr, optional): The parameter attribute for the learnable
                                         scale :math:`g`. If it is set to False, no scale will be added to the output units.
                                         If it is set to None, the bias is initialized one. Default: None.
        bias_attr(ParamAttr, optional): The parameter attribute for the learnable
                                        bias :math:`b`. If it is set to False, no bias will be added to the output units.
                                        If it is set to None, the bias is initialized zero. Default: None.
        act(str, optional): Activation to be applied to the output of group normalizaiton. Default: None.
        data_layout(str, optional): Specify the input data format. Only NCHW is supported. Default: NCHW.

    Returns:
        None

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid
          import numpy as np

          with fluid.dygraph.guard():
              x = np.random.random((8, 32, 32)).astype('float32')
2561
              groupNorm = fluid.dygraph.nn.GroupNorm(channels=32, groups=4)
2562
              ret = groupNorm(fluid.dygraph.base.to_variable(x))
L
lujun 已提交
2563 2564 2565 2566

    """

    def __init__(self,
2567
                 channels,
L
lujun 已提交
2568 2569 2570 2571 2572
                 groups,
                 epsilon=1e-05,
                 param_attr=None,
                 bias_attr=None,
                 act=None,
2573 2574 2575
                 data_layout='NCHW',
                 dtype='float32'):
        super(GroupNorm, self).__init__()
L
lujun 已提交
2576 2577 2578
        self._param_attr = param_attr
        self._bias_attr = bias_attr
        self._epsilon = epsilon
2579
        self._channels = channels
L
lujun 已提交
2580 2581
        self._groups = groups
        self._act = act
2582
        self._dtype = dtype
L
lujun 已提交
2583 2584 2585
        if data_layout != 'NCHW':
            raise ValueError("unsupported data layout:" + data_layout)

2586
        param_shape = [self._channels]
L
lujun 已提交
2587

2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598
        self.weight = self.create_parameter(
            attr=self._param_attr or False,
            shape=param_shape,
            dtype=self._dtype,
            default_initializer=Constant(1.0))

        self.bias = self.create_parameter(
            attr=self._bias_attr or False,
            shape=param_shape,
            dtype=self._dtype,
            is_bias=True)
L
lujun 已提交
2599 2600 2601

    def forward(self, input):
        inputs = {'X': input}
2602 2603 2604 2605
        if self.bias:
            inputs['Bias'] = self.bias
        if self.weight:
            inputs['Scale'] = self.weight
L
lujun 已提交
2606 2607

        # create output
2608
        mean_out = self._helper.create_variable_for_type_inference(
L
lujun 已提交
2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629
            dtype=self._dtype, stop_gradient=True)
        variance_out = self._helper.create_variable_for_type_inference(
            dtype=self._dtype, stop_gradient=True)
        group_norm_out = self._helper.create_variable_for_type_inference(
            dtype=self._dtype)

        self._helper.append_op(
            type="group_norm",
            inputs=inputs,
            outputs={
                "Y": group_norm_out,
                "Mean": mean_out,
                "Variance": variance_out,
            },
            attrs={"epsilon": self._epsilon,
                   "groups": self._groups})

        return self._helper.append_activation(group_norm_out, self._act)


class SpectralNorm(layers.Layer):
2630
    """
2631 2632
    This interface is used to construct a callable object of the ``SpectralNorm`` class.
    For more details, refer to code examples. It implements the function of the Spectral Normalization Layer.
2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663
    This layer calculates the spectral normalization value of weight parameters of
    fc, conv1d, conv2d, conv3d layers which should be 2-D, 3-D, 4-D, 5-D
    Parameters. Calculations are showed as follows.

    Step 1:
    Generate vector U in shape of [H], and V in shape of [W].
    While H is the :attr:`dim` th dimension of the input weights,
    and W is the product result of remaining dimensions.

    Step 2:
    :attr:`power_iters` shoule be a positive interger, do following
    calculations with U and V for :attr:`power_iters` rounds.

    .. math::

        \mathbf{v} := \\frac{\mathbf{W}^{T} \mathbf{u}}{\|\mathbf{W}^{T} \mathbf{u}\|_2}

        \mathbf{u} := \\frac{\mathbf{W}^{T} \mathbf{v}}{\|\mathbf{W}^{T} \mathbf{v}\|_2}

    Step 3:
    Calculate :math:`\sigma(\mathbf{W})` and normalize weight values.

    .. math::

        \sigma(\mathbf{W}) = \mathbf{u}^{T} \mathbf{W} \mathbf{v}

        \mathbf{W} = \\frac{\mathbf{W}}{\sigma(\mathbf{W})}


    Refer to `Spectral Normalization <https://arxiv.org/abs/1802.05957>`_ .

2664
    Parameters:
2665
        weight_shape(list or tuple): The shape of weight parameter.
2666 2667 2668 2669
        dim(int, optional): The index of dimension which should be permuted to the first before reshaping Input(Weight) to matrix, it should be set as 0 if Input(Weight) is the weight of fc layer, and should be set as 1 if Input(Weight) is the weight of conv layer. Default: 0.
        power_iters(int, optional): The number of power iterations to calculate spectral norm. Default: 1.
        eps(float, optional): The epsilon for numerical stability in calculating norms. Default: 1e-12.
        name (str, optional): The default value is None.  Normally there is no need for user to set this property.  For more information, please refer to :ref:`api_guide_Name` .
2670
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
2671 2672

    Returns:
2673
        None
2674 2675 2676 2677 2678

    Examples:
       .. code-block:: python

            import paddle.fluid as fluid
2679
            import numpy as np
2680 2681

            with fluid.dygraph.guard():
2682 2683 2684
                weight = np.random.random((2, 8, 32, 32)).astype('float32')
                spectralNorm = fluid.dygraph.nn.SpectralNorm(weight.shape, dim=1, power_iters=2)
                ret = spectralNorm(fluid.dygraph.base.to_variable(weight))
2685 2686 2687

    """

2688 2689 2690 2691 2692 2693 2694
    def __init__(self,
                 weight_shape,
                 dim=0,
                 power_iters=1,
                 eps=1e-12,
                 dtype='float32'):
        super(SpectralNorm, self).__init__()
L
lujun 已提交
2695 2696 2697
        self._power_iters = power_iters
        self._eps = eps
        self._dim = dim
2698
        self._dtype = dtype
L
lujun 已提交
2699

2700 2701 2702
        self._weight_shape = list(weight_shape)
        h = self._weight_shape[self._dim]
        w = np.prod(self._weight_shape) // h
L
lujun 已提交
2703

2704
        self.weight_u = self.create_parameter(
L
lujun 已提交
2705 2706 2707 2708
            attr=ParamAttr(),
            shape=[h],
            dtype=self._dtype,
            default_initializer=Normal(0., 1.))
2709
        self.weight_u.stop_gradient = True
L
lujun 已提交
2710

2711
        self.weight_v = self.create_parameter(
L
lujun 已提交
2712 2713 2714 2715
            attr=ParamAttr(),
            shape=[w],
            dtype=self._dtype,
            default_initializer=Normal(0., 1.))
2716
        self.weight_v.stop_gradient = True
L
lujun 已提交
2717 2718

    def forward(self, weight):
2719
        inputs = {'Weight': weight, 'U': self.weight_u, 'V': self.weight_v}
L
lujun 已提交
2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734
        out = self._helper.create_variable_for_type_inference(self._dtype)
        self._helper.append_op(
            type="spectral_norm",
            inputs=inputs,
            outputs={"Out": out, },
            attrs={
                "dim": self._dim,
                "power_iters": self._power_iters,
                "eps": self._eps,
            })

        return out


class TreeConv(layers.Layer):
2735
    """
2736 2737 2738 2739 2740 2741 2742 2743 2744 2745
    This interface is used to construct a callable object of the ``TreeConv`` class.
    For more details, refer to code examples.
    Tree-Based Convolution is a kind of convolution based on tree structure.
    Tree-Based Convolution is a part of Tree-Based Convolution Neural Network(TBCNN),
    which is used to classify tree structures, such as Abstract Syntax Tree.
    Tree-Based Convolution proposed a kind of data structure called continuous binary tree,
    which regards multiway tree as binary tree.
    The paper of Tree-Based Convolution Operator is here: `tree-based convolution <https://arxiv.org/abs/1409.5718v1/>`_ .
    
    Parameters:
2746
        feature_size(int): last dimension of nodes_vector.
2747 2748 2749 2750 2751 2752 2753
        output_size(int): output feature width.
        num_filters(int, optional): number of filters, Default: 1.
        max_depth(int, optional): max depth of filters, Default: 2.
        act(str, optional): activation function, Default: tanh.
        param_attr(ParamAttr, optional): the parameter attribute for the filters, Default: None.
        bias_attr(ParamAttr, optional): the parameter attribute for the bias of this layer, Default: None.
        name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` .
2754
        dtype (str, optional): Data type, it can be "float32" or "float64". Default: "float32".
2755

2756 2757
    Attribute:
        **weight** (Parameter): the learnable weights of filters of this layer.
2758

2759
        **bias** (Parameter or None): the learnable bias of this layer.
2760

2761 2762
    Returns:
        None
L
lujun 已提交
2763

2764
    Examples:
L
lujun 已提交
2765

2766
        .. code-block:: python
2767

2768 2769
          import paddle.fluid as fluid
          import numpy
2770

2771 2772 2773 2774
          with fluid.dygraph.guard():
              nodes_vector = numpy.random.random((1, 10, 5)).astype('float32')
              edge_set = numpy.random.random((1, 9, 2)).astype('int32')
              treeConv = fluid.dygraph.nn.TreeConv(
2775
                feature_size=5, output_size=6, num_filters=1, max_depth=2)
2776
              ret = treeConv(fluid.dygraph.base.to_variable(nodes_vector), fluid.dygraph.base.to_variable(edge_set))
2777 2778
    """

L
lujun 已提交
2779
    def __init__(self,
2780
                 feature_size,
L
lujun 已提交
2781 2782 2783 2784 2785 2786
                 output_size,
                 num_filters=1,
                 max_depth=2,
                 act='tanh',
                 param_attr=None,
                 bias_attr=None,
2787 2788 2789
                 name=None,
                 dtype='float32'):
        super(TreeConv, self).__init__()
L
lujun 已提交
2790
        self._name = name
2791
        self._feature_size = feature_size
L
lujun 已提交
2792 2793 2794 2795 2796 2797
        self._output_size = output_size
        self._act = act
        self._max_depth = max_depth
        self._num_filters = num_filters
        self._bias_attr = bias_attr
        self._param_attr = param_attr
2798 2799
        self._dtype = dtype
        w_shape = [self._feature_size, 3, self._output_size, self._num_filters]
L
lujun 已提交
2800
        if self._bias_attr:
2801
            self.bias = self.create_parameter(
L
lujun 已提交
2802 2803 2804 2805
                attr=self._bias_attr,
                shape=[self._num_filters],
                dtype=self._dtype,
                is_bias=True)
2806
        self.weight = self.create_parameter(
L
lujun 已提交
2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823
            attr=self._param_attr,
            shape=w_shape,
            dtype=self._dtype,
            is_bias=False)

    def forward(self, nodes_vector, edge_set):
        if self._name:
            out = self.create_variable(
                name=self._name, dtype=self._dtype, persistable=False)
        else:
            out = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
        self._helper.append_op(
            type='tree_conv',
            inputs={
                'NodesVector': nodes_vector,
                'EdgeSet': edge_set,
2824
                'Filter': self.weight
L
lujun 已提交
2825 2826 2827 2828 2829 2830 2831 2832 2833
            },
            outputs={'Out': out, },
            attrs={'max_depth': self._max_depth})
        if self._bias_attr:
            pre_activation = self._helper.create_variable_for_type_inference(
                dtype=self._dtype)
            self._helper.append_op(
                type='elementwise_add',
                inputs={'X': [out],
2834
                        'Y': [self.bias]},
L
lujun 已提交
2835 2836 2837 2838 2839
                outputs={'Out': [pre_activation]},
                attrs={'axis': 1})
        else:
            pre_activation = out
        return self._helper.append_activation(pre_activation, act=self._act)