nn.py 153.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Y
Yu Yang 已提交
14
"""
15
All layers just related to the neural network.
Y
Yu Yang 已提交
16
"""
P
peizhilin 已提交
17
import os
S
sneaxiy 已提交
18
import inspect
19 20 21 22 23
import warnings

import numpy as np

import paddle
Y
Yu Yang 已提交
24
from ..layer_helper import LayerHelper
25
from paddle.fluid.framework import _in_legacy_dygraph
26
from ..initializer import Normal, Constant
27 28 29 30 31 32 33 34 35 36 37 38 39
from ..framework import (
    Variable,
    OpProtoHolder,
    _non_static_mode,
    dygraph_only,
    _dygraph_tracer,
    default_main_program,
    _varbase_creator,
    static_only,
    _global_flags,
    _in_legacy_dygraph,
    in_dygraph_mode,
)
40
from ..framework import _current_expected_place
41
from .. import dygraph_utils
Y
yangyaming 已提交
42
from ..param_attr import ParamAttr
43 44 45 46 47
from .layer_function_generator import (
    autodoc,
    templatedoc,
    _generate_doc_string_,
)
48
from .tensor import concat, assign, fill_constant, zeros, tensor_array_to_tensor
49
from . import utils
F
fengjiayi 已提交
50
from .. import unique_name
51
from functools import reduce
52
from .. import core
53
from ...utils import deprecated
54 55 56 57 58 59
from ..data_feeder import (
    convert_dtype,
    check_variable_and_dtype,
    check_type,
    check_dtype,
)
60
from paddle.utils import deprecated
61
from paddle import _C_ops, _legacy_C_ops
62 63
from collections.abc import Iterable

Y
Yu Yang 已提交
64 65

__all__ = [
X
Xin Pan 已提交
66 67 68 69 70 71 72 73 74 75 76
    'fc',
    'embedding',
    'linear_chain_crf',
    'crf_decoding',
    'conv2d',
    'pool2d',
    'dropout',
    'split',
    'l2_normalize',
    'row_conv',
    'layer_norm',
D
dengkaipeng 已提交
77
    'spectral_norm',
X
Xin Pan 已提交
78 79 80 81 82 83 84 85 86 87 88 89 90
    'one_hot',
    'autoincreased_step_counter',
    'unsqueeze',
    'lod_reset',
    'relu',
    'elementwise_add',
    'elementwise_div',
    'elementwise_sub',
    'elementwise_mul',
    'clip',
    'clip_by_norm',
    'mean',
    'mul',
C
chengduo 已提交
91 92
    'merge_selected_rows',
    'get_tensor_from_selected_rows',
93
    'unfold',
C
cjt222 已提交
94
    'deformable_roi_pooling',
95
    'shard_index',
H
huangjun12 已提交
96
    'hard_swish',
K
Kaipeng Deng 已提交
97
    'mish',
98
    'uniform_random',
myq406450149's avatar
myq406450149 已提交
99
    'unbind',
Y
Yu Yang 已提交
100 101
]

102
OP_NAMEMAPPING = {
103 104 105 106 107 108 109 110
    'elementwise_max': 'maximum',
    'elementwise_min': 'minimum',
    'elementwise_pow': 'elementwise_pow',
    'elementwise_floordiv': 'floor_divide',
    'elementwise_add': 'add',
    'elementwise_sub': 'subtract',
    'elementwise_mul': 'multiply',
    'elementwise_div': 'divide',
C
Chen Weihang 已提交
111
    'elementwise_mod': 'remainder',
112 113
}

Y
Yu Yang 已提交
114

115 116
def _get_reduce_dim(dim, input):
    """
117
    Internal function for reduce_sum, reduce_mean, reduce_prod.
118 119 120 121 122 123 124 125 126
    It computes the attribute reduce_all value based on axis.
    """
    if dim is not None and not isinstance(dim, list):
        if isinstance(dim, (tuple, range)):
            dim = list(dim)
        elif isinstance(dim, int):
            dim = [dim]
        else:
            raise TypeError(
127
                "The type of dim must be int, list, tuple or range, but received {}".format(
128
                    type(dim)
129 130
                )
            )
131 132 133 134 135 136 137 138 139 140
    if dim is None:
        dim = []
    if dim == [] or len(dim) == len(input.shape):
        reduce_all = True
    else:
        reduce_all = False

    return reduce_all, dim


141
@dygraph_only
142 143 144
def _elementwise_op_in_dygraph(
    x, y, axis=-1, act=None, use_mkldnn=False, op_name=None
):
145 146 147 148
    def is_inplace(op_name):
        return op_name[-1] == "_"

    if op_name not in OP_NAMEMAPPING.keys() or axis != -1:
149
        op = getattr(_legacy_C_ops, op_name)
150 151 152
        out = op(x, y, 'axis', axis, 'use_mkldnn', use_mkldnn)
    else:
        if in_dygraph_mode():
153 154
            op = getattr(
                _C_ops,
155 156
                OP_NAMEMAPPING[op_name] if not is_inplace(op_name) else op_name,
            )
157 158 159
            out = op(x, y)

        if _in_legacy_dygraph():
160
            op = getattr(_legacy_C_ops, op_name)
161
            out = op(x, y, 'axis', axis, 'use_mkldnn', use_mkldnn)
162 163 164 165 166 167 168 169 170 171 172 173 174 175
    return dygraph_utils._append_activation_in_dygraph(
        out, act, use_mkldnn=use_mkldnn
    )


def fc(
    input,
    size,
    num_flatten_dims=1,
    param_attr=None,
    bias_attr=None,
    act=None,
    name=None,
):
176
    r"""
177 178
    :api_attr: Static Graph

179
    **Fully Connected Layer**
Y
Yu Yang 已提交
180

181 182 183
    This operator creates a fully connected layer in the network. It can take
    a Tensor(or LoDTensor) or a list of Tensor(or LoDTensor) as its inputs(see
    Args in detail). It creates a variable called weight for each input Tensor,
184
    which represents a fully connected weight matrix from each input unit to
185 186 187 188
    each output unit. The fully connected layer multiplies each input Tensor
    with its corresponding weight to produce an output Tensor with shape :math:`[M, size]` ,
    where M is batch size. If a list of Tensor is given, the results of
    multiple output Tensors with shape :math:`[M, size]` will be summed up. If :attr:`bias_attr`
189
    is not None, a bias variable will be created and added to the output.
190
    Finally, if :attr:`act` is not None, it will be applied to the output as well.
C
caoying03 已提交
191

192
    When the input is a single Tensor(or LoDTensor):
C
caoying03 已提交
193

194 195 196 197
    .. math::

        Out = Act({XW + b})

198
    When the input is a list of Tensor(or LoDTensor):
199 200 201

    .. math::

202
        Out = Act({\sum_{i=0}^{N-1}X_iW_i + b})
203 204 205

    In the above equation:

206 207 208
    * :math:`N`: Number of the input. N equals to len(input) if input is list of Variable.
    * :math:`X_i`: The i-th input tensor.
    * :math:`W_i`: The i-th weights matrix corresponding i-th input tensor.
C
caoying03 已提交
209
    * :math:`b`: The bias parameter created by this layer (if needed).
210
    * :math:`Act`: The activation function.
211
    * :math:`Out`: The output Tensor.
212 213 214

    .. code-block:: text

215 216 217 218 219 220 221 222 223 224 225 226 227 228
        Case 1:
        Given a single Tensor data_1, and num_flatten_dims = 2:
            data_1.data = [[[0.1, 0.2],
                            [0.3, 0.4]]]
            data_1.shape = (1, 2, 2) # 1 is batch_size

            out = fluid.layers.fc(input=data_1, size=1, num_flatten_dims=2)

        Then output is:
            out.data = [[0.83234344], [0.34936576]]
            out.shape = (1, 2, 1)

        Case 2:
        Given a list of Tensor:
229 230 231 232 233 234 235 236 237 238 239 240 241
            data_1.data = [[[0.1, 0.2],
                           [0.3, 0.4]]]
            data_1.shape = (1, 2, 2) # 1 is batch_size

            data_2 = [[[0.1, 0.2, 0.3]]]
            data_2.shape = (1, 1, 3)

            out = fluid.layers.fc(input=[data_1, data_2], size=2)

        Then:
            out.data = [[0.18669507, 0.1893476]]
            out.shape = (1, 2)

Y
Yu Yang 已提交
242
    Args:
243 244 245
        input (Variable|list of Variable): A Tensor(or LoDTensor) with shape :math:`[N_1, N_2,..., N_k]` or
            a list of Tensor(or LoDTensor). The dimensions of the input Tensor is at least 2 and the data
            type should be float32 or float64.
T
tianshuo78520a 已提交
246
        size(int): The number of output units in this layer, which also means the feature size of output
247 248
            Tensor(or LoDTensor).
        num_flatten_dims (int): The fc layer can accept an input Tensor with more than
R
ranqiu 已提交
249
            two dimensions. If this happens, the multidimensional tensor will first be flattened
250 251
            into a 2-D matrix. The parameter :attr:`num_flatten_dims` determines how the input
            Tensor is flattened: the first :attr:`num_flatten_dims` (inclusive, index starts from 1)
R
ranqiu 已提交
252
            dimensions will be flatten to form the first dimension of the final matrix (height of
253 254 255 256 257 258 259 260 261 262 263 264 265 266 267
            the matrix), and the rest :math:`rank(X) - num\_flatten\_dims` dimensions are flattened to
            form the second dimension of the final matrix (width of the matrix). For example, assuming that
            X is a 5-dimensional Tensor with a shape [2, 3, 4, 5, 6], and :attr:`num_flatten_dims` = 3.
            Then, the flattened matrix will have a shape [2 x 3 x 4, 5 x 6] = [24, 30]. Default: 1.
        param_attr (ParamAttr): To specify the weight parameter property. Default: None, which means the
            default weight parameter property is used. See usage for details in :ref:`api_fluid_ParamAttr` .
        bias_attr (ParamAttr): To specify the bias parameter property. Default: None, which means the
            default bias parameter property is used. See usage for details in :ref:`api_fluid_ParamAttr` .
        act (str): Activation to be applied to the output of this layer, such as tanh, softmax,
            sigmoid, relu. For more information, please refer to :ref:`api_guide_activations_en` . Default: None.
        name (str, optional): The default value is None.  Normally there is no need for user to set this property.
            For more information, please refer to :ref:`api_guide_Name` .

    Returns:
        Variable: Tensor or LoDTensor calculated by fc layer. The data type is same with input.
268 269

    Raises:
270
        ValueError: If dimensions of the input Tensor is less than 2.
271 272 273 274

    Examples:
        .. code-block:: python

275
          import paddle.fluid as fluid
276 277
          import paddle
          paddle.enable_static()
278
          # when input is single tensor
279
          data = fluid.data(name="data", shape=[-1, 32], dtype="float32")
280
          fc = fluid.layers.fc(input=data, size=1000, act="tanh")
281 282

          # when input are multiple tensors
283 284
          data_1 = fluid.data(name="data_1", shape=[-1, 32], dtype="float32")
          data_2 = fluid.data(name="data_2", shape=[-1, 36], dtype="float32")
285
          fc = fluid.layers.fc(input=[data_1, data_2], size=1000, act="tanh")
Y
Yu Yang 已提交
286
    """
C
caoying03 已提交
287
    helper = LayerHelper("fc", **locals())
288
    check_type(input, 'input', (list, tuple, Variable), 'fc')
289 290
    if isinstance(input, (list, tuple)):
        for i, input_x in enumerate(input):
291
            check_type(input_x, 'input[' + str(i) + ']', Variable, 'fc')
Y
Yu Yang 已提交
292
    dtype = helper.input_dtype()
293 294 295
    check_dtype(
        dtype, 'input', ['float16', 'uint16', 'float32', 'float64'], 'fc'
    )
Y
Yu Yang 已提交
296
    mul_results = []
297 298
    for input_var, param_attr in helper.iter_inputs_and_params():
        input_shape = input_var.shape
299 300
        if num_flatten_dims == -1:
            num_flatten_dims = len(input_shape) - 1
Y
Yu Yang 已提交
301 302 303
        param_shape = [
            reduce(lambda a, b: a * b, input_shape[num_flatten_dims:], 1)
        ] + [size]
Y
ying 已提交
304

305 306 307
        w = helper.create_parameter(
            attr=param_attr, shape=param_shape, dtype=dtype, is_bias=False
        )
X
Xin Pan 已提交
308
        tmp = helper.create_variable_for_type_inference(dtype)
309 310 311 312 313 314
        helper.append_op(
            type="mul",
            inputs={"X": input_var, "Y": w},
            outputs={"Out": tmp},
            attrs={"x_num_col_dims": num_flatten_dims, "y_num_col_dims": 1},
        )
315 316 317 318
        mul_results.append(tmp)

    if len(mul_results) == 1:
        pre_bias = mul_results[0]
319
    else:
X
Xin Pan 已提交
320
        pre_bias = helper.create_variable_for_type_inference(dtype)
321 322 323 324 325 326
        helper.append_op(
            type="sum",
            inputs={"X": mul_results},
            outputs={"Out": pre_bias},
            attrs={"use_mkldnn": False},
        )
327 328 329 330
    # add bias
    pre_activation = helper.append_bias_op(pre_bias, dim_start=num_flatten_dims)
    # add activation
    return helper.append_activation(pre_activation)
Y
Yu Yang 已提交
331 332


T
tangwei12 已提交
333
@deprecated(since="2.0.0", update_to="paddle.nn.functional.embedding")
334 335 336 337 338 339 340 341 342
def embedding(
    input,
    size,
    is_sparse=False,
    is_distributed=False,
    padding_idx=None,
    param_attr=None,
    dtype='float32',
):
343
    r"""
344
    :api_attr: Static Graph
345

346 347 348 349 350 351 352 353 354 355 356 357
    **WARING:** This OP will be deprecated in a future release. This OP requires the
    last dimension of Tensor shape must be equal to 1. It is recommended to use
    fluid. :ref:`api_fluid_embedding` .

    The operator is used to lookup embeddings vector of ids provided by :attr:`input` .
    It automatically constructs a 2D embedding matrix based on the
    input :attr:`size` (vocab_size, emb_size) and :attr:`dtype` .

    This OP requires the last dimension of Tensor shape must be equal to 1. The shape
    of output Tensor is generated by replacing the last dimension of the input Tensor shape
    with emb_size.

358
    **Note:** The id in :attr:`input` must satisfy :math:`0 =< id < size[0]` ,
359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
    otherwise the program will throw an exception and exit.

    .. code-block:: text

        Case 1:

        input is a Tensor. padding_idx = -1
            input.data = [[[1], [3]], [[2], [4]], [[4], [127]]]
            input.shape = [3, 2, 1]
        Given size = [128, 16]
        output is a Tensor:
            out.shape = [3, 2, 16]
            out.data = [[[0.129435295, 0.244512452, ..., 0.436322452],
                        [0.345421456, 0.524563927, ..., 0.144534654]],

                        [[0.345249859, 0.124939536, ..., 0.194353745],
                        [0.945345345, 0.435394634, ..., 0.435345365]],
376

377 378 379 380
                        [[0.945345345, 0.435394634, ..., 0.435345365],
                        [0.0,         0.0,         ..., 0.0        ]]]  # padding data
        The input padding_idx is less than 0, it is automatically converted to padding_idx = -1 + 128 = 127
        It will pad all-zero data when ids is 127.
381

382
        Case 2:
383

384 385 386 387 388 389 390 391 392 393 394 395 396 397
        input is a LoDTensor with 1-level LoD. padding_idx = 0
            input.lod = [[2, 3]]
            input.data = [[1], [3], [2], [4], [0]]
            input.shape = [5, 1]
        Given size = [128, 16]
        output is a LoDTensor:
            out.lod = [[2, 3]]
            out.shape = [5, 16]
            out.data = [[0.129435295, 0.244512452, ..., 0.436322452],
                        [0.345421456, 0.524563927, ..., 0.144534654],
                        [0.345249859, 0.124939536, ..., 0.194353745],
                        [0.945345345, 0.435394634, ..., 0.435345365],
                        [0.0,         0.0,         ..., 0.0        ]]  # padding data
        It will pad all-zero data when ids is 0.
Y
Yu Yang 已提交
398 399

    Args:
400 401 402 403 404 405
        input(Variable): A Tensor or LoDTensor with type int64, which contains the id information.
            The last dimension of Tensor shape must be equal to 1. The value of the input id should
            satisfy :math:`0<= id < size[0]` .
        size(tuple|list): The shape of lookup table parameter. It should have two elements which
            indicates the size of the dictionary of embeddings and the size of each embedding vector respectively.
        is_sparse(bool): The flag indicating whether to use sparse update. This parameter only
406
            affects the performance of the backwards gradient update. It is recommended to set
407
            True because sparse update is faster. But some optimizer does not support sparse update,
408
            such as :ref:`api_fluid_optimizer_AdadeltaOptimizer` , :ref:`api_fluid_optimizer_AdamaxOptimizer` ,
409 410 411 412 413
            :ref:`api_fluid_optimizer_DecayedAdagradOptimizer` , :ref:`api_fluid_optimizer_FtrlOptimizer` ,
            :ref:`api_fluid_optimizer_LambOptimizer` and :ref:`api_fluid_optimizer_LarsMomentumOptimizer` .
            In these case, is_sparse must be False. Default: False.
        is_distributed(bool): Whether to store the embedding matrix in a distributed manner. Only used
            in multi-machine distributed CPU training. Default: False.
414
        padding_idx(int|long|None): padding_idx needs to be in the interval [-vocab_size, vocab_size).
415 416 417 418 419 420
            If :math:`padding\_idx < 0`, the :math:`padding\_idx` will automatically be converted
            to :math:`vocab\_size + padding\_idx` . It will output all-zero padding data whenever lookup
            encounters :math:`padding\_idx` in id. And the padding data will not be updated while training.
            If set None, it makes no effect to output. Default: None.
        param_attr(ParamAttr): To specify the weight parameter property. Default: None, which means the
            default weight parameter property is used. See usage for details in :ref:`api_fluid_ParamAttr` . In addition,
421
            user-defined or pre-trained word vectors can be loaded with the :attr:`param_attr` parameter.
422
            The local word vector needs to be transformed into numpy format, and the shape of local word
T
tianshuo78520a 已提交
423
            vector should be consistent with :attr:`size` . Then :ref:`api_fluid_initializer_NumpyArrayInitializer`
424 425 426
            is used to load custom or pre-trained word vectors. See code example 2 for details.
        dtype(str|core.VarDesc.VarType): It refers to the data type of output Tensor.
            It must be float32 or float64. Default: float32.
Y
Yu Yang 已提交
427

428
    Returns:
429
        Variable: Embedding Tensor or LoDTensor mapped by input. The data type is the same as :attr:`dtype` .
Y
Yu Yang 已提交
430

431 432
    Examples:
        .. code-block:: python
Y
Yu Yang 已提交
433

B
bdzhuxiaoning 已提交
434
          import paddle.fluid as fluid
435
          import numpy as np
436 437
          import paddle
          paddle.enable_static()
438

439 440
          data = fluid.data(name='x', shape=[None, 1], dtype='int64')

T
tianshuo78520a 已提交
441
          # example 1
442 443 444 445 446 447 448 449 450
          emb_1 = fluid.embedding(input=data, size=[128, 64])

          # example 2: load custom or pre-trained word vectors
          weight_data = np.random.random(size=(128, 100))  # word vectors with numpy format
          w_param_attrs = fluid.ParamAttr(
              name="emb_weight",
              learning_rate=0.5,
              initializer=fluid.initializer.NumpyArrayInitializer(weight_data),
              trainable=True)
451
          emb_2 = fluid.layers.embedding(input=data, size=(128, 100), param_attr=w_param_attrs, dtype='float32')
Y
Yu Yang 已提交
452 453 454
    """

    helper = LayerHelper('embedding', **locals())
455 456 457 458 459 460 461 462 463
    check_variable_and_dtype(
        input, 'input', ['int64'], 'fluid.layers.embedding'
    )
    check_dtype(
        dtype,
        'dtype',
        ['uint16', 'float16', 'float32', 'float64'],
        'fluid.layers.embedding',
    )
464 465 466 467 468 469 470 471 472

    if is_distributed:
        is_distributed = False
        warnings.warn(
            "is_distributed is go out of use, `fluid.contrib.layers.sparse_embedding` is your needed"
        )

    remote_prefetch = True if is_sparse else False

473 474 475
    w = helper.create_parameter(
        attr=helper.param_attr, shape=size, dtype=dtype, is_bias=False
    )
X
Xin Pan 已提交
476
    tmp = helper.create_variable_for_type_inference(dtype)
477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494
    padding_idx = (
        -1
        if padding_idx is None
        else padding_idx
        if padding_idx >= 0
        else (size[0] + padding_idx)
    )
    helper.append_op(
        type='lookup_table',
        inputs={'Ids': input, 'W': w},
        outputs={'Out': tmp},
        attrs={
            'is_sparse': is_sparse,
            'is_distributed': is_distributed,
            'remote_prefetch': remote_prefetch,
            'padding_idx': padding_idx,
        },
    )
Y
Yu Yang 已提交
495 496 497
    return tmp


498 499 500 501 502 503 504 505 506 507 508
def _pull_sparse(
    input,
    size,
    table_id,
    accessor_class,
    name="embedding",
    ctr_label_name="",
    padding_id=0,
    dtype='float32',
    scale_sparse_grad=True,
):
509
    r"""
510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554
    **Pull Fleet Sparse Layer**

    This layer is used to lookup embeddings of IDs, provided by :attr:`input`, in
    Fleet lookup table. The result of this lookup is the embedding of each ID in the
    :attr:`input`.

    Args:
        input(Variable|list of Variable): Input is a Tensor<int64> Variable, which
            contains the IDs information.
        size(int): The embedding size parameter, which indicates the size of
            each embedding vector respectively.
        table_id(int): the fleet table id of this embedding.
        accessor_class(str): the pslib accessor of the table, default is DownpourCtrAccessor.
        ctr_label_name(str): the layer name of click.
        padding_id(int): the padding id during lookup, default is 0.
        dtype(str): The dtype refers to the data type of output tensor. Only supports
            float32 now.
        scale_sparse_grad(bool): whether to scale sparse gradient with batch size. default
            is True.

    Returns:
        Variable|list of Variable: The tensor variable storing the embeddings of the \
                  supplied inputs.

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid
          data = fluid.layers.data(name='sequence', shape=[1], dtype='int64', lod_level=1)
          emb = fluid.layers.nn._pull_sparse(
              input=data, size=11, table_id=0, accessor_class="DownpourCtrAccessor")
    """
    helper = LayerHelper(name, **locals())
    inputs = helper.multiple_input()
    outs = [helper.create_variable_for_type_inference(dtype)]
    input_names = [i.name for i in inputs]
    attrs = {
        'EmbeddingDim': size,
        'TableId': table_id,
        'AccessorClass': accessor_class,
        'CtrLabelName': ctr_label_name,
        'PaddingId': padding_id,
        'ScaleSparseGrad': scale_sparse_grad,
        'InputNames': input_names,
        # this is only for compatible with embedding op
555
        'is_distributed': True,
556 557
    }
    # this is only for compatible with embedding op
558 559 560 561 562 563 564 565 566
    w, _ = helper.create_or_get_global_variable(
        name=name, shape=[size], dtype=dtype, is_bias=False, persistable=True
    )
    helper.append_op(
        type='pull_sparse',
        inputs={'Ids': inputs, 'W': w},
        outputs={'Out': outs},
        attrs=attrs,
    )
567 568 569 570 571
    if len(outs) == 1:
        return outs[0]
    return outs


572 573 574 575 576 577 578 579 580 581 582
def _pull_sparse_v2(
    input,
    size,
    table_id,
    accessor_class,
    name="embedding",
    ctr_label_name="",
    padding_id=0,
    dtype='float32',
    scale_sparse_grad=True,
):
583
    r"""
584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628
    **Pull Fleet Sparse Layer**

    This layer is used to lookup embeddings of IDs, provided by :attr:`input`, in
    Fleet lookup table. The result of this lookup is the embedding of each ID in the
    :attr:`input`.

    Args:
        input(Variable|list of Variable): Input is a Tensor<int64> Variable, which
            contains the IDs information.
        size(int): The embedding size parameter, which indicates the size of
            each embedding vector respectively.
        table_id(int): the pslib table id of this embedding.
        accessor_class(str): the fleet accessor of the table, default is DownpourCtrAccessor.
        ctr_label_name(str): the layer name of click.
        padding_id(int): the padding id during lookup, default is 0.
        dtype(str): The dtype refers to the data type of output tensor. Only supports
            float32 now.
        scale_sparse_grad(bool): whether to scale sparse gradient with batch size. default
            is True.

    Returns:
        Variable|list of Variable: The tensor variable storing the embeddings of the \
                  supplied inputs.

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid
          data = fluid.layers.data(name='sequence', shape=[1], dtype='int64', lod_level=1)
          emb = fluid.layers.nn._pull_sparse_v2(
              input=data, size=11, table_id=0, accessor_class="DownpourCtrAccessor")
    """
    helper = LayerHelper(name, **locals())
    inputs = helper.multiple_input()
    outs = [helper.create_variable_for_type_inference(dtype)]
    input_names = [i.name for i in inputs]
    attrs = {
        'EmbeddingDim': size,
        'TableId': table_id,
        'AccessorClass': accessor_class,
        'CtrLabelName': ctr_label_name,
        'PaddingId': padding_id,
        'ScaleSparseGrad': scale_sparse_grad,
        'InputNames': input_names,
        # this is only for compatible with embedding op
629
        'is_distributed': True,
630 631
    }
    # this is only for compatible with embedding op
632 633 634 635 636 637 638 639 640
    w, _ = helper.create_or_get_global_variable(
        name=name, shape=[size], dtype=dtype, is_bias=False, persistable=True
    )
    helper.append_op(
        type='pull_sparse_v2',
        inputs={'Ids': inputs, 'W': w},
        outputs={'Out': outs},
        attrs=attrs,
    )
641
    if len(outs) == 1:
Y
yaoxuefeng 已提交
642 643 644 645
        return outs[0]
    return outs


646 647 648
def _pull_gpups_sparse(
    input, size, dtype='float32', is_distributed=False, is_sparse=False
):
Y
yaoxuefeng 已提交
649 650 651 652 653 654 655 656 657 658 659 660 661
    r"""
    **Pull GpuPS Sparse Layer**

    This layer is used to lookup embeddings of IDs, provided by :attr:`input`, in
    GpuPS lookup table. The result of this lookup is the embedding of each ID in the
    :attr:`input`.

    Args:
        input(Variable|list of Variable): Input is a Tensor<int64> Variable, which
            contains the IDs information.
        size(int|list of int): The embedding size parameter of each input, which indicates the size of
            each embedding vector respectively.
        dtype(str): The dtype refers to the data type of output tensor. Only supports
662
        float32 now.
Y
yaoxuefeng 已提交
663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681

    Returns:
        Variable|list of Variable: The tensor variable storing the embeddings of the \
                  supplied inputs, whose size are indicated by size respectively.

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid
          slots = []
          data_1 = fluid.layers.data(name='sequence', shape=[1], dtype='int64', lod_level=1)
          slots.append(data_1)
          data_2 = fluid.layers.data(name='sequence', shape=[1], dtype='int64', lod_level=1)
          slots.append(data_2)
          embs = fluid.layers.pull_gpups_sparse(input=slots, size=[11, 35])
    """
    helper = LayerHelper('pull_gpups_sparse', **locals())
    if dtype != 'float32':
        raise ValueError(
682 683 684
            "GpuPS only support float type embedding now, and your type is: "
            + dtype
        )
Y
yaoxuefeng 已提交
685 686 687 688 689 690
    helper.input_dtype()
    inputs = helper.multiple_input()
    outs = [
        helper.create_variable_for_type_inference(dtype)
        for i in range(len(inputs))
    ]
691 692 693 694 695 696 697 698 699 700 701 702 703
    w = helper.create_parameter(
        attr=helper.param_attr, shape=[size[0]], dtype=dtype, is_bias=False
    )
    helper.append_op(
        type='pull_gpups_sparse',
        inputs={'Ids': inputs, 'W': w},
        outputs={'Out': outs},
        attrs={
            'size': size,
            'is_distributed': is_distributed,
            'is_sparse': is_sparse,
        },
    )
Y
yaoxuefeng 已提交
704
    if len(outs) == 1:
705 706 707 708
        return outs[0]
    return outs


709 710 711
def _pull_box_sparse(
    input, size, dtype='float32', is_distributed=False, is_sparse=False
):
712
    r"""
H
hutuxian 已提交
713 714 715 716 717 718 719
    **Pull Box Sparse Layer**

    This layer is used to lookup embeddings of IDs, provided by :attr:`input`, in
    BoxPS lookup table. The result of this lookup is the embedding of each ID in the
    :attr:`input`.

    Args:
720
        input(Variable|list of Variable): Input is a Tensor<int64> Variable, which
H
hutuxian 已提交
721
            contains the IDs information.
722
        size(int): The embedding size parameter, which indicates the size of
H
hutuxian 已提交
723
            each embedding vector respectively.
724
        dtype(str): The dtype refers to the data type of output tensor. Only supports
725
        float32 now.
H
hutuxian 已提交
726 727 728 729 730 731 732 733 734 735

    Returns:
        Variable|list of Variable: The tensor variable storing the embeddings of the \
                  supplied inputs.

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid
          data = fluid.layers.data(name='sequence', shape=[1], dtype='int64', lod_level=1)
736
          emb = fluid.layers.pull_box_sparse(input=data, size=[11])
H
hutuxian 已提交
737 738 739 740
    """
    helper = LayerHelper('pull_box_sparse', **locals())
    if dtype != 'float32':
        raise ValueError(
741 742 743
            "BoxPS only support float type embedding now, and your type is: "
            + dtype
        )
H
hutuxian 已提交
744 745 746 747 748 749
    helper.input_dtype()
    inputs = helper.multiple_input()
    outs = [
        helper.create_variable_for_type_inference(dtype)
        for i in range(len(inputs))
    ]
750 751 752 753 754 755 756 757 758 759 760 761 762
    w = helper.create_parameter(
        attr=helper.param_attr, shape=[size], dtype=dtype, is_bias=False
    )
    helper.append_op(
        type='pull_box_sparse',
        inputs={'Ids': inputs, 'W': w},
        outputs={'Out': outs},
        attrs={
            'size': size,
            'is_distributed': is_distributed,
            'is_sparse': is_sparse,
        },
    )
H
hutuxian 已提交
763 764 765 766 767
    if len(outs) == 1:
        return outs[0]
    return outs


Y
yuyang18 已提交
768
@templatedoc()
769
def linear_chain_crf(input, label, param_attr=None, length=None):
Y
yuyang18 已提交
770
    """
771 772
    :api_attr: Static Graph

Y
yuyang18 已提交
773 774 775 776 777
    Linear Chain CRF.

    ${comment}

    Args:
778
        input(${emission_type}): ${emission_comment}
Y
yuyang18 已提交
779
        label(${label_type}): ${label_comment}
780
        Length(${length_type}): ${length_comment}
781
        param_attr(ParamAttr): The attribute of the learnable parameter for transition parameter.
Y
yuyang18 已提交
782 783

    Returns:
D
dzhwinter 已提交
784 785
        output(${emission_exps_type}): ${emission_exps_comment} \n
        output(${transition_exps_type}): ${transition_exps_comment} \n
786
        output(${log_likelihood_type}): ${log_likelihood_comment} \n
Y
yuyang18 已提交
787

J
JesseyXujin 已提交
788 789 790
    Examples:
        .. code-block:: python

791 792
            import paddle.fluid as fluid
            import numpy as np
793 794
            import paddle
            paddle.enable_static()
795 796 797 798 799

            #define net structure, using LodTensor
            train_program = fluid.Program()
            startup_program = fluid.Program()
            with fluid.program_guard(train_program, startup_program):
800 801
                input_data = fluid.data(name='input_data', shape=[-1,10], dtype='float32')
                label = fluid.data(name='label', shape=[-1,1], dtype='int')
802 803 804 805 806 807
                emission= fluid.layers.fc(input=input_data, size=10, act="tanh")
                crf_cost = fluid.layers.linear_chain_crf(
                    input=emission,
                    label=label,
                    param_attr=fluid.ParamAttr(
                    name='crfw',
808
                    learning_rate=0.01))
809 810 811
            use_cuda = False
            place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
            exe = fluid.Executor(place)
812
            exe.run(startup_program)
813 814 815 816 817
            #define data, using LoDTensor
            a = fluid.create_lod_tensor(np.random.rand(12,10).astype('float32'), [[3,3,4,2]], place)
            b = fluid.create_lod_tensor(np.array([[1],[1],[2],[3],[1],[1],[1],[3],[1],[1],[1],[1]]),[[3,3,4,2]] , place)
            feed1 = {'input_data':a,'label':b}
            loss= exe.run(train_program,feed=feed1, fetch_list=[crf_cost])
818
            print(loss)
819 820 821 822 823

            #define net structure, using padding
            train_program = fluid.Program()
            startup_program = fluid.Program()
            with fluid.program_guard(train_program, startup_program):
824 825 826
                input_data2 = fluid.data(name='input_data2', shape=[-1,10,10], dtype='float32')
                label2 = fluid.data(name='label2', shape=[-1,10,1], dtype='int')
                label_length = fluid.data(name='length', shape=[-1,1], dtype='int')
827 828 829 830 831 832
                emission2= fluid.layers.fc(input=input_data2, size=10, act="tanh", num_flatten_dims=2)
                crf_cost2 = fluid.layers.linear_chain_crf(
                    input=emission2,
                    label=label2,
                    length=label_length,
                    param_attr=fluid.ParamAttr(
J
JesseyXujin 已提交
833
                     name='crfw',
834 835 836 837 838 839
                     learning_rate=0.01))

            use_cuda = False
            place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
            exe = fluid.Executor(place)
            exe.run(startup_program)
J
JesseyXujin 已提交
840

841 842 843
            #define data, using padding
            cc=np.random.rand(4,10,10).astype('float32')
            dd=np.random.rand(4,10,1).astype('int64')
844
            ll=np.array([[3],[3],[4],[2]])
845 846
            feed2 = {'input_data2':cc,'label2':dd,'length':ll}
            loss2= exe.run(train_program,feed=feed2, fetch_list=[crf_cost2])
847
            print(loss2)
848 849 850 851 852
            #[array([[ 7.8902354],
            #        [ 7.3602567],
            #        [ 10.004011],
            #        [ 5.86721  ]], dtype=float32)]

853 854 855
            #you can use find_var to get transition parameter.
            transition=np.array(fluid.global_scope().find_var('crfw').get_tensor())
            print(transition)
856

Y
yuyang18 已提交
857
    """
858 859 860
    check_variable_and_dtype(
        input, 'input', ['float32', 'float64'], 'linear_chain_crf'
    )
861
    check_variable_and_dtype(label, 'label', ['int64'], 'linear_chain_crf')
Y
Yu Yang 已提交
862
    helper = LayerHelper('linear_chain_crf', **locals())
863
    size = input.shape[2] if length else input.shape[1]
864 865 866 867 868
    transition = helper.create_parameter(
        attr=helper.param_attr,
        shape=[size + 2, size],
        dtype=helper.input_dtype(),
    )
X
Xin Pan 已提交
869
    alpha = helper.create_variable_for_type_inference(
870 871
        dtype=helper.input_dtype()
    )
X
Xin Pan 已提交
872
    emission_exps = helper.create_variable_for_type_inference(
873 874
        dtype=helper.input_dtype()
    )
X
Xin Pan 已提交
875
    transition_exps = helper.create_variable_for_type_inference(
876 877
        dtype=helper.input_dtype()
    )
X
Xin Pan 已提交
878
    log_likelihood = helper.create_variable_for_type_inference(
879 880
        dtype=helper.input_dtype()
    )
881 882 883
    this_inputs = {
        "Emission": [input],
        "Transition": transition,
884
        "Label": [label],
885 886
    }
    if length:
887
        this_inputs['Length'] = [length]
888 889 890 891 892 893 894 895 896 897
    helper.append_op(
        type='linear_chain_crf',
        inputs=this_inputs,
        outputs={
            "Alpha": [alpha],
            "EmissionExps": [emission_exps],
            "TransitionExps": transition_exps,
            "LogLikelihood": log_likelihood,
        },
    )
Y
Yu Yang 已提交
898 899 900 901

    return log_likelihood


W
wopeizl 已提交
902
@templatedoc()
903
def crf_decoding(input, param_attr, label=None, length=None):
W
wopeizl 已提交
904
    """
905
    :api_attr: Static Graph
906

W
wopeizl 已提交
907
    ${comment}
Y
yi.wu 已提交
908

W
wopeizl 已提交
909
    Args:
910
        input(Tensor): ${emission_comment}
Y
yi.wu 已提交
911

912 913
        param_attr (ParamAttr|None): To specify the weight parameter attribute.
            Default: None, which means the default weight parameter property is
914
            used. See usage for details in :ref:`api_paddle_fluid_param_attr_ParamAttr` .
Y
yuyang18 已提交
915

Y
Yibing Liu 已提交
916
        label(${label_type}, optional): ${label_comment}
917

Y
Yibing Liu 已提交
918
        length(${length_type}, optional): ${length_comment}
919

W
wopeizl 已提交
920
    Returns:
921
        Tensor: ${viterbi_path_comment}
Y
yi.wu 已提交
922

W
wopeizl 已提交
923 924
    Examples:
        .. code-block:: python
Y
yi.wu 已提交
925

926 927
           import paddle
           paddle.enable_static()
928 929 930

           # LoDTensor-based example
           num_labels = 10
931 932 933
           feature = paddle.static.data(name='word_emb', shape=[-1, 784], dtype='float32', lod_level=1)
           label = paddle.static.data(name='label', shape=[-1, 1], dtype='int64', lod_level=1)
           emission = paddle.static.nn.fc(feature, size=num_labels)
934

935 936 937 938
           crf_cost = paddle.fluid.layers.linear_chain_crf(input=emission, label=label,
                     param_attr=paddle.ParamAttr(name="crfw"))
           crf_decode = paddle.static.nn.crf_decoding(input=emission,
                     param_attr=paddle.ParamAttr(name="crfw"))
939 940 941

           # Common tensor example
           num_labels, max_len = 10, 20
942 943 944 945
           feature = paddle.static.data(name='word_emb_pad', shape=[-1, max_len, 784], dtype='float32')
           label = paddle.static.data(name='label_pad', shape=[-1, max_len, 1], dtype='int64')
           length = paddle.static.data(name='length', shape=[-1, 1], dtype='int64')
           emission = paddle.static.nn.fc(feature, size=num_labels,
946
                                      num_flatten_dims=2)
947

948 949 950 951
           crf_cost = paddle.fluid.layers.linear_chain_crf(input=emission, label=label, length=length,
                     param_attr=paddle.ParamAttr(name="crfw_pad"))
           crf_decode = paddle.static.nn.crf_decoding(input=emission, length=length,
                     param_attr=paddle.ParamAttr(name="crfw_pad"))
W
wopeizl 已提交
952
    """
953 954 955
    check_variable_and_dtype(
        input, 'input', ['float32', 'float64'], 'crf_decoding'
    )
W
wopeizl 已提交
956 957 958
    helper = LayerHelper('crf_decoding', **locals())
    transition = helper.get_parameter(param_attr.name)
    viterbi_path = helper.create_variable_for_type_inference(
959 960
        dtype=core.VarDesc.VarType.INT64
    )
961 962 963
    inputs = {"Emission": [input], "Transition": transition, "Label": label}
    if length:
        inputs['Length'] = length
964 965 966 967 968
    helper.append_op(
        type='crf_decoding',
        inputs=inputs,
        outputs={"ViterbiPath": [viterbi_path]},
    )
Y
Yu Yang 已提交
969

W
wopeizl 已提交
970
    return viterbi_path
Y
Yu Yang 已提交
971 972


973
@deprecated(since="2.0.0", update_to="paddle.nn.functional.dropout")
974 975 976 977 978 979 980 981
def dropout(
    x,
    dropout_prob,
    is_test=None,
    seed=None,
    name=None,
    dropout_implementation="downgrade_in_infer",
):
982
    """
983

984 985 986 987
    Computes dropout.

    Drop or keep each element of `x` independently. Dropout is a regularization
    technique for reducing overfitting by preventing neuron co-adaption during
988
    training. The dropout operator randomly sets (according to the given dropout
989 990 991
    probability) the outputs of some units to zero, while others are remain
    unchanged.

H
haowang101779990 已提交
992 993
    dropout op can be removed from the program to make the program more efficient.

994
    Args:
L
lvmengsi 已提交
995
        x (Variable): The input tensor variable. The data type is float16 or float32 or float64.
996
        dropout_prob (float): Probability of setting units to zero.
997
        is_test (bool): A flag indicating whether it is in test phrase or not.
998
                        Default None, in dynamic graph, it use global tracer mode; in static graph, it means False.
999 1000 1001
        seed (int): A Python integer used to create random seeds. If this
                    parameter is set to None, a random seed is used.
                    NOTE: If an integer seed is given, always the same output
L
lvmengsi 已提交
1002
                    units will be dropped. DO NOT use a fixed seed in training.Default: None.
1003 1004
        name (str|None): A name for this layer(optional). If set None, the layer
                         will be named automatically.
H
haowang101779990 已提交
1005 1006
        dropout_implementation(string): ['downgrade_in_infer'(default)|'upscale_in_train']

P
phlrain 已提交
1007
                                        1. downgrade_in_infer(default), downgrade the outcome at inference
H
haowang101779990 已提交
1008 1009

                                           - train: out = input * mask
C
ceci3 已提交
1010
                                           - inference: out = input * (1.0 - dropout_prob)
H
haowang101779990 已提交
1011 1012 1013

                                           (mask is a tensor same shape with input, value is 0 or 1
                                           ratio of 0 is dropout_prob)
P
phlrain 已提交
1014
                                        2. upscale_in_train, upscale the outcome at training time
1015

H
haowang101779990 已提交
1016 1017
                                           - train: out = input * mask / ( 1.0 - dropout_prob )
                                           - inference: out = input
P
phlrain 已提交
1018

H
haowang101779990 已提交
1019 1020
                                           (mask is a tensor same shape with input, value is 0 or 1
                                           ratio of 0 is dropout_prob)
1021

M
minqiyang 已提交
1022

1023
    Returns:
L
lvmengsi 已提交
1024
        A Variable holding Tensor representing the dropout, has same shape and data type with `x`.
1025 1026

    Examples:
1027

1028 1029
        .. code-block:: python

1030
            import paddle
1031
            import paddle.fluid as fluid
1032

1033
            paddle.enable_static()
L
lvmengsi 已提交
1034
            x = fluid.data(name="data", shape=[None, 32, 32], dtype="float32")
T
tianshuo78520a 已提交
1035
            dropped = fluid.layers.dropout(x, dropout_prob=0.5)
1036
    """
1037 1038
    if not isinstance(dropout_prob, (float, int, Variable)):
        raise TypeError(
1039 1040
            "dropout_prob argument should be a number(int|float) or Variable"
        )
1041
    # fast return for p == 0
1042
    if isinstance(dropout_prob, (int, float)) and dropout_prob == 0:
1043
        return x
1044

J
Jiabin Yang 已提交
1045
    if _non_static_mode():
1046 1047 1048
        if (
            seed is None or seed == 0
        ) and default_main_program().random_seed != 0:
1049
            seed = default_main_program().random_seed
1050 1051
        if is_test is None:
            is_test = not _dygraph_tracer()._train_mode
1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
        out, mask = _legacy_C_ops.dropout(
            x,
            'dropout_prob',
            dropout_prob,
            'is_test',
            is_test,
            'fix_seed',
            seed is not None,
            'seed',
            seed if seed is not None else 0,
            'dropout_implementation',
            dropout_implementation,
        )
1065
        return out
1066

W
wanghuancoder 已提交
1067 1068 1069
    def get_attrs(prog, dropout_prob, is_test, seed):
        if (seed is None or seed == 0) and prog.random_seed != 0:
            seed = prog.random_seed
1070 1071
        if isinstance(dropout_prob, Variable) and not dropout_prob.shape != [1]:
            raise TypeError(
1072 1073 1074 1075
                "Required dropout_prob.shape == [1] if type(dropout_prob) is Variable, but received dropout_prob.shape = {}".format(
                    dropout_prob.shape
                )
            )
W
wanghuancoder 已提交
1076 1077 1078 1079 1080 1081 1082 1083 1084
        attrs = {
            'dropout_prob': dropout_prob,
            'is_test': is_test,
            'fix_seed': seed is not None,
            'seed': seed if seed is not None else 0,
            'dropout_implementation': dropout_implementation,
        }
        return attrs

F
fengjiayi 已提交
1085
    helper = LayerHelper('dropout', **locals())
1086 1087 1088
    check_variable_and_dtype(
        x, 'x', ['float16', 'float32', 'float64'], 'dropout'
    )
1089

X
Xin Pan 已提交
1090 1091
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
    mask = helper.create_variable_for_type_inference(
1092 1093
        dtype=core.VarDesc.VarType.UINT8, stop_gradient=True
    )
C
chengduo 已提交
1094

1095
    attrs = get_attrs(helper.main_program, dropout_prob, is_test, seed)
C
chengduo 已提交
1096

1097 1098 1099 1100 1101 1102
    helper.append_op(
        type='dropout',
        inputs={'X': [x]},
        outputs={'Out': [out], 'Mask': [mask]},
        attrs=attrs,
    )
1103 1104 1105
    return out


1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120
def conv2d(
    input,
    num_filters,
    filter_size,
    stride=1,
    padding=0,
    dilation=1,
    groups=None,
    param_attr=None,
    bias_attr=None,
    use_cudnn=True,
    act=None,
    name=None,
    data_format="NCHW",
):
1121
    r"""
1122 1123
    :api_attr: Static Graph

C
chengduoZH 已提交
1124
    The convolution2D layer calculates the output based on the input, filter
T
tensor-tang 已提交
1125
    and strides, paddings, dilations, groups parameters. Input and
L
liym27 已提交
1126
    Output are in NCHW or NHWC format, where N is batch size, C is the number of
1127
    channels, H is the height of the feature, and W is the width of the feature.
T
tensor-tang 已提交
1128 1129 1130 1131 1132 1133
    Filter is in MCHW format, where M is the number of output image channels,
    C is the number of input image channels, H is the height of the filter,
    and W is the width of the filter. If the groups is greater than 1,
    C will equal the number of input image channels divided by the groups.
    Please refer to UFLDL's `convolution
    <http://ufldl.stanford.edu/tutorial/supervised/FeatureExtractionUsingConvolution/>`_
1134
    for more details.
1135 1136 1137
    If bias attribution and activation type are provided, bias is added to the
    output of the convolution, and the corresponding activation function is
    applied to the final result.
C
chengduoZH 已提交
1138

1139
    For each input :math:`X`, the equation is:
C
refine  
chengduoZH 已提交
1140

C
chengduoZH 已提交
1141 1142
    .. math::

C
refine  
chengduoZH 已提交
1143
        Out = \sigma (W \\ast X + b)
C
chengduoZH 已提交
1144

T
tensor-tang 已提交
1145
    Where:
C
chengduoZH 已提交
1146

L
liym27 已提交
1147
    * :math:`X`: Input value, a tensor with NCHW or NHWC format.
1148 1149 1150 1151
    * :math:`W`: Filter value, a tensor with MCHW format.
    * :math:`\\ast`: Convolution operation.
    * :math:`b`: Bias value, a 2-D tensor with shape [M, 1].
    * :math:`\\sigma`: Activation function.
T
tensor-tang 已提交
1152
    * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.
C
chengduoZH 已提交
1153 1154 1155

    Example:

1156 1157
        - Input:

W
weixing02 已提交
1158
          Input shape: :math:`(N, C_{in}, H_{in}, W_{in})`
C
refine  
chengduoZH 已提交
1159

W
weixing02 已提交
1160
          Filter shape: :math:`(C_{out}, C_{in}, H_f, W_f)`
C
refine  
chengduoZH 已提交
1161

1162
        - Output:
T
tensor-tang 已提交
1163

W
weixing02 已提交
1164
          Output shape: :math:`(N, C_{out}, H_{out}, W_{out})`
C
refine  
chengduoZH 已提交
1165

C
chengduoZH 已提交
1166
        Where
1167 1168

        .. math::
C
chengduoZH 已提交
1169

W
weixing02 已提交
1170 1171
            H_{out}&= \\frac{(H_{in} + 2 * paddings[0] - (dilations[0] * (H_f - 1) + 1))}{strides[0]} + 1 \\\\
            W_{out}&= \\frac{(W_{in} + 2 * paddings[1] - (dilations[1] * (W_f - 1) + 1))}{strides[1]} + 1
C
chengduoZH 已提交
1172 1173

    Args:
1174
        input (Tensor): The input is 4-D Tensor with shape [N, C, H, W], the data type
L
lvmengsi 已提交
1175
            of input is float16 or float32 or float64.
T
tensor-tang 已提交
1176
        num_filters(int): The number of filter. It is as same as the output
1177
            image channel.
1178 1179
        filter_size (int|tuple): The filter size. If filter_size
            is a tuple, it must contain two integers, (filter_size_height,
L
lvmengsi 已提交
1180 1181
            filter_size_width). Otherwise, filter_size_height = filter_size_width =\
            filter_size.
1182 1183
        stride (int|tuple): The stride size. It means the stride in convolution.
            If stride is a tuple, it must contain two integers, (stride_height, stride_width).
L
lvmengsi 已提交
1184 1185
            Otherwise, stride_height = stride_width = stride. Default: stride = 1.
        padding (string|int|list|tuple): The padding size. It means the number of zero-paddings
T
tianshuo78520a 已提交
1186
            on both sides for each dimension.If `padding` is a string, either 'VALID' or
L
liym27 已提交
1187 1188
            'SAME' which is the padding algorithm. If padding size is a tuple or list,
            it could be in three forms: `[pad_height, pad_width]` or
1189 1190
            `[pad_height_top, pad_height_bottom, pad_width_left, pad_width_right]`, and when
            `data_format` is `"NCHW"`, `padding` can be in the form `[[0,0], [0,0],
L
lvmengsi 已提交
1191
            [pad_height_top, pad_height_bottom], [pad_width_left, pad_width_right]]`.
L
liym27 已提交
1192 1193 1194
            when `data_format` is `"NHWC"`, `pool_padding` can be in the form
            `[[0,0], [pad_height_top, pad_height_bottom], [pad_width_left, pad_width_right], [0,0]]`.
            Default: padding = 0.
L
lvmengsi 已提交
1195
        dilation (int|tuple): The dilation size. It means the spacing between the kernel
1196 1197
            points. If dilation is a tuple, it must contain two integers, (dilation_height,
            dilation_width). Otherwise, dilation_height = dilation_width = dilation.
L
lvmengsi 已提交
1198
            Default: dilation = 1.
1199 1200 1201 1202
        groups (int): The groups number of the Conv2d Layer. According to grouped
            convolution in Alex Krizhevsky's Deep CNN paper: when group=2,
            the first half of the filters is only connected to the first half
            of the input channels, while the second half of the filters is only
C
chengduo 已提交
1203 1204 1205 1206 1207
            connected to the second half of the input channels. Default: groups=1.
        param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights
            of conv2d. If it is set to None or one attribute of ParamAttr, conv2d
            will create ParamAttr as param_attr. If the Initializer of the param_attr
            is not set, the parameter is initialized with :math:`Normal(0.0, std)`,
H
haowang101779990 已提交
1208
            and the :math:`std` is :math:`(\\frac{2.0 }{filter\_elem\_num})^{0.5}`. Default: None.
C
chengduo 已提交
1209 1210 1211 1212 1213
        bias_attr (ParamAttr|bool|None): The parameter attribute for the bias of conv2d.
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, conv2d
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
            is not set, the bias is initialized zero. Default: None.
1214 1215
        use_cudnn (bool): Use cudnn kernel or not, it is valid only when the cudnn
            library is installed. Default: True
C
chengduo 已提交
1216 1217
        act (str): Activation type, if it is set to None, activation is not appended.
            Default: None
1218 1219
        name(str|None): For detailed information, please refer
           to :ref:`api_guide_Name`. Usually name is no need to set and
L
lvmengsi 已提交
1220
           None by default.
1221
        data_format (str, optional): Specify the data format of the input, and the data format of the output
1222
            will be consistent with that of the input. An optional string from: `"NCHW"`, `"NHWC"`.
L
liym27 已提交
1223 1224
            The default is `"NCHW"`. When it is `"NCHW"`, the data is stored in the order of:
            `[batch_size, input_channels, input_height, input_width]`.
C
chengduoZH 已提交
1225 1226

    Returns:
1227 1228 1229
        A Tensor representing the conv2d, whose data type is the
        same with input. If act is None, the tensor storing the convolution
        result, and if act is not None, the tensor storing convolution
L
lvmengsi 已提交
1230
        and non-linearity activation result.
C
refine  
chengduoZH 已提交
1231

1232 1233 1234 1235 1236
    Raises:
        ValueError: If the type of `use_cudnn` is not bool.
        ValueError: If `data_format` is not "NCHW" or "NHWC".
        ValueError: If the channel dimmention of the input is less than or equal to zero.
        ValueError: If `padding` is a string, but not "SAME" or "VALID".
1237
        ValueError: If `padding` is a tuple, but the element corresponding to the input's batch size is not 0
1238 1239 1240 1241 1242 1243 1244
            or the element corresponding to the input's channel is not 0.
        ShapeError: If the input is not 4-D Tensor.
        ShapeError: If the input's dimension size and filter's dimension size not equal.
        ShapeError: If the dimension size of input minus the size of `stride` is not 2.
        ShapeError: If the number of input channels is not equal to filter's channels * groups.
        ShapeError: If the number of output channels is not be divided by groups.

C
chengduoZH 已提交
1245 1246 1247
    Examples:
        .. code-block:: python

1248 1249
          import paddle
          paddle.enable_static()
1250

1251 1252 1253
          data = paddle.static.data(name='data', shape=[None, 3, 32, 32], dtype='float32')
          conv2d = paddle.static.nn.conv2d(input=data, num_filters=2, filter_size=3, act="relu")
          print(conv2d.shape) # [-1, 2, 30, 30]
Y
Yu Yang 已提交
1254 1255
    """

1256 1257 1258
    check_variable_and_dtype(
        input, 'input', ['float16', 'float32', 'float64'], 'conv2d'
    )
1259
    if len(input.shape) != 4:
1260 1261 1262 1263
        raise ValueError(
            "Input size should be 4, "
            "but received {}".format(len(input.shape))
        )
1264
    num_channels = input.shape[1]
L
liym27 已提交
1265
    if not isinstance(use_cudnn, bool):
1266 1267 1268 1269
        raise ValueError(
            "Attr(use_cudnn) should be True or False. Received "
            "Attr(use_cudnn): %s. " % str(use_cudnn)
        )
L
liym27 已提交
1270 1271 1272 1273

    if data_format not in ["NCHW", "NHWC"]:
        raise ValueError(
            "Attr(data_format) should be 'NCHW' or 'NHWC'. Received "
1274 1275
            "Attr(data_format): %s." % str(data_format)
        )
L
liym27 已提交
1276

1277
    channel_last = data_format == "NHWC"
L
liym27 已提交
1278 1279 1280 1281
    num_channels = input.shape[3] if channel_last else input.shape[1]
    if num_channels < 0:
        raise ValueError(
            "The channel dimmention of the input(%s) should be defined. "
1282 1283
            "Received: %s." % (str(input.shape), str(num_channels))
        )
C
chengduo 已提交
1284
    assert param_attr is not False, "param_attr should not be False here."
L
liym27 已提交
1285

1286 1287 1288
    if groups is None:
        num_filter_channels = num_channels
    elif groups <= 0:
1289 1290
        raise ValueError(
            "the groups of input must be greater than 0, "
1291 1292
            "but received the groups of input is {}".format(groups)
        )
1293 1294 1295 1296 1297
    else:
        if num_channels % groups != 0:
            raise ValueError(
                "the channel of input must be divisible by groups,"
                "received: the channel of input is {}, the shape of input is {}"
1298 1299
                ", the groups is {}".format(num_channels, input.shape, groups)
            )
1300 1301
        num_filter_channels = num_channels // groups

1302
    l_type = 'conv2d'
1303 1304 1305 1306 1307
    if (
        num_channels == groups
        and num_filters % num_channels == 0
        and not use_cudnn
    ):
1308
        l_type = 'depthwise_conv2d'
1309

1310 1311 1312 1313 1314
    if (
        num_channels == groups
        and num_filters % num_channels == 0
        and core.is_compiled_with_rocm()
    ):
1315 1316
        l_type = 'depthwise_conv2d'

1317 1318
    # NPU only supports depthwise_conv2d when  "input_channel = output_channel = groups"
    if core.is_compiled_with_npu():
1319
        if num_channels == groups and num_channels == num_filters:
1320 1321 1322 1323
            l_type = 'depthwise_conv2d'
        else:
            l_type = 'conv2d'

1324 1325 1326
    helper = LayerHelper(l_type, **locals())
    dtype = helper.input_dtype()

C
chengduoZH 已提交
1327 1328
    filter_size = utils.convert_to_list(filter_size, 2, 'filter_size')
    stride = utils.convert_to_list(stride, 2, 'stride')
1329
    dilation = utils.convert_to_list(dilation, 2, 'dilation')
C
chengduoZH 已提交
1330

L
liym27 已提交
1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342
    # padding
    def _update_padding(padding, data_format):
        def is_list_or_tuple(ele):
            if isinstance(ele, list) or isinstance(ele, tuple):
                return True
            return False

        if is_list_or_tuple(padding) and len(padding) == 4:
            if is_list_or_tuple(padding[0]) and (data_format == "NCHW"):
                if not (padding[0] == [0, 0] and padding[1] == [0, 0]):
                    raise ValueError(
                        "Non-zero padding(%s) in the batch or channel dimensions "
1343 1344
                        "is not supported." % str(padding)
                    )
L
liym27 已提交
1345 1346 1347 1348 1349 1350
                padding = padding[2:4]
                padding = [ele for a_list in padding for ele in a_list]
            elif is_list_or_tuple(padding[0]) and (data_format == "NHWC"):
                if not (padding[0] == [0, 0] and padding[3] == [0, 0]):
                    raise ValueError(
                        "Non-zero padding(%s) in the batch or channel dimensions "
1351 1352
                        "is not supported." % str(padding)
                    )
L
liym27 已提交
1353 1354 1355
                padding = padding[1:3]
                padding = [ele for a_list in padding for ele in a_list]
            padding = utils.convert_to_list(padding, 4, 'padding')
1356 1357 1358
            if utils._is_symmetric_padding(padding, 2):
                padding = [padding[0], padding[2]]

L
liym27 已提交
1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
        else:
            padding = utils.convert_to_list(padding, 2, 'padding')

        return padding

    padding_algorithm = "EXPLICIT"
    if isinstance(padding, str):
        padding = padding.upper()
        if padding not in ["SAME", "VALID"]:
            raise ValueError(
1369 1370 1371
                "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'."
                % str(padding)
            )
L
liym27 已提交
1372 1373
        if padding == "VALID":
            padding_algorithm = "VALID"
1374
            padding = [0, 0]
L
liym27 已提交
1375 1376
        elif padding == "SAME":
            padding_algorithm = "SAME"
1377
            padding = [0, 0]
L
liym27 已提交
1378 1379

    padding = _update_padding(padding, data_format)
Y
Yu Yang 已提交
1380

M
minqiyang 已提交
1381
    filter_shape = [num_filters, int(num_filter_channels)] + filter_size
Y
Yu Yang 已提交
1382 1383

    def _get_default_param_initializer():
C
chengduo 已提交
1384
        filter_elem_num = filter_size[0] * filter_size[1] * num_channels
1385 1386 1387 1388
        if filter_elem_num <= 0:
            raise ValueError(
                "Invalid filter number, excepted number is larger than 0, but"
                " received {}, please check the input shape and "
1389 1390 1391
                "filter size.".format(filter_elem_num)
            )
        std = (2.0 / filter_elem_num) ** 0.5
Y
Yu Yang 已提交
1392 1393 1394 1395 1396 1397
        return Normal(0.0, std, 0)

    filter_param = helper.create_parameter(
        attr=helper.param_attr,
        shape=filter_shape,
        dtype=dtype,
1398 1399
        default_initializer=_get_default_param_initializer(),
    )
Y
Yu Yang 已提交
1400

X
Xin Pan 已提交
1401
    pre_bias = helper.create_variable_for_type_inference(dtype)
Y
Yu Yang 已提交
1402

1403 1404 1405 1406 1407 1408
    if (
        core.is_compiled_with_cuda()
        and paddle.fluid.get_flags("FLAGS_conv2d_disable_cudnn")[
            "FLAGS_conv2d_disable_cudnn"
        ]
    ):
1409 1410
        use_cudnn = False

1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429
    helper.append_op(
        type=l_type,
        inputs={
            'Input': input,
            'Filter': filter_param,
        },
        outputs={"Output": pre_bias},
        attrs={
            'strides': stride,
            'paddings': padding,
            'dilations': dilation,
            'groups': groups,
            'use_cudnn': use_cudnn,
            'use_mkldnn': False,
            'fuse_relu_before_depthwise_conv': False,
            "padding_algorithm": padding_algorithm,
            "data_format": data_format,
        },
    )
Y
Yu Yang 已提交
1430

1431 1432 1433 1434
    if data_format == 'NCHW':
        pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2)
    else:
        pre_act = helper.append_bias_op(pre_bias, dim_start=3, dim_end=4)
Y
Yu Yang 已提交
1435 1436 1437 1438

    return helper.append_activation(pre_act)


F
fengjiayi 已提交
1439
@templatedoc()
1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452
def pool2d(
    input,
    pool_size=-1,
    pool_type="max",
    pool_stride=1,
    pool_padding=0,
    global_pooling=False,
    use_cudnn=True,
    ceil_mode=False,
    name=None,
    exclusive=True,
    data_format="NCHW",
):
Y
Yu Yang 已提交
1453
    """
1454

F
fengjiayi 已提交
1455
    ${comment}
1456 1457

    Args:
K
Kaipeng Deng 已提交
1458 1459 1460 1461 1462
        input (Variable): The input tensor of pooling operator which is a 4-D tensor with
                          shape [N, C, H, W]. The format of input tensor is `"NCHW"` or
                          `"NHWC"`, where `N` is batch size, `C` is the number of channels,
                          `H` is the height of the feature, and `W` is the width of the
                          feature. The data type if float32 or float64.
J
JiabinYang 已提交
1463
        pool_size (int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list,
J
JiabinYang 已提交
1464 1465
            it must contain two integers, (pool_size_Height, pool_size_Width).
            Otherwise, the pool kernel size will be a square of an int.
F
fengjiayi 已提交
1466
        pool_type: ${pooling_type_comment}
J
JiabinYang 已提交
1467 1468 1469
        pool_stride (int|list|tuple): The pool stride size. If pool stride size is a tuple or list,
            it must contain two integers, (pool_stride_Height, pool_stride_Width).
            Otherwise, the pool stride size will be a square of an int.
1470 1471 1472 1473 1474 1475 1476
        pool_padding (string|int|list|tuple): The pool padding. If `pool_padding` is a string, either 'VALID' or
            'SAME' which is the padding algorithm. If pool padding size is a tuple or list,
            it could be in three forms: `[pad_height, pad_width]` or
            `[pad_height_top, pad_height_bottom, pad_width_left, pad_width_right]`, and when `data_format` is `"NCHW"`,
            `pool_padding` can be in the form `[[0,0], [0,0], [pad_height_top, pad_height_bottom], [pad_width_left, pad_width_right]]`.
            when `data_format` is `"NHWC"`, `pool_padding` can be in the form
            `[[0,0], [pad_height_top, pad_height_bottom], [pad_width_left, pad_width_right], [0,0]]`.
J
JiabinYang 已提交
1477
            Otherwise, the pool padding size will be a square of an int.
1478 1479 1480
        global_pooling (bool): ${global_pooling_comment}
        use_cudnn (bool): ${use_cudnn_comment}
        ceil_mode (bool): ${ceil_mode_comment}
K
Kaipeng Deng 已提交
1481 1482 1483
        name(str, optional): For detailed information, please refer
                             to :ref:`api_guide_Name`. Usually name is no need to set and
                             None by default.
1484
        exclusive (bool): Whether to exclude padding points in average pooling
1485
                          mode, default is `true`.
1486
        data_format (string): The data format of the input and output data. An optional string from: `"NCHW"`, `"NHWC"`.
1487 1488
                The default is `"NCHW"`. When it is `"NCHW"`, the data is stored in the order of:
                `[batch_size, input_channels, input_height, input_width]`.
F
fengjiayi 已提交
1489

1490
    Returns:
K
Kaipeng Deng 已提交
1491
        Variable: The output tensor of pooling result. The data type is same as input tensor.
F
fengjiayi 已提交
1492 1493

    Raises:
1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505
        ValueError: If `pool_type` is not "max" nor "avg".
        ValueError: If `global_pooling` is False and `pool_size` is -1.
        TypeError: If `use_cudnn` is not a bool value.
        ValueError: If `data_format` is not "NCHW" or "NHWC".
        ValueError: If `pool_padding` is a string, but not "SAME" or "VALID".
        ValueError: If `pool_padding` is "VALID", but `ceil_mode` is True.
        ValueError: If `pool_padding` is a list or tuple, but the elements in the batch or channel dimensions are non-zero.
        ShapeError: If the input is not a 4-D or 5-D Tensor.
        ShapeError: If the dimension of input minus the size of `pool_stride` is not 2.
        ShapeError: If the size of `pool_size` and `pool_stride` is not equal.
        ShapeError: If the output's shape calculated is not greater than 0.

F
fengjiayi 已提交
1506 1507 1508 1509 1510

    Examples:

        .. code-block:: python

1511
          import paddle.fluid as fluid
1512 1513 1514
          import paddle

          paddle.enable_static()
1515

K
Kaipeng Deng 已提交
1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540
          data = fluid.data(name='data', shape=[None, 3, 32, 32], dtype='float32')

          # max pool2d
          pool2d = fluid.layers.pool2d(
            input = data,
            pool_size = 2,
            pool_type = "max",
            pool_stride = 1,
            global_pooling=False)

          # average pool2d
          pool2d = fluid.layers.pool2d(
            input = data,
            pool_size = 2,
            pool_type = "avg",
            pool_stride = 1,
            global_pooling=False)

          # global average pool2d
          pool2d = fluid.layers.pool2d(
            input = data,
            pool_size = 2,
            pool_type = "avg",
            pool_stride = 1,
            global_pooling=True)
1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558

          # Attr(pool_padding) is a list with 4 elements, Attr(data_format) is "NCHW".
          out_1 = fluid.layers.pool2d(
            input = data,
            pool_size = 3,
            pool_type = "avg",
            pool_stride = 1,
            pool_padding = [1, 2, 1, 0],
            data_format = "NCHW")

          # Attr(pool_padding) is a string, Attr(data_format) is "NCHW".
          out_2 = fluid.layers.pool2d(
            input = data,
            pool_size = 3,
            pool_type = "avg",
            pool_stride = 1,
            pool_padding = "VALID",
            data_format = "NCHW")
Y
Yu Yang 已提交
1559 1560 1561
    """
    if pool_type not in ["max", "avg"]:
        raise ValueError(
1562
            "Unknown Attr(pool_type): '%s'. It can only be 'max' or 'avg'.",
1563 1564
            str(pool_type),
        )
C
chengduoZH 已提交
1565

C
chengduoZH 已提交
1566 1567
    if global_pooling is False and pool_size == -1:
        raise ValueError(
1568
            "When Attr(global_pooling) is False, Attr(pool_size) must be passed "
1569 1570
            "and be a valid value. Received pool_size: %s." % str(pool_size)
        )
1571 1572

    if not isinstance(use_cudnn, bool):
1573 1574 1575 1576
        raise TypeError(
            "Attr(use_cudnn) should be True or False. Received "
            "Attr(use_cudnn): %s." % str(use_cudnn)
        )
1577 1578 1579 1580

    if data_format not in ["NCHW", "NHWC"]:
        raise ValueError(
            "Attr(data_format) should be 'NCHW' or 'NHWC'. Received "
1581 1582
            "Attr(data_format): %s." % str(data_format)
        )
C
chengduoZH 已提交
1583

C
chengduoZH 已提交
1584 1585 1586
    pool_size = utils.convert_to_list(pool_size, 2, 'pool_size')
    pool_stride = utils.convert_to_list(pool_stride, 2, 'pool_stride')

1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597
    def update_padding(padding, data_format):
        def is_list_or_tuple(ele):
            if isinstance(ele, list) or isinstance(ele, tuple):
                return True
            return False

        if is_list_or_tuple(padding) and len(padding) == 4:
            if is_list_or_tuple(padding[0]) and (data_format == "NCHW"):
                if not (padding[0] == [0, 0] and padding[1] == [0, 0]):
                    raise ValueError(
                        "Non-zero pool_padding(%s) in the batch or channel dimensions "
1598 1599
                        "is not supported." % str(padding)
                    )
1600 1601 1602 1603 1604 1605
                padding = padding[2:4]
                padding = [ele for a_list in padding for ele in a_list]
            elif is_list_or_tuple(padding[0]) and (data_format == "NHWC"):
                if not (padding[0] == [0, 0] and padding[3] == [0, 0]):
                    raise ValueError(
                        "Non-zero pool_padding(%s) in the batch or channel dimensions "
1606 1607
                        "is not supported." % str(padding)
                    )
1608 1609 1610
                padding = padding[1:3]
                padding = [ele for a_list in padding for ele in a_list]
            padding = utils.convert_to_list(padding, 4, 'padding')
1611

1612 1613
            if utils._is_symmetric_padding(padding, 2):
                padding = [padding[0], padding[2]]
1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624
        else:
            padding = utils.convert_to_list(padding, 2, 'padding')

        return padding

    padding_algorithm = "EXPLICIT"
    if isinstance(pool_padding, str):
        pool_padding = pool_padding.upper()
        if pool_padding not in ["SAME", "VALID"]:
            raise ValueError(
                "Unknown Attr(pool_padding): '%s'. It can only be 'SAME' or 'VALID'."
1625 1626
                % str(pool_padding)
            )
1627 1628
        if pool_padding == "VALID":
            padding_algorithm = "VALID"
1629
            pool_padding = [0, 0]
1630
            if ceil_mode is not False:
1631 1632
                raise ValueError(
                    "When Attr(pool_padding) is \"VALID\", Attr(ceil_mode) must be False. "
1633 1634
                    "Received ceil_mode: True."
                )
1635 1636
        elif pool_padding == "SAME":
            padding_algorithm = "SAME"
1637
            pool_padding = [0, 0]
1638 1639

    pool_padding = update_padding(pool_padding, data_format)
1640
    if in_dygraph_mode():
1641
        input = input._use_gpudnn(use_cudnn)
1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654
        return _C_ops.pool2d(
            input,
            pool_size,
            pool_stride,
            pool_padding,
            ceil_mode,
            exclusive,
            data_format,
            pool_type,
            global_pooling,
            False,
            padding_algorithm,
        )
1655 1656
    op_type = 'pool2d'
    helper = LayerHelper(op_type, **locals())
Y
Yu Yang 已提交
1657
    dtype = helper.input_dtype()
X
Xin Pan 已提交
1658
    pool_out = helper.create_variable_for_type_inference(dtype)
Y
Yu Yang 已提交
1659

1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677
    helper.append_op(
        type=op_type,
        inputs={"X": input},
        outputs={"Out": pool_out},
        attrs={
            "pooling_type": pool_type,
            "ksize": pool_size,
            "global_pooling": global_pooling,
            "strides": pool_stride,
            "paddings": pool_padding,
            "padding_algorithm": padding_algorithm,
            "use_cudnn": use_cudnn,
            "ceil_mode": ceil_mode,
            "use_mkldnn": False,
            "exclusive": exclusive,
            "data_format": data_format,
        },
    )
1678 1679 1680 1681

    return pool_out


Y
yuyang18 已提交
1682
@templatedoc()
1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693
def layer_norm(
    input,
    scale=True,
    shift=True,
    begin_norm_axis=1,
    epsilon=1e-05,
    param_attr=None,
    bias_attr=None,
    act=None,
    name=None,
):
1694
    r"""
1695 1696
    :api_attr: Static Graph

1697 1698 1699 1700
    **Layer Normalization Layer**

    The API implements the function of the Layer Normalization Layer and can be applied to mini-batch input data.
    Refer to `Layer Normalization <https://arxiv.org/pdf/1607.06450v1.pdf>`_
G
guosheng 已提交
1701 1702 1703

    The formula is as follows:

Y
yuyang18 已提交
1704
    ..  math::
G
guosheng 已提交
1705

1706
        \\mu & = \\frac{1}{H}\\sum_{i=1}^{H} x_i
G
guosheng 已提交
1707

1708
        \\sigma & = \\sqrt{\\frac{1}{H}\sum_{i=1}^{H}{(x_i - \\mu)^2} + \\epsilon}
Y
yuyang18 已提交
1709

1710
        y & = f(\\frac{g}{\\sigma}(x - \\mu) + b)
Y
yuyang18 已提交
1711

1712 1713 1714 1715 1716
    - :math:`x`: the vector representation of the summed inputs to the neurons in that layer.
    - :math:`H`: the number of hidden units in a layers
    - :math:`\\epsilon`: the small value added to the variance to prevent division by zero.
    - :math:`g`: the trainable scale parameter.
    - :math:`b`: the trainable bias parameter.
Y
yuyang18 已提交
1717

G
guosheng 已提交
1718
    Args:
1719
        input(Tensor): A multi-dimension ``Tensor`` , and the data type is float32 or float64.
1720 1721 1722 1723 1724
        scale(bool, optional): Whether to learn the adaptive gain :math:`g` after
            normalization. Default: True.
        shift(bool, optional): Whether to learn the adaptive bias :math:`b` after
            normalization. Default: True.
        begin_norm_axis(int, optional): The normalization will be performed along
G
guosheng 已提交
1725
            dimensions from :attr:`begin_norm_axis` to :attr:`rank(input)`.
1726 1727 1728 1729
            Default: 1.
        epsilon(float, optional): The small value added to the variance to prevent
            division by zero. Default: 1e-05.
        param_attr(ParamAttr, optional): The parameter attribute for the learnable
S
sneaxiy 已提交
1730 1731
            gain :math:`g`. If :attr:`scale` is False, :attr:`param_attr` is
            omitted. If :attr:`scale` is True and :attr:`param_attr` is None,
1732
            a default :code:`ParamAttr` would be added as scale. The
1733 1734
            :attr:`param_attr` is initialized as 1 if it is added. Default: None.
        bias_attr(ParamAttr, optional): The parameter attribute for the learnable
S
sneaxiy 已提交
1735 1736
            bias :math:`b`. If :attr:`shift` is False, :attr:`bias_attr` is
            omitted. If :attr:`shift` is True and :attr:`param_attr` is None,
1737
            a default :code:`ParamAttr` would be added as bias. The
1738
            :attr:`bias_attr` is initialized as 0 if it is added. Default: None.
T
tianshuo78520a 已提交
1739
        act(str, optional): Activation to be applied to the output of layer normalization.
1740 1741
                  Default: None.
        name(str): The default value is None.  Normally there is no need for user to set this property.  For more information, please refer to :ref:`api_guide_Name` .
G
guosheng 已提交
1742 1743

    Returns:
1744
        Tensor: ``Tensor``  indicating the normalized result, the data type is the same as  ``input`` , and the return dimension is the same as  ``input`` .
G
guosheng 已提交
1745 1746 1747

    Examples:

1748 1749
        .. code-block:: python

1750 1751
            import paddle
            paddle.enable_static()
1752 1753 1754
            x = paddle.static.data(name='x', shape=[8, 32, 32], dtype='float32')
            output = paddle.static.nn.layer_norm(input=x, begin_norm_axis=1)
            print(output.shape)  # [8, 32, 32]
G
guosheng 已提交
1755
    """
1756 1757 1758
    assert (
        _non_static_mode() is not True
    ), "please use LayerNorm instead of layer_norm in dygraph mode!"
G
guosheng 已提交
1759
    helper = LayerHelper('layer_norm', **locals())
1760 1761 1762
    check_variable_and_dtype(
        input, 'input', ['float32', 'float64'], 'layer_norm'
    )
G
guosheng 已提交
1763 1764 1765 1766 1767 1768 1769
    dtype = helper.input_dtype()

    # create intput and parameters
    inputs = {'X': input}
    input_shape = input.shape
    param_shape = [reduce(lambda x, y: x * y, input_shape[begin_norm_axis:])]
    if scale:
1770 1771 1772 1773 1774 1775 1776 1777 1778
        assert (
            param_attr is not False
        ), "param_attr should not be False when using scale."
        scale = helper.create_parameter(
            attr=helper.param_attr,
            shape=param_shape,
            dtype=dtype,
            default_initializer=Constant(1.0),
        )
G
guosheng 已提交
1779
        inputs['Scale'] = scale
1780 1781
    else:
        if param_attr:
T
tianshuo78520a 已提交
1782
            warnings.warn("param_attr is only available with scale is True.")
G
guosheng 已提交
1783
    if shift:
1784 1785 1786 1787 1788 1789
        assert (
            bias_attr is not False
        ), "bias_attr should not be False when using shift."
        bias = helper.create_parameter(
            attr=helper.bias_attr, shape=param_shape, dtype=dtype, is_bias=True
        )
G
guosheng 已提交
1790
        inputs['Bias'] = bias
1791 1792
    else:
        if bias_attr:
T
tianshuo78520a 已提交
1793
            warnings.warn("bias_attr is only available with shift is True.")
G
guosheng 已提交
1794 1795

    # create output
1796 1797 1798 1799 1800 1801
    mean_out = helper.create_variable_for_type_inference(
        dtype=dtype, stop_gradient=True
    )
    variance_out = helper.create_variable_for_type_inference(
        dtype=dtype, stop_gradient=True
    )
X
Xin Pan 已提交
1802
    layer_norm_out = helper.create_variable_for_type_inference(dtype)
G
guosheng 已提交
1803

1804 1805 1806 1807 1808 1809 1810 1811 1812 1813
    helper.append_op(
        type="layer_norm",
        inputs=inputs,
        outputs={
            "Y": layer_norm_out,
            "Mean": mean_out,
            "Variance": variance_out,
        },
        attrs={"epsilon": epsilon, "begin_norm_axis": begin_norm_axis},
    )
G
guosheng 已提交
1814 1815 1816 1817

    return helper.append_activation(layer_norm_out)


D
dengkaipeng 已提交
1818
@templatedoc()
1819
def spectral_norm(weight, dim=0, power_iters=1, eps=1e-12, name=None):
1820
    r"""
1821 1822
    :api_attr: Static Graph

D
dengkaipeng 已提交
1823 1824
    **Spectral Normalization Layer**

K
Kaipeng Deng 已提交
1825
    This operation calculates the spectral normalization value of weight parameters of
1826
    fc, conv1d, conv2d, conv3d layers which should be 2-D, 3-D, 4-D, 5-D
K
Kaipeng Deng 已提交
1827 1828
    Parameters. Output tensor will be in same shape with input tensor.
    Calculations are showed as follows.
1829

D
dengkaipeng 已提交
1830 1831 1832
    Step 1:
    Generate vector U in shape of [H], and V in shape of [W].
    While H is the :attr:`dim` th dimension of the input weights,
D
dengkaipeng 已提交
1833
    and W is the product result of remaining dimensions.
D
dengkaipeng 已提交
1834 1835

    Step 2:
T
tianshuo78520a 已提交
1836
    :attr:`power_iters` should be a positive integer, do following
K
Kaipeng Deng 已提交
1837 1838
    calculations with U and V for :attr:`power_iters` rounds. Calculations
    as follows:
D
dengkaipeng 已提交
1839

1840
    .. math::
D
dengkaipeng 已提交
1841 1842 1843 1844 1845 1846

        \mathbf{v} := \\frac{\mathbf{W}^{T} \mathbf{u}}{\|\mathbf{W}^{T} \mathbf{u}\|_2}

        \mathbf{u} := \\frac{\mathbf{W}^{T} \mathbf{v}}{\|\mathbf{W}^{T} \mathbf{v}\|_2}

    Step 3:
D
dengkaipeng 已提交
1847
    Calculate :math:`\sigma(\mathbf{W})` and normalize weight values.
D
dengkaipeng 已提交
1848 1849 1850 1851

    .. math::

        \sigma(\mathbf{W}) = \mathbf{u}^{T} \mathbf{W} \mathbf{v}
1852

D
dengkaipeng 已提交
1853
        \mathbf{W} = \\frac{\mathbf{W}}{\sigma(\mathbf{W})}
1854

1855

D
dengkaipeng 已提交
1856 1857 1858
    Refer to `Spectral Normalization <https://arxiv.org/abs/1802.05957>`_ .

    Args:
C
Chen Long 已提交
1859
        weight(Tensor): ${weight_comment}
D
dengkaipeng 已提交
1860 1861 1862
        dim(int): ${dim_comment}
        power_iters(int): ${power_iters_comment}
        eps(float): ${eps_comment}
K
Kaipeng Deng 已提交
1863 1864 1865
        name(str, optional): For detailed information, please refer
                             to :ref:`api_guide_Name`. Usually name is no need to set and
                             None by default.
D
dengkaipeng 已提交
1866 1867

    Returns:
C
Chen Long 已提交
1868
        Tensor: A tensor of weight parameters after spectral normalization.
K
Kaipeng Deng 已提交
1869
                  The data type and shape is same as input tensor.
D
dengkaipeng 已提交
1870 1871

    Examples:
K
Kaipeng Deng 已提交
1872
       .. code-block:: python
D
dengkaipeng 已提交
1873

1874
            import paddle
K
Kaipeng Deng 已提交
1875

1876
            paddle.enable_static()
C
Chen Long 已提交
1877
            weight = paddle.static.data(name='weight', shape=[2, 8, 32, 32], dtype='float32')
1878
            x = paddle.static.nn.spectral_norm(weight=weight, dim=1, power_iters=2)
C
Chen Long 已提交
1879
            print(x.shape) # [2, 8, 32, 32]
D
dengkaipeng 已提交
1880 1881
    """
    helper = LayerHelper('spectral_norm', **locals())
1882 1883 1884
    check_variable_and_dtype(
        weight, 'weight', ['float32', 'float64'], 'spectral_norm'
    )
1885 1886 1887
    check_type(dim, 'dim', int, 'spectral_norm')
    check_type(power_iters, 'power_iters', int, 'spectral_norm')
    check_type(eps, 'eps', float, 'spectral_norm')
1888
    dtype = weight.dtype
D
dengkaipeng 已提交
1889 1890

    # create intput and parameters
1891
    input_shape = weight.shape
1892
    assert weight.numel() > 0, "Any dimension of input cannot be equal to 0."
1893 1894 1895 1896 1897
    assert dim < len(input_shape), (
        "The input `dim` should be less than the "
        "rank of `weight`, but received dim="
        "{}".format(dim)
    )
1898 1899 1900
    h = input_shape[dim]
    w = np.prod(input_shape) // h

1901 1902 1903 1904 1905 1906
    u = helper.create_parameter(
        attr=ParamAttr(),
        shape=[h],
        dtype=dtype,
        default_initializer=Normal(0.0, 1.0),
    )
1907
    u.stop_gradient = True
1908 1909 1910 1911 1912 1913
    v = helper.create_parameter(
        attr=ParamAttr(),
        shape=[w],
        dtype=dtype,
        default_initializer=Normal(0.0, 1.0),
    )
1914
    v.stop_gradient = True
D
dengkaipeng 已提交
1915

1916 1917 1918 1919 1920 1921 1922
    if in_dygraph_mode():
        return _C_ops.spectral_norm(weight, u, v, dim, power_iters, eps)

    inputs = {'Weight': weight}
    inputs['U'] = u
    inputs['V'] = v

D
dengkaipeng 已提交
1923
    # create output
1924
    out = helper.create_variable(dtype=dtype)
D
Dun 已提交
1925

1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937
    helper.append_op(
        type="spectral_norm",
        inputs=inputs,
        outputs={
            "Out": out,
        },
        attrs={
            "dim": dim,
            "power_iters": power_iters,
            "eps": eps,
        },
    )
D
Dun 已提交
1938

1939
    return out
D
Dun 已提交
1940 1941


C
caoying03 已提交
1942
def reduce_sum(input, dim=None, keep_dim=False, name=None):
G
guosheng 已提交
1943
    """
1944

Y
yangyaming 已提交
1945
    Computes the sum of tensor elements over the given dimension.
G
guosheng 已提交
1946 1947

    Args:
1948 1949 1950
        input (Variable): The input variable which is a Tensor, the data type is float32,
            float64, int32, int64.
        dim (list|int, optional): The dimensions along which the sum is performed. If
Y
yangyaming 已提交
1951 1952
            :attr:`None`, sum all elements of :attr:`input` and return a
            Tensor variable with a single element, otherwise must be in the
W
whs 已提交
1953 1954
            range :math:`[-rank(input), rank(input))`. If :math:`dim[i] < 0`,
            the dimension to reduce is :math:`rank + dim[i]`.
1955
        keep_dim (bool, optional): Whether to reserve the reduced dimension in the
Y
yangyaming 已提交
1956
            output Tensor. The result tensor will have one fewer dimension
1957 1958 1959 1960
            than the :attr:`input` unless :attr:`keep_dim` is true, default
            value is False.
        name(str, optional): The default value is None.  Normally there is no need for
            user to set this property.  For more information, please refer to :ref:`api_guide_Name`
G
guosheng 已提交
1961 1962

    Returns:
1963 1964
        Variable: Tensor, results of summation operation on the specified dim of input tensor,
        it's data type is the same as input's Tensor.
F
fengjiayi 已提交
1965

1966 1967
    Raises:
        TypeError, if out data type is different with the input data type.
1968

G
guosheng 已提交
1969 1970 1971
    Examples:
        .. code-block:: python

1972
            import paddle.fluid as fluid
1973 1974
            import paddle
            paddle.enable_static()
G
guosheng 已提交
1975 1976 1977
            # x is a Tensor variable with following elements:
            #    [[0.2, 0.3, 0.5, 0.9]
            #     [0.1, 0.2, 0.6, 0.7]]
Q
qiaolongfei 已提交
1978
            # Each example is followed by the corresponding output tensor.
1979
            x = fluid.data(name='x', shape=[2, 4], dtype='float32')
G
guosheng 已提交
1980 1981 1982 1983
            fluid.layers.reduce_sum(x)  # [3.5]
            fluid.layers.reduce_sum(x, dim=0)  # [0.3, 0.5, 1.1, 1.6]
            fluid.layers.reduce_sum(x, dim=-1)  # [1.9, 1.6]
            fluid.layers.reduce_sum(x, dim=1, keep_dim=True)  # [[1.9], [1.6]]
W
whs 已提交
1984

1985
            # y is a Tensor variable with shape [2, 2, 2] and elements as below:
W
whs 已提交
1986 1987
            #      [[[1, 2], [3, 4]],
            #      [[5, 6], [7, 8]]]
Q
qiaolongfei 已提交
1988
            # Each example is followed by the corresponding output tensor.
1989
            y = fluid.data(name='y', shape=[2, 2, 2], dtype='float32')
1990 1991
            fluid.layers.reduce_sum(y, dim=[1, 2]) # [10, 26]
            fluid.layers.reduce_sum(y, dim=[0, 1]) # [16, 20]
W
whs 已提交
1992

G
guosheng 已提交
1993
    """
1994 1995
    reduce_all, dim = _get_reduce_dim(dim, input)

1996
    if in_dygraph_mode():
1997
        return _C_ops.sum(input, dim, None, keep_dim)
1998
    elif _in_legacy_dygraph():
1999 2000 2001
        return _legacy_C_ops.reduce_sum(
            input, 'dim', dim, 'keep_dim', keep_dim, 'reduce_all', reduce_all
        )
2002
    attrs = {'dim': dim, 'keep_dim': keep_dim, 'reduce_all': reduce_all}
2003
    check_variable_and_dtype(
2004 2005 2006 2007 2008
        input,
        'input',
        ['float16', 'float32', 'float64', 'int32', 'int64'],
        'reduce_sum',
    )
2009
    helper = LayerHelper('reduce_sum', **locals())
X
Xin Pan 已提交
2010
    out = helper.create_variable_for_type_inference(dtype=helper.input_dtype())
2011 2012 2013 2014 2015 2016
    helper.append_op(
        type='reduce_sum',
        inputs={'X': input},
        outputs={'Out': out},
        attrs=attrs,
    )
G
guosheng 已提交
2017
    return out
G
guosheng 已提交
2018 2019


C
caoying03 已提交
2020
def split(input, num_or_sections, dim=-1, name=None):
G
guosheng 已提交
2021
    """
2022
    Split the input tensor into multiple sub-Tensors.
G
guosheng 已提交
2023 2024

    Args:
2025
        input (Tensor): A N-D Tensor. The data type is bool, float16, float32, float64, int32 or int64.
2026
        num_or_sections (int|list|tuple): If ``num_or_sections`` is int, then the ``num_or_sections``
2027
            indicates the number of equal sized sub-Tensors that the ``input``
2028
            will be divided into. If ``num_or_sections`` is a list or tuple, the length of it
2029 2030 2031 2032 2033
            indicates the number of sub-Tensors and the elements in it indicate the sizes of sub-Tensors'
            dimension orderly. The length of the list mustn't be larger than the ``input`` 's size of specified dim.
        dim (int|Tensor, optional): The dimension along which to split, it can be a scalar with type ``int`` or
            a ``Tensor`` with shape [1] and data type ``int32`` or ``int64``. If :math:`dim < 0`,
            the dimension to split along is :math:`rank(input) + dim`. Default is -1.
2034
        name (str, optional): The default value is None.  Normally there is no need for user to set this property.
2035
            For more information, please refer to :ref:`api_guide_Name` .
G
guosheng 已提交
2036 2037

    Returns:
2038
        list(Tensor): The list of segmented Tensors.
G
guosheng 已提交
2039

2040
    Example:
G
guosheng 已提交
2041 2042
        .. code-block:: python

2043 2044
            import paddle.fluid as fluid

2045
            # input is a Tensor which shape is [3, 9, 5]
2046
            input = fluid.data(
2047 2048
                 name="input", shape=[3, 9, 5], dtype="float32")

2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062
            out0, out1, out2 = fluid.layers.split(input, num_or_sections=3, dim=1)
            # out0.shape [3, 3, 5]
            # out1.shape [3, 3, 5]
            # out2.shape [3, 3, 5]

            out0, out1, out2 = fluid.layers.split(input, num_or_sections=[2, 3, 4], dim=1)
            # out0.shape [3, 2, 5]
            # out1.shape [3, 3, 5]
            # out2.shape [3, 4, 5]

            out0, out1, out2 = fluid.layers.split(input, num_or_sections=[2, 3, -1], dim=1)
            # out0.shape [3, 2, 5]
            # out1.shape [3, 3, 5]
            # out2.shape [3, 4, 5]
2063

2064 2065 2066 2067 2068 2069
            # dim is negative, the real dim is (rank(input) + axis) which real
            # value is 1.
            out0, out1, out2 = fluid.layers.split(input, num_or_sections=3, dim=-2)
            # out0.shape [3, 3, 5]
            # out1.shape [3, 3, 5]
            # out2.shape [3, 3, 5]
2070

G
guosheng 已提交
2071
    """
J
Jiabin Yang 已提交
2072
    if _non_static_mode():
2073 2074 2075
        num = None
        attrs = ()

S
songyouwei 已提交
2076 2077
        if isinstance(dim, Variable):
            dim = dim.numpy()
2078
            dim = dim.item(0)
W
wangzhen38 已提交
2079
        assert len(input.shape) + dim >= 0, "(rank(x) + axis) must >= 0"
S
songyouwei 已提交
2080
        dim = (len(input.shape) + dim) if dim < 0 else dim
2081
        attrs += ('axis', dim)
2082 2083 2084

        if isinstance(num_or_sections, int):
            num = num_or_sections
2085
            attrs += ('num', num_or_sections)
L
Leo Chen 已提交
2086
        elif isinstance(num_or_sections, (list, tuple)):
2087
            num = len(num_or_sections)
L
Leo Chen 已提交
2088
            if utils._contain_var(num_or_sections):
2089 2090
                for index, item in enumerate(num_or_sections):
                    if isinstance(item, Variable):
2091 2092 2093
                        num_or_sections[index] = num_or_sections[index].numpy()[
                            0
                        ]
2094
                attrs += ('sections', list(num_or_sections))
L
Leo Chen 已提交
2095
            else:
2096
                attrs += ('sections', list(num_or_sections))
2097 2098
        else:
            raise TypeError(
2099
                "The type of 'num_or_sections' in split must be int, list or tuple in imperative mode, but "
2100 2101
                "received %s." % (type(num_or_sections))
            )
2102
        if in_dygraph_mode():
C
Charles-hit 已提交
2103 2104 2105 2106
            if isinstance(num_or_sections, int):
                return _C_ops.split_with_num(input, num_or_sections, dim)
            else:
                return _C_ops.split(input, num_or_sections, dim)
2107 2108
        elif _in_legacy_dygraph():
            out = [_varbase_creator() for n in range(num)]
2109
            _legacy_C_ops.split(input, out, *attrs)
2110
            return out
L
Leo Chen 已提交
2111

2112
    check_variable_and_dtype(
2113 2114 2115 2116 2117
        input,
        'input',
        ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'],
        'split',
    )
2118 2119 2120 2121
    check_type(num_or_sections, 'num_or_sections', (list, int, tuple), 'split')
    check_type(dim, 'dim', (int, Variable), 'split')
    if isinstance(dim, Variable):
        check_dtype(dim.dtype, 'dim', ['int32', 'int64'], 'split')
2122

G
guosheng 已提交
2123
    helper = LayerHelper('split', **locals())
2124

G
guosheng 已提交
2125
    input_shape = input.shape
2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136
    inputs = {'X': input}
    attrs = {'num': num_or_sections if isinstance(num_or_sections, int) else 0}

    def _get_SectionsTensorList(one_list):
        tensor_list = []
        unk_dim_idx = -1
        for idx, dim_size in enumerate(one_list):
            if isinstance(dim_size, Variable):
                dim_size.stop_gradient = True
                tensor_list.append(dim_size)
            else:
2137
                assert isinstance(dim_size, int)
2138 2139 2140
                if dim_size == -1:
                    assert unk_dim_idx == -1, (
                        "Only one value of 'num_or_section' in split can "
2141 2142 2143
                        "be -1. But received num_or_section[%d] is also -1."
                        % idx
                    )
2144 2145
                    unk_dim_idx = idx
                temp_out = helper.create_variable_for_type_inference('int32')
2146 2147 2148
                fill_constant(
                    [1], 'int32', dim_size, force_cpu=True, out=temp_out
                )
2149 2150 2151 2152 2153 2154 2155
                tensor_list.append(temp_out)
        return tensor_list

    if isinstance(dim, Variable):
        dim.stop_gradient = True
        inputs['AxisTensor'] = dim
    else:
W
wangzhen38 已提交
2156
        assert len(input.shape) + dim >= 0, "(rank(x) + axis) must >= 0"
2157 2158 2159
        dim = (len(input_shape) + dim) if dim < 0 else dim
        attrs['axis'] = dim

G
guosheng 已提交
2160 2161
    if isinstance(num_or_sections, int):
        assert num_or_sections > 1, 'num_or_sections must be more than 1.'
2162
        if isinstance(dim, int) and input_shape[dim] > 0:
2163 2164 2165 2166 2167 2168
            assert input_shape[dim] % num_or_sections == 0, (
                "The input's size along the split dimension "
                "must be evenly divisible by Attr(num_or_sections). "
                "But %d is not evenly divisible by %d. "
                % (num_or_sections, input_shape[dim])
            )
G
guosheng 已提交
2169 2170
        num = num_or_sections
    else:
2171
        if isinstance(dim, int) and input_shape[dim] > 0:
2172 2173 2174
            assert (
                len(num_or_sections) <= input_shape[dim]
            ), 'len(num_or_sections) must not be more than input.shape[dim].'
G
guosheng 已提交
2175
        num = len(num_or_sections)
2176
        attrs['sections'] = list(
2177 2178 2179 2180 2181
            map(
                lambda ele: -1 if isinstance(ele, Variable) else ele,
                num_or_sections,
            )
        )
L
Leo Chen 已提交
2182
        if utils._contain_var(num_or_sections):
2183
            inputs['SectionsTensorList'] = _get_SectionsTensorList(
2184 2185
                num_or_sections
            )
2186

G
guosheng 已提交
2187
    outs = [
X
Xin Pan 已提交
2188
        helper.create_variable_for_type_inference(dtype=helper.input_dtype())
G
guosheng 已提交
2189 2190
        for i in range(num)
    ]
2191 2192 2193
    helper.append_op(
        type='split', inputs=inputs, outputs={'Out': outs}, attrs=attrs
    )
G
guosheng 已提交
2194
    return outs
C
caoying03 已提交
2195 2196 2197


def l2_normalize(x, axis, epsilon=1e-12, name=None):
2198
    r"""
2199

R
ruri 已提交
2200
    This op normalizes `x` along dimension `axis` using an L2
C
caoying03 已提交
2201 2202
    norm. For a 1-D tensor (`dim` is fixed to 0), this layer computes

2203
    .. math::
2204 2205

        y = \\frac{x}{ \sqrt{\sum {x^2} + epsion }}
C
caoying03 已提交
2206 2207 2208 2209 2210

    For `x` with more dimensions, this layer independently normalizes each 1-D
    slice along dimension `axis`.

    Args:
2211
        x(Variable|list): The input tensor could be N-D tensor, and the input data type could be float16, float32 or float64.
2212
        axis(int): The axis on which to apply normalization. If `axis < 0`, \
2213 2214
            the dimension to normalization is rank(X) + axis. -1 is the
            last dimension.
2215
        epsilon(float): The epsilon value is used to avoid division by zero, \
翟飞跃 已提交
2216
            the default value is 1e-12.
2217
    name(str, optional): The default value is None.  Normally there is no need for user to set this property.  For more information, please refer to :ref:`api_guide_Name`
2218

C
caoying03 已提交
2219
    Returns:
R
ruri 已提交
2220
        Variable: The output has the same shape and data type with `x`.
C
caoying03 已提交
2221 2222

    Examples:
2223

2224 2225
    .. code-block:: python
        :name: code-example1
2226

2227
        import paddle
2228

2229 2230
        X = paddle.randn(shape=[3, 5], dtype='float64')
        out = paddle.fluid.layers.l2_normalize(X, axis=-1)
G
Guoxia Wang 已提交
2231
        print(out)
R
ruri 已提交
2232

2233 2234 2235
        # [[ 0.21558504  0.56360189  0.47466096  0.46269539 -0.44326736]
        #  [-0.70602414 -0.52745777  0.37771788 -0.2804768  -0.04449922]
        #  [-0.33972208 -0.43014923  0.31772556  0.76617881 -0.10761525]]
2236

C
caoying03 已提交
2237
    """
F
fengjiayi 已提交
2238 2239
    if len(x.shape) == 1:
        axis = 0
J
Jiabin Yang 已提交
2240
    if _non_static_mode():
2241 2242 2243
        if in_dygraph_mode():
            out, _ = _C_ops.norm(x, 1 if axis is None else axis, epsilon, False)
        elif _in_legacy_dygraph():
2244 2245 2246
            _, out = _legacy_C_ops.norm(
                x, 'axis', 1 if axis is None else axis, 'epsilon', epsilon
            )
2247 2248 2249
        return out

    check_variable_and_dtype(x, "X", ("float16", "float32", "float64"), "norm")
C
caoying03 已提交
2250

2251
    helper = LayerHelper("l2_normalize", **locals())
X
Xin Pan 已提交
2252 2253
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
    norm = helper.create_variable_for_type_inference(dtype=x.dtype)
2254 2255 2256 2257 2258 2259 2260 2261 2262
    helper.append_op(
        type="norm",
        inputs={"X": x},
        outputs={"Out": out, "Norm": norm},
        attrs={
            "axis": 1 if axis is None else axis,
            "epsilon": epsilon,
        },
    )
C
caoying03 已提交
2263
    return out
2264 2265


Y
yuyang18 已提交
2266
@templatedoc()
2267
def row_conv(input, future_context_size, param_attr=None, act=None):
Y
yuyang18 已提交
2268
    """
2269 2270
    :api_attr: Static Graph

Y
yuyang18 已提交
2271
    ${comment}
2272 2273

    Args:
Y
yuyang18 已提交
2274
        input (${x_type}): ${x_comment}.
Y
yangyaming 已提交
2275 2276
        future_context_size (int): Future context size. Please note, the shape
            of convolution kernel is [future_context_size + 1, D].
2277 2278 2279 2280 2281
        param_attr (ParamAttr): Attributes of parameters, including
            name, initializer etc.
        act (str): Non-linear activation to be applied to output variable.

    Returns:
Y
yuyang18 已提交
2282
        ${out_comment}.
2283 2284

    Examples:
B
Bai Yifan 已提交
2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296

      .. code-block:: python

        # for LodTensor inputs
        import paddle
        paddle.enable_static()
        x = paddle.static.data(name='x', shape=[9, 16],
                               dtype='float32', lod_level=1)
        out = paddle.static.nn.row_conv(input=x, future_context_size=2)
        # for Tensor inputs
        x = paddle.static.data(name='x', shape=[9, 4, 16], dtype='float32')
        out = paddle.static.nn.row_conv(input=x, future_context_size=2)
2297 2298
    """
    helper = LayerHelper('row_conv', **locals())
2299
    check_variable_and_dtype(input, 'input', ['float32'], 'row_conv')
2300
    dtype = helper.input_dtype()
2301
    filter_shape = [future_context_size + 1, input.shape[-1]]
2302 2303 2304
    filter_param = helper.create_parameter(
        attr=helper.param_attr, shape=filter_shape, dtype=dtype
    )
X
Xin Pan 已提交
2305
    out = helper.create_variable_for_type_inference(dtype)
2306 2307 2308 2309 2310
    helper.append_op(
        type='row_conv',
        inputs={'X': [input], 'Filter': [filter_param]},
        outputs={'Out': [out]},
    )
Y
yangyaming 已提交
2311
    return helper.append_activation(out)
2312 2313


2314
@deprecated(since='2.0.0', update_to='paddle.nn.functional.one_hot')
2315
def one_hot(input, depth, allow_out_of_range=False):
2316
    """
2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354

    **WARING:** This OP requires the last dimension of Tensor shape must be equal to 1.
    This OP will be deprecated in a future release. It is recommended to use fluid. :ref:`api_fluid_one_hot` .

    The operator converts each id in the input to an one-hot vector with a
    :attr:`depth` length. The value in the vector dimension corresponding to the id
    is 1, and the value in the remaining dimension is 0.

    The shape of output Tensor or LoDTensor is generated by adding :attr:`depth` dimension
    behind the last dimension of the input shape.

    .. code-block:: text

        Example 1 (allow_out_of_range=False):

        input:
            X.shape = [4, 1]
            X.data = [[1], [1], [3], [0]]
            depth = 4

        output:
            Out.shape = [4, 4]
            Out.data = [[0., 1., 0., 0.],
                        [0., 1., 0., 0.],
                        [0., 0., 0., 1.],
                        [1., 0., 0., 0.]]

        Example 2 (allow_out_of_range=True):

        input:
            X.shape = [4, 1]
            X.data = [[1], [1], [5], [0]]
            depth = 4
            allow_out_of_range = True

        output:
            Out.shape = [4, 4]
            Out.data = [[0., 1., 0., 0.],
2355
                        [0., 1., 0., 0.],
2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367
                        [0., 0., 0., 0.], # This id is 5, which goes beyond depth, so set it all-zeros data.
                        [1., 0., 0., 0.]]

        Example 3 (allow_out_of_range=False):

        input:
            X.shape = [4, 1]
            X.data = [[1], [1], [5], [0]]
            depth = 4
            allow_out_of_range = False

        output: Throw an exception for Illegal value
2368
            The second dimension in X is 5, which is greater than depth.
2369 2370
            Allow_out_of_range =False means that does not allow the word id to exceed depth,
            so it throws an exception.
2371 2372

    Args:
2373 2374 2375
        input(Variable): Tensor or LoDTensor with shape :math:`[N_1, N_2, ..., N_k, 1]` ,
            which contains at least one dimension and the last dimension must be 1.
            The data type is int32 or int64.
2376
        depth(scalar): An integer defining the :attr:`depth` of the one hot dimension. If input
2377
            is word id, depth is generally the dictionary size.
2378
        allow_out_of_range(bool): A bool value indicating whether the input
2379 2380 2381 2382
            indices could be out of range :math:`[0, depth)` . When input indices are
            out of range, exceptions :code:`Illegal value` is raised if :attr:`allow_out_of_range`
            is False, or zero-filling representations is created if it is set True.
            Default: False.
2383 2384

    Returns:
2385
        Variable: The one-hot representations of input. A Tensor or LoDTensor with type float32.
2386 2387

    Examples:
C
caoying03 已提交
2388
        .. code-block:: python
2389

2390
            import paddle
2391
            import paddle.fluid as fluid
2392 2393
            paddle.enable_static()

2394 2395 2396
            # Correspond to the first example above, where label.shape is [4, 1] and one_hot_label.shape is [4, 4].
            label = fluid.data(name="label", shape=[4, 1], dtype="int64")
            one_hot_label = fluid.layers.one_hot(input=label, depth=4)
2397
    """
J
Jiabin Yang 已提交
2398
    if _non_static_mode():
S
songyouwei 已提交
2399 2400 2401
        if isinstance(depth, Variable):
            depth = depth.numpy()
            assert depth.shape == (
2402 2403
                1,
            ), "depth of type Variable should have shape [1]"
2404
            depth = depth.item(0)
2405 2406 2407
        out = _legacy_C_ops.one_hot(
            input, 'depth', depth, 'allow_out_of_range', allow_out_of_range
        )
2408 2409
        out.stop_gradient = True
        return out
2410

2411
    helper = LayerHelper("one_hot", **locals())
2412
    check_variable_and_dtype(input, 'input', ['int32', 'int64'], 'one_hot')
2413
    check_type(depth, 'depth', (int, Variable), 'one_hot')
X
Xin Pan 已提交
2414
    one_hot_out = helper.create_variable_for_type_inference(dtype='float32')
2415

2416 2417
    if not isinstance(depth, Variable):
        # user attribute
2418
        inputs = {'X': input}
Y
Yi Liu 已提交
2419
        attrs = {'depth': depth, 'allow_out_of_range': allow_out_of_range}
2420
    else:
2421 2422 2423
        depth.stop_gradient = True
        inputs = {'X': input, 'depth_tensor': depth}
        attrs = {'allow_out_of_range': allow_out_of_range}
2424 2425 2426
    helper.append_op(
        type="one_hot", inputs=inputs, attrs=attrs, outputs={'Out': one_hot_out}
    )
2427
    one_hot_out.stop_gradient = True
2428
    return one_hot_out
Y
Yu Yang 已提交
2429 2430


Y
Yu Yang 已提交
2431
def autoincreased_step_counter(counter_name=None, begin=1, step=1):
Y
Yu Yang 已提交
2432
    """
2433 2434
    :api_attr: Static Graph

2435 2436
    Create an auto-increase variable. which will be automatically increased
    by 1 in every iteration. By default, the first return of this counter is 1,
Y
Yibing Liu 已提交
2437
    and the step size is 1.
Y
Yu Yang 已提交
2438 2439

    Args:
Y
Yibing Liu 已提交
2440 2441 2442
        counter_name(str, optional): The counter name. Default '@STEP_COUNTER@'.
        begin(int, optional): The first return value of this counter. Default 1.
        step(int, optional): The step size. Default 1.
Y
Yu Yang 已提交
2443

2444
    Returns:
Y
Yibing Liu 已提交
2445
        Variable: The auto-increased Variable with data type int64.
Y
yi.wu 已提交
2446 2447 2448 2449

    Examples:
        .. code-block:: python

2450
           import paddle.fluid as fluid
2451 2452
           import paddle
           paddle.enable_static()
Y
yi.wu 已提交
2453
           global_step = fluid.layers.autoincreased_step_counter(
Y
Yibing Liu 已提交
2454
               counter_name='@LR_DECAY_COUNTER@', begin=0, step=1)
Y
Yu Yang 已提交
2455 2456
    """
    helper = LayerHelper('global_step_counter')
Y
Yu Yang 已提交
2457 2458
    if counter_name is None:
        counter_name = '@STEP_COUNTER@'
Y
Yu Yang 已提交
2459
    counter, is_new_var = helper.create_or_get_global_variable(
H
hong 已提交
2460 2461 2462 2463
        name=counter_name,
        dtype='int64',
        shape=[1],
        persistable=True,
2464 2465
        belong_to_optimizer=True,
    )
Y
Yu Yang 已提交
2466
    if is_new_var:
2467 2468 2469
        helper.set_variable_initializer(
            counter, initializer=Constant(value=begin - 1, force_cpu=True)
        )
W
Wu Yi 已提交
2470
        helper.main_program.global_block()._prepend_op(
Y
Yu Yang 已提交
2471 2472
            type='increment',
            inputs={'X': [counter]},
Y
Yu Yang 已提交
2473
            outputs={'Out': [counter]},
2474 2475
            attrs={'step': float(step)},
        )
Y
Yu Yang 已提交
2476 2477 2478
        counter.stop_gradient = True

    return counter
Y
yangyaming 已提交
2479 2480


2481
def unsqueeze(input, axes, name=None):
Y
Yibing Liu 已提交
2482
    """
2483
    Insert single-dimensional entries to the shape of a Tensor. Takes one
M
minqiyang 已提交
2484 2485
    required argument axes, a list of dimensions that will be inserted.
    Dimension indices in axes are as seen in the output tensor.
Y
Yibing Liu 已提交
2486

M
minqiyang 已提交
2487
    For example:
H
haowang101779990 已提交
2488 2489 2490

    .. code-block:: text

M
minqiyang 已提交
2491
      Given a tensor such that tensor with shape [3, 4, 5],
Y
Yibing Liu 已提交
2492
      then Unsqueezed tensor with axes=[0, 4] has shape [1, 3, 4, 5, 1].
M
minqiyang 已提交
2493

Y
Yibing Liu 已提交
2494
    Args:
2495
        input (Variable): The input Tensor to be unsqueezed. Supported data type: float32, float64, bool, int8, int32, int64.
2496
        axes (int|list|tuple|Variable): Indicates the dimensions to be inserted. The data type is ``int32`` . If ``axes`` is a list or tuple, the elements of it should be integers or Tensors with shape [1]. If ``axes`` is an Variable, it should be an 1-D Tensor .
2497
        name (str|None): Name for this layer.
Y
Yibing Liu 已提交
2498 2499

    Returns:
2500
        Variable: Unsqueezed Tensor, with the same data type as input.
Y
Yibing Liu 已提交
2501 2502 2503 2504

    Examples:
        .. code-block:: python

2505 2506 2507
            import paddle.fluid as fluid
            x = fluid.layers.data(name='x', shape=[5, 10])
            y = fluid.layers.unsqueeze(input=x, axes=[1])
2508

Y
Yibing Liu 已提交
2509
    """
J
Jiabin Yang 已提交
2510
    if _non_static_mode():
L
Leo Chen 已提交
2511 2512 2513
        if isinstance(axes, int):
            axes = [axes]
        elif isinstance(axes, Variable):
2514
            axes = axes.numpy().tolist()
L
Leo Chen 已提交
2515 2516 2517 2518 2519
        elif isinstance(axes, (list, tuple)):
            axes = [
                item.numpy().item(0) if isinstance(item, Variable) else item
                for item in axes
            ]
2520
        if _in_legacy_dygraph():
2521
            out, _ = _legacy_C_ops.unsqueeze2(input, 'axes', axes)
2522
            return out
2523
        return _C_ops.unsqueeze(input, axes)
2524 2525

    check_type(axes, 'axis/axes', (int, list, tuple, Variable), 'unsqueeze')
2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542
    check_variable_and_dtype(
        input,
        'input',
        [
            'float16',
            'float32',
            'float64',
            'bool',
            'int8',
            'int16',
            'int32',
            'int64',
            'complex64',
            'complex128',
        ],
        'unsqueeze',
    )
2543 2544 2545 2546 2547 2548 2549 2550 2551 2552
    helper = LayerHelper("unsqueeze2", **locals())
    inputs = {"X": input}
    attrs = {}

    if isinstance(axes, int):
        axes = [axes]
    if isinstance(axes, Variable):
        axes.stop_gradient = True
        inputs["AxesTensor"] = axes
    elif isinstance(axes, (list, tuple)):
L
Leo Chen 已提交
2553
        if utils._contain_var(axes):
2554
            inputs["AxesTensorList"] = utils._convert_to_tensor_list(axes)
2555 2556 2557
        else:
            attrs["axes"] = axes

X
Xin Pan 已提交
2558 2559
    out = helper.create_variable_for_type_inference(dtype=input.dtype)
    x_shape = helper.create_variable_for_type_inference(dtype=input.dtype)
2560 2561 2562 2563 2564 2565
    helper.append_op(
        type="unsqueeze2",
        inputs=inputs,
        attrs=attrs,
        outputs={"Out": out, "XShape": x_shape},
    )
Y
Yibing Liu 已提交
2566

2567 2568
    return out

2569

Y
yangyaming 已提交
2570
def lod_reset(x, y=None, target_lod=None):
Y
yangyaming 已提交
2571
    """
Y
Yibing Liu 已提交
2572
    Set LoD of :attr:`x` to a new one specified by :attr:`y` or
2573 2574 2575 2576
    :attr:`target_lod`. When :attr:`y` provided, :attr:`y.lod` would be
    considered as target LoD first, otherwise :attr:`y.data` would be
    considered as target LoD. If :attr:`y` is not provided, target LoD should
    be specified by :attr:`target_lod`. If target LoD is specified by
2577
    :attr:`y.data` or :attr:`target_lod`, only one level LoD is supported.
Y
yangyaming 已提交
2578 2579 2580 2581 2582 2583

    .. code-block:: text

        * Example 1:

            Given a 1-level LoDTensor x:
2584
                x.lod =  [[ 2,           3,                   1 ]]
Y
yangyaming 已提交
2585 2586 2587
                x.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]]
                x.dims = [6, 1]

2588
            target_lod: [4, 2]
Y
yangyaming 已提交
2589 2590

            then we get a 1-level LoDTensor:
2591
                out.lod =  [[4,                          2]]
Y
yangyaming 已提交
2592 2593 2594 2595 2596 2597
                out.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]]
                out.dims = [6, 1]

        * Example 2:

            Given a 1-level LoDTensor x:
2598
                x.lod =  [[2,            3,                   1]]
Y
yangyaming 已提交
2599 2600 2601 2602
                x.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]]
                x.dims = [6, 1]

            y is a Tensor:
2603
                y.data = [[2, 4]]
Y
yangyaming 已提交
2604 2605 2606
                y.dims = [1, 3]

            then we get a 1-level LoDTensor:
2607
                out.lod =  [[2,            4]]
Y
yangyaming 已提交
2608 2609 2610 2611 2612 2613
                out.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]]
                out.dims = [6, 1]

        * Example 3:

            Given a 1-level LoDTensor x:
2614
                x.lod =  [[2,            3,                   1]]
Y
yangyaming 已提交
2615 2616 2617 2618
                x.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]]
                x.dims = [6, 1]

            y is a 2-level LoDTensor:
2619
                y.lod =  [[2, 2], [2, 2, 1, 1]]
Y
yangyaming 已提交
2620 2621 2622 2623
                y.data = [[1.1], [2.1], [3.1], [4.1], [5.1], [6.1]]
                y.dims = [6, 1]

            then we get a 2-level LoDTensor:
2624
                out.lod =  [[2, 2], [2, 2, 1, 1]]
Y
yangyaming 已提交
2625 2626 2627 2628
                out.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]]
                out.dims = [6, 1]

    Args:
2629
        x (Variable): Input variable which could be a Tensor or LoDTensor.
2630
                      The data type should be int32, int64, float32 or float64.
2631 2632
        y (Variable, optional): If provided, output's LoD would be derived from :attr:`y`.
                                If y's lod level>0, the data type can be any type.
2633 2634
                                If y's lod level=0, the data type should be int32.
        target_lod (list|tuple, optional): One level LoD which should be considered
Y
Yibing Liu 已提交
2635
                                      as target LoD when :attr:`y` not provided.
Y
yangyaming 已提交
2636 2637

    Returns:
Y
Yibing Liu 已提交
2638
        Variable: Output variable with LoD specified by this layer.
Y
yangyaming 已提交
2639 2640

    Raises:
Y
Yibing Liu 已提交
2641
        ValueError: If :attr:`y` and :attr:`target_lod` are both None.
Y
yangyaming 已提交
2642 2643 2644 2645

    Examples:
        .. code-block:: python

2646
            import paddle.fluid as fluid
2647 2648 2649
            x = fluid.layers.data(name='x', shape=[10])
            y = fluid.layers.data(name='y', shape=[10, 20], lod_level=2)
            out = fluid.layers.lod_reset(x=x, y=y)
Y
yangyaming 已提交
2650
    """
2651 2652 2653
    check_variable_and_dtype(
        x, 'x', ['float32', 'float64', 'int32', 'int64'], 'lod_reset'
    )
Y
yangyaming 已提交
2654
    helper = LayerHelper("lod_reset", **locals())
X
Xin Pan 已提交
2655
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
Y
yangyaming 已提交
2656
    if y is not None:
2657
        check_type(y, 'y', (Variable), 'lod_reset')
2658 2659 2660 2661
        # TODO: check y.lod_level = 0 dtype
        helper.append_op(
            type="lod_reset", inputs={'X': x, 'Y': y}, outputs={'Out': out}
        )
Y
yangyaming 已提交
2662
    elif target_lod is not None:
2663 2664 2665 2666 2667 2668
        helper.append_op(
            type="lod_reset",
            inputs={'X': x},
            attrs={'target_lod': target_lod},
            outputs={'Out': out},
        )
Y
yangyaming 已提交
2669
    else:
2670 2671 2672 2673
        raise ValueError("y and target_lod should not be both none.")
    return out


2674
@deprecated(since="2.0.0", update_to="paddle.nn.functional.relu")
2675
def relu(x, name=None):
W
wanghaoshuang 已提交
2676
    """
Z
zhupengyang 已提交
2677
    ${comment}
W
wanghaoshuang 已提交
2678 2679

    Args:
Z
zhupengyang 已提交
2680 2681 2682 2683
        x(Variable): ${x_comment}
        name(str, optional): The default value is None. Normally there is no
            need for user to set this property. For more information, please
            refer to :ref:`api_guide_Name`.
W
wanghaoshuang 已提交
2684 2685

    Returns:
Z
zhupengyang 已提交
2686
        Variable: ${out_comment}
W
wanghaoshuang 已提交
2687 2688 2689 2690 2691

    Examples:

        .. code-block:: python

2692
            import paddle.fluid as fluid
Z
zhupengyang 已提交
2693 2694 2695 2696 2697 2698 2699
            import numpy as np
            in1 = np.array([[-1,0],[1,2.6]])
            with fluid.dygraph.guard():
                x1 = fluid.dygraph.to_variable(in1)
                out1 = fluid.layers.relu(x1)
                print(out1.numpy())
                # [[0.  0. ]
2700
                #  [1.  2.6]]"""
2701 2702

    if in_dygraph_mode():
W
wanghuancoder 已提交
2703
        return _C_ops.relu(x)
2704 2705
    if _in_legacy_dygraph():
        return _legacy_C_ops.relu(x)
2706

2707 2708
    check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'relu')

2709
    inputs = {'X': [x]}
W
wanghaoshuang 已提交
2710
    helper = LayerHelper('relu', **locals())
W
wanghaoshuang 已提交
2711
    dtype = helper.input_dtype(input_param_name='x')
X
Xin Pan 已提交
2712
    out = helper.create_variable_for_type_inference(dtype)
2713 2714 2715
    helper.append_op(
        type="relu", inputs={"X": helper.input('x')}, outputs={"Out": out}
    )
W
wanghaoshuang 已提交
2716
    return out
2717 2718


G
fix  
gongweibao 已提交
2719 2720 2721
from paddle.fluid.framework import convert_np_dtype_to_dtype_


S
sneaxiy 已提交
2722 2723 2724 2725
def _elementwise_op(helper):
    op_type = helper.layer_type
    x = helper.kwargs.get('x', None)
    y = helper.kwargs.get('y', None)
X
Xin Pan 已提交
2726

S
sneaxiy 已提交
2727 2728
    assert x is not None, 'x cannot be None in {}'.format(op_type)
    assert y is not None, 'y cannot be None in {}'.format(op_type)
2729
    check_variable_and_dtype(
2730 2731 2732 2733 2734
        x,
        'x',
        ['float16', 'uint16', 'float32', 'float64', 'int32', 'int64'],
        op_type,
    )
2735
    check_variable_and_dtype(
2736 2737 2738 2739 2740
        y,
        'y',
        ['float16', 'uint16', 'float32', 'float64', 'int32', 'int64'],
        op_type,
    )
2741

S
sneaxiy 已提交
2742 2743
    axis = helper.kwargs.get('axis', -1)
    use_mkldnn = helper.kwargs.get('use_mkldnn', False)
S
sneaxiy 已提交
2744
    name = helper.kwargs.get('name', None)
2745
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
S
sneaxiy 已提交
2746

2747 2748 2749 2750 2751 2752
    helper.append_op(
        type=op_type,
        inputs={'X': x, 'Y': y},
        outputs={'Out': out},
        attrs={'axis': axis, 'use_mkldnn': use_mkldnn},
    )
S
sneaxiy 已提交
2753 2754 2755
    return helper.append_activation(out)


X
Xin Pan 已提交
2756
def elementwise_add(x, y, axis=-1, act=None, name=None):
2757
    """
2758

2759
    Examples:
2760

2761
        .. code-block:: python
2762

2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775
            import paddle.fluid as fluid
            import numpy as np
            import paddle
            def gen_data():
                return {
                    "x": np.array([2, 3, 4]).astype('float32'),
                    "y": np.array([1, 5, 2]).astype('float32')
                }
            paddle.enable_static()
            x = fluid.data(name="x", shape=[3], dtype='float32')
            y = fluid.data(name="y", shape=[3], dtype='float32')
            z = fluid.layers.elementwise_add(x, y)
            # z = x + y
2776

2777 2778 2779 2780
            place = fluid.CPUPlace()
            exe = fluid.Executor(place)
            z_value = exe.run(feed=gen_data(),
                                fetch_list=[z.name])
2781

2782
            print(z_value) # [3., 8., 6.]
2783 2784


2785
        .. code-block:: python
2786

2787 2788 2789
            import paddle.fluid as fluid
            import numpy as np
            import paddle
2790

2791 2792 2793 2794 2795 2796 2797 2798 2799 2800
            def gen_data():
                return {
                    "x": np.ones((2, 3, 4, 5)).astype('float32'),
                    "y": np.zeros((3, 4)).astype('float32')
                }
            paddle.enable_static()
            x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32')
            y = fluid.data(name="y", shape=[3,4], dtype='float32')
            z = fluid.layers.elementwise_add(x, y, axis=1)
            # z = x + y
2801

2802 2803
            place = fluid.CPUPlace()
            exe = fluid.Executor(place)
2804

2805 2806
            z_value = exe.run(feed=gen_data(),
                                fetch_list=[z.name])
2807

2808
            print(z_value) # z.shape=[2,3,4,5]
2809 2810


2811
        ..  code-block:: python
2812

2813 2814 2815
            import paddle.fluid as fluid
            import numpy as np
            import paddle
2816

2817 2818 2819 2820 2821 2822 2823 2824 2825 2826
            def gen_data():
                return {
                    "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'),
                    "y": np.random.randint(1, 5, size=[5]).astype('float32')
                }
            paddle.enable_static()
            x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32')
            y = fluid.data(name="y", shape=[5], dtype='float32')
            z = fluid.layers.elementwise_add(x, y, axis=3)
            # z = x + y
2827

2828 2829
            place = fluid.CPUPlace()
            exe = fluid.Executor(place)
2830

2831 2832 2833
            z_value = exe.run(feed=gen_data(),
                                fetch_list=[z.name])
            print(z_value) # z.shape=[2,3,4,5]
2834 2835

    """
J
Jiabin Yang 已提交
2836
    if _non_static_mode():
2837
        return _elementwise_op_in_dygraph(
2838 2839 2840 2841 2842
            x,
            y,
            axis=axis,
            act=act,
            op_name='elementwise_add',
2843 2844
            use_mkldnn=_global_flags()["FLAGS_use_mkldnn"],
        )
2845

S
sneaxiy 已提交
2846 2847 2848
    return _elementwise_op(LayerHelper('elementwise_add', **locals()))


2849
@deprecated(since="2.0.0", update_to="paddle.divide")
X
Xin Pan 已提交
2850
def elementwise_div(x, y, axis=-1, act=None, name=None):
2851
    """
2852

2853
    Examples:
2854

2855
        .. code-block:: python
2856

2857 2858 2859
            import paddle.fluid as fluid
            import numpy as np
            import paddle
2860

2861 2862 2863 2864 2865 2866 2867 2868 2869 2870
            def gen_data():
                return {
                    "x": np.array([2, 3, 4]).astype('float32'),
                    "y": np.array([1, 5, 2]).astype('float32')
                }
            paddle.enable_static()
            x = fluid.data(name="x", shape=[3], dtype='float32')
            y = fluid.data(name="y", shape=[3], dtype='float32')
            z = fluid.layers.elementwise_div(x, y)
            # z = x / y
2871

2872 2873 2874 2875
            place = fluid.CPUPlace()
            exe = fluid.Executor(place)
            z_value = exe.run(feed=gen_data(),
                                fetch_list=[z.name])
2876

2877
            print(z_value) # [2., 0.6, 2.]
2878 2879


2880
        .. code-block:: python
2881

2882 2883 2884
            import paddle.fluid as fluid
            import numpy as np
            import paddle
2885

2886 2887 2888 2889 2890 2891 2892 2893 2894 2895
            def gen_data():
                return {
                    "x": np.ones((2, 3, 4, 5)).astype('float32'),
                    "y": np.zeros((3, 4)).astype('float32')
                }
            paddle.enable_static()
            x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32')
            y = fluid.data(name="y", shape=[3,4], dtype='float32')
            z = fluid.layers.elementwise_div(x, y, axis=1)
            # z = x / y
2896

2897 2898
            place = fluid.CPUPlace()
            exe = fluid.Executor(place)
2899

2900 2901
            z_value = exe.run(feed=gen_data(),
                                fetch_list=[z.name])
2902

2903
            print(z_value) # z.shape=[2,3,4,5]
2904 2905


2906
        ..  code-block:: python
2907

2908 2909 2910
            import paddle.fluid as fluid
            import numpy as np
            import paddle
2911

2912 2913 2914 2915 2916 2917 2918 2919 2920 2921
            def gen_data():
                return {
                    "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'),
                    "y": np.random.randint(1, 5, size=[5]).astype('float32')
                }
            paddle.enable_static()
            x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32')
            y = fluid.data(name="y", shape=[5], dtype='float32')
            z = fluid.layers.elementwise_div(x, y, axis=3)
            # z = x / y
2922

2923 2924
            place = fluid.CPUPlace()
            exe = fluid.Executor(place)
2925

2926 2927 2928
            z_value = exe.run(feed=gen_data(),
                                fetch_list=[z.name])
            print(z_value) # z.shape=[2,3,4,5]
2929 2930

    """
J
Jiabin Yang 已提交
2931
    if _non_static_mode():
2932 2933 2934
        return _elementwise_op_in_dygraph(
            x, y, axis=axis, act=act, op_name='elementwise_div'
        )
2935

S
sneaxiy 已提交
2936 2937 2938
    return _elementwise_op(LayerHelper('elementwise_div', **locals()))


X
Xin Pan 已提交
2939
def elementwise_sub(x, y, axis=-1, act=None, name=None):
2940
    """
2941

2942
    Examples:
2943

2944
        .. code-block:: python
2945

2946 2947 2948
            import paddle.fluid as fluid
            import numpy as np
            import paddle
2949

2950 2951 2952 2953 2954 2955 2956 2957 2958 2959
            def gen_data():
                return {
                    "x": np.array([2, 3, 4]).astype('float32'),
                    "y": np.array([1, 5, 2]).astype('float32')
                }
            paddle.enable_static()
            x = fluid.data(name="x", shape=[3], dtype='float32')
            y = fluid.data(name="y", shape=[3], dtype='float32')
            z = fluid.layers.elementwise_sub(x, y)
            # z = x - y
2960

2961 2962 2963 2964
            place = fluid.CPUPlace()
            exe = fluid.Executor(place)
            z_value = exe.run(feed=gen_data(),
                                fetch_list=[z.name])
2965

2966
            print(z_value) # [1., -2., 2.]
2967 2968


2969
        .. code-block:: python
2970

2971 2972 2973
            import paddle.fluid as fluid
            import numpy as np
            import paddle
2974

2975 2976 2977 2978 2979 2980 2981 2982 2983 2984
            def gen_data():
                return {
                    "x": np.ones((2, 3, 4, 5)).astype('float32'),
                    "y": np.zeros((3, 4)).astype('float32')
                }
            paddle.enable_static()
            x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32')
            y = fluid.data(name="y", shape=[3,4], dtype='float32')
            z = fluid.layers.elementwise_sub(x, y, axis=1)
            # z = x - y
2985

2986 2987
            place = fluid.CPUPlace()
            exe = fluid.Executor(place)
2988

2989 2990
            z_value = exe.run(feed=gen_data(),
                                fetch_list=[z.name])
2991

2992
            print(z_value) # z.shape=[2,3,4,5]
2993 2994


2995
        ..  code-block:: python
2996

2997 2998 2999
            import paddle.fluid as fluid
            import numpy as np
            import paddle
3000

3001 3002 3003 3004 3005 3006 3007 3008 3009 3010
            def gen_data():
                return {
                    "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'),
                    "y": np.random.randint(1, 5, size=[5]).astype('float32')
                }
            paddle.enable_static()
            x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32')
            y = fluid.data(name="y", shape=[5], dtype='float32')
            z = fluid.layers.elementwise_sub(x, y, axis=3)
            # z = x - y
3011

3012 3013
            place = fluid.CPUPlace()
            exe = fluid.Executor(place)
3014

3015 3016 3017
            z_value = exe.run(feed=gen_data(),
                                fetch_list=[z.name])
            print(z_value) # z.shape=[2,3,4,5]
3018 3019

    """
J
Jiabin Yang 已提交
3020
    if _non_static_mode():
3021 3022 3023
        return _elementwise_op_in_dygraph(
            x, y, axis=axis, act=act, op_name='elementwise_sub'
        )
3024

S
sneaxiy 已提交
3025 3026 3027
    return _elementwise_op(LayerHelper('elementwise_sub', **locals()))


3028
@deprecated(since="2.0.0", update_to="paddle.multiply")
X
Xin Pan 已提交
3029
def elementwise_mul(x, y, axis=-1, act=None, name=None):
3030
    """
3031

3032
    Examples:
3033

3034
        .. code-block:: python
3035

3036 3037 3038
            import paddle.fluid as fluid
            import numpy as np
            import paddle
3039

3040 3041 3042 3043 3044 3045 3046 3047 3048 3049
            def gen_data():
                return {
                    "x": np.array([2, 3, 4]).astype('float32'),
                    "y": np.array([1, 5, 2]).astype('float32')
                }
            paddle.enable_static()
            x = fluid.data(name="x", shape=[3], dtype='float32')
            y = fluid.data(name="y", shape=[3], dtype='float32')
            z = fluid.layers.elementwise_mul(x, y)
            # z = x * y
3050

3051 3052 3053 3054
            place = fluid.CPUPlace()
            exe = fluid.Executor(place)
            z_value = exe.run(feed=gen_data(),
                                fetch_list=[z.name])
3055

3056
            print(z_value) # [2., 15., 8.]
3057 3058


3059
        .. code-block:: python
3060

3061 3062 3063
            import paddle.fluid as fluid
            import numpy as np
            import paddle
3064

3065 3066 3067 3068 3069 3070 3071 3072 3073 3074
            def gen_data():
                return {
                    "x": np.ones((2, 3, 4, 5)).astype('float32'),
                    "y": np.zeros((3, 4)).astype('float32')
                }
            paddle.enable_static()
            x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32')
            y = fluid.data(name="y", shape=[3,4], dtype='float32')
            z = fluid.layers.elementwise_mul(x, y, axis=1)
            # z = x * y
3075

3076 3077
            place = fluid.CPUPlace()
            exe = fluid.Executor(place)
3078

3079 3080
            z_value = exe.run(feed=gen_data(),
                                fetch_list=[z.name])
3081

3082
            print(z_value) # z.shape=[2,3,4,5]
3083 3084


3085
        ..  code-block:: python
3086

3087 3088 3089
            import paddle.fluid as fluid
            import numpy as np
            import paddle
3090

3091 3092 3093 3094 3095 3096 3097 3098 3099 3100
            def gen_data():
                return {
                    "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'),
                    "y": np.random.randint(1, 5, size=[5]).astype('float32')
                }
            paddle.enable_static()
            x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32')
            y = fluid.data(name="y", shape=[5], dtype='float32')
            z = fluid.layers.elementwise_mul(x, y, axis=3)
            # z = x * y
3101

3102 3103
            place = fluid.CPUPlace()
            exe = fluid.Executor(place)
3104

3105 3106 3107
            z_value = exe.run(feed=gen_data(),
                                fetch_list=[z.name])
            print(z_value) # z.shape=[2,3,4,5]
3108

3109
    """
J
Jiabin Yang 已提交
3110
    if _non_static_mode():
3111 3112 3113
        return _elementwise_op_in_dygraph(
            x, y, axis=axis, act=act, op_name='elementwise_mul'
        )
3114

S
sneaxiy 已提交
3115 3116 3117 3118
    return _elementwise_op(LayerHelper('elementwise_mul', **locals()))


for func in [
3119 3120 3121 3122
    elementwise_add,
    elementwise_div,
    elementwise_sub,
    elementwise_mul,
3123 3124
]:
    op_proto = OpProtoHolder.instance().get_op_proto(func.__name__)
3125 3126

    # insert the c++ doc string on top of python doc string
3127 3128 3129 3130 3131
    func.__doc__ = (
        _generate_doc_string_(
            op_proto,
            additional_args_lines=[
                "axis (int32, optional): If X.dimension != Y.dimension, \
3132 3133
            Y.dimension must be a subsequence of x.dimension. \
            And axis is the start dimension index for broadcasting Y onto X. ",
3134
                "act (string, optional): Activation applied to the output. \
3135
            Default is None. Details: :ref:`api_guide_activations_en` ",
3136
                "name (string, optional): Name of the output. \
3137
            Default is None. It's used to print debug info for developers. Details: \
3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153
            :ref:`api_guide_Name` ",
            ],
            skip_attrs_set={
                "x_data_format",
                "y_data_format",
                "axis",
                "use_quantizer",
                "mkldnn_data_type",
                "Scale_x",
                "Scale_y",
                "Scale_out",
            },
        )
        + """\n"""
        + str(func.__doc__)
    )
3154

3155 3156 3157
    doc_list = func.__doc__.splitlines()

    for idx, val in enumerate(doc_list):
3158 3159 3160 3161 3162
        if (
            val.startswith("Warning: ")
            and val.endswith(" instead.")
            and "and will be removed in future versions." in val
        ):
3163 3164 3165 3166
            doc_list.insert(0, doc_list.pop(idx))
            func.__doc__ = "\n" + "\n".join(i for i in doc_list)
            break

3167
for func in []:
S
sneaxiy 已提交
3168 3169 3170 3171
    op_proto = OpProtoHolder.instance().get_op_proto(func.__name__)
    func.__doc__ = _generate_doc_string_(
        op_proto,
        additional_args_lines=[
S
sneaxiy 已提交
3172
            "act (basestring|None): Activation applied to the output.",
3173 3174 3175 3176 3177 3178
            "name (basestring|None): Name of the output.",
        ],
    )
    func.__doc__ = (
        func.__doc__
        + """
3179 3180 3181

Examples:
  .. code-block:: python
3182

3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212
    import paddle.fluid as fluid
    # example 1: shape(x) = (2, 3, 4, 5), shape(y) = (2, 3, 4, 5)
    x0 = fluid.layers.data(name="x0", shape=[2, 3, 4, 5], dtype='float32')
    y0 = fluid.layers.data(name="y0", shape=[2, 3, 4, 5], dtype='float32')
    z0 = fluid.layers.%s(x0, y0)

    # example 2: shape(X) = (2, 3, 4, 5), shape(Y) = (5)
    x1 = fluid.layers.data(name="x1", shape=[2, 3, 4, 5], dtype='float32')
    y1 = fluid.layers.data(name="y1", shape=[5], dtype='float32')
    z1 = fluid.layers.%s(x1, y1)

    # example 3: shape(X) = (2, 3, 4, 5), shape(Y) = (4, 5), with axis=-1(default) or axis=2
    x2 = fluid.layers.data(name="x2", shape=[2, 3, 4, 5], dtype='float32')
    y2 = fluid.layers.data(name="y2", shape=[4, 5], dtype='float32')
    z2 = fluid.layers.%s(x2, y2, axis=2)

    # example 4: shape(X) = (2, 3, 4, 5), shape(Y) = (3, 4), with axis=1
    x3 = fluid.layers.data(name="x3", shape=[2, 3, 4, 5], dtype='float32')
    y3 = fluid.layers.data(name="y3", shape=[3, 4], dtype='float32')
    z3 = fluid.layers.%s(x3, y3, axis=1)

    # example 5: shape(X) = (2, 3, 4, 5), shape(Y) = (2), with axis=0
    x4 = fluid.layers.data(name="x4", shape=[2, 3, 4, 5], dtype='float32')
    y4 = fluid.layers.data(name="y4", shape=[2], dtype='float32')
    z4 = fluid.layers.%s(x4, y4, axis=0)

    # example 6: shape(X) = (2, 3, 4, 5), shape(Y) = (2, 1), with axis=0
    x5 = fluid.layers.data(name="x5", shape=[2, 3, 4, 5], dtype='float32')
    y5 = fluid.layers.data(name="y5", shape=[2], dtype='float32')
    z5 = fluid.layers.%s(x5, y5, axis=0)
3213 3214 3215 3216 3217 3218 3219 3220 3221 3222
    """
        % (
            func.__name__,
            func.__name__,
            func.__name__,
            func.__name__,
            func.__name__,
            func.__name__,
        )
    )
M
minqiyang 已提交
3223 3224


3225
def _logical_op(op_name, x, y, out=None, name=None, binary_op=True):
J
Jiabin Yang 已提交
3226
    if _non_static_mode():
3227
        op = getattr(_legacy_C_ops, op_name)
3228 3229 3230 3231
        if binary_op:
            return op(x, y)
        else:
            return op(x)
3232
    check_variable_and_dtype(
3233 3234
        x,
        "x",
3235
        ["bool", "int8", "int16", "int32", "int64", "float32", "float64"],
3236 3237
        op_name,
    )
3238
    if y is not None:
3239
        check_variable_and_dtype(
3240 3241
            y,
            "y",
3242
            ["bool", "int8", "int16", "int32", "int64", "float32", "float64"],
3243 3244
            op_name,
        )
3245
    if out is not None:
3246
        check_type(out, "out", Variable, op_name)
3247

M
minqiyang 已提交
3248 3249
    helper = LayerHelper(op_name, **locals())

3250 3251 3252
    if binary_op and x.dtype != y.dtype:
        raise ValueError(
            "(InvalidArgument) The DataType of %s Op's Variable must be consistent, but received %s and %s."
3253 3254
            % (op_name, x.dtype, y.dtype)
        )
M
minqiyang 已提交
3255 3256

    if out is None:
3257
        out = helper.create_variable_for_type_inference(dtype=x.dtype)
M
minqiyang 已提交
3258 3259

    if binary_op:
3260 3261 3262
        helper.append_op(
            type=op_name, inputs={"X": x, "Y": y}, outputs={"Out": out}
        )
M
minqiyang 已提交
3263 3264 3265 3266 3267 3268
    else:
        helper.append_op(type=op_name, inputs={"X": x}, outputs={"Out": out})

    return out


3269 3270 3271
@templatedoc()
def clip(x, min, max, name=None):
    """
3272
        :old_api: paddle.fluid.layers.clip
3273

3274 3275 3276 3277
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
S
SunGaofeng 已提交
3278 3279
        min(float): ${min_comment}
        max(float): ${max_comment}
3280 3281
        name(str, optional): The default value is None.
                             Normally there is no need for user to set this property.
S
SunGaofeng 已提交
3282
                             For more information, please refer to :ref:`api_guide_Name`
3283 3284

    Returns:
S
SunGaofeng 已提交
3285 3286 3287 3288
        ${out_comment}

    Return Type:
        ${out_type}
3289 3290 3291 3292

    Examples:
        .. code-block:: python

S
SunGaofeng 已提交
3293
            import paddle.fluid as fluid
S
SunGaofeng 已提交
3294
            input = fluid.data(
3295 3296
                name='data', shape=[1], dtype='float32')
            reward = fluid.layers.clip(x=input, min=-1.0, max=1.0)
3297 3298 3299
    """

    helper = LayerHelper("clip", **locals())
3300
    check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'clip')
3301 3302

    if name is None:
3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316
        name = unique_name.generate_with_ignorable_key(
            ".".join([helper.name, 'tmp'])
        )

    out = helper.create_variable(
        type=x.type, name=name, dtype=x.dtype, persistable=False
    )

    helper.append_op(
        type="clip",
        inputs={"X": x},
        attrs={"min": min, "max": max},
        outputs={"Out": out},
    )
3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328

    return out


@templatedoc()
def clip_by_norm(x, max_norm, name=None):
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        max_norm(${max_norm_type}): ${max_norm_comment}
3329 3330 3331
        name(str, optional): For detailed information, please refer
            to :ref:`api_guide_Name`. Usually name is no need to set and
            None by default.
3332 3333

    Returns:
3334
        Tensor:
W
wangguanzhong 已提交
3335

3336
        out(${out_type}): ${out_comment}
3337

W
wangguanzhong 已提交
3338

3339 3340 3341
    Examples:
        .. code-block:: python

3342
            import paddle
3343
            import paddle.fluid as fluid
3344

3345 3346 3347
            input = paddle.to_tensor([[2.0, 2.0], [2.0, 2.0]], dtype='float32')
            reward = fluid.layers.clip_by_norm(x=input, max_norm=1.0)
            # [[0.5, 0.5], [0.5, 0.5]]
3348 3349
    """

L
lyq 已提交
3350
    if in_dygraph_mode():
3351
        return _C_ops.clip_by_norm(x, max_norm)
J
Jiabin Yang 已提交
3352
    if _non_static_mode():
3353
        return _legacy_C_ops.clip_by_norm(x, 'max_norm', max_norm)
3354

3355
    helper = LayerHelper("clip_by_norm", **locals())
3356
    check_variable_and_dtype(x, 'X', ['float32', 'float16'], 'clip_by_norm')
3357
    check_type(max_norm, 'max_norm', (float), 'clip_by_norm')
3358 3359

    if name is None:
3360 3361 3362
        name = unique_name.generate_with_ignorable_key(
            ".".join([helper.name, 'tmp'])
        )
S
sneaxiy 已提交
3363

3364 3365 3366
    out = helper.create_variable(
        type=x.type, name=name, dtype=x.dtype, persistable=False
    )
3367

3368 3369 3370 3371 3372 3373
    helper.append_op(
        type="clip_by_norm",
        inputs={"X": x},
        attrs={"max_norm": max_norm},
        outputs={"Out": out},
    )
3374 3375

    return out
X
Xin Pan 已提交
3376 3377


3378
@deprecated(since="2.0.0", update_to="paddle.mean")
X
Xin Pan 已提交
3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389
@templatedoc()
def mean(x, name=None):
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        name(basestring|None): Name of the output.

    Returns:
        out(${out_type}): ${out_comment}
3390 3391 3392 3393

    Examples:
        .. code-block:: python

3394
            import paddle
3395
            import paddle.fluid as fluid
3396 3397
            paddle.enable_static()

3398 3399
            input = fluid.layers.data(
                name='data', shape=[2, 3], dtype='float32')
3400
            mean = paddle.mean(input)
X
Xin Pan 已提交
3401
    """
3402

3403
    if _in_legacy_dygraph():
3404
        return _legacy_C_ops.mean(x)
3405
    if in_dygraph_mode():
3406
        return _C_ops.mean_all(x)
X
Xin Pan 已提交
3407 3408

    helper = LayerHelper("mean", **locals())
3409
    check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'mean')
3410
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
X
Xin Pan 已提交
3411

3412 3413 3414
    helper.append_op(
        type="mean", inputs={"X": x}, attrs={}, outputs={"Out": out}
    )
X
Xin Pan 已提交
3415 3416 3417 3418

    return out


C
chengduo 已提交
3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429
@templatedoc()
def merge_selected_rows(x, name=None):
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        name(basestring|None): Name of the output.

    Returns:
        out(${out_type}): ${out_comment}
3430 3431 3432 3433

    Examples:
        .. code-block:: python

3434
            import paddle.fluid as fluid
3435 3436 3437 3438 3439
            b = fluid.default_main_program().global_block()
            var = b.create_var(
                name="X", dtype="float32", persistable=True,
                type=fluid.core.VarDesc.VarType.SELECTED_ROWS)
            y = fluid.layers.merge_selected_rows(var)
C
chengduo 已提交
3440
    """
3441 3442 3443
    if in_dygraph_mode():
        return _C_ops.merge_selected_rows(x)

3444
    if _non_static_mode():
3445
        return _legacy_C_ops.merge_selected_rows(x)
C
chengduo 已提交
3446 3447 3448

    helper = LayerHelper("merge_selected_rows", **locals())
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
3449 3450 3451 3452 3453 3454
    helper.append_op(
        type="merge_selected_rows",
        inputs={"X": x},
        attrs={},
        outputs={"Out": out},
    )
C
chengduo 已提交
3455 3456 3457
    return out


X
Xin Pan 已提交
3458 3459
def mul(x, y, x_num_col_dims=1, y_num_col_dims=1, name=None):
    """
L
liu zhengxi 已提交
3460 3461 3462 3463 3464 3465 3466 3467
    Mul Operator.
    This operator is used to perform matrix multiplication for input $x$ and $y$.
    The equation is:

    ..  math::
        Out = x * y

    Both the input $x$ and $y$ can carry the LoD (Level of Details) information, or not. But the output only shares the LoD information with input $x$.
X
Xin Pan 已提交
3468 3469

    Args:
L
liu zhengxi 已提交
3470 3471
        x (Variable): The first input Tensor/LoDTensor of mul_op.
        y (Variable): The second input Tensor/LoDTensor of mul_op.
3472 3473 3474
        x_num_col_dims (int, optional): The mul_op can take tensors with more than two dimensions as its inputs. If the input $x$ is a tensor with more than two dimensions, $x$ will be flattened into a two-dimensional matrix first. The flattening rule is: the first `num_col_dims` will be flattened to form the first dimension of the final matrix (the height of the matrix), and the rest `rank(x) - num_col_dims` dimensions are flattened to form the second dimension of the final matrix (the width of the matrix). As a result, height of the flattened matrix is equal to the product of $x$'s first `x_num_col_dims` dimensions' sizes, and width of the flattened matrix is equal to the product of $x$'s last `rank(x) - num_col_dims` dimensions' size. For example, suppose $x$ is a 6-dimensional tensor with the shape [2, 3, 4, 5, 6], and `x_num_col_dims` = 3. Thus, the flattened matrix will have a shape [2 x 3 x 4, 5 x 6] = [24, 30]. Default is 1.
        y_num_col_dims (int, optional): The mul_op can take tensors with more than two dimensions as its inputs. If the input $y$ is a tensor with more than two dimensions, $y$ will be flattened into a two-dimensional matrix first. The attribute `y_num_col_dims` determines how $y$ is flattened. See comments of `x_num_col_dims` for more details. Default is 1.
        name (str, optional): Name of the output. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. Default is None.
X
Xin Pan 已提交
3475 3476

    Returns:
L
liu zhengxi 已提交
3477
        Variable(Tensor/LoDTensor): The output Tensor/LoDTensor of mul op.
3478 3479

    Examples:
L
liu zhengxi 已提交
3480
        ..  code-block:: python
3481

3482
            import paddle.fluid as fluid
3483 3484
            import paddle
            paddle.enable_static()
3485 3486 3487 3488 3489
            dataX = fluid.layers.data(name="dataX", append_batch_size = False, shape=[2, 5], dtype="float32")
            dataY = fluid.layers.data(name="dataY", append_batch_size = False, shape=[5, 3], dtype="float32")
            output = fluid.layers.mul(dataX, dataY,
                                      x_num_col_dims = 1,
                                      y_num_col_dims = 1)
3490

3491

X
Xin Pan 已提交
3492
    """
J
Jiabin Yang 已提交
3493
    if _non_static_mode():
3494 3495 3496 3497 3498 3499 3500 3501
        return _legacy_C_ops.mul(
            x,
            y,
            'x_num_col_dims',
            x_num_col_dims,
            'y_num_col_dims',
            y_num_col_dims,
        )
X
Xin Pan 已提交
3502

3503 3504
    inputs = {"X": [x], "Y": [y]}
    attrs = {"x_num_col_dims": x_num_col_dims, "y_num_col_dims": y_num_col_dims}
X
Xin Pan 已提交
3505
    helper = LayerHelper("mul", **locals())
3506 3507
    check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'mul')
    check_variable_and_dtype(y, 'y', ['float16', 'float32', 'float64'], 'mul')
3508
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
X
Xin Pan 已提交
3509

3510 3511 3512
    helper.append_op(
        type="mul", inputs={"X": x, "Y": y}, attrs=attrs, outputs={"Out": out}
    )
X
Xin Pan 已提交
3513 3514 3515
    return out


C
chengduo 已提交
3516 3517 3518
@templatedoc()
def get_tensor_from_selected_rows(x, name=None):
    """
3519 3520 3521 3522 3523 3524 3525 3526 3527
    This operator gets tensor data from input with SelectedRows type, and outputs a LoDTensor.

    .. code-block:: text

        input x is SelectedRows:
           x.rows = [0, 5, 5, 4, 19]
           x.height = 20
           x.value = [[1, 1] [2, 2] [2, 2] [3, 3] [6, 6]]

3528
        Output is LoDTensor:
3529 3530 3531 3532 3533 3534
           out.shape = [5, 2]
           out.data = [[1, 1],
                       [2, 2],
                       [2, 2],
                       [3, 3],
                       [6, 6]]
C
chengduo 已提交
3535 3536

    Args:
3537 3538 3539
        x(SelectedRows): Input with SelectedRows type. The data type is float32, float64, int32 or int64.
        name(str, optional): The default value is None.  Normally there is no need for user to set this property.
            For more information, please refer to :ref:`api_guide_Name` .
C
chengduo 已提交
3540 3541

    Returns:
3542
        Variable: LoDTensor transformed from SelectedRows. The data type is same with input.
B
bdzhuxiaoning 已提交
3543 3544 3545

    Examples:
        .. code-block:: python
3546

B
bdzhuxiaoning 已提交
3547 3548 3549 3550
            import paddle.fluid as fluid
            b = fluid.default_main_program().global_block()
            input = b.create_var(name="X", dtype="float32", persistable=True, type=fluid.core.VarDesc.VarType.SELECTED_ROWS)
            out = fluid.layers.get_tensor_from_selected_rows(input)
C
chengduo 已提交
3551 3552
    """

3553 3554 3555 3556 3557
    check_type(x, 'x', Variable, 'get_tensor_from_selected_rows')
    if x.type != core.VarDesc.VarType.SELECTED_ROWS:
        raise TypeError(
            "The type of 'x' in get_tensor_from_selected_rows must be SELECTED_ROWS."
        )
C
chengduo 已提交
3558 3559
    helper = LayerHelper('get_tensor_from_selected_rows', **locals())
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
3560 3561 3562 3563 3564 3565
    helper.append_op(
        type='get_tensor_from_selected_rows',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={},
    )
C
chengduo 已提交
3566
    return out
3567 3568


3569
def unfold(x, kernel_sizes, strides=1, paddings=0, dilations=1, name=None):
3570
    r"""
3571

S
SunGaofeng 已提交
3572
    This op returns a col buffer of sliding local blocks of input x, also known
3573
    as im2col for batched 2D image tensors. For each block under the convolution filter,
T
tianshuo78520a 已提交
3574
    all element will be rearranged as a column. While the convolution filter sliding over
3575 3576
    the input feature map, a series of such columns will be formed.

S
SunGaofeng 已提交
3577
    For each input :math:`x` with shape [N, C, H, W], the output shape [N, Cout, Lout]
3578 3579 3580 3581
    can be calculated as following.

    .. math::

3582
        dkernel[0] &= dilations[0] \times (kernel\_sizes[0] - 1) + 1
3583

3584
        dkernel[1] &= dilations[1] \times (kernel\_sizes[1] - 1) + 1
3585

3586
        hout &= \frac{H + paddings[0] + paddings[2] - dkernel[0]}{strides[0]} + 1
3587

3588
        wout &= \frac{W + paddings[1] + paddings[3] - dkernel[1]}{strides[1]} + 1
3589

3590
        Cout &= C \times kernel\_sizes[0] \times kernel\_sizes[1]
3591

3592
        Lout &= hout \times wout
3593 3594


S
SunGaofeng 已提交
3595
    Parameters:
3596
        x(Tensor):              4-D Tensor, input tensor of format [N, C, H, W],
S
SunGaofeng 已提交
3597
                                  data type can be float32 or float64
3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609
        kernel_sizes(int|list):   The size of convolution kernel, should be [k_h, k_w]
                                  or an integer k treated as [k, k].
        strides(int|list):        The strides, should be [stride_h, stride_w]
                                  or an integer stride treated as [sride, stride].
                                  For default, strides will be [1, 1].
        paddings(int|list):       The paddings of each dimension, should be
                                  [padding_top, padding_left, padding_bottom, padding_right]
                                  or [padding_h, padding_w] or an integer padding.
                                  If [padding_h, padding_w] was given, it will expanded to
                                  [padding_h, padding_w, padding_h, padding_w]. If an integer
                                  padding was given, [padding, padding, padding, padding] will
                                  be used. For default, paddings will be [0, 0, 0, 0]
T
tianshuo78520a 已提交
3610
        dilations(int|list):      the dilations of convolution kernel, should be
T
tianshuo78520a 已提交
3611
                                  [dilation_h, dilation_w], or an integer dilation treated as
3612
                                  [dilation, dilation]. For default, it will be [1, 1].
3613 3614
        name(str, optional): The default value is None.
                             Normally there is no need for user to set this property.
S
SunGaofeng 已提交
3615
                             For more information, please refer to :ref:`api_guide_Name`
3616

3617

3618
    Returns:
3619
        The tensor corresponding to the sliding local blocks.
3620 3621 3622
        The output shape is [N, Cout, Lout] as decriabled above.
        Cout is the  total number of values within each block,
        and Lout is the total number of such blocks.
S
SunGaofeng 已提交
3623 3624 3625
        The data type of output is the same as the input :math:`x`

    Return Type:
3626
        Tensor
3627 3628 3629 3630 3631

    Examples:

        .. code-block:: python

3632 3633 3634 3635 3636
            import paddle
            import paddle.nn.functional as F

            x = paddle.randn((100,3,224,224))
            y = F.unfold(x, [3, 3], 1, 1, 1)
3637 3638
    """

3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658
    return paddle.nn.functional.unfold(
        x, kernel_sizes, strides, paddings, dilations, name
    )


def deformable_roi_pooling(
    input,
    rois,
    trans,
    no_trans=False,
    spatial_scale=1.0,
    group_size=[1, 1],
    pooled_height=1,
    pooled_width=1,
    part_size=None,
    sample_per_part=1,
    trans_std=0.1,
    position_sensitive=False,
    name=None,
):
3659
    r"""
3660

3661
    Deformable ROI Pooling Layer
3662

3663
    Performs deformable region-of-interest pooling on inputs. As described
3664
    in `Deformable Convolutional Networks <https://arxiv.org/abs/1703.06211>`_, it will get offset for each bin after
3665
    roi pooling so that pooling at correct region. Batch_size will change to the number of region bounding boxes after deformable_roi_pooling.
3666

3667
    The operation has three steps:
3668

3669
    1. Dividing each region proposal into equal-sized sections with the pooled_width and pooled_height.
3670

3671 3672
    2. Add offset to pixel in ROI to get new location and the new value which are computed directly through
       bilinear interpolation with four nearest pixel.
3673

3674
    3. Sample several points in each bin to get average values as output.
3675 3676


3677 3678 3679 3680 3681 3682 3683 3684 3685
    Args:
        input (Variable):The input of deformable roi pooling and it is tensor which value type is float32. The shape of input is
                         [N, C, H, W]. Where N is batch size, C is number of input channels,
                         H is height of the feature, and W is the width of the feature.
        rois (Variable): ROIs (Regions of Interest) with type float32 to pool over. It should be
                         a 2-D LoDTensor of shape (num_rois, 4), and the lod level
                         is 1. Given as [[x1, y1, x2, y2], ...], (x1, y1) is
                         the top left coordinates, and (x2, y2) is the bottom
                         right coordinates, which value type is float32.
3686 3687 3688
        trans (Variable): Offset of features on ROIs while pooling which value type is float32. The format is [N, C, H, W], where
                          N is number of ROIs, C is number of channels, which indicate the offset distance
                          in the x and y directions, H is pooled height, and W is pooled width.
3689 3690 3691 3692
        no_trans (bool): Whether to add offset to get new value or not while roi pooling, which value with type bool is True or False.
                         If value is True, no offset will be added in operation. Default: False.
        spatial_scale (float): Ratio of input feature map height (or width) to raw image height (or width), which value type is float32.
                         Equals the reciprocal of total stride in convolutional layers, Default: 1.0.
3693
        group_size (list|tuple): The number of groups which input channels are divided and the input is list or tuple, which value type is int32. (eg.number of input channels
3694
                          is k1 * k2 * (C + 1), which k1 and k2 are group width and height and C+1 is number of output
T
tianshuo78520a 已提交
3695
                          channels.) eg.(4, 6), which 4 is height of group and 6 is width of group. Default: [1, 1].
3696 3697 3698 3699 3700 3701 3702
        pooled_height (int): The pooled output height which value type is int32. Default: 1.
        pooled_width (int): The pooled output width which value type is int32. Default: 1.
        part_size (list|tuple): The height and width of offset which values in list or tuple is int32, eg.(4, 6), which height is 4 and width is 6, and values always equal to pooled_height \
                         and pooled_width. Default: if None, default value is [pooled_height, pooled_width].
        sample_per_part (int): The number of samples in each bin which value type is int32. If value is bigger, it will consume more performance. Default: 1.
        trans_std (float): Coefficient of offset which value type is float32. It controls weight of offset. Default: 0.1.
        position_sensitive (bool): Whether to choose deformable psroi pooling mode or not, and value type is bool(True or False). If value is False, input dimension equals to output dimension. \
T
tianshuo78520a 已提交
3703
                                   If value is True, input dimension should be output dimension * pooled_height * pooled_width. Default: False.
3704 3705 3706 3707
        name (str|None): Name of layer. Default: None.
    Returns:
        Variable: Output of deformable roi pooling is that, if position sensitive is False, input dimension equals to output dimension. If position sensitive is True,\
                  input dimension should be the result of output dimension divided by pooled height and pooled width.
C
cjt222 已提交
3708 3709 3710 3711

    Examples:
      .. code-block:: python

3712 3713
        # position_sensitive=True
        import paddle.fluid as fluid
C
chengjuntao 已提交
3714
        input = fluid.data(name="input",
3715 3716
                           shape=[2, 192, 64, 64],
                           dtype='float32')
C
chengjuntao 已提交
3717 3718
        rois = fluid.data(name="rois",
                          shape=[-1, 4],
3719
                          dtype='float32',
C
chengjuntao 已提交
3720 3721
                          lod_level=1)
        trans = fluid.data(name="trans",
3722 3723 3724 3725 3726
                           shape=[2, 384, 64, 64],
                           dtype='float32')
        x = fluid.layers.deformable_roi_pooling(input=input,
                                                rois=rois,
                                                trans=trans,
C
chengjuntao 已提交
3727
                                                no_trans=False,
3728
                                                spatial_scale=1.0,
C
chengjuntao 已提交
3729 3730 3731 3732
                                                group_size=(1, 1),
                                                pooled_height=8,
                                                pooled_width=8,
                                                part_size=(8, 8),
3733
                                                sample_per_part=4,
C
chengjuntao 已提交
3734 3735
                                                trans_std=0.1,
                                                position_sensitive=True)
3736

3737
        # position_sensitive=False
3738
        import paddle.fluid as fluid
C
chengjuntao 已提交
3739
        input = fluid.data(name="input",
3740 3741
                           shape=[2, 192, 64, 64],
                           dtype='float32')
C
chengjuntao 已提交
3742 3743
        rois = fluid.data(name="rois",
                          shape=[-1, 4],
3744
                          dtype='float32',
C
chengjuntao 已提交
3745 3746
                          lod_level=1)
        trans = fluid.data(name="trans",
3747 3748 3749 3750 3751
                           shape=[2, 384, 64, 64],
                           dtype='float32')
        x = fluid.layers.deformable_roi_pooling(input=input,
                                                rois=rois,
                                                trans=trans,
C
chengjuntao 已提交
3752
                                                no_trans=False,
3753
                                                spatial_scale=1.0,
C
chengjuntao 已提交
3754 3755 3756 3757
                                                group_size=(1, 1),
                                                pooled_height=8,
                                                pooled_width=8,
                                                part_size=(8, 8),
3758
                                                sample_per_part=4,
C
chengjuntao 已提交
3759 3760
                                                trans_std=0.1,
                                                position_sensitive=False)
C
cjt222 已提交
3761 3762
    """

3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774
    check_variable_and_dtype(
        input, 'input', ['float32', 'float64'], 'deformable_roi_pooling'
    )
    check_variable_and_dtype(
        rois, 'rois', ['float32', 'float64'], 'deformable_roi_pooling'
    )
    check_variable_and_dtype(
        trans, 'trans', ['float32', 'float64'], 'deformable_roi_pooling'
    )
    check_type(
        group_size, 'group_size', (list, tuple), 'deformable_roi_pooling'
    )
3775
    if part_size is not None:
3776 3777 3778
        check_type(
            part_size, 'part_size', (list, tuple), 'deformable_roi_pooling'
        )
3779

C
cjt222 已提交
3780
    input_channels = input.shape[1]
3781
    if position_sensitive is False:
C
cjt222 已提交
3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795
        output_channels = input_channels
    else:
        output_channels = input_channels / pooled_height / pooled_width

    if part_size is None:
        part_height = pooled_height
        part_width = pooled_width
        part_size = [part_height, part_width]
    part_size = utils.convert_to_list(part_size, 2, 'part_size')
    group_size = utils.convert_to_list(group_size, 2, 'group_size')
    helper = LayerHelper('deformable_psroi_pooling', **locals())
    dtype = helper.input_dtype()
    output = helper.create_variable_for_type_inference(dtype)
    top_count = helper.create_variable_for_type_inference(dtype='int32')
3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811
    helper.append_op(
        type="deformable_psroi_pooling",
        inputs={"Input": input, "ROIs": rois, "Trans": trans},
        outputs={"Output": output, "TopCount": top_count},
        attrs={
            "no_trans": no_trans,
            "spatial_scale": spatial_scale,
            "output_dim": output_channels,
            "group_size": group_size,
            "pooled_height": pooled_height,
            "pooled_width": pooled_width,
            "part_size": part_size,
            "sample_per_part": sample_per_part,
            "trans_std": trans_std,
        },
    )
C
cjt222 已提交
3812
    return output
3813 3814


3815
@deprecated(since="2.0.0", update_to="paddle.shard_index")
3816 3817
def shard_index(input, index_num, nshards, shard_id, ignore_value=-1):
    """
L
lilong12 已提交
3818 3819 3820 3821 3822 3823 3824 3825 3826
    Reset the values of `input` according to the shard it beloning to.
    Every value in `input` must be a non-negative integer, and
    the parameter `index_num` represents the integer above the maximum
    value of `input`. Thus, all values in `input` must be in the range
    [0, index_num) and each value can be regarded as the offset to the beginning
    of the range. The range is further split into multiple shards. Specifically,
    we first compute the `shard_size` according to the following formula,
    which represents the number of integers each shard can hold. So for the
    i'th shard, it can hold values in the range [i*shard_size, (i+1)*shard_size).
3827 3828
    ::

3829
        shard_size = (index_num + nshards - 1) // nshards
3830

L
lilong12 已提交
3831 3832 3833
    For each value `v` in `input`, we reset it to a new value according to the
    following formula:
    ::
3834

L
lilong12 已提交
3835 3836 3837 3838
        v = v - shard_id * shard_size if shard_id * shard_size <= v < (shard_id+1) * shard_size else ignore_value

    That is, the value `v` is set to the new offset within the range represented by the shard `shard_id`
    if it in the range. Otherwise, we reset it to be `ignore_value`.
3839 3840

    Args:
L
lilong12 已提交
3841 3842
        input (Tensor): Input tensor with data type int64 or int32. It's last dimension must be 1.
        index_num (int): An integer represents the integer above the maximum value of `input`.
3843 3844 3845
        nshards (int): The number of shards.
        shard_id (int): The index of the current shard.
        ignore_value (int): An integer value out of sharded index range.
3846 3847

    Returns:
L
lilong12 已提交
3848
        Tensor.
3849 3850 3851 3852

    Examples:
        .. code-block:: python

3853 3854 3855 3856 3857 3858 3859 3860
            import paddle
            label = paddle.to_tensor([[16], [1]], "int64")
            shard_label = paddle.shard_index(input=label,
                                             index_num=20,
                                             nshards=2,
                                             shard_id=0)
            print(shard_label)
            # [[-1], [1]]
3861
    """
H
hong 已提交
3862
    if in_dygraph_mode():
3863 3864 3865
        return _C_ops.shard_index(
            input, index_num, nshards, shard_id, ignore_value
        )
H
hong 已提交
3866

B
Baibaifan 已提交
3867
    check_variable_and_dtype(input, 'input', ['int64', 'int32'], 'shard_index')
3868 3869 3870
    op_type = 'shard_index'
    helper = LayerHelper(op_type, **locals())
    if shard_id < 0 or shard_id >= nshards:
3871 3872 3873
        raise ValueError(
            'The shard_id(%d) should be in [0, %d)' % (shard_id, nshards)
        )
3874 3875

    out = helper.create_variable_for_type_inference(dtype=input.dtype)
3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887
    helper.append_op(
        type=op_type,
        inputs={'X': [input]},
        outputs={'Out': out},
        attrs={
            'index_num': index_num,
            'nshards': nshards,
            'shard_id': shard_id,
            'ignore_value': ignore_value,
        },
        stop_gradient=True,
    )
3888
    return out
H
huangjun12 已提交
3889 3890 3891 3892


@templatedoc()
def hard_swish(x, threshold=6.0, scale=6.0, offset=3.0, name=None):
3893
    r"""
3894 3895 3896
    This operator implements the hard_swish activation function.
    Hard_swish is proposed in MobileNetV3, and performs better in computational stability and efficiency compared to swish function.
    For more details please refer to: https://arxiv.org/pdf/1905.02244.pdf
H
huangjun12 已提交
3897

3898
    The formula is as follows:
H
huangjun12 已提交
3899

3900
    .. math::
H
huangjun12 已提交
3901

3902
        out = \\frac{x * (min(max(0, x+offset), threshold))}{scale}
H
huangjun12 已提交
3903

3904 3905 3906 3907 3908 3909 3910 3911 3912
    In the above equation:

    ``threshold`` and ``scale`` should be positive, ``offset`` can be positive or negative. It is recommended to use default parameters.

    Args:
        x (Variable): Input feature, multi-dimensional Tensor. The data type should be float32 or float64.
        threshold (float, optional): The threshold in Relu function. Default: 6.0
        scale (float, optional): The scale factor. Default: 6.0
        offset (float, optional): The offset factor. Default: 3.0
3913 3914
        name (str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`

3915 3916
    Returns:
        Variable: The output tensor with the same shape and data type as input.
3917 3918


3919
    Examples:
3920

3921
    .. code-block:: python
3922

3923
        import paddle.fluid as fluid
3924
        import paddle
3925
        import numpy as np
3926
        paddle.enable_static()
3927

3928
        DATATYPE='float32'
3929

3930
        x_data = np.array([i for i in range(1,5)]).reshape([1,1,4]).astype(DATATYPE)
3931

3932 3933
        x = fluid.data(name="x", shape=[None,1,4], dtype=DATATYPE)
        y = fluid.layers.hard_swish(x)
3934

3935 3936 3937 3938 3939
        place = fluid.CPUPlace()
        #place = fluid.CUDAPlace(0)
        exe = fluid.Executor(place)
        out, = exe.run(feed={'x':x_data}, fetch_list=[y.name])
        print(out)  # [[0.66666667, 1.66666667,3., 4.]]
H
huangjun12 已提交
3940
    """
J
Jiabin Yang 已提交
3941
    if _non_static_mode():
3942 3943 3944
        return _legacy_C_ops.hard_swish(
            x, 'threshold', threshold, 'scale', scale, 'offset', offset
        )
3945

3946 3947 3948
    check_variable_and_dtype(
        x, 'x', ['float16', 'float32', 'float64'], 'hard_swish'
    )
3949

H
huangjun12 已提交
3950 3951
    helper = LayerHelper('hard_swish', **locals())
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
3952 3953 3954 3955 3956 3957
    helper.append_op(
        type='hard_swish',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={'threshold': threshold, 'scale': scale, 'offset': offset},
    )
H
huangjun12 已提交
3958
    return out
R
ruri 已提交
3959 3960


K
Kaipeng Deng 已提交
3961 3962
@templatedoc()
def mish(x, threshold=20, name=None):
3963
    r"""
K
Kaipeng Deng 已提交
3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978
    This operator implements the mish activation function.
    Refer to `Mish: A Self Regularized Non-Monotonic Neural
    Activation Function <https://arxiv.org/abs/1908.08681>`_


    The formula is as follows if :attr:`threshold` is :code:`None` or negative:

    .. math::

        out = x * \\tanh(\\ln(1 + e^{x}))

    The formula is as follows if :attr:`threshold` is set as positive value:

    .. math::

3979 3980 3981 3982 3983
    out = \\begin{cases}
        x \\ast \\tanh(x), \\text{if } x > \\text{threshold} \\\\
        x \\ast \\tanh(e^{x}), \\text{if } x < -\\text{threshold} \\\\
        x \\ast \\tanh(\\ln(1 + e^{x})),  \\text{otherwise}
          \\end{cases}
K
Kaipeng Deng 已提交
3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004

    Args:
        x (Variable): Input feature, multi-dimensional Tensor. The data type
                      should be float16, float32 or float64.
        threshold (float|None): threshold for softplus in Mish operator.
                Approximate value of softplus will be used if absolute value
                of input is greater than :attr:threshold and :attr:threshold
                is set as positive value. For none or negative threshold,
                approximate value is not used. Default 20.
        name (str, optional): The default value is None. Normally there is no
                need for user to set this property. For more information, please
                refer to :ref:`api_guide_Name`

    Returns:
        Variable: The output tensor with the same shape and data type as input.


    Examples:

    .. code-block:: python

4005
        import paddle
K
Kaipeng Deng 已提交
4006 4007 4008
        import paddle.fluid as fluid
        import numpy as np

4009
        paddle.enable_static()
K
Kaipeng Deng 已提交
4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022
        DATATYPE='float32'

        x_data = np.array([i for i in range(1,5)]).reshape([1,1,4]).astype(DATATYPE)

        x = fluid.data(name="x", shape=[None,1,4], dtype=DATATYPE)
        y = fluid.layers.mish(x)

        place = fluid.CPUPlace()
        # place = fluid.CUDAPlace(0)
        exe = fluid.Executor(place)
        out, = exe.run(feed={'x':x_data}, fetch_list=[y.name])
        print(out)  # [[0.66666667, 1.66666667, 3., 4.]]
    """
4023
    if in_dygraph_mode():
4024
        return _C_ops.mish(x, threshold)
4025
    if _in_legacy_dygraph():
4026
        return _legacy_C_ops.mish(x, 'threshold', threshold)
4027

K
Kaipeng Deng 已提交
4028 4029
    check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'mish')
    check_type(threshold, 'threshold', (float, int), 'mish')
4030 4031 4032 4033 4034
    assert (
        threshold > 0
    ), "threshold of mish should be greater than 0, " "but got {}".format(
        threshold
    )
K
Kaipeng Deng 已提交
4035 4036 4037

    helper = LayerHelper('mish', **locals())
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
4038 4039 4040 4041 4042 4043
    helper.append_op(
        type='mish',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={'threshold': threshold},
    )
K
Kaipeng Deng 已提交
4044 4045 4046
    return out


4047
@deprecated(since="2.0.0", update_to="paddle.uniform")
4048
@templatedoc()
4049 4050 4051
def uniform_random(
    shape, dtype='float32', min=-1.0, max=1.0, seed=0, name=None
):
4052
    """
4053 4054
    This OP returns a Tensor filled with random values sampled from a uniform
    distribution in the range [``min``, ``max``), with ``shape`` and ``dtype``.
4055 4056 4057

    Examples:
    ::
4058

4059 4060
        Input:
          shape = [1, 2]
4061

4062 4063 4064 4065
        Output:
          result=[[0.8505902, 0.8397286]]

    Args:
4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078
        shape(list|tuple|Tensor): The shape of the output Tensor. If ``shape``
            is a list or tuple, the elements of it should be integers or Tensors
            (with the shape [1], and the data type int32 or int64). If ``shape``
            is a Tensor, it should be a 1-D Tensor(with the data type int32 or
            int64).
        dtype(str|np.dtype|core.VarDesc.VarType, optional): The data type of
            the output Tensor. Supported data types: float32, float64.
            Default is float32.
        min(float|int, optional): The lower bound on the range of random values
            to generate, ``min`` is included in the range. Default is -1.0.
        max(float|int, optional): The upper bound on the range of random values
            to generate, ``max`` is excluded in the range. Default is 1.0.
        seed(int, optional): Random seed used for generating samples. 0 means
4079 4080
            use a seed generated by the system. Note that if seed is not 0,
            this operator will always generate the same random numbers every
4081
            time. Default is 0.
4082 4083 4084
        name(str, optional): The default value is None. Normally there is no
            need for user to set this property. For more information, please
            refer to :ref:`api_guide_Name`.
4085

4086
    Returns:
4087 4088
        Tensor: A Tensor filled with random values sampled from a uniform
        distribution in the range [``min``, ``max``), with ``shape`` and ``dtype``.
4089

4090
    Raises:
4091 4092
        TypeError: If ``shape`` is not list, tuple, Tensor.
        TypeError: If ``dtype`` is not float32, float64.
4093

4094 4095 4096
    Examples:
        .. code-block:: python

4097
            import paddle
4098
            import paddle.fluid as fluid
4099
            paddle.enable_static()
4100 4101

            # example 1:
4102
            # attr shape is a list which doesn't contain Tensor.
4103
            result_1 = fluid.layers.uniform_random(shape=[3, 4])
4104 4105 4106
            # [[ 0.84524226,  0.6921872,   0.56528175,  0.71690357],
            #  [-0.34646994, -0.45116323, -0.09902662, -0.11397249],
            #  [ 0.433519,    0.39483607, -0.8660099,   0.83664286]]
4107 4108

            # example 2:
4109 4110 4111
            # attr shape is a list which contains Tensor.
            dim_1 = fluid.layers.fill_constant([1], "int64", 2)
            dim_2 = fluid.layers.fill_constant([1], "int32", 3)
4112
            result_2 = fluid.layers.uniform_random(shape=[dim_1, dim_2])
4113 4114
            # [[-0.9951253,   0.30757582, 0.9899647 ],
            #  [ 0.5864527,   0.6607096,  -0.8886161 ]]
4115 4116

            # example 3:
4117
            # attr shape is a Tensor, the data type must be int64 or int32.
4118
            var_shape = fluid.data(name='var_shape', shape=[2], dtype="int64")
4119
            result_3 = fluid.layers.uniform_random(var_shape)
4120 4121 4122 4123
            # if var_shape's value is [2, 3]
            # result_3 is:
            # [[-0.8517412,  -0.4006908,   0.2551912 ],
            #  [ 0.3364414,   0.36278176, -0.16085452]]
4124

4125 4126 4127
    """
    if not isinstance(dtype, core.VarDesc.VarType):
        dtype = convert_np_dtype_to_dtype_(dtype)
4128

4129 4130
    if in_dygraph_mode():
        shape = utils.convert_shape_to_list(shape)
4131
        return _C_ops.uniform(
4132 4133 4134 4135 4136 4137 4138
            shape,
            dtype,
            float(min),
            float(max),
            seed,
            _current_expected_place(),
        )
4139
    elif _in_legacy_dygraph():
4140
        shape = utils.convert_shape_to_list(shape)
4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152
        return _legacy_C_ops.uniform_random(
            'shape',
            shape,
            'min',
            float(min),
            'max',
            float(max),
            'seed',
            seed,
            'dtype',
            dtype,
        )
4153

4154
    check_type(shape, 'shape', (list, tuple, Variable), 'uniform_random/rand')
4155 4156 4157
    check_dtype(
        dtype, 'dtype', ('float32', 'float64', 'uint16'), 'uniform_random/rand'
    )
4158 4159
    check_type(min, 'min', (float, int, Variable), 'uniform_random/rand')
    check_type(max, 'max', (float, int, Variable), 'uniform_random/rand')
4160 4161

    inputs = dict()
4162
    attrs = {'seed': seed, 'min': min, 'max': max, 'dtype': dtype}
4163 4164 4165
    utils.get_shape_tensor_inputs(
        inputs=inputs, attrs=attrs, shape=shape, op_type='uniform_random/rand'
    )
4166

4167
    helper = LayerHelper("uniform_random", **locals())
4168
    out = helper.create_variable_for_type_inference(dtype)
4169 4170 4171
    helper.append_op(
        type="uniform_random", inputs=inputs, attrs=attrs, outputs={"Out": out}
    )
4172
    utils.try_set_static_shape_tensor(out, shape)
4173
    return out
myq406450149's avatar
myq406450149 已提交
4174 4175 4176 4177 4178 4179 4180


def unbind(input, axis=0):
    """
    Removes a tensor dimension, then split the input tensor into multiple sub-Tensors.
    Args:
        input (Variable): The input variable which is an N-D Tensor, data type being float32, float64, int32 or int64.
4181

myq406450149's avatar
myq406450149 已提交
4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206
        axis (int32|int64, optional): A scalar with type ``int32|int64`` shape [1]. The dimension along which to unbind. If :math:`axis < 0`, the
            dimension to unbind along is :math:`rank(input) + axis`. Default is 0.
    Returns:
        list(Variable): The list of segmented Tensor variables.

    Example:
        .. code-block:: python
            import paddle
            # input is a variable which shape is [3, 4, 5]
            input = paddle.fluid.data(
                 name="input", shape=[3, 4, 5], dtype="float32")
            [x0, x1, x2] = paddle.tensor.unbind(input, axis=0)
            # x0.shape [4, 5]
            # x1.shape [4, 5]
            # x2.shape [4, 5]
            [x0, x1, x2, x3] = paddle.tensor.unbind(input, axis=1)
            # x0.shape [3, 5]
            # x1.shape [3, 5]
            # x2.shape [3, 5]
            # x3.shape [3, 5]

    """
    helper = LayerHelper("unbind", **locals())
    check_type(input, 'input', (Variable), 'unbind')
    dtype = helper.input_dtype()
4207 4208 4209
    check_dtype(
        dtype, 'unbind', ['float32', 'float64', 'int32', 'int64'], 'unbind'
    )
myq406450149's avatar
myq406450149 已提交
4210
    if not isinstance(axis, (int)):
4211 4212 4213
        raise TypeError(
            "The type of 'axis'  must be int, but received %s." % (type(axis))
        )
myq406450149's avatar
myq406450149 已提交
4214 4215 4216 4217 4218 4219 4220 4221 4222 4223
    if isinstance(axis, np.generic):
        axis = np.asscalar(axis)
    input_shape = input.shape
    axis_ = axis if axis >= 0 else len(input_shape) + axis
    num = input_shape[axis_]
    outs = [
        helper.create_variable_for_type_inference(dtype=helper.input_dtype())
        for i in range(num)
    ]

4224 4225 4226 4227 4228 4229
    helper.append_op(
        type="unbind",
        inputs={"X": input},
        outputs={"Out": outs},
        attrs={"axis": axis},
    )
myq406450149's avatar
myq406450149 已提交
4230
    return outs