nn.py 415.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Y
Yu Yang 已提交
14
"""
15
All layers just related to the neural network.
Y
Yu Yang 已提交
16 17
"""

18 19
from __future__ import print_function

20
import numpy as np
21
import warnings
S
sneaxiy 已提交
22
import six
P
peizhilin 已提交
23
import os
S
sneaxiy 已提交
24
import inspect
Y
Yu Yang 已提交
25
from ..layer_helper import LayerHelper
26
from ..initializer import Normal, Constant, NumpyArrayInitializer
L
lujun 已提交
27
from ..framework import Variable, OpProtoHolder, in_dygraph_mode
L
lujun 已提交
28
from ..dygraph import base
Y
yangyaming 已提交
29
from ..param_attr import ParamAttr
S
sneaxiy 已提交
30
from .layer_function_generator import autodoc, templatedoc, _generate_doc_string_
31
from .tensor import concat, assign
32
from . import utils
F
fengjiayi 已提交
33
from .. import unique_name
34
from functools import reduce
35
from .. import core
L
lujun 已提交
36
from ..dygraph import layers
Y
Yu Yang 已提交
37 38

__all__ = [
X
Xin Pan 已提交
39 40 41 42 43 44 45 46 47 48
    'fc',
    'embedding',
    'dynamic_lstm',
    'dynamic_lstmp',
    'dynamic_gru',
    'gru_unit',
    'linear_chain_crf',
    'crf_decoding',
    'cos_sim',
    'cross_entropy',
49
    'bpr_loss',
X
Xin Pan 已提交
50 51 52 53 54 55 56 57 58 59
    'square_error_cost',
    'chunk_eval',
    'sequence_conv',
    'conv2d',
    'conv3d',
    'sequence_pool',
    'sequence_softmax',
    'softmax',
    'pool2d',
    'pool3d',
60 61
    'adaptive_pool2d',
    'adaptive_pool3d',
X
Xin Pan 已提交
62
    'batch_norm',
H
heqiaozhi 已提交
63
    'data_norm',
X
Xin Pan 已提交
64 65 66 67 68 69
    'beam_search_decode',
    'conv2d_transpose',
    'conv3d_transpose',
    'sequence_expand',
    'sequence_expand_as',
    'sequence_pad',
Y
Yibing Liu 已提交
70
    'sequence_unpad',
X
Xin Pan 已提交
71 72 73 74 75 76
    'lstm_unit',
    'reduce_sum',
    'reduce_mean',
    'reduce_max',
    'reduce_min',
    'reduce_prod',
Z
zhoukunsheng 已提交
77 78
    'reduce_all',
    'reduce_any',
X
Xin Pan 已提交
79 80
    'sequence_first_step',
    'sequence_last_step',
Y
Yibing Liu 已提交
81
    'sequence_slice',
X
Xin Pan 已提交
82 83 84 85 86 87 88 89 90 91 92 93
    'dropout',
    'split',
    'ctc_greedy_decoder',
    'edit_distance',
    'l2_normalize',
    'matmul',
    'topk',
    'warpctc',
    'sequence_reshape',
    'transpose',
    'im2sequence',
    'nce',
94
    'sampled_softmax_with_cross_entropy',
X
Xin Pan 已提交
95 96 97 98 99
    'hsigmoid',
    'beam_search',
    'row_conv',
    'multiplex',
    'layer_norm',
D
Dun 已提交
100
    'group_norm',
D
dengkaipeng 已提交
101
    'spectral_norm',
X
Xin Pan 已提交
102 103 104 105 106 107 108 109 110 111 112 113 114
    'softmax_with_cross_entropy',
    'smooth_l1',
    'one_hot',
    'autoincreased_step_counter',
    'reshape',
    'squeeze',
    'unsqueeze',
    'lod_reset',
    'lrn',
    'pad',
    'pad_constant_like',
    'label_smooth',
    'roi_pool',
J
jerrywgz 已提交
115
    'roi_align',
X
Xin Pan 已提交
116 117 118 119
    'dice_loss',
    'image_resize',
    'image_resize_short',
    'resize_bilinear',
120
    'resize_nearest',
X
Xin Pan 已提交
121 122 123 124 125 126
    'gather',
    'scatter',
    'sequence_scatter',
    'random_crop',
    'mean_iou',
    'relu',
C
chengduo 已提交
127
    'selu',
X
Xin Pan 已提交
128 129 130
    'log',
    'crop',
    'rank_loss',
M
minqiyang 已提交
131
    'margin_rank_loss',
X
Xin Pan 已提交
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
    'elu',
    'relu6',
    'pow',
    'stanh',
    'hard_sigmoid',
    'swish',
    'prelu',
    'brelu',
    'leaky_relu',
    'soft_relu',
    'flatten',
    'sequence_mask',
    'stack',
    'pad2d',
    'unstack',
    'sequence_enumerate',
    'expand',
    'sequence_concat',
    'scale',
    'elementwise_add',
    'elementwise_div',
    'elementwise_sub',
    'elementwise_mul',
    'elementwise_max',
    'elementwise_min',
    'elementwise_pow',
Z
zhoukunsheng 已提交
158 159
    'elementwise_mod',
    'elementwise_floordiv',
X
Xin Pan 已提交
160 161 162 163 164 165 166
    'uniform_random_batch_size_like',
    'gaussian_random',
    'sampling_id',
    'gaussian_random_batch_size_like',
    'sum',
    'slice',
    'shape',
Z
zhoukunsheng 已提交
167
    'rank',
X
Xin Pan 已提交
168 169 170 171 172 173 174 175 176 177
    'logical_and',
    'logical_or',
    'logical_xor',
    'logical_not',
    'clip',
    'clip_by_norm',
    'mean',
    'mul',
    'sigmoid_cross_entropy_with_logits',
    'maxout',
J
JiabinYang 已提交
178
    'space_to_depth',
W
whs 已提交
179
    'affine_grid',
S
sneaxiy 已提交
180
    'sequence_reverse',
181
    'affine_channel',
B
barrierye 已提交
182
    'similarity_focus',
M
minqiyang 已提交
183
    'hash',
D
dengkaipeng 已提交
184
    'grid_sampler',
G
gmcather 已提交
185 186
    'log_loss',
    'add_position_encoding',
Q
Qiao Longfei 已提交
187
    'bilinear_tensor_product',
C
chengduo 已提交
188 189
    'merge_selected_rows',
    'get_tensor_from_selected_rows',
P
phlrain 已提交
190
    'lstm',
S
shippingwang 已提交
191
    'shuffle_channel',
192
    'temporal_shift',
S
sneaxiy 已提交
193
    'py_func',
194
    'psroi_pool',
H
heqiaozhi 已提交
195
    'teacher_student_sigmoid_loss',
M
minqiyang 已提交
196
    'huber_loss',
D
dengkaipeng 已提交
197
    'kldiv_loss',
Z
zhaozhehao 已提交
198
    'tree_conv',
C
ceci3 已提交
199
    'npair_loss',
R
ruri 已提交
200
    'pixel_shuffle',
201
    'fsp_matrix',
H
heqiaozhi 已提交
202
    'continuous_value_model',
Z
zhoukunsheng 已提交
203
    'where',
Z
zhoukunsheng 已提交
204
    'sign',
Y
Yu Yang 已提交
205 206
]

J
jerrywgz 已提交
207 208
kIgnoreIndex = -100

Y
Yu Yang 已提交
209 210 211 212 213 214 215

def fc(input,
       size,
       num_flatten_dims=1,
       param_attr=None,
       bias_attr=None,
       act=None,
J
Jacek Czaja 已提交
216
       is_test=False,
217
       name=None):
Y
Yu Yang 已提交
218
    """
219
    **Fully Connected Layer**
Y
Yu Yang 已提交
220

221
    This function creates a fully connected layer in the network. It can take
222
    one or multiple tensors as its inputs(input can be a list of Variable, see
A
Aurelius84 已提交
223
    Args in detail). It creates a variable called weights for each input tensor,
224 225 226 227
    which represents a fully connected weight matrix from each input unit to
    each output unit. The fully connected layer multiplies each input tensor
    with its corresponding weight to produce an output Tensor with shape [M, `size`],
    where M is batch size. If multiple input tensors are given, the results of
A
Aurelius84 已提交
228
    multiple output tensors with shape [M, `size`] will be summed up. If bias_attr
229 230
    is not None, a bias variable will be created and added to the output.
    Finally, if activation is not None, it will be applied to the output as well.
C
caoying03 已提交
231

232
    When the input is single tensor:
C
caoying03 已提交
233

234 235 236 237 238
    .. math::

        Out = Act({XW + b})

    When the input are multiple tensors:
239 240 241

    .. math::

242
        Out = Act({\sum_{i=0}^{N-1}X_iW_i + b})
243 244 245

    In the above equation:

246 247 248
    * :math:`N`: Number of the input. N equals to len(input) if input is list of Variable.
    * :math:`X_i`: The i-th input tensor.
    * :math:`W_i`: The i-th weights matrix corresponding i-th input tensor.
C
caoying03 已提交
249
    * :math:`b`: The bias parameter created by this layer (if needed).
250
    * :math:`Act`: The activation function.
C
caoying03 已提交
251
    * :math:`Out`: The output tensor.
Y
Yu Yang 已提交
252

253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
    See below for an example.

    .. code-block:: text

        Given:
            data_1.data = [[[0.1, 0.2],
                           [0.3, 0.4]]]
            data_1.shape = (1, 2, 2) # 1 is batch_size

            data_2 = [[[0.1, 0.2, 0.3]]]
            data_2.shape = (1, 1, 3)

            out = fluid.layers.fc(input=[data_1, data_2], size=2)

        Then:
            out.data = [[0.18669507, 0.1893476]]
            out.shape = (1, 2)

Y
Yu Yang 已提交
271
    Args:
R
ranqiu 已提交
272 273 274 275 276 277 278 279 280 281
        input (Variable|list of Variable): The input tensor(s) of this layer, and the dimension of
            the input tensor(s) is at least 2.
        size(int): The number of output units in this layer.
        num_flatten_dims (int, default 1): The fc layer can accept an input tensor with more than
            two dimensions. If this happens, the multidimensional tensor will first be flattened
            into a 2-dimensional matrix. The parameter `num_flatten_dims` determines how the input
            tensor is flattened: the first `num_flatten_dims` (inclusive, index starts from 1)
            dimensions will be flatten to form the first dimension of the final matrix (height of
            the matrix), and the rest `rank(X) - num_flatten_dims` dimensions are flattened to
            form the second dimension of the final matrix (width of the matrix). For example, suppose
H
haowang101779990 已提交
282
            `X` is a 5-dimensional tensor with a shape [2, 3, 4, 5, 6], and `num_flatten_dims` = 3.
R
ranqiu 已提交
283 284 285 286
            Then, the flattened matrix will have a shape [2 x 3 x 4, 5 x 6] = [24, 30].
        param_attr (ParamAttr|list of ParamAttr, default None): The parameter attribute for learnable
            parameters/weights of this layer.
        bias_attr (ParamAttr|list of ParamAttr, default None): The parameter attribute for the bias
287 288
            of this layer. If it is set to False, no bias will be added to the output units.
            If it is set to None, the bias is initialized zero. Default: None.
R
ranqiu 已提交
289
        act (str, default None): Activation to be applied to the output of this layer.
J
Jacek Czaja 已提交
290
        is_test(bool): A flag indicating whether execution is in test phase.
R
ranqiu 已提交
291
        name (str, default None): The name of this layer.
Y
Yu Yang 已提交
292

293
    Returns:
F
fengjiayi 已提交
294
        Variable: The transformation result.
295 296

    Raises:
C
caoying03 已提交
297
        ValueError: If rank of the input tensor is less than 2.
298 299 300 301

    Examples:
        .. code-block:: python

302
          # when input is single tensor
F
fengjiayi 已提交
303
          data = fluid.layers.data(name="data", shape=[32, 32], dtype="float32")
304
          fc = fluid.layers.fc(input=data, size=1000, act="tanh")
305 306 307 308 309

          # when input are multiple tensors
          data_1 = fluid.layers.data(name="data_1", shape=[32, 32], dtype="float32")
          data_2 = fluid.layers.data(name="data_2", shape=[24, 36], dtype="float32")
          fc = fluid.layers.fc(input=[data_1, data_2], size=1000, act="tanh")
Y
Yu Yang 已提交
310
    """
C
caoying03 已提交
311
    helper = LayerHelper("fc", **locals())
Y
Yu Yang 已提交
312 313 314 315

    dtype = helper.input_dtype()

    mul_results = []
316 317
    for input_var, param_attr in helper.iter_inputs_and_params():
        input_shape = input_var.shape
Y
Yu Yang 已提交
318 319 320
        param_shape = [
            reduce(lambda a, b: a * b, input_shape[num_flatten_dims:], 1)
        ] + [size]
Y
ying 已提交
321

Y
Yu Yang 已提交
322
        w = helper.create_parameter(
323
            attr=param_attr, shape=param_shape, dtype=dtype, is_bias=False)
X
Xin Pan 已提交
324
        tmp = helper.create_variable_for_type_inference(dtype)
325
        helper.append_op(
326 327 328
            type="mul",
            inputs={"X": input_var,
                    "Y": w},
329
            outputs={"Out": tmp},
M
mozga-intel 已提交
330 331
            attrs={"x_num_col_dims": num_flatten_dims,
                   "y_num_col_dims": 1})
332 333 334 335
        mul_results.append(tmp)

    if len(mul_results) == 1:
        pre_bias = mul_results[0]
336
    else:
X
Xin Pan 已提交
337
        pre_bias = helper.create_variable_for_type_inference(dtype)
338
        helper.append_op(
339 340 341
            type="sum",
            inputs={"X": mul_results},
            outputs={"Out": pre_bias},
X
Xin Pan 已提交
342
            attrs={"use_mkldnn": False})
343 344 345 346
    # add bias
    pre_activation = helper.append_bias_op(pre_bias, dim_start=num_flatten_dims)
    # add activation
    return helper.append_activation(pre_activation)
Y
Yu Yang 已提交
347 348


349 350 351
def embedding(input,
              size,
              is_sparse=False,
352
              is_distributed=False,
353 354 355
              padding_idx=None,
              param_attr=None,
              dtype='float32'):
Y
Yu Yang 已提交
356
    """
357 358
    **Embedding Layer**

359
    This layer is used to lookup embeddings of IDs, provided by :attr:`input`, in
360 361
    a lookup table. The result of this lookup is the embedding of each ID in the
    :attr:`input`.
362 363 364

    All the input variables are passed in as local variables to the LayerHelper
    constructor.
Y
Yu Yang 已提交
365 366

    Args:
367 368 369 370 371
        input(Variable): The tensor variable containing the IDs.
        size(tuple|list): The shape of the look up table parameter. It should
            have two elements which indicate the size of the dictionary of
            embeddings and the size of each embedding vector respectively.
        is_sparse(bool): The flag indicating whether to use sparse update.
372
        is_distributed(bool): Whether to run lookup table from remote parameter server.
373 374
        padding_idx(int|long|None): If :attr:`None`, it makes no effect to lookup.
            Otherwise the given :attr:`padding_idx` indicates padding the output
375
            with zeros whenever lookup encounters it in :attr:`input`. If
376
            :math:`padding_idx < 0`, the :attr:`padding_idx` to use in lookup is
377 378
            :math:`size[0] + dim`.
        param_attr(ParamAttr): Parameters for this layer
379
        dtype(np.dtype|core.VarDesc.VarType|str): The type of data : float32, float_16, int etc
Y
Yu Yang 已提交
380

381 382 383
    Returns:
        Variable: The tensor variable storing the embeddings of the \
                  supplied inputs.
Y
Yu Yang 已提交
384

385 386
    Examples:
        .. code-block:: python
Y
Yu Yang 已提交
387

B
bdzhuxiaoning 已提交
388 389 390
          import paddle.fluid as fluid
          data = fluid.layers.data(name='sequence', shape=[1], dtype='int64', lod_level=1)
          emb = fluid.layers.embedding(input=data, size=[128, 64])    
Y
Yu Yang 已提交
391 392 393
    """

    helper = LayerHelper('embedding', **locals())
394
    remote_prefetch = is_sparse and (not is_distributed)
Q
Qiao Longfei 已提交
395 396
    if remote_prefetch:
        assert is_sparse is True and is_distributed is False
Y
Yu Yang 已提交
397 398
    w = helper.create_parameter(
        attr=helper.param_attr, shape=size, dtype=dtype, is_bias=False)
X
Xin Pan 已提交
399
    tmp = helper.create_variable_for_type_inference(dtype)
400 401
    padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else (
        size[0] + padding_idx)
Y
Yu Yang 已提交
402 403 404 405 406
    helper.append_op(
        type='lookup_table',
        inputs={'Ids': input,
                'W': w},
        outputs={'Out': tmp},
407 408 409
        attrs={
            'is_sparse': is_sparse,
            'is_distributed': is_distributed,
Q
Qiao Longfei 已提交
410
            'remote_prefetch': remote_prefetch,
411 412
            'padding_idx': padding_idx
        })
Y
Yu Yang 已提交
413 414 415
    return tmp


W
wopeizl 已提交
416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
@templatedoc(op_type="lstm")
def dynamic_lstm(input,
                 size,
                 h_0=None,
                 c_0=None,
                 param_attr=None,
                 bias_attr=None,
                 use_peepholes=True,
                 is_reverse=False,
                 gate_activation='sigmoid',
                 cell_activation='tanh',
                 candidate_activation='tanh',
                 dtype='float32',
                 name=None):
    """
    ${comment}
Y
Yibing Liu 已提交
432

W
wopeizl 已提交
433 434 435 436 437 438 439 440 441 442 443
    Args:
        input (Variable): ${input_comment}
        size (int): 4 * hidden size.
        h_0(Variable): The initial hidden state is an optional input, default is zero.
                       This is a tensor with shape (N x D), where N is the
                       batch size and D is the hidden size.
        c_0(Variable): The initial cell state is an optional input, default is zero.
                       This is a tensor with shape (N x D), where N is the
                       batch size. `h_0` and `c_0` can be NULL but only at the same time.
        param_attr(ParamAttr|None): The parameter attribute for the learnable
                               hidden-hidden weights.
Y
Yu Yang 已提交
444

W
wopeizl 已提交
445 446 447 448
                               - Weights = {:math:`W_{ch}, W_{ih}, \
                                                W_{fh}, W_{oh}`}
                               - The shape is (D x 4D), where D is the hidden
                                 size.
Y
Yu Yang 已提交
449

W
wopeizl 已提交
450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485
                               If it is set to None or one attribute of ParamAttr,
                               dynamic_lstm will create ParamAttr as param_attr.
                               If the Initializer of the param_attr is not set, the
                               parameter is initialized with Xavier. Default: None.
        bias_attr (ParamAttr|None): The bias attribute for the learnable bias
                              weights, which contains two parts, input-hidden
                              bias weights and peephole connections weights if
                              setting `use_peepholes` to `True`.

                              1. `use_peepholes = False`
                                 - Biases = {:math:`b_c, b_i, b_f, b_o`}.
                                 - The shape is (1 x 4D).
                              2. `use_peepholes = True`
                                 - Biases = { :math:`b_c, b_i, b_f, b_o, W_{ic}, \
                                                 W_{fc}, W_{oc}`}.
                                 - The shape is (1 x 7D).

                              If it is set to None or one attribute of ParamAttr,
                              dynamic_lstm will create ParamAttr as bias_attr.
                              If the Initializer of the bias_attr is not set,
                              the bias is initialized zero. Default: None.
        use_peepholes (bool): ${use_peepholes_comment}
        is_reverse (bool): ${is_reverse_comment}
        gate_activation (str): ${gate_activation_comment}
        cell_activation (str): ${cell_activation_comment}
        candidate_activation (str): ${candidate_activation_comment}
        dtype (str): Data type. Choices = ["float32", "float64"], default "float32".
        name (str|None): A name for this layer(optional). If set None, the layer
                         will be named automatically.

    Returns:
        tuple: The hidden state, and cell state of LSTM. The shape of both \
        is (T x D), and lod is the same with the `input`.

    Examples:
        .. code-block:: python
486 487 488
            
            emb_dim = 256
            vocab_size = 10000
W
wopeizl 已提交
489
            hidden_dim = 512
490 491 492 493 494 495
            
            data = fluid.layers.data(name='x', shape=[1],
                         dtype='int32', lod_level=1)
            emb = fluid.layers.embedding(input=data, size=[vocab_size, emb_dim], is_sparse=True)

            forward_proj = fluid.layers.fc(input=emb, size=hidden_dim * 4,
W
wopeizl 已提交
496
                                           bias_attr=False)
497

W
wopeizl 已提交
498 499 500
            forward, _ = fluid.layers.dynamic_lstm(
                input=forward_proj, size=hidden_dim * 4, use_peepholes=False)
    """
L
lujun 已提交
501
    assert in_dygraph_mode(
502
    ) is not True, "please use lstm instead of dynamic_lstm in dygraph mode!"
W
wopeizl 已提交
503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545
    assert bias_attr is not False, "bias_attr should not be False in dynamic_lstmp."
    helper = LayerHelper('lstm', **locals())
    size = size // 4
    weight = helper.create_parameter(
        attr=helper.param_attr, shape=[size, 4 * size], dtype=dtype)
    bias_size = [1, 7 * size]
    if not use_peepholes:
        bias_size[1] = 4 * size
    bias = helper.create_parameter(
        attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True)

    hidden = helper.create_variable_for_type_inference(dtype)
    cell = helper.create_variable_for_type_inference(dtype)
    batch_gate = helper.create_variable_for_type_inference(dtype)
    batch_cell_pre_act = helper.create_variable_for_type_inference(dtype)
    inputs = {'Input': input, 'Weight': weight, 'Bias': bias}
    batch_size = input.shape[0]
    if h_0:
        assert h_0.shape == (batch_size, size), \
            'The shape of h0 should be (batch_size, %d)' % size
        inputs['H0'] = h_0
    if c_0:
        assert c_0.shape == (batch_size, size), \
            'The shape of c0 should be (batch_size, %d)' % size
        inputs['C0'] = c_0

    helper.append_op(
        type='lstm',
        inputs=inputs,
        outputs={
            'Hidden': hidden,
            'Cell': cell,
            'BatchGate': batch_gate,
            'BatchCellPreAct': batch_cell_pre_act
        },
        attrs={
            'use_peepholes': use_peepholes,
            'is_reverse': is_reverse,
            'gate_activation': gate_activation,
            'cell_activation': cell_activation,
            'candidate_activation': candidate_activation
        })
    return hidden, cell
Y
Yu Yang 已提交
546 547


P
phlrain 已提交
548 549 550 551 552 553
def lstm(input,
         init_h,
         init_c,
         max_len,
         hidden_size,
         num_layers,
P
phlrain 已提交
554
         dropout_prob=0.0,
P
phlrain 已提交
555 556 557 558 559
         is_bidirec=False,
         is_test=False,
         name=None,
         default_initializer=None,
         seed=-1):
L
liuhongyu 已提交
560
    """
P
phlrain 已提交
561
    If Device is GPU, This op will use cudnn LSTM implementation
L
liuhongyu 已提交
562 563

    A four-gate Long Short-Term Memory network with no peephole connections.
M
minqiyang 已提交
564
    In the forward pass the output ht and cell output ct for a given iteration can be computed from the recurrent input ht-1,
L
liuhongyu 已提交
565 566
    the cell input ct-1 and the previous layer input xt given matrices W, R and biases bW, bR from the following equations:

H
haowang101779990 已提交
567
    .. math::
M
minqiyang 已提交
568 569 570 571 572 573 574

       i_t &= \sigma(W_{ix}x_{t} + W_{ih}h_{t-1} + bx_i + bh_i)

       f_t &= \sigma(W_{fx}x_{t} + W_{fh}h_{t-1} + bx_f + bh_f)

       o_t &= \sigma(W_{ox}x_{t} + W_{oh}h_{t-1} + bx_o + bh_o)

H
haowang101779990 已提交
575
       \\tilde{c_t} &= tanh(W_{cx}x_t + W_{ch}h_{t-1} + bx_c + bh_c)
M
minqiyang 已提交
576 577 578 579

       c_t &= f_t \odot c_{t-1} + i_t \odot \\tilde{c_t}

       h_t &= o_t \odot tanh(c_t)
H
haowang101779990 已提交
580 581

    - $W$ terms denote weight matrices (e.g. $W_{ix}$ is the matrix
P
phlrain 已提交
582 583 584 585 586 587
      of weights from the input gate to the input)
    - The b terms denote bias vectors ($bx_i$ and $bh_i$ are the input gate bias vector).
    - sigmoid is the logistic sigmoid function.
    - $i, f, o$ and $c$ are the input gate, forget gate, output gate,
      and cell activation vectors, respectively, all of which have the same size as
      the cell output activation vector $h$.
H
haowang101779990 已提交
588 589 590
    - The :math:`\odot` is the element-wise product of the vectors.
    - :math:`tanh` is the activation functions.
    - :math:`\\tilde{c_t}` is also called candidate hidden state,
P
phlrain 已提交
591
      which is computed based on the current input and the previous hidden state.
L
liuhongyu 已提交
592

M
minqiyang 已提交
593
    Where sigmoid is the sigmoid operator: :math:`sigmoid(x) = 1 / (1 + e^{-x})` , * represents a point-wise multiplication,
L
liuhongyu 已提交
594 595 596 597 598
    X represensts a matrix multiplication


    Args:
        input (Variable): LSTM input tensor, shape MUST be ( seq_len x batch_size x input_size )
M
minqiyang 已提交
599
        init_h(Variable): The initial hidden state of the LSTM
L
liuhongyu 已提交
600 601 602 603 604
                       This is a tensor with shape ( num_layers x batch_size x hidden_size)
                       if is_bidirec = True, shape should be ( num_layers*2 x batch_size x hidden_size)
        init_c(Variable): The initial cell state of the LSTM.
                       This is a tensor with shape ( num_layers x batch_size x hidden_size )
                       if is_bidirec = True, shape should be ( num_layers*2 x batch_size x hidden_size)
M
minqiyang 已提交
605
        max_len (int): max length of LSTM. the first dim of input tensor CAN NOT greater than max_len
L
liuhongyu 已提交
606 607
        hidden_size (int): hidden size of the LSTM
        num_layers (int): total layers number of the LSTM
P
phlrain 已提交
608 609
        dropout_prob(float|0.0): dropout prob, dropout ONLY work between rnn layers, NOT between time steps
                             There is NO dropout work on rnn output of the last RNN layers
L
liuhongyu 已提交
610 611 612 613 614 615
        is_bidirec (bool): If it is bidirectional
        is_test (bool): If it is in test phrase
        name (str|None): A name for this layer(optional). If set None, the layer
                         will be named automatically.
        default_initializer(Initialize|None): Where use initializer to initialize the Weight
                         If set None, defaule initializer will be used
P
phlrain 已提交
616
        seed(int): Seed for dropout in LSTM, If it's -1, dropout will use random seed
P
phlrain 已提交
617

L
liuhongyu 已提交
618 619

    Returns:
M
minqiyang 已提交
620 621
        rnn_out(Tensor),last_h(Tensor),last_c(Tensor):

H
haowang101779990 已提交
622
                        Three tensors, rnn_out, last_h, last_c:
M
minqiyang 已提交
623

H
haowang101779990 已提交
624 625 626 627
                        - rnn_out is result of LSTM hidden, shape is (seq_len x batch_size x hidden_size) \
                          if is_bidirec set to True, shape will be ( seq_len x batch_sze x hidden_size*2)
                        - last_h is the hidden state of the last step of LSTM \
                          shape is ( num_layers x batch_size x hidden_size ) \
M
minqiyang 已提交
628
                          if is_bidirec set to True, shape will be ( num_layers*2 x batch_size x hidden_size)
H
haowang101779990 已提交
629 630
                        - last_c(Tensor): the cell state of the last step of LSTM \
                          shape is ( num_layers x batch_size x hidden_size ) \
M
minqiyang 已提交
631
                          if is_bidirec set to True, shape will be ( num_layers*2 x batch_size x hidden_size)
L
liuhongyu 已提交
632 633 634 635


    Examples:
        .. code-block:: python
636 637 638 639 640 641
            
            emb_dim = 256
            vocab_size = 10000
            data = fluid.layers.data(name='x', shape=[-1, 100, 1],
                         dtype='int32')
            emb = fluid.layers.embedding(input=data, size=[vocab_size, emb_dim], is_sparse=True)
L
liuhongyu 已提交
642 643 644 645 646 647
            batch_size = 20
            max_len = 100
            dropout_prob = 0.2
            input_size = 100
            hidden_size = 150
            num_layers = 1
648 649 650 651 652
            init_h = layers.fill_constant( [num_layers, batch_size, hidden_size], 'float32', 0.0 )
            init_c = layers.fill_constant( [num_layers, batch_size, hidden_size], 'float32', 0.0 )
            rnn_out, last_h, last_c = layers.lstm( emb, init_h, init_c, \
                    max_len, hidden_size, num_layers, \
                    dropout_prob=dropout_prob)
L
liuhongyu 已提交
653 654 655 656
    """

    helper = LayerHelper('cudnn_lstm', **locals())

P
phlrain 已提交
657 658 659
    dtype = input.dtype
    input_shape = list(input.shape)
    input_size = input_shape[-1]
L
liuhongyu 已提交
660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
    weight_size = 0
    for i in range(num_layers):
        if i == 0:
            input_weight_size = (input_size * hidden_size) * 4
        else:
            if is_bidirec:
                input_weight_size = (hidden_size * 2 * hidden_size) * 4
            else:
                input_weight_size = (hidden_size * hidden_size) * 4

        hidden_weight_size = (hidden_size * hidden_size) * 4

        if is_bidirec:
            weight_size += (input_weight_size + hidden_weight_size) * 2
            weight_size += hidden_size * 8 * 2
        else:
            weight_size += input_weight_size + hidden_weight_size
            weight_size += hidden_size * 8

    weight = helper.create_parameter(
        attr=helper.param_attr,
        shape=[weight_size],
        dtype=dtype,
        default_initializer=default_initializer)

    out = helper.create_variable_for_type_inference(dtype)
    last_h = helper.create_variable_for_type_inference(dtype)
    last_c = helper.create_variable_for_type_inference(dtype)

    cache = helper.create_variable(
        persistable=True, type=core.VarDesc.VarType.RAW, stop_gradient=True)

    helper.append_op(
        type='cudnn_lstm',
        inputs={
            'Input': input,
            'InitH': init_h,
            'InitC': init_c,
            'W': weight,
            'Cache': cache,
        },
        outputs={
            'Out': out,
            'last_h': last_h,
            'last_c': last_c,
        },
        attrs={
            'max_len': max_len,
            'is_bidirec': is_bidirec,
            'input_size': input_size,
            'hidden_size': hidden_size,
            'num_layers': num_layers,
            'is_test': is_test,
            'dropout_prob': dropout_prob,
            'seed': seed,
        })
    return out, last_h, last_c


Y
Yibing Liu 已提交
719 720 721 722 723 724 725 726 727 728
def dynamic_lstmp(input,
                  size,
                  proj_size,
                  param_attr=None,
                  bias_attr=None,
                  use_peepholes=True,
                  is_reverse=False,
                  gate_activation='sigmoid',
                  cell_activation='tanh',
                  candidate_activation='tanh',
X
xuezhong 已提交
729
                  proj_activation='tanh',
730
                  dtype='float32',
X
xuezhong 已提交
731 732 733 734 735
                  name=None,
                  h_0=None,
                  c_0=None,
                  cell_clip=None,
                  proj_clip=None):
Y
Yibing Liu 已提交
736 737 738
    """
    **Dynamic LSTMP Layer**

739 740 741 742 743 744
    LSTMP (LSTM with recurrent projection) layer has a separate projection
    layer after the LSTM layer, projecting the original hidden state to a
    lower-dimensional one, which is proposed to reduce the number of total
    parameters and furthermore computational complexity for the LSTM,
    espeacially for the case that the size of output units is relative
    large (https://research.google.com/pubs/archive/43905.pdf).
Y
Yibing Liu 已提交
745 746 747 748 749

    The formula is as follows:

    .. math::

750
        i_t & = \sigma(W_{ix}x_{t} + W_{ir}r_{t-1} + W_{ic}c_{t-1} + b_i)
Y
Yibing Liu 已提交
751

752
        f_t & = \sigma(W_{fx}x_{t} + W_{fr}r_{t-1} + W_{fc}c_{t-1} + b_f)
Y
Yibing Liu 已提交
753

754
        \\tilde{c_t} & = act_g(W_{cx}x_t + W_{cr}r_{t-1} + b_c)
Y
Yibing Liu 已提交
755

756
        o_t & = \sigma(W_{ox}x_{t} + W_{or}r_{t-1} + W_{oc}c_t + b_o)
Y
Yibing Liu 已提交
757

758
        c_t & = f_t \odot c_{t-1} + i_t \odot \\tilde{c_t}
Y
Yibing Liu 已提交
759

760
        h_t & = o_t \odot act_h(c_t)
Y
Yibing Liu 已提交
761

762
        r_t & = \overline{act_h}(W_{rh}h_t)
Y
Yibing Liu 已提交
763

Y
Yibing Liu 已提交
764 765 766 767 768 769
    In the above formula:

    * :math:`W`: Denotes weight matrices (e.g. :math:`W_{xi}` is \
          the matrix of weights from the input gate to the input).
    * :math:`W_{ic}`, :math:`W_{fc}`, :math:`W_{oc}`: Diagonal weight \
          matrices for peephole connections. In our implementation, \
770
          we use vectors to reprenset these diagonal weight matrices.
Y
Yibing Liu 已提交
771
    * :math:`b`: Denotes bias vectors (e.g. :math:`b_i` is the input gate \
772
          bias vector).
Y
Yibing Liu 已提交
773 774 775
    * :math:`\sigma`: The activation, such as logistic sigmoid function.
    * :math:`i, f, o` and :math:`c`: The input gate, forget gate, output \
          gate, and cell activation vectors, respectively, all of which have \
776
          the same size as the cell output activation vector :math:`h`.
Y
Yibing Liu 已提交
777
    * :math:`h`: The hidden state.
778
    * :math:`r`: The recurrent projection of the hidden state.
Y
Yibing Liu 已提交
779 780
    * :math:`\\tilde{c_t}`: The candidate hidden state, whose \
          computation is based on the current input and previous hidden state.
781
    * :math:`\odot`: The element-wise product of the vectors.
Y
Yibing Liu 已提交
782
    * :math:`act_g` and :math:`act_h`: The cell input and cell output \
783
          activation functions and `tanh` is usually used for them.
Y
Yibing Liu 已提交
784 785
    * :math:`\overline{act_h}`: The activation function for the projection \
          output, usually using `identity` or same as :math:`act_h`.
Y
Yibing Liu 已提交
786 787 788 789

    Set `use_peepholes` to `False` to disable peephole connection. The formula
    is omitted here, please refer to the paper
    http://www.bioinf.jku.at/publications/older/2604.pdf for details.
790

Y
Yibing Liu 已提交
791 792 793 794 795 796 797 798 799 800 801 802
    Note that these :math:`W_{xi}x_{t}, W_{xf}x_{t}, W_{xc}x_{t}, W_{xo}x_{t}`
    operations on the input :math:`x_{t}` are NOT included in this operator.
    Users can choose to use fully-connected layer before LSTMP layer.

    Args:
        input(Variable): The input of dynamic_lstmp layer, which supports
                         variable-time length input sequence. The underlying
                         tensor in this Variable is a matrix with shape
                         (T X 4D), where T is the total time steps in this
                         mini-batch, D is the hidden size.
        size(int): 4 * hidden size.
        proj_size(int): The size of projection output.
803
        param_attr(ParamAttr|None): The parameter attribute for the learnable
Y
Yibing Liu 已提交
804 805
                               hidden-hidden weight and projection weight.

806 807
                               - Hidden-hidden weight = {:math:`W_{ch}, W_{ih}, \
                                                W_{fh}, W_{oh}`}.
808 809
                               - The shape of hidden-hidden weight is (P x 4D),
                                 where P is the projection size and D the hidden
Y
Yibing Liu 已提交
810 811
                                 size.
                               - Projection weight = {:math:`W_{rh}`}.
812
                               - The shape of projection weight is (D x P).
C
chengduo 已提交
813 814 815 816 817

                               If it is set to None or one attribute of ParamAttr,
                               dynamic_lstm will create ParamAttr as param_attr.
                               If the Initializer of the param_attr is not set, the
                               parameter is initialized with Xavier. Default: None.
818
        bias_attr(ParamAttr|None): The bias attribute for the learnable bias
Y
Yibing Liu 已提交
819 820 821 822 823 824
                              weights, which contains two parts, input-hidden
                              bias weights and peephole connections weights if
                              setting `use_peepholes` to `True`.

                              1. `use_peepholes = False`
                                - Biases = {:math:`b_c, b_i, b_f, b_o`}.
825
                                - The shape is (1 x 4D).
Y
Yibing Liu 已提交
826 827 828
                              2. `use_peepholes = True`
                                - Biases = { :math:`b_c, b_i, b_f, b_o, W_{ic}, \
                                                 W_{fc}, W_{oc}`}.
829
                                - The shape is (1 x 7D).
C
chengduo 已提交
830 831 832 833 834

                              If it is set to None or one attribute of ParamAttr,
                              dynamic_lstm will create ParamAttr as bias_attr.
                              If the Initializer of the bias_attr is not set,
                              the bias is initialized zero. Default: None.
Y
Yibing Liu 已提交
835 836 837 838 839 840 841 842 843
        use_peepholes(bool): Whether to enable diagonal/peephole connections,
                             default `True`.
        is_reverse(bool): Whether to compute reversed LSTM, default `False`.
        gate_activation(str): The activation for input gate, forget gate and
                              output gate. Choices = ["sigmoid", "tanh", "relu",
                              "identity"], default "sigmoid".
        cell_activation(str): The activation for cell output. Choices = ["sigmoid",
                              "tanh", "relu", "identity"], default "tanh".
        candidate_activation(str): The activation for candidate hidden state.
844
                              Choices = ["sigmoid", "tanh", "relu", "identity"],
Y
Yibing Liu 已提交
845 846
                              default "tanh".
        proj_activation(str): The activation for projection output.
847
                              Choices = ["sigmoid", "tanh", "relu", "identity"],
X
xuezhong 已提交
848
                              default "tanh".
Y
Yibing Liu 已提交
849
        dtype(str): Data type. Choices = ["float32", "float64"], default "float32".
850 851
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.
X
xuezhong 已提交
852 853 854 855 856 857 858 859 860 861 862
        h_0(Variable): The initial hidden state is an optional input, default is zero.
                       This is a tensor with shape (N x D), where N is the
                       batch size and D is the projection size.
        c_0(Variable): The initial cell state is an optional input, default is zero.
                       This is a tensor with shape (N x D), where N is the
                       batch size. `h_0` and `c_0` can be NULL but only at the same time.
        cell_clip(float): If provided the cell state is clipped
                             by this value prior to the cell output activation.
        proj_clip(float): If `num_proj > 0` and `proj_clip` is
                            provided, then the projected values are clipped elementwise to within
                            `[-proj_clip, proj_clip]`.
Y
Yibing Liu 已提交
863 864

    Returns:
865 866 867 868
        tuple: A tuple of two output variable: the projection of hidden state, \
               and cell state of LSTMP. The shape of projection is (T x P), \
               for the cell state which is (T x D), and both LoD is the same \
               with the `input`.
Y
Yibing Liu 已提交
869 870

    Examples:
871

Y
Yibing Liu 已提交
872 873
        .. code-block:: python

874 875 876 877
            dict_dim, emb_dim = 128, 64
            data = fluid.layers.data(name='sequence', shape=[1],
                                     dtype='int32', lod_level=1)
            emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim])
Y
Yibing Liu 已提交
878
            hidden_dim, proj_dim = 512, 256
879
            fc_out = fluid.layers.fc(input=emb, size=hidden_dim * 4,
Y
Yibing Liu 已提交
880
                                     act=None, bias_attr=None)
881 882 883
            proj_out, _ = fluid.layers.dynamic_lstmp(input=fc_out,
                                                     size=hidden_dim * 4,
                                                     proj_size=proj_dim,
Y
Yibing Liu 已提交
884 885 886 887
                                                     use_peepholes=False,
                                                     is_reverse=True,
                                                     cell_activation="tanh",
                                                     proj_activation="tanh")
Y
Yibing Liu 已提交
888
    """
889

L
lujun 已提交
890
    assert in_dygraph_mode(
891 892
    ) is not True, "please use lstm instead of dynamic_lstmp in dygraph mode!"

C
chengduo 已提交
893
    assert bias_attr is not False, "bias_attr should not be False in dynamic_lstmp."
Y
Yibing Liu 已提交
894
    helper = LayerHelper('lstmp', **locals())
M
minqiyang 已提交
895
    size = size // 4
Y
Yibing Liu 已提交
896 897 898 899 900 901 902 903 904 905
    weight = helper.create_parameter(
        attr=helper.param_attr, shape=[proj_size, 4 * size], dtype=dtype)
    proj_weight = helper.create_parameter(
        attr=helper.param_attr, shape=[size, proj_size], dtype=dtype)
    bias_size = [1, 7 * size]
    if not use_peepholes:
        bias_size[1] = 4 * size
    bias = helper.create_parameter(
        attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True)

X
Xin Pan 已提交
906 907 908 909 910 911
    projection = helper.create_variable_for_type_inference(dtype)
    cell = helper.create_variable_for_type_inference(dtype)
    ordered_proj0 = helper.create_variable_for_type_inference(dtype)
    batch_hidden = helper.create_variable_for_type_inference(dtype)
    batch_gate = helper.create_variable_for_type_inference(dtype)
    batch_cell_pre_act = helper.create_variable_for_type_inference(dtype)
912 913 914 915 916 917 918 919 920 921 922 923 924 925 926
    inputs = {
        'Input': input,
        'Weight': weight,
        'ProjWeight': proj_weight,
        'Bias': bias
    }
    batch_size = input.shape[0]
    if h_0:
        assert h_0.shape == (batch_size, proj_size), \
            'The shape of h0 should be (batch_size, %d)' % proj_size
        inputs['H0'] = h_0
    if c_0:
        assert c_0.shape == (batch_size, size), \
            'The shape of c0 should be (batch_size, %d)' % size
        inputs['C0'] = c_0
Y
Yibing Liu 已提交
927

X
xuezhong 已提交
928 929 930 931 932
    if cell_clip:
        assert cell_clip >= 0, "cell_clip should not be negtive."
    if proj_clip:
        assert proj_clip >= 0, "proj_clip should not be negtive."

Y
Yibing Liu 已提交
933 934
    helper.append_op(
        type='lstmp',
935
        inputs=inputs,
Y
Yibing Liu 已提交
936 937 938 939 940 941 942 943 944
        outputs={
            'Projection': projection,
            'Cell': cell,
            'BatchHidden': batch_hidden,
            'BatchGate': batch_gate,
            'BatchCellPreAct': batch_cell_pre_act
        },
        attrs={
            'use_peepholes': use_peepholes,
945 946
            'cell_clip': cell_clip,
            'proj_clip': proj_clip,
Y
Yibing Liu 已提交
947 948 949 950 951 952 953 954 955
            'is_reverse': is_reverse,
            'gate_activation': gate_activation,
            'cell_activation': cell_activation,
            'candidate_activation': candidate_activation,
            'proj_activation': proj_activation
        })
    return projection, cell


G
guosheng 已提交
956 957 958 959 960 961 962
def dynamic_gru(input,
                size,
                param_attr=None,
                bias_attr=None,
                is_reverse=False,
                gate_activation='sigmoid',
                candidate_activation='tanh',
963 964
                h_0=None,
                origin_mode=False):
G
guosheng 已提交
965
    """
966
    **Gated Recurrent Unit (GRU) Layer**
G
guosheng 已提交
967

968 969 970
    if origin_mode is False, then the equation of a gru step is from paper
    `Empirical Evaluation of Gated Recurrent Neural Networks on Sequence
    Modeling <https://arxiv.org/pdf/1412.3555.pdf>`_ .
971

G
guosheng 已提交
972 973 974 975 976 977 978 979 980
    The formula is as follows:

    .. math::

        u_t & = act_g(W_{ux}x_{t} + W_{uh}h_{t-1} + b_u)

        r_t & = act_g(W_{rx}x_{t} + W_{rh}h_{t-1} + b_r)

        \\tilde{h_t} & = act_c(W_{cx}x_{t} + W_{ch}(r_t \odot h_{t-1}) + b_c)
981

G
guosheng 已提交
982
        h_t & = (1-u_t) \odot h_{t-1} + u_t \odot \\tilde{h_t}
983

Q
Qiao Longfei 已提交
984 985 986

    if origin_mode is True then the equation is from paper
    Learning Phrase Representations using RNN Encoder-Decoder for Statistical
987 988 989 990 991 992 993 994 995 996 997 998
    Machine Translation <https://arxiv.org/pdf/1406.1078.pdf>`_

    .. math::

        u_t & = act_g(W_{ux}x_{t} + W_{uh}h_{t-1} + b_u)

        r_t & = act_g(W_{rx}x_{t} + W_{rh}h_{t-1} + b_r)

        \\tilde{h_t} & = act_c(W_{cx}x_{t} + W_{ch}(r_t \odot h_{t-1}) + b_c)

        h_t & = u_t \odot h_{t-1} + (1-u_t) \odot \\tilde{h_t}

G
guosheng 已提交
999
    The :math:`\odot` is the element-wise product of the vectors. :math:`act_g`
1000 1001
    is the update gate and reset gate activation function and :math:`sigmoid`
    is usually used for it. :math:`act_c` is the activation function for
G
guosheng 已提交
1002 1003 1004 1005
    candidate hidden state and :math:`tanh` is usually used for it.

    Note that these :math:`W_{ux}x_{t}, W_{rx}x_{t}, W_{cx}x_{t}` operations on
    the input :math:`x_{t}` are NOT included in this operator. Users can choose
1006
    to use fully-connect layer before GRU layer.
G
guosheng 已提交
1007 1008

    Args:
1009 1010
        input(Variable): The input of dynamic_gru layer, which supports
            variable-time length input sequence. The underlying tensor in this
G
guosheng 已提交
1011
            Variable is a matrix with shape :math:`(T \\times 3D)`, where
1012
            :math:`T` is the total time steps in this mini-batch, :math:`D`
G
guosheng 已提交
1013 1014
            is the hidden size.
        size(int): The dimension of the gru cell.
1015
        param_attr(ParamAttr|None): The parameter attribute for the learnable
G
guosheng 已提交
1016 1017
            hidden-hidden weight matrix. Note:

1018
            - The shape of the weight matrix is :math:`(T \\times 3D)`, where
G
guosheng 已提交
1019
              :math:`D` is the hidden size.
1020
            - All elements in the weight matrix can be divided into two parts.
G
guosheng 已提交
1021
              The first part are weights of the update gate and reset gate with
1022
              shape :math:`(D \\times 2D)`, and the second part are weights for
G
guosheng 已提交
1023
              candidate hidden state with shape :math:`(D \\times D)`.
1024 1025 1026 1027 1028

            If it is set to None or one attribute of ParamAttr, dynamic_gru will
            create ParamAttr as param_attr. If the Initializer of the param_attr
            is not set, the parameter is initialized with Xavier. Default: None.
        bias_attr (ParamAttr|bool|None): The parameter attribute for the bias
1029
            of GRU.Note that the bias with :math:`(1 \\times 3D)` concatenates
1030
            the bias in the update gate, reset gate and candidate calculations.
1031 1032 1033
            If it is set to False, no bias will be applied to the update gate,
            reset gate and candidate calculations. If it is set to None or one
            attribute of ParamAttr, dynamic_gru will create ParamAttr as
1034 1035
            bias_attr. If the Initializer of the bias_attr is not set, the bias
            is initialized zero. Default: None.
1036
        is_reverse(bool): Whether to compute reversed GRU, default
G
guosheng 已提交
1037 1038 1039
            :attr:`False`.
        gate_activation(str): The activation for update gate and reset gate.
            Choices = ["sigmoid", "tanh", "relu", "identity"], default "sigmoid".
1040
        candidate_activation(str): The activation for candidate hidden state.
G
guosheng 已提交
1041
            Choices = ["sigmoid", "tanh", "relu", "identity"], default "tanh".
1042 1043 1044 1045
        h_0 (Variable): This is initial hidden state. If not set, default is
            zero. This is a tensor with shape (N x D), where N is the number of
            total time steps of input mini-batch feature and D is the hidden
            size.
G
guosheng 已提交
1046 1047

    Returns:
G
guosheng 已提交
1048
        Variable: The hidden state of GRU. The shape is :math:`(T \\times D)`, \
1049
            and sequence length is the same with the input.
1050

G
guosheng 已提交
1051
    Examples:
1052

G
guosheng 已提交
1053 1054
        .. code-block:: python

1055 1056
            import paddle.fluid as fluid

1057 1058 1059 1060
            dict_dim, emb_dim = 128, 64
            data = fluid.layers.data(name='sequence', shape=[1],
                                     dtype='int32', lod_level=1)
            emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim])
G
guosheng 已提交
1061
            hidden_dim = 512
1062
            x = fluid.layers.fc(input=emb, size=hidden_dim * 3)
T
Tink_Y 已提交
1063
            hidden = fluid.layers.dynamic_gru(input=x, size=hidden_dim)
G
guosheng 已提交
1064 1065
    """

L
lujun 已提交
1066
    assert in_dygraph_mode(
1067 1068
    ) is not True, "please use gru instead of dynamic_gru in dygraph mode!"

G
guosheng 已提交
1069 1070 1071 1072 1073 1074 1075
    helper = LayerHelper('gru', **locals())
    dtype = helper.input_dtype()

    weight = helper.create_parameter(
        attr=helper.param_attr, shape=[size, 3 * size], dtype=dtype)
    bias = helper.create_parameter(
        attr=helper.bias_attr, shape=[1, 3 * size], dtype=dtype, is_bias=True)
Y
Yancey 已提交
1076
    batch_size = input.shape[0]
G
guosheng 已提交
1077
    inputs = {'Input': input, 'Weight': weight, 'Bias': bias}
S
sneaxiy 已提交
1078
    if h_0:
G
guosheng 已提交
1079
        assert h_0.shape == (
Y
Yancey 已提交
1080 1081 1082
            batch_size, size
        ), 'The shape of h0 should be(batch_size, %d)' % size
        inputs['H0'] = h_0
G
guosheng 已提交
1083

X
Xin Pan 已提交
1084 1085 1086 1087
    hidden = helper.create_variable_for_type_inference(dtype)
    batch_gate = helper.create_variable_for_type_inference(dtype)
    batch_reset_hidden_prev = helper.create_variable_for_type_inference(dtype)
    batch_hidden = helper.create_variable_for_type_inference(dtype)
G
guosheng 已提交
1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100

    helper.append_op(
        type='gru',
        inputs=inputs,
        outputs={
            'Hidden': hidden,
            'BatchGate': batch_gate,
            'BatchResetHiddenPrev': batch_reset_hidden_prev,
            'BatchHidden': batch_hidden
        },
        attrs={
            'is_reverse': is_reverse,
            'gate_activation': gate_activation,
1101 1102
            'activation': candidate_activation,
            'origin_mode': origin_mode
G
guosheng 已提交
1103 1104 1105 1106
        })
    return hidden


Y
Yu Yang 已提交
1107 1108 1109
def gru_unit(input,
             hidden,
             size,
1110 1111
             param_attr=None,
             bias_attr=None,
Y
Yu Yang 已提交
1112
             activation='tanh',
Q
Qiao Longfei 已提交
1113 1114
             gate_activation='sigmoid',
             origin_mode=False):
Y
Yu Yang 已提交
1115
    """
1116 1117 1118
    **GRU unit layer**

    if origin_mode is True, then the equation of a gru step is from paper
Q
Qiao Longfei 已提交
1119
    `Learning Phrase Representations using RNN Encoder-Decoder for Statistical
1120
    Machine Translation <https://arxiv.org/pdf/1406.1078.pdf>`_
Y
Yu Yang 已提交
1121

1122 1123
        .. math::
            u_t & = actGate(xu_{t} + W_u h_{t-1} + b_u)
Y
Yu Yang 已提交
1124

1125
            r_t & = actGate(xr_{t} + W_r h_{t-1} + b_r)
Y
Yu Yang 已提交
1126

1127
            m_t & = actNode(xm_t + W_c dot(r_t, h_{t-1}) + b_m)
1128

1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143
            h_t & = dot(u_t, h_{t-1}) + dot((1-u_t), m_t)

    if origin_mode is False, then the equation of a gru step is from paper
    `Empirical Evaluation of Gated Recurrent Neural Networks on Sequence
    Modeling <https://arxiv.org/pdf/1412.3555.pdf>`_

        .. math::
            u_t & = actGate(xu_{t} + W_u h_{t-1} + b_u)

            r_t & = actGate(xr_{t} + W_r h_{t-1} + b_r)

            m_t & = actNode(xm_t + W_c dot(r_t, h_{t-1}) + b_m)

            h_t & = dot((1-u_t), h_{t-1}) + dot(u_t, m_t)

1144 1145

    The inputs of gru unit includes :math:`z_t`, :math:`h_{t-1}`. In terms
1146 1147 1148
    of the equation above, the :math:`z_t` is split into 3 parts -
    :math:`xu_t`, :math:`xr_t` and :math:`xm_t`. This means that in order to
    implement a full GRU unit operator for an input, a fully
1149 1150
    connected layer has to be applied, such that :math:`z_t = W_{fc}x_t`.

1151 1152
    The terms :math:`u_t` and :math:`r_t` represent the update and reset gates
    of the GRU cell. Unlike LSTM, GRU has one lesser gate. However, there is
1153 1154 1155
    an intermediate candidate hidden output, which is denoted by :math:`m_t`.
    This layer has three outputs :math:`h_t`, :math:`dot(r_t, h_{t-1})`
    and concatenation of :math:`u_t`, :math:`r_t` and :math:`m_t`.
1156 1157 1158

    Args:
        input (Variable): The fc transformed input value of current step.
1159
        hidden (Variable): The hidden value of gru unit from previous step.
1160
        size (integer): The input dimension value.
1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
        param_attr(ParamAttr|None): The parameter attribute for the learnable
            hidden-hidden weight matrix. Note:

            - The shape of the weight matrix is :math:`(T \\times 3D)`, where
              :math:`D` is the hidden size.
            - All elements in the weight matrix can be divided into two parts.
              The first part are weights of the update gate and reset gate with
              shape :math:`(D \\times 2D)`, and the second part are weights for
              candidate hidden state with shape :math:`(D \\times D)`.

            If it is set to None or one attribute of ParamAttr, gru_unit will
            create ParamAttr as param_attr. If the Initializer of the param_attr
            is not set, the parameter is initialized with Xavier. Default: None.
        bias_attr (ParamAttr|bool|None): The parameter attribute for the bias
1175
            of GRU.Note that the bias with :math:`(1 \\times 3D)` concatenates
1176
            the bias in the update gate, reset gate and candidate calculations.
1177 1178 1179
            If it is set to False, no bias will be applied to the update gate,
            reset gate and candidate calculations. If it is set to None or one
            attribute of ParamAttr, gru_unit will create ParamAttr as
1180 1181
            bias_attr. If the Initializer of the bias_attr is not set, the bias
            is initialized zero. Default: None.
1182 1183 1184 1185
        activation (string): The activation type for cell (actNode).
                             Default: 'tanh'
        gate_activation (string): The activation type for gates (actGate).
                                  Default: 'sigmoid'
Y
Yu Yang 已提交
1186

1187 1188 1189 1190 1191 1192
    Returns:
        tuple: The hidden value, reset-hidden value and gate values.

    Examples:

        .. code-block:: python
Y
Yu Yang 已提交
1193

1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204
            import paddle.fluid as fluid

            dict_dim, emb_dim = 128, 64
            data = fluid.layers.data(name='step_data', shape=[1], dtype='int32')
            emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim])
            hidden_dim = 512
            x = fluid.layers.fc(input=emb, size=hidden_dim * 3)
            pre_hidden = fluid.layers.data(
                name='pre_hidden', shape=[hidden_dim], dtype='float32')
            hidden = fluid.layers.gru_unit(
                input=x, hidden=pre_hidden, size=hidden_dim * 3)
Y
Yu Yang 已提交
1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216

    """
    activation_dict = dict(
        identity=0,
        sigmoid=1,
        tanh=2,
        relu=3, )
    activation = activation_dict[activation]
    gate_activation = activation_dict[gate_activation]

    helper = LayerHelper('gru_unit', **locals())
    dtype = helper.input_dtype()
M
minqiyang 已提交
1217
    size = size // 3
Y
Yu Yang 已提交
1218 1219

    # create weight
1220 1221
    weight = helper.create_parameter(
        attr=helper.param_attr, shape=[size, 3 * size], dtype=dtype)
Y
Yu Yang 已提交
1222

X
Xin Pan 已提交
1223 1224 1225
    gate = helper.create_variable_for_type_inference(dtype)
    reset_hidden_pre = helper.create_variable_for_type_inference(dtype)
    updated_hidden = helper.create_variable_for_type_inference(dtype)
1226
    inputs = {'Input': input, 'HiddenPrev': hidden, 'Weight': weight}
Y
Yu Yang 已提交
1227
    # create bias
1228
    if helper.bias_attr:
Y
Yu Yang 已提交
1229 1230 1231
        bias_size = [1, 3 * size]
        bias = helper.create_parameter(
            attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True)
1232
        inputs['Bias'] = bias
Y
Yu Yang 已提交
1233 1234 1235

    helper.append_op(
        type='gru_unit',
1236
        inputs=inputs,
Y
Yu Yang 已提交
1237 1238 1239 1240 1241 1242
        outputs={
            'Gate': gate,
            'ResetHiddenPrev': reset_hidden_pre,
            'Hidden': updated_hidden,
        },
        attrs={
1243 1244
            'activation': 2,  # tanh
            'gate_activation': 1,  # sigmoid
Y
Yu Yang 已提交
1245 1246 1247 1248 1249
        })

    return updated_hidden, reset_hidden_pre, gate


Y
yuyang18 已提交
1250
@templatedoc()
1251
def linear_chain_crf(input, label, param_attr=None):
Y
yuyang18 已提交
1252 1253 1254 1255 1256 1257 1258
    """
    Linear Chain CRF.

    ${comment}

    Args:
        input(${emission_type}): ${emission_comment}
D
dzhwinter 已提交
1259
        input(${transition_type}): ${transition_comment}
Y
yuyang18 已提交
1260 1261 1262 1263
        label(${label_type}): ${label_comment}
        param_attr(ParamAttr): The attribute of the learnable parameter.

    Returns:
D
dzhwinter 已提交
1264 1265 1266
        output(${emission_exps_type}): ${emission_exps_comment} \n
        output(${transition_exps_type}): ${transition_exps_comment} \n
        output(${log_likelihood_type}): ${log_likelihood_comment}
Y
yuyang18 已提交
1267

J
JesseyXujin 已提交
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280
    Examples:
        .. code-block:: python

             import paddle.fluid as fluid
             emission = fluid.layers.data(name='emission', shape=[1000], dtype='float32')
             target = fluid.layers.data(name='target', shape=[1], dtype='int32')
             crf_cost = fluid.layers.linear_chain_crf(
                 input=emission,
                 label=target,
                 param_attr=fluid.ParamAttr(
                     name='crfw',
                     learning_rate=0.2))

Y
yuyang18 已提交
1281
    """
Y
Yu Yang 已提交
1282 1283 1284 1285 1286 1287
    helper = LayerHelper('linear_chain_crf', **locals())
    size = input.shape[1]
    transition = helper.create_parameter(
        attr=helper.param_attr,
        shape=[size + 2, size],
        dtype=helper.input_dtype())
X
Xin Pan 已提交
1288 1289 1290 1291 1292 1293 1294 1295
    alpha = helper.create_variable_for_type_inference(
        dtype=helper.input_dtype())
    emission_exps = helper.create_variable_for_type_inference(
        dtype=helper.input_dtype())
    transition_exps = helper.create_variable_for_type_inference(
        dtype=helper.input_dtype())
    log_likelihood = helper.create_variable_for_type_inference(
        dtype=helper.input_dtype())
Y
Yu Yang 已提交
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310
    helper.append_op(
        type='linear_chain_crf',
        inputs={"Emission": [input],
                "Transition": transition,
                "Label": label},
        outputs={
            "Alpha": [alpha],
            "EmissionExps": [emission_exps],
            "TransitionExps": transition_exps,
            "LogLikelihood": log_likelihood
        })

    return log_likelihood


W
wopeizl 已提交
1311 1312 1313 1314
@templatedoc()
def crf_decoding(input, param_attr, label=None):
    """
    ${comment}
Y
yi.wu 已提交
1315

W
wopeizl 已提交
1316 1317
    Args:
        input(${emission_type}): ${emission_comment}
Y
yi.wu 已提交
1318

W
wopeizl 已提交
1319
        param_attr(ParamAttr): The parameter attribute for training.
Y
yuyang18 已提交
1320

W
wopeizl 已提交
1321
        label(${label_type}): ${label_comment}
1322

W
wopeizl 已提交
1323 1324
    Returns:
        Variable: ${viterbi_path_comment}
Y
yi.wu 已提交
1325

W
wopeizl 已提交
1326 1327
    Examples:
        .. code-block:: python
Y
yi.wu 已提交
1328

Y
Yibing Liu 已提交
1329 1330 1331 1332 1333 1334 1335
           images = fluid.layers.data(name='pixel', shape=[784], dtype='float32')
           label = fluid.layers.data(name='label', shape=[1], dtype='int32')
           hidden = fluid.layers.fc(input=images, size=2)
           crf = fluid.layers.linear_chain_crf(input=hidden, label=label, 
                     param_attr=fluid.ParamAttr(name="crfw"))
           crf_decode = fluid.layers.crf_decoding(input=hidden, 
                     param_attr=fluid.ParamAttr(name="crfw"))
W
wopeizl 已提交
1336 1337 1338 1339 1340 1341 1342 1343
    """
    helper = LayerHelper('crf_decoding', **locals())
    transition = helper.get_parameter(param_attr.name)
    viterbi_path = helper.create_variable_for_type_inference(
        dtype=helper.input_dtype())
    helper.append_op(
        type='crf_decoding',
        inputs={"Emission": [input],
Y
Yu Yang 已提交
1344
                "Transition": transition,
W
wopeizl 已提交
1345 1346
                "Label": label},
        outputs={"ViterbiPath": [viterbi_path]})
Y
Yu Yang 已提交
1347

W
wopeizl 已提交
1348
    return viterbi_path
Y
Yu Yang 已提交
1349 1350


Y
yi.wu 已提交
1351
@templatedoc()
F
fengjiayi 已提交
1352
def cos_sim(X, Y):
Y
Yu Yang 已提交
1353
    """
Y
yi.wu 已提交
1354 1355 1356
    ${comment}

    Args:
1357 1358
        X (Variable): ${x_comment}.
        Y (Variable): ${y_comment}.
F
fengjiayi 已提交
1359

Y
yi.wu 已提交
1360
    Returns:
1361
        Variable: the output of cosine(X, Y).
L
lvmengsi 已提交
1362 1363 1364 1365 1366 1367 1368

    Examples:
        .. code-block:: python

            x = fluid.layers.data(name='x', shape=[3, 7], dtype='float32', append_batch_size=False)
            y = fluid.layers.data(name='y', shape=[1, 7], dtype='float32', append_batch_size=False)
            out = fluid.layers.cos_sim(x, y)
Y
Yu Yang 已提交
1369
    """
F
fengjiayi 已提交
1370
    helper = LayerHelper('cos_sim', **locals())
X
Xin Pan 已提交
1371 1372 1373
    out = helper.create_variable_for_type_inference(dtype=X.dtype)
    xnorm = helper.create_variable_for_type_inference(dtype=X.dtype)
    ynorm = helper.create_variable_for_type_inference(dtype=X.dtype)
Y
Yu Yang 已提交
1374 1375 1376 1377 1378 1379 1380 1381 1382 1383
    helper.append_op(
        type='cos_sim',
        inputs={'X': [X],
                'Y': [Y]},
        outputs={'Out': [out],
                 'XNorm': [xnorm],
                 'YNorm': [ynorm]})
    return out


P
phlrain 已提交
1384 1385 1386 1387 1388
def dropout(x,
            dropout_prob,
            is_test=False,
            seed=None,
            name=None,
P
phlrain 已提交
1389
            dropout_implementation="downgrade_in_infer"):
1390 1391 1392 1393 1394
    """
    Computes dropout.

    Drop or keep each element of `x` independently. Dropout is a regularization
    technique for reducing overfitting by preventing neuron co-adaption during
1395
    training. The dropout operator randomly sets (according to the given dropout
1396 1397 1398
    probability) the outputs of some units to zero, while others are remain
    unchanged.

H
haowang101779990 已提交
1399 1400
    dropout op can be removed from the program to make the program more efficient.

1401
    Args:
1402 1403
        x (Variable): The input tensor variable.
        dropout_prob (float): Probability of setting units to zero.
1404 1405 1406 1407 1408 1409 1410
        is_test (bool): A flag indicating whether it is in test phrase or not.
        seed (int): A Python integer used to create random seeds. If this
                    parameter is set to None, a random seed is used.
                    NOTE: If an integer seed is given, always the same output
                    units will be dropped. DO NOT use a fixed seed in training.
        name (str|None): A name for this layer(optional). If set None, the layer
                         will be named automatically.
H
haowang101779990 已提交
1411 1412
        dropout_implementation(string): ['downgrade_in_infer'(default)|'upscale_in_train']

P
phlrain 已提交
1413
                                        1. downgrade_in_infer(default), downgrade the outcome at inference
H
haowang101779990 已提交
1414 1415

                                           - train: out = input * mask
C
ceci3 已提交
1416
                                           - inference: out = input * (1.0 - dropout_prob)
H
haowang101779990 已提交
1417 1418 1419

                                           (mask is a tensor same shape with input, value is 0 or 1
                                           ratio of 0 is dropout_prob)
P
phlrain 已提交
1420
                                        2. upscale_in_train, upscale the outcome at training time
1421

H
haowang101779990 已提交
1422 1423
                                           - train: out = input * mask / ( 1.0 - dropout_prob )
                                           - inference: out = input
P
phlrain 已提交
1424

H
haowang101779990 已提交
1425 1426
                                           (mask is a tensor same shape with input, value is 0 or 1
                                           ratio of 0 is dropout_prob)
1427

M
minqiyang 已提交
1428

1429
    Returns:
1430
        Variable: A tensor variable is the shape with `x`.
1431 1432

    Examples:
1433

1434 1435
        .. code-block:: python

1436 1437
            x = fluid.layers.data(name="data", shape=[32, 32], dtype="float32")
            droped = fluid.layers.dropout(x, dropout_prob=0.5)
1438 1439
    """

F
fengjiayi 已提交
1440
    helper = LayerHelper('dropout', **locals())
X
Xin Pan 已提交
1441 1442
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
    mask = helper.create_variable_for_type_inference(
Z
Zeng Jinle 已提交
1443
        dtype=core.VarDesc.VarType.UINT8, stop_gradient=True)
C
chengduo 已提交
1444 1445 1446 1447

    if (seed is None or seed == 0) and helper.main_program.random_seed != 0:
        seed = helper.main_program.random_seed

1448 1449 1450 1451 1452
    helper.append_op(
        type='dropout',
        inputs={'X': [x]},
        outputs={'Out': [out],
                 'Mask': [mask]},
1453 1454 1455 1456
        attrs={
            'dropout_prob': dropout_prob,
            'is_test': is_test,
            'fix_seed': seed is not None,
P
phlrain 已提交
1457 1458
            'seed': seed if seed is not None else 0,
            'dropout_implementation': dropout_implementation,
1459
        })
1460 1461 1462
    return out


J
jerrywgz 已提交
1463
def cross_entropy(input, label, soft_label=False, ignore_index=kIgnoreIndex):
Y
Yu Yang 已提交
1464
    """
Y
Yibing Liu 已提交
1465 1466
    **Cross Entropy Layer**

1467 1468 1469
    This layer computes the cross entropy between `input` and `label`. It
    supports both standard cross-entropy and soft-label cross-entropy loss
    computation.
Y
Yibing Liu 已提交
1470 1471

    1) One-hot cross-entropy:
F
fengjiayi 已提交
1472
        `soft_label = False`, `Label[i, 0]` indicates the class index for sample i:
Y
yangyaming 已提交
1473

Y
Yibing Liu 已提交
1474
        .. math::
Y
yangyaming 已提交
1475

Y
Yibing Liu 已提交
1476 1477 1478
            Y[i] = -\log(X[i, Label[i]])

    2) Soft-label cross-entropy:
F
fengjiayi 已提交
1479 1480
        `soft_label = True`, `Label[i, j]` indicates the soft label of class j
        for sample i:
Y
Yibing Liu 已提交
1481 1482 1483 1484 1485

        .. math::

            Y[i] = \sum_j{-Label[i, j] * log(X[i, j])}

Y
Yibing Liu 已提交
1486
       Please make sure that in this case the summation of each row of `label`
Y
Yibing Liu 已提交
1487 1488 1489
       equals one.

    3) One-hot cross-entropy with vecterized `label`:
F
fengjiayi 已提交
1490 1491
         As a special case of 2), when each row of 'label' has only one
         non-zero element which is equal to 1, soft-label cross-entropy degenerates
Y
Yibing Liu 已提交
1492
         to a one-hot cross-entropy with one-hot label representation.
Y
yangyaming 已提交
1493

Y
Yibing Liu 已提交
1494
    Args:
Y
yangyaming 已提交
1495
        input (Variable|list):  a 2-D tensor with shape [N x D], where N is the
1496 1497 1498 1499
                                batch size and D is the number of classes. This
                                input is a probability computed by the previous
                                operator, which is almost always the result of
                                a softmax operator.
Y
yangyaming 已提交
1500
        label (Variable|list): the ground truth which is a 2-D tensor. When
1501 1502 1503 1504
                               `soft_label` is set to `False`, `label` is a
                               tensor<int64> with shape [N x 1]. When
                               `soft_label` is set to `True`, `label` is a
                               tensor<float/double> with shape [N x D].
F
fengjiayi 已提交
1505
        soft_label (bool): a flag indicating whether to
1506
                                           interpretate the given labels as soft
1507
                                           labels. Default: `False`.
M
minqiyang 已提交
1508 1509
        ignore_index (int): Specifies a target value that is ignored and does
                            not contribute to the input gradient. Only valid
J
jerrywgz 已提交
1510
                            if soft_label is set to False. Default: kIgnoreIndex
Y
Yibing Liu 已提交
1511 1512 1513 1514 1515

    Returns:
         A 2-D tensor with shape [N x 1], the cross entropy loss.

    Raises:
H
haowang101779990 已提交
1516 1517 1518
         ValueError:

                      1. the 1st dimension of ``input`` and ``label`` are not equal.
M
minqiyang 已提交
1519

H
haowang101779990 已提交
1520 1521
                      2. when ``soft_label == True``, and the 2nd dimension of
                         ``input`` and ``label`` are not equal.
M
minqiyang 已提交
1522

H
haowang101779990 已提交
1523 1524
                      3. when ``soft_label == False``, and the 2nd dimension of
                         ``label`` is not 1.
Y
Yibing Liu 已提交
1525 1526 1527 1528

    Examples:
        .. code-block:: python

L
lvmengsi 已提交
1529 1530 1531 1532
          classdim = 7
          x = fluid.layers.data(name='x', shape=[3, 7], dtype='float32', append_batch_size=False)
          label = fluid.layers.data(name='label', shape=[3, 1], dtype='float32', append_batch_size=False)
          predict = fluid.layers.fc(input=x, size=classdim, act='softmax')
Y
Yibing Liu 已提交
1533
          cost = fluid.layers.cross_entropy(input=predict, label=label)
Y
Yu Yang 已提交
1534
    """
S
sneaxiy 已提交
1535 1536
    if not soft_label:
        return cross_entropy2(input, label, ignore_index)
F
fengjiayi 已提交
1537
    helper = LayerHelper('cross_entropy', **locals())
X
Xin Pan 已提交
1538
    out = helper.create_variable_for_type_inference(dtype=input.dtype)
Y
Yu Yang 已提交
1539 1540 1541 1542 1543
    helper.append_op(
        type='cross_entropy',
        inputs={'X': [input],
                'Label': [label]},
        outputs={'Y': [out]},
1544 1545
        attrs={"soft_label": soft_label,
               "ignore_index": ignore_index})
Y
Yu Yang 已提交
1546 1547 1548
    return out


S
sneaxiy 已提交
1549 1550 1551 1552
def cross_entropy2(input, label, ignore_index=kIgnoreIndex):
    helper = LayerHelper('cross_entropy2', **locals())
    out = helper.create_variable_for_type_inference(dtype=input.dtype)
    xshape = helper.create_variable_for_type_inference(dtype=input.dtype)
S
sneaxiy 已提交
1553
    match_x = helper.create_variable_for_type_inference(dtype=input.dtype)
S
sneaxiy 已提交
1554 1555 1556 1557 1558
    helper.append_op(
        type='cross_entropy2',
        inputs={'X': [input],
                'Label': [label]},
        outputs={'Y': [out],
S
sneaxiy 已提交
1559
                 'MatchX': [match_x],
S
sneaxiy 已提交
1560 1561 1562 1563 1564
                 'XShape': [xshape]},
        attrs={'ignore_index': ignore_index})
    return out


F
frankwhzhang 已提交
1565
def bpr_loss(input, label, name=None):
F
frankwhzhang 已提交
1566
    """
1567
    **Bayesian Personalized Ranking Loss Operator**
F
frankwhzhang 已提交
1568

1569
    This operator belongs to pairwise ranking loss. Label is the desired item.
F
frankwhzhang 已提交
1570
    The loss at a given point in one session is defined as:
1571 1572 1573

    .. math::
        Y[i] = 1/(N[i] - 1) * \sum_j{\log(\sigma(X[i, Label[i]]-X[i, j]))}
F
frankwhzhang 已提交
1574 1575

    Learn more details by reading paper <session-based recommendations with recurrent
1576
    neural networks>.
F
frankwhzhang 已提交
1577

1578 1579 1580 1581 1582 1583
    Args:
        input (Variable|list):  a 2-D tensor with shape [N x D], where N is the
                                batch size and D is the number of classes.
                                This input is not probability but logits.
        label (Variable|list):  the ground truth which is a 2-D tensor.  `label`
                                is a tensor<int64> with shape [N x 1].
F
frankwhzhang 已提交
1584 1585
        name (str|None):        A name for this layer(optional). If set None, the
                                layer will be named automatically. Default: None.
1586 1587 1588
    Returns:
        A 2-D tensor with shape [N x 1], the bpr loss.

F
frankwhzhang 已提交
1589 1590 1591
    Examples:
        .. code-block:: python

1592 1593 1594 1595 1596 1597 1598
          import paddle.fluid as fluid

          neg_size = 10
          label = fluid.layers.data(
                    name="label", shape=[1], dtype="int64")
          predict = fluid.layers.data(
                    name="predict", shape=[neg_size + 1], dtype="float32")
1599
          cost = fluid.layers.bpr_loss(input=predict, label=label)
F
frankwhzhang 已提交
1600
    """
1601 1602 1603 1604 1605
    helper = LayerHelper('bpr_loss', **locals())
    out = helper.create_variable_for_type_inference(dtype=input.dtype)
    helper.append_op(
        type='bpr_loss',
        inputs={'X': [input],
1606
                'Label': [label]},
1607 1608 1609 1610
        outputs={'Y': [out]})
    return out


F
fengjiayi 已提交
1611
def square_error_cost(input, label):
Y
Yu Yang 已提交
1612
    """
1613 1614
    **Square error cost layer**

1615 1616
    This layer accepts input predictions and target label and returns the
    squared error cost.
Y
ying 已提交
1617

1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630
    For predictions, :math:`X`, and target labels, :math:`Y`, the equation is:

    .. math::

        Out = (X - Y)^2

    In the above equation:

        * :math:`X`: Input predictions, a tensor.
        * :math:`Y`: Input labels, a tensor.
        * :math:`Out`: Output value, same shape with :math:`X`.

    Args:
1631 1632
        input (Variable): Input tensor, has predictions.
        label (Variable): Label tensor, has target labels.
1633 1634

    Returns:
G
guosheng 已提交
1635
        Variable: The tensor variable storing the element-wise squared error \
1636
                  difference of input and label.
1637 1638 1639 1640

    Examples:
        .. code-block:: python

R
ruri 已提交
1641 1642 1643
          y = fluid.layers.data(name='y', shape=[1], dtype='float32')
          y_predict = fluid.layers.data(name='y_predict', shape=[1], dtype='float32')
          cost = fluid.layers.square_error_cost(input=y_predict, label=y)
1644

Y
Yu Yang 已提交
1645
    """
F
fengjiayi 已提交
1646
    helper = LayerHelper('square_error_cost', **locals())
X
Xin Pan 已提交
1647
    minus_out = helper.create_variable_for_type_inference(dtype=input.dtype)
Y
Yu Yang 已提交
1648 1649 1650 1651 1652 1653
    helper.append_op(
        type='elementwise_sub',
        inputs={'X': [input],
                'Y': [label]},
        outputs={'Out': [minus_out]})

X
Xin Pan 已提交
1654
    square_out = helper.create_variable_for_type_inference(dtype=input.dtype)
Y
Yu Yang 已提交
1655
    helper.append_op(
F
fengjiayi 已提交
1656 1657
        type='square', inputs={'X': [minus_out]},
        outputs={'Out': [square_out]})
Y
Yu Yang 已提交
1658 1659 1660
    return square_out


Y
yi.wu 已提交
1661
@templatedoc()
Y
Yu Yang 已提交
1662 1663 1664 1665
def chunk_eval(input,
               label,
               chunk_scheme,
               num_chunk_types,
F
fengjiayi 已提交
1666
               excluded_chunk_types=None):
Y
Yu Yang 已提交
1667
    """
Y
yi.wu 已提交
1668
    **Chunk Evaluator**
Y
yi.wu 已提交
1669

Y
yangyaming 已提交
1670
    This function computes and outputs the precision, recall and
1671
    F1-score of chunk detection.
Y
yi.wu 已提交
1672

M
minqiyang 已提交
1673
    For some basics of chunking, please refer to
H
haowang101779990 已提交
1674
    `Chunking with Support Vector Machines <https://aclanthology.info/pdf/N/N01/N01-1025.pdf>`_ .
Y
yi.wu 已提交
1675 1676 1677 1678 1679 1680

    ChunkEvalOp computes the precision, recall, and F1-score of chunk detection,
    and supports IOB, IOE, IOBES and IO (also known as plain) tagging schemes.
    Here is a NER example of labeling for these tagging schemes:

    .. code-block:: python
1681

Y
yi.wu 已提交
1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706
       ====== ====== ======  =====  ==  ============   =====  ===== =====  ==  =========
              Li     Ming    works  at  Agricultural   Bank   of    China  in  Beijing.
       ====== ====== ======  =====  ==  ============   =====  ===== =====  ==  =========
       IO     I-PER  I-PER   O      O   I-ORG          I-ORG  I-ORG I-ORG  O   I-LOC
       IOB    B-PER  I-PER   O      O   B-ORG          I-ORG  I-ORG I-ORG  O   B-LOC
       IOE    I-PER  E-PER   O      O   I-ORG          I-ORG  I-ORG E-ORG  O   E-LOC
       IOBES  B-PER  E-PER   O      O   I-ORG          I-ORG  I-ORG E-ORG  O   S-LOC
       ====== ====== ======  =====  ==  ============   =====  ===== =====  ==  =========

    There are three chunk types(named entity types) including PER(person), ORG(organization)
    and LOC(LOCATION), and we can see that the labels have the form <tag type>-<chunk type>.

    Since the calculations actually use label ids rather than labels, extra attention
    should be paid when mapping labels to ids to make CheckEvalOp work. The key point
    is that the listed equations are satisfied by ids.

    .. code-block:: python

       tag_type = label % num_tag_type
       chunk_type = label / num_tag_type

    where `num_tag_type` is the num of tag types in the tagging scheme, `num_chunk_type`
    is the num of chunk types, and `tag_type` get its value from the following table.

    .. code-block:: python
1707

Y
yi.wu 已提交
1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731
       Scheme Begin Inside End   Single
        plain   0     -      -     -
        IOB     0     1      -     -
        IOE     -     0      1     -
        IOBES   0     1      2     3

    Still use NER as example, assuming the tagging scheme is IOB while chunk types are ORG,
    PER and LOC. To satisfy the above equations, the label map can be like this:

    .. code-block:: python

       B-ORG  0
       I-ORG  1
       B-PER  2
       I-PER  3
       B-LOC  4
       I-LOC  5
       O      6

    It's not hard to verify the equations noting that the num of chunk types
    is 3 and the num of tag types in IOB scheme is 2. For example, the label
    id of I-LOC is 5, the tag type id of I-LOC is 1, and the chunk type id of
    I-LOC is 2, which consistent with the results from the equations.

Y
yi.wu 已提交
1732
    Args:
1733 1734 1735 1736 1737
        input (Variable): prediction output of the network.
        label (Variable): label of the test data set.
        chunk_scheme (str): ${chunk_scheme_comment}
        num_chunk_types (int): ${num_chunk_types_comment}
        excluded_chunk_types (list): ${excluded_chunk_types_comment}
F
fengjiayi 已提交
1738

Y
yi.wu 已提交
1739
    Returns:
Y
update  
yi.wu 已提交
1740 1741 1742
        tuple: tuple containing: precision, recall, f1_score,
        num_infer_chunks, num_label_chunks,
        num_correct_chunks
1743

Y
yi.wu 已提交
1744 1745 1746
    Examples:
        .. code-block:: python

1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757
            import paddle.fluid as fluid

            dict_size = 10000
            label_dict_len = 7
            sequence = fluid.layers.data(
                name='id', shape=[1], lod_level=1, dtype='int64')
            embedding = fluid.layers.embedding(
                input=sequence, size=[dict_size, 512])
            hidden = fluid.layers.fc(input=embedding, size=512)
            label = fluid.layers.data(
                name='label', shape=[1], lod_level=1, dtype='int32')
Y
yi.wu 已提交
1758
            crf = fluid.layers.linear_chain_crf(
1759
                input=hidden, label=label, param_attr=fluid.ParamAttr(name="crfw"))
Y
yi.wu 已提交
1760
            crf_decode = fluid.layers.crf_decoding(
1761
                input=hidden, param_attr=fluid.ParamAttr(name="crfw"))
Y
yi.wu 已提交
1762 1763 1764 1765 1766
            fluid.layers.chunk_eval(
                input=crf_decode,
                label=label,
                chunk_scheme="IOB",
                num_chunk_types=(label_dict_len - 1) / 2)
Y
Yu Yang 已提交
1767
    """
F
fengjiayi 已提交
1768
    helper = LayerHelper("chunk_eval", **locals())
Y
Yu Yang 已提交
1769 1770

    # prepare output
X
Xin Pan 已提交
1771 1772 1773 1774 1775 1776 1777
    precision = helper.create_variable_for_type_inference(dtype="float32")
    recall = helper.create_variable_for_type_inference(dtype="float32")
    f1_score = helper.create_variable_for_type_inference(dtype="float32")
    num_infer_chunks = helper.create_variable_for_type_inference(dtype="int64")
    num_label_chunks = helper.create_variable_for_type_inference(dtype="int64")
    num_correct_chunks = helper.create_variable_for_type_inference(
        dtype="int64")
Y
Yu Yang 已提交
1778 1779 1780 1781 1782 1783 1784 1785

    helper.append_op(
        type="chunk_eval",
        inputs={"Inference": [input],
                "Label": [label]},
        outputs={
            "Precision": [precision],
            "Recall": [recall],
1786 1787 1788 1789
            "F1-Score": [f1_score],
            "NumInferChunks": [num_infer_chunks],
            "NumLabelChunks": [num_label_chunks],
            "NumCorrectChunks": [num_correct_chunks]
Y
Yu Yang 已提交
1790 1791 1792
        },
        attrs={
            "num_chunk_types": num_chunk_types,
G
guosheng 已提交
1793 1794
            "chunk_scheme": chunk_scheme,
            "excluded_chunk_types": excluded_chunk_types or []
Y
Yu Yang 已提交
1795
        })
1796 1797
    return (precision, recall, f1_score, num_infer_chunks, num_label_chunks,
            num_correct_chunks)
Y
Yu Yang 已提交
1798 1799


1800
@templatedoc()
Y
Yu Yang 已提交
1801 1802 1803 1804 1805 1806 1807
def sequence_conv(input,
                  num_filters,
                  filter_size=3,
                  filter_stride=1,
                  padding=None,
                  bias_attr=None,
                  param_attr=None,
C
chengduo 已提交
1808 1809
                  act=None,
                  name=None):
Y
Yu Yang 已提交
1810 1811 1812 1813
    """
    This function creates the op for sequence_conv, using the inputs and
    other convolutional configurations for the filters and stride as given
    in the input parameters to the function.
1814 1815 1816 1817 1818 1819 1820

    Args:
        input (Variable): ${x_comment}
        num_filters (int): number of filters.
        filter_size (int): the filter size (H and W).
        filter_stride (int): stride of the filter.
        padding (bool): if True, add paddings.
C
chengduo 已提交
1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833
        bias_attr (ParamAttr|bool|None): The parameter attribute for the bias of sequence_conv.
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, sequence_conv
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
            is not set, the bias is initialized zero. Default: None.
        param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights
            of sequence_conv. If it is set to None or one attribute of ParamAttr, sequence_conv
            will create ParamAttr as param_attr. If the Initializer of the param_attr
            is not set, the parameter is initialized with Xavier. Default: None.
        act (str): Activation type, if it is set to None, activation is not appended.
            Default: None.
        name (str|None): A name for this layer(optional). If set None, the layer
            will be named automatically. Default: None.
F
fengjiayi 已提交
1834

1835 1836
    Returns:
        Variable: output of sequence_conv
B
bdzhuxiaoning 已提交
1837 1838 1839 1840 1841 1842 1843

    Examples:
        .. code-block:: python

             import paddle.fluid as fluid
             x = fluid.layers.data(name='x', shape=[10,10], append_batch_size=False, dtype='float32')
             x_conved = fluid.layers.sequence_conv(x,2)
Y
Yu Yang 已提交
1844 1845
    """

L
lujun 已提交
1846
    assert not in_dygraph_mode(), (
1847
        "sequence layer is not supported in dygraph mode yet.")
Y
Yu Yang 已提交
1848 1849 1850 1851 1852
    helper = LayerHelper('sequence_conv', **locals())
    dtype = helper.input_dtype()
    filter_shape = [filter_size * input.shape[1], num_filters]
    filter_param = helper.create_parameter(
        attr=helper.param_attr, shape=filter_shape, dtype=dtype)
X
Xin Pan 已提交
1853
    pre_bias = helper.create_variable_for_type_inference(dtype)
Y
Yu Yang 已提交
1854 1855 1856 1857 1858 1859 1860 1861 1862 1863

    helper.append_op(
        type='sequence_conv',
        inputs={
            'X': [input],
            'Filter': [filter_param],
        },
        outputs={"Out": pre_bias},
        attrs={
            'contextStride': filter_stride,
M
minqiyang 已提交
1864
            'contextStart': -int(filter_size // 2),
Y
Yu Yang 已提交
1865 1866 1867 1868 1869 1870
            'contextLength': filter_size
        })
    pre_act = helper.append_bias_op(pre_bias)
    return helper.append_activation(pre_act)


C
chengduo 已提交
1871
def sequence_softmax(input, use_cudnn=False, name=None):
1872 1873 1874
    """
    This function computes the softmax activation among all time-steps for each
    sequence. The dimension of each time-step should be 1. Thus, the shape of
1875
    input Tensor can be either :math:`[N, 1]` or :math:`[N]`, where :math:`N`
1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891
    is the sum of the length of all sequences.

    For i-th sequence in a mini-batch:

    .. math::

        Out(X[lod[i]:lod[i+1]], :) = \\frac{\exp(X[lod[i]:lod[i+1], :])}{\sum(\exp(X[lod[i]:lod[i+1], :]))}

    For example, for a mini-batch of 3 sequences with variable-length,
    each containing 2, 3, 2 time-steps, the lod of which is [0, 2, 5, 7],
    then softmax will be computed among :math:`X[0:2, :]`, :math:`X[2:5, :]`,
    :math:`X[5:7, :]`, and :math:`N` turns out to be 7.

    Args:
        input (Variable): The input variable which is a LoDTensor.
        use_cudnn (bool): Use cudnn kernel or not, it is valid only when the cudnn \
C
chengduo 已提交
1892 1893 1894
            library is installed. Default: False.
        name (str|None): A name for this layer(optional). If set None, the layer
            will be named automatically. Default: None.
1895

1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906
    Returns:
        Variable: output of sequence_softmax

    Examples:

        .. code-block:: python

             x = fluid.layers.data(name='x', shape=[7, 1],
                              dtype='float32', lod_level=1)
             x_sequence_softmax = fluid.layers.sequence_softmax(input=x)
    """
L
lujun 已提交
1907
    assert not in_dygraph_mode(), (
1908
        "sequence layer is not supported in dygraph mode yet.")
1909 1910
    helper = LayerHelper('sequence_softmax', **locals())
    dtype = helper.input_dtype()
X
Xin Pan 已提交
1911
    softmax_out = helper.create_variable_for_type_inference(dtype)
1912 1913 1914 1915 1916 1917 1918 1919
    helper.append_op(
        type="sequence_softmax",
        inputs={"X": input},
        outputs={"Out": softmax_out},
        attrs={"use_cudnn": use_cudnn})
    return softmax_out


D
dengkaipeng 已提交
1920
def softmax(input, use_cudnn=False, name=None, axis=-1):
Q
qiaolongfei 已提交
1921
    """
1922
    The input of the softmax operator is a tensor of any rank. The output tensor
F
fengjiayi 已提交
1923
    has the same shape as the input.
Q
qiaolongfei 已提交
1924

D
dengkaipeng 已提交
1925
    The dimension :attr:`axis` of the input tensor will be permuted to the last.
D
dengkaipeng 已提交
1926
    Then the input tensor will be logically flattened to a 2-D matrix. The matrix's
D
dengkaipeng 已提交
1927
    second dimension(row length) is the same as the dimension :attr:`axis` of the input
1928 1929 1930
    tensor, and the first dimension(column length) is the product of all other
    dimensions of the input tensor. For each row of the matrix, the softmax operator
    squashes the K-dimensional(K is the width of the matrix, which is also the size
D
dengkaipeng 已提交
1931
    of the input tensor's dimension :attr:`axis`) vector of arbitrary real values to a
F
fengjiayi 已提交
1932
    K-dimensional vector of real values in the range [0, 1] that add up to 1.
Q
qiaolongfei 已提交
1933 1934 1935 1936 1937 1938 1939

    It computes the exponential of the given dimension and the sum of exponential
    values of all the other dimensions in the K-dimensional vector input.
    Then the ratio of the exponential of the given dimension and the sum of
    exponential values of all the other dimensions is the output of the softmax
    operator.

F
fengjiayi 已提交
1940
    For each row :math:`i` and each column :math:`j` in the matrix, we have:
Q
qiaolongfei 已提交
1941 1942 1943 1944 1945 1946 1947 1948

    .. math::

        Out[i, j] = \\frac{\exp(X[i, j])}{\sum_j(exp(X[i, j])}

    Args:
        input (Variable): The input variable.
        use_cudnn (bool): Use cudnn kernel or not, it is valid only when the cudnn \
J
jerrywgz 已提交
1949 1950
            library is installed. To improve numerical stablity, set use_cudnn to \
            False by default. Default: False
C
chengduo 已提交
1951 1952
        name (str|None): A name for this layer(optional). If set None, the layer
            will be named automatically. Default: None.
D
dengkaipeng 已提交
1953 1954 1955
        axis (int): The index of dimension to perform softmax calculations, it should
            be in range :math:`[-1, rank - 1]`, while :math:`rank` is the rank of
            input variable. Default: -1.
Q
qiaolongfei 已提交
1956 1957 1958 1959 1960 1961 1962 1963

    Returns:
        Variable: output of softmax

    Examples:

        .. code-block:: python

J
JesseyXujin 已提交
1964 1965
             import paddle.fluid as fluid
             x = fluid.layers.data(name='x', shape=[2], dtype='float32')
Q
qiaolongfei 已提交
1966
             fc = fluid.layers.fc(input=x, size=10)
D
dengkaipeng 已提交
1967
             # perform softmax in the second dimension
D
dengkaipeng 已提交
1968
             softmax = fluid.layers.softmax(input=fc, axis=1)
D
dengkaipeng 已提交
1969 1970
             # perform softmax in the last dimension
             softmax = fluid.layers.softmax(input=fc, axis=-1)
Q
qiaolongfei 已提交
1971 1972

    """
1973 1974
    helper = LayerHelper('softmax', **locals())
    dtype = helper.input_dtype()
X
Xin Pan 已提交
1975
    softmax_out = helper.create_variable_for_type_inference(dtype)
1976 1977 1978 1979
    helper.append_op(
        type="softmax",
        inputs={"X": input},
        outputs={"Out": softmax_out},
D
dengkaipeng 已提交
1980 1981
        attrs={"axis": axis,
               "use_cudnn": use_cudnn})
1982 1983 1984
    return softmax_out


Y
Yu Yang 已提交
1985 1986 1987
def conv2d(input,
           num_filters,
           filter_size,
C
chengduoZH 已提交
1988 1989
           stride=1,
           padding=0,
1990
           dilation=1,
Y
Yu Yang 已提交
1991 1992 1993
           groups=None,
           param_attr=None,
           bias_attr=None,
C
chengduoZH 已提交
1994
           use_cudnn=True,
1995 1996
           act=None,
           name=None):
Y
Yu Yang 已提交
1997
    """
C
chengduoZH 已提交
1998
    The convolution2D layer calculates the output based on the input, filter
T
tensor-tang 已提交
1999 2000
    and strides, paddings, dilations, groups parameters. Input and
    Output are in NCHW format, where N is batch size, C is the number of
2001
    channels, H is the height of the feature, and W is the width of the feature.
T
tensor-tang 已提交
2002 2003 2004 2005 2006 2007 2008
    Filter is in MCHW format, where M is the number of output image channels,
    C is the number of input image channels, H is the height of the filter,
    and W is the width of the filter. If the groups is greater than 1,
    C will equal the number of input image channels divided by the groups.
    Please refer to UFLDL's `convolution
    <http://ufldl.stanford.edu/tutorial/supervised/FeatureExtractionUsingConvolution/>`_
    for more detials.
2009 2010 2011
    If bias attribution and activation type are provided, bias is added to the
    output of the convolution, and the corresponding activation function is
    applied to the final result.
C
chengduoZH 已提交
2012

2013
    For each input :math:`X`, the equation is:
C
refine  
chengduoZH 已提交
2014

C
chengduoZH 已提交
2015 2016
    .. math::

C
refine  
chengduoZH 已提交
2017
        Out = \sigma (W \\ast X + b)
C
chengduoZH 已提交
2018

T
tensor-tang 已提交
2019
    Where:
C
chengduoZH 已提交
2020

2021 2022 2023 2024 2025
    * :math:`X`: Input value, a tensor with NCHW format.
    * :math:`W`: Filter value, a tensor with MCHW format.
    * :math:`\\ast`: Convolution operation.
    * :math:`b`: Bias value, a 2-D tensor with shape [M, 1].
    * :math:`\\sigma`: Activation function.
T
tensor-tang 已提交
2026
    * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.
C
chengduoZH 已提交
2027 2028 2029

    Example:

2030 2031
        - Input:

W
weixing02 已提交
2032
          Input shape: :math:`(N, C_{in}, H_{in}, W_{in})`
C
refine  
chengduoZH 已提交
2033

W
weixing02 已提交
2034
          Filter shape: :math:`(C_{out}, C_{in}, H_f, W_f)`
C
refine  
chengduoZH 已提交
2035

2036
        - Output:
T
tensor-tang 已提交
2037

W
weixing02 已提交
2038
          Output shape: :math:`(N, C_{out}, H_{out}, W_{out})`
C
refine  
chengduoZH 已提交
2039

C
chengduoZH 已提交
2040
        Where
2041 2042

        .. math::
C
chengduoZH 已提交
2043

W
weixing02 已提交
2044 2045
            H_{out}&= \\frac{(H_{in} + 2 * paddings[0] - (dilations[0] * (H_f - 1) + 1))}{strides[0]} + 1 \\\\
            W_{out}&= \\frac{(W_{in} + 2 * paddings[1] - (dilations[1] * (W_f - 1) + 1))}{strides[1]} + 1
C
chengduoZH 已提交
2046 2047

    Args:
2048
        input (Variable): The input image with [N, C, H, W] format.
T
tensor-tang 已提交
2049
        num_filters(int): The number of filter. It is as same as the output
2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066
            image channel.
        filter_size (int|tuple|None): The filter size. If filter_size is a tuple,
            it must contain two integers, (filter_size_H, filter_size_W).
            Otherwise, the filter will be a square.
        stride (int|tuple): The stride size. If stride is a tuple, it must
            contain two integers, (stride_H, stride_W). Otherwise, the
            stride_H = stride_W = stride. Default: stride = 1.
        padding (int|tuple): The padding size. If padding is a tuple, it must
            contain two integers, (padding_H, padding_W). Otherwise, the
            padding_H = padding_W = padding. Default: padding = 0.
        dilation (int|tuple): The dilation size. If dilation is a tuple, it must
            contain two integers, (dilation_H, dilation_W). Otherwise, the
            dilation_H = dilation_W = dilation. Default: dilation = 1.
        groups (int): The groups number of the Conv2d Layer. According to grouped
            convolution in Alex Krizhevsky's Deep CNN paper: when group=2,
            the first half of the filters is only connected to the first half
            of the input channels, while the second half of the filters is only
C
chengduo 已提交
2067 2068 2069 2070 2071
            connected to the second half of the input channels. Default: groups=1.
        param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights
            of conv2d. If it is set to None or one attribute of ParamAttr, conv2d
            will create ParamAttr as param_attr. If the Initializer of the param_attr
            is not set, the parameter is initialized with :math:`Normal(0.0, std)`,
H
haowang101779990 已提交
2072
            and the :math:`std` is :math:`(\\frac{2.0 }{filter\_elem\_num})^{0.5}`. Default: None.
C
chengduo 已提交
2073 2074 2075 2076 2077
        bias_attr (ParamAttr|bool|None): The parameter attribute for the bias of conv2d.
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, conv2d
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
            is not set, the bias is initialized zero. Default: None.
2078 2079
        use_cudnn (bool): Use cudnn kernel or not, it is valid only when the cudnn
            library is installed. Default: True
C
chengduo 已提交
2080 2081
        act (str): Activation type, if it is set to None, activation is not appended.
            Default: None
2082
        name (str|None): A name for this layer(optional). If set None, the layer
C
chengduo 已提交
2083
            will be named automatically. Default: None
C
chengduoZH 已提交
2084 2085

    Returns:
G
guosheng 已提交
2086
        Variable: The tensor variable storing the convolution and \
C
chengduoZH 已提交
2087 2088
                  non-linearity activation result.

C
refine  
chengduoZH 已提交
2089
    Raises:
2090 2091
        ValueError: If the shapes of input, filter_size, stride, padding and
                    groups mismatch.
C
refine  
chengduoZH 已提交
2092

C
chengduoZH 已提交
2093 2094 2095
    Examples:
        .. code-block:: python

2096 2097
          data = fluid.layers.data(name='data', shape=[3, 32, 32], dtype='float32')
          conv2d = fluid.layers.conv2d(input=data, num_filters=2, filter_size=3, act="relu")
Y
Yu Yang 已提交
2098 2099 2100
    """

    num_channels = input.shape[1]
C
chengduo 已提交
2101
    assert param_attr is not False, "param_attr should not be False here."
2102
    l_type = 'conv2d'
X
xzl 已提交
2103 2104
    if (num_channels == groups and num_filters % num_channels == 0 and
            not use_cudnn):
2105
        l_type = 'depthwise_conv2d'
2106 2107 2108 2109

    helper = LayerHelper(l_type, **locals())
    dtype = helper.input_dtype()

Y
Yu Yang 已提交
2110 2111 2112 2113 2114
    if groups is None:
        num_filter_channels = num_channels
    else:
        if num_channels % groups != 0:
            raise ValueError("num_channels must be divisible by groups.")
M
minqiyang 已提交
2115
        num_filter_channels = num_channels // groups
Y
Yu Yang 已提交
2116

C
chengduoZH 已提交
2117 2118 2119
    filter_size = utils.convert_to_list(filter_size, 2, 'filter_size')
    stride = utils.convert_to_list(stride, 2, 'stride')
    padding = utils.convert_to_list(padding, 2, 'padding')
2120
    dilation = utils.convert_to_list(dilation, 2, 'dilation')
C
chengduoZH 已提交
2121

C
chengduoZH 已提交
2122 2123
    if not isinstance(use_cudnn, bool):
        raise ValueError("use_cudnn should be True or False")
Y
Yu Yang 已提交
2124 2125

    input_shape = input.shape
M
minqiyang 已提交
2126
    filter_shape = [num_filters, int(num_filter_channels)] + filter_size
Y
Yu Yang 已提交
2127 2128

    def _get_default_param_initializer():
C
chengduo 已提交
2129 2130
        filter_elem_num = filter_size[0] * filter_size[1] * num_channels
        std = (2.0 / filter_elem_num)**0.5
Y
Yu Yang 已提交
2131 2132 2133 2134 2135 2136 2137 2138
        return Normal(0.0, std, 0)

    filter_param = helper.create_parameter(
        attr=helper.param_attr,
        shape=filter_shape,
        dtype=dtype,
        default_initializer=_get_default_param_initializer())

X
Xin Pan 已提交
2139
    pre_bias = helper.create_variable_for_type_inference(dtype)
Y
Yu Yang 已提交
2140

2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154
    if use_cudnn:
        helper.create_variable(
            name="kCUDNNFwdAlgoCache",
            persistable=True,
            type=core.VarDesc.VarType.RAW)
        helper.create_variable(
            name="kCUDNNBwdDataAlgoCache",
            persistable=True,
            type=core.VarDesc.VarType.RAW)
        helper.create_variable(
            name="kCUDNNBwdFilterAlgoCache",
            persistable=True,
            type=core.VarDesc.VarType.RAW)

Y
Yu Yang 已提交
2155
    helper.append_op(
2156
        type=l_type,
Y
Yu Yang 已提交
2157 2158 2159 2160 2161
        inputs={
            'Input': input,
            'Filter': filter_param,
        },
        outputs={"Output": pre_bias},
C
chengduoZH 已提交
2162 2163 2164
        attrs={
            'strides': stride,
            'paddings': padding,
2165
            'dilations': dilation,
C
chengduoZH 已提交
2166
            'groups': groups,
2167
            'use_cudnn': use_cudnn,
2168
            'use_mkldnn': False,
2169
            'fuse_relu_before_depthwise_conv': False
C
chengduoZH 已提交
2170
        })
Y
Yu Yang 已提交
2171 2172 2173 2174 2175 2176

    pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2)

    return helper.append_activation(pre_act)


C
chengduoZH 已提交
2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193
def conv3d(input,
           num_filters,
           filter_size,
           stride=1,
           padding=0,
           dilation=1,
           groups=None,
           param_attr=None,
           bias_attr=None,
           use_cudnn=True,
           act=None,
           name=None):
    """
    **Convlution3D Layer**

    The convolution3D layer calculates the output based on the input, filter
    and strides, paddings, dilations, groups parameters. Input(Input) and
2194 2195 2196 2197 2198 2199
    Output(Output) are in NCDHW format. Where N is batch size C is the number of
    channels, D is the depth of the feature, H is the height of the feature,
    and W is the width of the feature. Convlution3D is similar with Convlution2D
    but adds one dimension(depth). If bias attribution and activation type are
    provided, bias is added to the output of the convolution, and the
    corresponding activation function is applied to the final result.
C
chengduoZH 已提交
2200 2201 2202 2203 2204 2205 2206 2207 2208

    For each input :math:`X`, the equation is:

    .. math::

        Out = \sigma (W \\ast X + b)

    In the above equation:

2209 2210
    * :math:`X`: Input value, a tensor with NCDHW format.
    * :math:`W`: Filter value, a tensor with MCDHW format.
C
chengduoZH 已提交
2211 2212 2213
    * :math:`\\ast`: Convolution operation.
    * :math:`b`: Bias value, a 2-D tensor with shape [M, 1].
    * :math:`\\sigma`: Activation function.
2214
    * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.
C
chengduoZH 已提交
2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239

    Example:

        - Input:

          Input shape: :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})`

          Filter shape: :math:`(C_{out}, C_{in}, D_f, H_f, W_f)`

        - Output:
          Output shape: :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})`

        Where

        .. math::

            D_{out}&= \\frac{(D_{in} + 2 * paddings[0] - (dilations[0] * (D_f - 1) + 1))}{strides[0]} + 1 \\\\
            H_{out}&= \\frac{(H_{in} + 2 * paddings[1] - (dilations[1] * (H_f - 1) + 1))}{strides[1]} + 1 \\\\
            W_{out}&= \\frac{(W_{in} + 2 * paddings[2] - (dilations[2] * (W_f - 1) + 1))}{strides[2]} + 1

    Args:
        input (Variable): The input image with [N, C, D, H, W] format.
            num_filters(int): The number of filter. It is as same as the output
            image channel.
        filter_size (int|tuple|None): The filter size. If filter_size is a tuple,
2240
            it must contain three integers, (filter_size_D, filter_size_H, filter_size_W).
C
chengduoZH 已提交
2241 2242
            Otherwise, the filter will be a square.
        stride (int|tuple): The stride size. If stride is a tuple, it must
2243
            contain three integers, (stride_D, stride_H, stride_W). Otherwise, the
C
chengduoZH 已提交
2244 2245
            stride_D = stride_H = stride_W = stride. Default: stride = 1.
        padding (int|tuple): The padding size. If padding is a tuple, it must
2246
            contain three integers, (padding_D, padding_H, padding_W). Otherwise, the
C
chengduoZH 已提交
2247 2248
            padding_D = padding_H = padding_W = padding. Default: padding = 0.
        dilation (int|tuple): The dilation size. If dilation is a tuple, it must
2249
            contain three integers, (dilation_D, dilation_H, dilation_W). Otherwise, the
C
chengduoZH 已提交
2250 2251 2252 2253 2254 2255
            dilation_D = dilation_H = dilation_W = dilation. Default: dilation = 1.
        groups (int): The groups number of the Conv3d Layer. According to grouped
            convolution in Alex Krizhevsky's Deep CNN paper: when group=2,
            the first half of the filters is only connected to the first half
            of the input channels, while the second half of the filters is only
            connected to the second half of the input channels. Default: groups=1
C
chengduo 已提交
2256 2257 2258 2259 2260 2261 2262 2263 2264 2265
        param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights
            of conv3d. If it is set to None or one attribute of ParamAttr, conv3d
            will create ParamAttr as param_attr. If it is set to None, the parameter
            is initialized with :math:`Normal(0.0, std)`, and the :math:`std` is
            :math:`(\\frac{2.0 }{filter\_elem\_num})^{0.5}`. Default: None.
        bias_attr (ParamAttr|bool|None): The parameter attribute for the bias of conv3d.
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, conv3d
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
            is not set, the bias is initialized zero. Default: None.
C
chengduoZH 已提交
2266 2267
        use_cudnn (bool): Use cudnn kernel or not, it is valid only when the cudnn
            library is installed. Default: True
C
chengduo 已提交
2268 2269
        act (str): Activation type, if it is set to None, activation is not appended.
            Default: None.
C
chengduoZH 已提交
2270
        name (str|None): A name for this layer(optional). If set None, the layer
C
chengduo 已提交
2271
            will be named automatically. Default: None.
C
chengduoZH 已提交
2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283

    Returns:
        Variable: The tensor variable storing the convolution and \
                  non-linearity activation result.

    Raises:
        ValueError: If the shapes of input, filter_size, stride, padding and
                    groups mismatch.

    Examples:
        .. code-block:: python

2284 2285
          data = fluid.layers.data(name='data', shape=[3, 12, 32, 32], dtype='float32')
          conv3d = fluid.layers.conv3d(input=data, num_filters=2, filter_size=3, act="relu")
C
chengduoZH 已提交
2286 2287 2288
    """

    l_type = 'conv3d'
C
chengduo 已提交
2289
    assert param_attr is not False, "param_attr should not be False here."
C
chengduoZH 已提交
2290 2291 2292 2293 2294 2295 2296 2297 2298 2299
    helper = LayerHelper(l_type, **locals())
    dtype = helper.input_dtype()

    num_channels = input.shape[1]

    if groups is None:
        num_filter_channels = num_channels
    else:
        if num_channels % groups != 0:
            raise ValueError("num_channels must be divisible by groups.")
M
minqiyang 已提交
2300
        num_filter_channels = num_channels // groups
C
chengduoZH 已提交
2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313

    filter_size = utils.convert_to_list(filter_size, 3, 'filter_size')
    stride = utils.convert_to_list(stride, 3, 'stride')
    padding = utils.convert_to_list(padding, 3, 'padding')
    dilation = utils.convert_to_list(dilation, 3, 'dilation')

    if not isinstance(use_cudnn, bool):
        raise ValueError("use_cudnn should be True or False")

    input_shape = input.shape
    filter_shape = [num_filters, num_filter_channels] + filter_size

    def _get_default_param_initializer():
C
chengduo 已提交
2314 2315 2316
        filter_elem_num = filter_size[0] * filter_size[1] * filter_size[
            2] * num_channels
        std = (2.0 / filter_elem_num)**0.5
C
chengduoZH 已提交
2317 2318 2319 2320 2321 2322 2323 2324
        return Normal(0.0, std, 0)

    filter_param = helper.create_parameter(
        attr=helper.param_attr,
        shape=filter_shape,
        dtype=dtype,
        default_initializer=_get_default_param_initializer())

X
Xin Pan 已提交
2325
    pre_bias = helper.create_variable_for_type_inference(dtype)
C
chengduoZH 已提交
2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339

    helper.append_op(
        type=l_type,
        inputs={
            'Input': input,
            'Filter': filter_param,
        },
        outputs={"Output": pre_bias},
        attrs={
            'strides': stride,
            'paddings': padding,
            'dilations': dilation,
            'groups': groups,
            'use_cudnn': use_cudnn,
X
Xin Pan 已提交
2340
            'use_mkldnn': False
C
chengduoZH 已提交
2341 2342
        })

2343
    pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2)
C
chengduoZH 已提交
2344 2345 2346 2347

    return helper.append_activation(pre_act)


J
Jacek Czaja 已提交
2348
def sequence_pool(input, pool_type, is_test=False):
Y
Yu Yang 已提交
2349
    """
Y
yangyaming 已提交
2350 2351 2352
    This function add the operator for sequence pooling.
    It pools features of all time-steps of each instance, and is applied
    on top of the input using pool_type mentioned in the parameters.
L
Luo Tao 已提交
2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363

    It supports four pool_type:

    - average: :math:`Out[i] = \\frac{\sum_i X_i}{N}`
    - sum:     :math:`Out[i] = \sum_jX_{ij}`
    - sqrt:    :math:`Out[i] = \\frac{\sum_jX_{ij}}{\sqrt{len(X_i)}}`
    - max:     :math:`Out[i] = max(X_i)`

    .. code-block:: text

       x is a 1-level LoDTensor:
2364
         x.lod = [[2, 3, 2]]
L
Luo Tao 已提交
2365 2366 2367 2368 2369
         x.data = [1, 3, 2, 4, 6, 5, 1]
         x.dims = [7, 1]

       then output is a Tensor:
         out.dim = [3, 1]
2370
         with condition len(x.lod[-1]) == out.dims[0]
L
Luo Tao 已提交
2371 2372 2373 2374 2375 2376 2377

       for different pool_type:
         average: out.data = [2, 4, 3], where 2=(1+3)/2, 4=(2+4+6)/3, 3=(5+1)/2
         sum    : out.data = [4, 12, 6], where 4=1+3, 12=2+4+6, 6=5+1
         sqrt   : out.data = [2.82, 6.93, 4.24], where 2.82=(1+3)/sqrt(2),
                    6.93=(2+4+6)/sqrt(3), 4.24=(5+1)/sqrt(2)
         max    : out.data = [3, 6, 5], where 3=max(1,3), 6=max(2,4,6), 5=max(5,1)
2378 2379
         last   : out.data = [3, 6, 1], where 3=last(1,3), 6=last(2,4,6), 1=last(5,1)
         first  : out.data = [1, 2, 5], where 1=first(1,3), 2=first(2,4,6), 5=first(5,1)
F
fengjiayi 已提交
2380

L
Luo Tao 已提交
2381 2382
    Args:
        input(variable): The input variable which is a LoDTensor.
Y
yangyaming 已提交
2383
        pool_type (string): The pooling type of sequence_pool.
L
Luo Tao 已提交
2384
            It supports average, sum, sqrt and max.
J
Jacek Czaja 已提交
2385
        is_test(bool, Default False): Used distinguish training from scoring mode.
L
Luo Tao 已提交
2386 2387 2388 2389 2390 2391 2392

    Returns:
        The sequence pooling variable which is a Tensor.

    Examples:

        .. code-block:: python
F
fengjiayi 已提交
2393

Y
yangyaming 已提交
2394
             x = fluid.layers.data(name='x', shape=[7, 1],
L
Luo Tao 已提交
2395 2396 2397 2398 2399
                              dtype='float32', lod_level=1)
             avg_x = fluid.layers.sequence_pool(input=x, pool_type='average')
             sum_x = fluid.layers.sequence_pool(input=x, pool_type='sum')
             sqrt_x = fluid.layers.sequence_pool(input=x, pool_type='sqrt')
             max_x = fluid.layers.sequence_pool(input=x, pool_type='max')
2400 2401
             last_x = fluid.layers.sequence_pool(input=x, pool_type='last')
             first_x = fluid.layers.sequence_pool(input=x, pool_type='first')
Y
Yu Yang 已提交
2402
    """
L
lujun 已提交
2403
    assert not in_dygraph_mode(), (
2404
        "sequence layer is not supported in dygraph mode yet.")
F
fengjiayi 已提交
2405
    helper = LayerHelper('sequence_pool', **locals())
Y
Yu Yang 已提交
2406
    dtype = helper.input_dtype()
X
Xin Pan 已提交
2407 2408
    pool_out = helper.create_variable_for_type_inference(dtype)
    max_index = helper.create_variable_for_type_inference(dtype)
Y
Yu Yang 已提交
2409 2410 2411 2412 2413 2414

    helper.append_op(
        type="sequence_pool",
        inputs={"X": input},
        outputs={"Out": pool_out,
                 "MaxIndex": max_index},
J
Jacek Czaja 已提交
2415 2416
        attrs={"pooltype": pool_type.upper(),
               "is_test": is_test})
Y
Yu Yang 已提交
2417

Y
yangyaming 已提交
2418 2419 2420 2421 2422
    # when pool_type is max, variable max_index is initialized,
    # so we stop the gradient explicitly here
    if pool_type == 'max':
        max_index.stop_gradient = True

Y
Yu Yang 已提交
2423 2424 2425
    return pool_out


C
add doc  
chengduoZH 已提交
2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441
@templatedoc()
def sequence_concat(input, name=None):
    """
    ${comment}

    Args:
        input(list): List of Variables to be concatenated.
        name(str|None): A name for this layer(optional). If set None, the layer
                       will be named automatically.

    Returns:
        Variable: Output variable of the concatenation.

    Examples:
        .. code-block:: python

B
bdzhuxiaoning 已提交
2442 2443 2444 2445
           import paddle.fluid as fluid
           x = fluid.layers.data(name='x', shape=[10], dtype='float32')
           y = fluid.layers.data(name='y', shape=[10], dtype='float32')
           out = fluid.layers.sequence_concat(input=[x, y])
C
add doc  
chengduoZH 已提交
2446
    """
L
lujun 已提交
2447
    assert not in_dygraph_mode(), (
2448
        "sequence layer is not supported in dygraph mode yet.")
C
add doc  
chengduoZH 已提交
2449
    helper = LayerHelper('sequence_concat', **locals())
X
Xin Pan 已提交
2450
    out = helper.create_variable_for_type_inference(dtype=helper.input_dtype())
C
add doc  
chengduoZH 已提交
2451 2452 2453 2454 2455
    helper.append_op(
        type='sequence_concat', inputs={'X': input}, outputs={'Out': [out]})
    return out


F
fengjiayi 已提交
2456
def sequence_first_step(input):
L
Luo Tao 已提交
2457
    """
L
Luo Tao 已提交
2458
    This function gets the first step of sequence.
L
Luo Tao 已提交
2459 2460 2461 2462

    .. code-block:: text

       x is a 1-level LoDTensor:
2463
         x.lod = [[2, 3, 2]]
L
Luo Tao 已提交
2464 2465 2466 2467 2468
         x.data = [1, 3, 2, 4, 6, 5, 1]
         x.dims = [7, 1]

       then output is a Tensor:
         out.dim = [3, 1]
2469
         with condition len(x.lod[-1]) == out.dims[0]
L
Luo Tao 已提交
2470
         out.data = [1, 2, 5], where 1=first(1,3), 2=first(2,4,6), 5=first(5,1)
F
fengjiayi 已提交
2471

L
Luo Tao 已提交
2472 2473 2474 2475 2476 2477 2478 2479 2480
    Args:
        input(variable): The input variable which is a LoDTensor.

    Returns:
        The sequence's first step variable which is a Tensor.

    Examples:

        .. code-block:: python
F
fengjiayi 已提交
2481

Y
yangyaming 已提交
2482
             x = fluid.layers.data(name='x', shape=[7, 1],
L
Luo Tao 已提交
2483 2484 2485
                              dtype='float32', lod_level=1)
             x_first_step = fluid.layers.sequence_first_step(input=x)
    """
2486 2487 2488
    return sequence_pool(input=input, pool_type="first")


F
fengjiayi 已提交
2489
def sequence_last_step(input):
L
Luo Tao 已提交
2490
    """
L
Luo Tao 已提交
2491
    This function gets the last step of sequence.
L
Luo Tao 已提交
2492 2493 2494 2495

    .. code-block:: text

       x is a 1-level LoDTensor:
2496
         x.lod = [[2, 3, 2]]
L
Luo Tao 已提交
2497 2498 2499 2500 2501
         x.data = [1, 3, 2, 4, 6, 5, 1]
         x.dims = [7, 1]

       then output is a Tensor:
         out.dim = [3, 1]
2502
         with condition len(x.lod[-1]) == out.dims[0]
L
Luo Tao 已提交
2503
         out.data = [3, 6, 1], where 3=last(1,3), 6=last(2,4,6), 1=last(5,1)
F
fengjiayi 已提交
2504

L
Luo Tao 已提交
2505 2506 2507 2508 2509 2510 2511 2512 2513
    Args:
        input(variable): The input variable which is a LoDTensor.

    Returns:
        The sequence's last step variable which is a Tensor.

    Examples:

        .. code-block:: python
F
fengjiayi 已提交
2514

Y
yangyaming 已提交
2515
             x = fluid.layers.data(name='x', shape=[7, 1],
L
Luo Tao 已提交
2516 2517 2518
                              dtype='float32', lod_level=1)
             x_last_step = fluid.layers.sequence_last_step(input=x)
    """
2519 2520 2521
    return sequence_pool(input=input, pool_type="last")


Y
Yibing Liu 已提交
2522 2523 2524 2525
def sequence_slice(input, offset, length, name=None):
    """
    **Sequence Slice Layer**

2526
    The layer crops a subsequence from given sequence with given start
Y
Yibing Liu 已提交
2527 2528 2529 2530 2531
    offset and subsequence length.

    It only supports sequence data (LoDTensor with lod_level equal to 1).

    .. code-block:: text
2532

H
haowang101779990 已提交
2533
              - Case:
Y
Yibing Liu 已提交
2534

2535
            Given the input Variable **input**:
2536

2537 2538 2539
                input.data = [[a1, a2], [b1, b2], [c1, c2], [d1, d2], [e1, e2]],
                input.lod = [[3, 2]],
                input.dims = (5, 2),
Y
Yibing Liu 已提交
2540

2541
            with offset.data = [[0], [1]] and length.data = [[2], [1]],
Y
Yibing Liu 已提交
2542

2543
            the output Variable will be
2544

2545 2546 2547
                out.data = [[a1, a2], [b1, b2], [e1, e2]],
                out.lod = [[2, 1]],
                out.dims = (3, 2).
2548

M
minqiyang 已提交
2549
    Note:
H
haowang101779990 已提交
2550
          The first dimension size of **input**, **offset** and **length**
2551
          should be equal. The **offset** should start from 0.
2552

Y
Yibing Liu 已提交
2553
    Args:
2554
        input(Variable): The input Variable which consists of the complete
Y
Yibing Liu 已提交
2555
                         sequences.
Y
Yibing Liu 已提交
2556 2557 2558 2559 2560 2561
        offset(Variable): The offset to slice each sequence.
        length(Variable): The length of each subsequence.
        name(str|None): A name for this layer(optional). If set None, the
                        layer will be named automatically.

    Returns:
Y
Yibing Liu 已提交
2562
        Variable: The output subsequences.
Y
Yibing Liu 已提交
2563 2564 2565 2566 2567 2568 2569 2570 2571 2572

    Examples:

        .. code-block:: python

             import numpy as np
             seqs = fluid.layers.data(name='x', shape=[10, 5],
                              dtype='float32', lod_level=1)
             offset = fluid.layers.assign(input=np.array([[0, 1]]).astype("int32"))
             length = fluid.layers.assign(input=np.array([[2, 1]]).astype("int32"))
2573
             subseqs = fluid.layers.sequence_slice(input=seqs, offset=offset,
Y
Yibing Liu 已提交
2574 2575
                                                   length=length)
    """
L
lujun 已提交
2576
    assert not in_dygraph_mode(), (
2577
        "sequence layer is not supported in dygraph mode yet.")
Y
Yibing Liu 已提交
2578 2579
    helper = LayerHelper("sequence_slice", **locals())
    dtype = helper.input_dtype()
X
Xin Pan 已提交
2580
    out = helper.create_variable_for_type_inference(dtype)
Y
Yibing Liu 已提交
2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594

    offset.stop_gradient = True
    length.stop_gradient = True

    helper.append_op(
        type="sequence_slice",
        inputs={"X": input,
                "Offset": offset,
                "Length": length},
        outputs={"Out": out})

    return out


F
fengjiayi 已提交
2595
@templatedoc()
Y
Yu Yang 已提交
2596
def pool2d(input,
C
chengduoZH 已提交
2597 2598
           pool_size=-1,
           pool_type="max",
C
chengduoZH 已提交
2599 2600
           pool_stride=1,
           pool_padding=0,
C
caoying03 已提交
2601
           global_pooling=False,
C
chengduoZH 已提交
2602
           use_cudnn=True,
2603
           ceil_mode=False,
2604 2605
           name=None,
           exclusive=True):
Y
Yu Yang 已提交
2606
    """
F
fengjiayi 已提交
2607
    ${comment}
2608 2609

    Args:
2610 2611 2612
        input (Variable): The input tensor of pooling operator. The format of
                          input tensor is NCHW, where N is batch size, C is
                          the number of channels, H is the height of the
F
fengjiayi 已提交
2613
                          feature, and W is the width of the feature.
J
JiabinYang 已提交
2614
        pool_size (int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list,
J
JiabinYang 已提交
2615 2616
            it must contain two integers, (pool_size_Height, pool_size_Width).
            Otherwise, the pool kernel size will be a square of an int.
F
fengjiayi 已提交
2617
        pool_type: ${pooling_type_comment}
J
JiabinYang 已提交
2618 2619 2620 2621 2622 2623
        pool_stride (int|list|tuple): The pool stride size. If pool stride size is a tuple or list,
            it must contain two integers, (pool_stride_Height, pool_stride_Width).
            Otherwise, the pool stride size will be a square of an int.
        pool_padding (int|list|tuple): The pool padding size. If pool padding size is a tuple,
            it must contain two integers, (pool_padding_on_Height, pool_padding_on_Width).
            Otherwise, the pool padding size will be a square of an int.
2624 2625 2626
        global_pooling (bool): ${global_pooling_comment}
        use_cudnn (bool): ${use_cudnn_comment}
        ceil_mode (bool): ${ceil_mode_comment}
2627
        name (str|None): A name for this layer(optional). If set None, the
F
fengjiayi 已提交
2628
                        layer will be named automatically.
2629
        exclusive (bool): Whether to exclude padding points in average pooling
2630
                          mode, default is true
F
fengjiayi 已提交
2631

2632
    Returns:
F
fengjiayi 已提交
2633
        Variable: The pooling result.
F
fengjiayi 已提交
2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645

    Raises:
        ValueError: If 'pool_type' is not "max" nor "avg"
        ValueError: If 'global_pooling' is False and 'pool_size' is -1
        ValueError: If 'use_cudnn' is not a bool value.

    Examples:

        .. code-block:: python

          data = fluid.layers.data(
              name='data', shape=[3, 32, 32], dtype='float32')
D
dengkaipeng 已提交
2646
          pool2d = fluid.layers.pool2d(
2647 2648 2649 2650
                            input=data,
                            pool_size=2,
                            pool_type='max',
                            pool_stride=1,
F
fengjiayi 已提交
2651
                            global_pooling=False)
Y
Yu Yang 已提交
2652 2653 2654 2655 2656
    """
    if pool_type not in ["max", "avg"]:
        raise ValueError(
            "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.",
            str(pool_type))
C
chengduoZH 已提交
2657

C
chengduoZH 已提交
2658 2659 2660 2661 2662
    if global_pooling is False and pool_size == -1:
        raise ValueError(
            "When the global_pooling is False, pool_size must be passed "
            "and be a valid value. Received pool_size: " + str(pool_size))

C
chengduoZH 已提交
2663 2664 2665 2666
    pool_size = utils.convert_to_list(pool_size, 2, 'pool_size')
    pool_padding = utils.convert_to_list(pool_padding, 2, 'pool_padding')
    pool_stride = utils.convert_to_list(pool_stride, 2, 'pool_stride')

C
chengduoZH 已提交
2667 2668
    if not isinstance(use_cudnn, bool):
        raise ValueError("use_cudnn should be True or False")
Y
Yu Yang 已提交
2669

C
Add doc  
chengduoZH 已提交
2670
    l_type = 'pool2d'
2671 2672

    helper = LayerHelper(l_type, **locals())
Y
Yu Yang 已提交
2673
    dtype = helper.input_dtype()
X
Xin Pan 已提交
2674
    pool_out = helper.create_variable_for_type_inference(dtype)
Y
Yu Yang 已提交
2675 2676

    helper.append_op(
2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687
        type=l_type,
        inputs={"X": input},
        outputs={"Out": pool_out},
        attrs={
            "pooling_type": pool_type,
            "ksize": pool_size,
            "global_pooling": global_pooling,
            "strides": pool_stride,
            "paddings": pool_padding,
            "use_cudnn": use_cudnn,
            "ceil_mode": ceil_mode,
2688 2689
            "use_mkldnn": False,
            "exclusive": exclusive,
2690 2691 2692 2693 2694
        })

    return pool_out


D
dengkaipeng 已提交
2695
@templatedoc()
2696 2697 2698 2699 2700 2701 2702 2703
def pool3d(input,
           pool_size=-1,
           pool_type="max",
           pool_stride=1,
           pool_padding=0,
           global_pooling=False,
           use_cudnn=True,
           ceil_mode=False,
2704 2705
           name=None,
           exclusive=True):
2706
    """
2707
    ${comment}
2708 2709

    Args:
D
dengkaipeng 已提交
2710 2711 2712 2713 2714
        input (Variable): The input tensor of pooling operator. The format of
                          input tensor is NCDHW, where N is batch size, C is
                          the number of channels, D is the depth of the feature,
                          H is the height of the feature, and W is the width
                          of the feature.
D
dengkaipeng 已提交
2715 2716 2717 2718 2719
        pool_size (int|list|tuple): The pool kernel size. If pool kernel size 
            is a tuple or list, it must contain three integers, 
            (pool_size_Depth, pool_size_Height, pool_size_Width).
            Otherwise, the pool kernel size will be the cube of an int.
        pool_type (string): ${pooling_type_comment}
2720 2721 2722 2723 2724 2725 2726
        pool_stride (int): stride of the pooling layer.
        pool_padding (int): padding size.
        global_pooling (bool): ${global_pooling_comment}
        use_cudnn (bool): ${use_cudnn_comment}
        ceil_mode (bool): ${ceil_mode_comment}
        name (str): A name for this layer(optional). If set None, the layer
            will be named automatically.
2727
        exclusive (bool): Whether to exclude padding points in average pooling
2728
                          mode, default is true
2729

2730
    Returns:
2731
        Variable: output of pool3d layer.
D
dengkaipeng 已提交
2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744

    Examples:

        .. code-block:: python

          data = fluid.layers.data(
              name='data', shape=[3, 32, 32, 32], dtype='float32')
          pool3d = fluid.layers.pool3d(
                            input=data,
                            pool_size=2,
                            pool_type='max',
                            pool_stride=1,
                            global_pooling=False)
Y
Yu Yang 已提交
2745 2746 2747 2748 2749
    """
    if pool_type not in ["max", "avg"]:
        raise ValueError(
            "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.",
            str(pool_type))
C
chengduoZH 已提交
2750

C
chengduoZH 已提交
2751 2752 2753 2754 2755
    if global_pooling is False and pool_size == -1:
        raise ValueError(
            "When the global_pooling is False, pool_size must be passed "
            "and be a valid value. Received pool_size: " + str(pool_size))

2756 2757 2758
    pool_size = utils.convert_to_list(pool_size, 3, 'pool_size')
    pool_padding = utils.convert_to_list(pool_padding, 3, 'pool_padding')
    pool_stride = utils.convert_to_list(pool_stride, 3, 'pool_stride')
C
chengduoZH 已提交
2759

C
chengduoZH 已提交
2760 2761
    if not isinstance(use_cudnn, bool):
        raise ValueError("use_cudnn should be True or False")
Y
Yu Yang 已提交
2762

2763 2764
    l_type = "pool3d"
    helper = LayerHelper(l_type, **locals())
Y
Yu Yang 已提交
2765
    dtype = helper.input_dtype()
X
Xin Pan 已提交
2766
    pool_out = helper.create_variable_for_type_inference(dtype)
Y
Yu Yang 已提交
2767 2768

    helper.append_op(
2769
        type=l_type,
Y
Yu Yang 已提交
2770 2771 2772 2773 2774 2775 2776
        inputs={"X": input},
        outputs={"Out": pool_out},
        attrs={
            "pooling_type": pool_type,
            "ksize": pool_size,
            "global_pooling": global_pooling,
            "strides": pool_stride,
C
chengduoZH 已提交
2777
            "paddings": pool_padding,
2778
            "use_cudnn": use_cudnn,
2779
            "ceil_mode": ceil_mode,
2780 2781
            "use_mkldnn": False,
            "exclusive": exclusive,
Y
Yu Yang 已提交
2782 2783 2784 2785 2786
        })

    return pool_out


2787 2788 2789 2790 2791 2792 2793
@templatedoc(op_type="pool2d")
def adaptive_pool2d(input,
                    pool_size,
                    pool_type="max",
                    require_index=False,
                    name=None):
    """
D
dengkaipeng 已提交
2794 2795 2796 2797 2798 2799 2800
    **Adaptive Pool2d Operator**
    The adaptive_pool2d operation calculates the output based on the input, pool_size,
    pool_type parameters. Input(X) and output(Out) are in NCHW format, where N is batch
    size, C is the number of channels, H is the height of the feature, and W is
    the width of the feature. Parameters(pool_size) should contain two elements which
    represent height and width, respectively. Also the H and W dimensions of output(Out)
    is same as Parameter(pool_size).
2801

2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814
    For average adaptive pool2d:

    ..  math::

       hstart &= floor(i * H_{in} / H_{out})

       hend &= ceil((i + 1) * H_{in} / H_{out})

       wstart &= floor(j * W_{in} / W_{out})

       wend &= ceil((j + 1) * W_{in} / W_{out})

       Output(i ,j) &= \\frac{sum(Input[hstart:hend, wstart:wend])}{(hend - hstart) * (wend - wstart)}
2815 2816 2817 2818 2819 2820 2821 2822 2823

    Args:
        input (Variable): The input tensor of pooling operator. The format of
                          input tensor is NCHW, where N is batch size, C is
                          the number of channels, H is the height of the
                          feature, and W is the width of the feature.
        pool_size (int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list,
            it must contain two integers, (pool_size_Height, pool_size_Width).
        pool_type: ${pooling_type_comment}
D
dengkaipeng 已提交
2824 2825
        require_index (bool): If true, the index of max pooling point will be returned along
            with outputs. It cannot be set in average pooling type.
2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839
        name (str|None): A name for this layer(optional). If set None, the
                        layer will be named automatically.

    Returns:
        Variable: The pooling result.

    Raises:
        ValueError: 'pool_type' is not 'max' nor 'avg'.
        ValueError: invalid setting 'require_index' true when 'pool_type' is 'avg'.
        ValueError: 'pool_size' should be a list or tuple with length as 2.

    Examples:
        .. code-block:: python

M
minqiyang 已提交
2840
          # suppose input data in shape of [N, C, H, W], `pool_size` is [m, n],
2841
          # output shape is [N, C, m, n], adaptive pool divide H and W dimentions
M
minqiyang 已提交
2842
          # of input data into m * n grids averagely and performs poolings in each
2843 2844
          # grid to get output.
          # adaptive average pool performs calculations as follow:
M
minqiyang 已提交
2845
          #
2846 2847 2848 2849 2850 2851 2852 2853
          #     for i in range(m):
          #         for j in range(n):
          #             hstart = floor(i * H / m)
          #             hend = ceil((i + 1) * H / m)
          #             wstart = floor(i * W / n)
          #             wend = ceil((i + 1) * W / n)
          #             output[:, :, i, j] = avg(input[:, :, hstart: hend, wstart: wend])
          #
2854 2855
          data = fluid.layers.data(
              name='data', shape=[3, 32, 32], dtype='float32')
D
dengkaipeng 已提交
2856
          pool_out = fluid.layers.adaptive_pool2d(
2857 2858
                            input=data,
                            pool_size=[3, 3],
2859
                            pool_type='avg')
2860 2861 2862 2863 2864 2865 2866 2867 2868 2869
    """
    if pool_type not in ["max", "avg"]:
        raise ValueError(
            "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.",
            str(pool_type))

    if pool_type == "avg" and require_index:
        raise ValueError(
            "invalid setting 'require_index' true when 'pool_type' is 'avg'.")

2870
    pool_size = utils.convert_to_list(pool_size, 2, 'pool_size')
2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895

    if pool_type == "max":
        l_type = 'max_pool2d_with_index'
    else:
        l_type = "pool2d"

    helper = LayerHelper(l_type, **locals())
    dtype = helper.input_dtype()
    pool_out = helper.create_variable_for_type_inference(dtype)

    outputs = {"Out": pool_out}
    if pool_type == "max":
        mask = helper.create_variable_for_type_inference(dtype)
        outputs["Mask"] = mask

    helper.append_op(
        type=l_type,
        inputs={"X": input},
        outputs=outputs,
        attrs={
            "pooling_type": pool_type,
            "ksize": pool_size,
            "adaptive": True,
        })

D
dengkaipeng 已提交
2896
    return (pool_out, mask) if require_index else pool_out
2897 2898 2899 2900 2901 2902 2903 2904 2905


@templatedoc(op_type="pool3d")
def adaptive_pool3d(input,
                    pool_size,
                    pool_type="max",
                    require_index=False,
                    name=None):
    """
D
dengkaipeng 已提交
2906 2907 2908 2909 2910 2911 2912
    **Adaptive Pool3d Operator**
    The adaptive_pool3d operation calculates the output based on the input, pool_size,
    pool_type parameters. Input(X) and output(Out) are in NCDHW format, where N is batch
    size, C is the number of channels, D is the depth of the feature, H is the height of
    the feature, and W is the width of the feature. Parameters(pool_size) should contain
    three elements which represent height and width, respectively. Also the D, H and W
    dimensions of output(Out) is same as Parameter(pool_size).
2913

2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930
    For average adaptive pool3d:

    ..  math::

      dstart &= floor(i * D_{in} / D_{out})

      dend &= ceil((i + 1) * D_{in} / D_{out})

      hstart &= floor(j * H_{in} / H_{out})

      hend &= ceil((j + 1) * H_{in} / H_{out})

      wstart &= floor(k * W_{in} / W_{out})

      wend &= ceil((k + 1) * W_{in} / W_{out})

      Output(i ,j, k) &= \\frac{sum(Input[dstart:dend, hstart:hend, wstart:wend])}{(dend - dstart) * (hend - hstart) * (wend - wstart)}
2931 2932 2933

    Args:
        input (Variable): The input tensor of pooling operator. The format of
D
dengkaipeng 已提交
2934 2935 2936
                          input tensor is NCDHW, where N is batch size, C is
                          the number of channels, D is the depth of the feature,
                          H is the height of the feature, and W is the width of the feature.
2937
        pool_size (int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list,
D
dengkaipeng 已提交
2938
            it must contain three integers, (Depth, Height, Width).
2939
        pool_type: ${pooling_type_comment}
D
dengkaipeng 已提交
2940 2941
        require_index (bool): If true, the index of max pooling point will be returned along
            with outputs. It cannot be set in average pooling type.
2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955
        name (str|None): A name for this layer(optional). If set None, the
                        layer will be named automatically.

    Returns:
        Variable: The pooling result.

    Raises:
        ValueError: 'pool_type' is not 'max' nor 'avg'.
        ValueError: invalid setting 'require_index' true when 'pool_type' is 'avg'.
        ValueError: 'pool_size' should be a list or tuple with length as 2.

    Examples:
        .. code-block:: python

2956 2957
          # suppose input data in shape of [N, C, D, H, W], `pool_size` is [l, m, n],
          # output shape is [N, C, l, m, n], adaptive pool divide D, H and W dimentions
M
minqiyang 已提交
2958
          # of input data into l * m * n grids averagely and performs poolings in each
2959 2960
          # grid to get output.
          # adaptive average pool performs calculations as follow:
M
minqiyang 已提交
2961
          #
2962 2963 2964 2965 2966 2967 2968 2969 2970
          #     for i in range(l):
          #         for j in range(m):
          #             for k in range(n):
          #                 dstart = floor(i * D / l)
          #                 dend = ceil((i + 1) * D / l)
          #                 hstart = floor(j * H / m)
          #                 hend = ceil((j + 1) * H / m)
          #                 wstart = floor(k * W / n)
          #                 wend = ceil((k + 1) * W / n)
M
minqiyang 已提交
2971
          #                 output[:, :, i, j, k] =
2972 2973
          #                     avg(input[:, :, dstart:dend, hstart: hend, wstart: wend])
          #
K
Kaipeng Deng 已提交
2974 2975 2976

          import paddle.fluid as fluid

2977
          data = fluid.layers.data(
K
Kaipeng Deng 已提交
2978 2979
              name='data', shape=[3, 32, 32, 32], dtype='float32')
          pool_out = fluid.layers.adaptive_pool3d(
2980
                            input=data,
D
dengkaipeng 已提交
2981
                            pool_size=[3, 3, 3],
2982
                            pool_type='avg')
2983 2984 2985 2986 2987 2988 2989 2990 2991 2992
    """
    if pool_type not in ["max", "avg"]:
        raise ValueError(
            "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.",
            str(pool_type))

    if pool_type == "avg" and require_index:
        raise ValueError(
            "invalid setting 'require_index' true when 'pool_type' is 'avg'.")

2993
    pool_size = utils.convert_to_list(pool_size, 3, 'pool_size')
2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018

    if pool_type == "max":
        l_type = 'max_pool3d_with_index'
    else:
        l_type = "pool3d"

    helper = LayerHelper(l_type, **locals())
    dtype = helper.input_dtype()
    pool_out = helper.create_variable_for_type_inference(dtype)

    outputs = {"Out": pool_out}
    if pool_type == "max":
        mask = helper.create_variable_for_type_inference(dtype)
        outputs["Mask"] = mask

    helper.append_op(
        type=l_type,
        inputs={"X": input},
        outputs=outputs,
        attrs={
            "pooling_type": pool_type,
            "ksize": pool_size,
            "adaptive": True,
        })

D
dengkaipeng 已提交
3019
    return (pool_out, mask) if require_index else pool_out
3020 3021


Y
Yu Yang 已提交
3022 3023 3024 3025 3026 3027 3028
def batch_norm(input,
               act=None,
               is_test=False,
               momentum=0.9,
               epsilon=1e-05,
               param_attr=None,
               bias_attr=None,
C
caoying03 已提交
3029
               data_layout='NCHW',
Y
Yang Yang 已提交
3030
               in_place=False,
3031 3032
               name=None,
               moving_mean_name=None,
W
wanghaoshuang 已提交
3033
               moving_variance_name=None,
3034
               do_model_average_for_mean_and_var=False,
3035 3036
               fuse_with_relu=False,
               use_global_stats=False):
Y
Yu Yang 已提交
3037
    """
Q
qiaolongfei 已提交
3038 3039 3040 3041
    **Batch Normalization Layer**

    Can be used as a normalizer function for conv2d and fully_connected operations.
    The required data format for this layer is one of the following:
Q
qiaolongfei 已提交
3042

Q
qiaolongfei 已提交
3043
    1. NHWC `[batch, in_height, in_width, in_channels]`
Q
qiaolongfei 已提交
3044

Q
qiaolongfei 已提交
3045 3046
    2. NCHW `[batch, in_channels, in_height, in_width]`

Q
qiaolongfei 已提交
3047 3048 3049
    Refer to `Batch Normalization: Accelerating Deep Network Training by Reducing
    Internal Covariate Shift <https://arxiv.org/pdf/1502.03167.pdf>`_
    for more details.
Q
qiaolongfei 已提交
3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061

    :math:`input` is the input features over a mini-batch.

    ..  math::

        \\mu_{\\beta} &\\gets \\frac{1}{m} \\sum_{i=1}^{m} x_i \\qquad &//\\
        \ mini-batch\ mean \\\\
        \\sigma_{\\beta}^{2} &\\gets \\frac{1}{m} \\sum_{i=1}^{m}(x_i - \\
        \\mu_{\\beta})^2 \\qquad &//\ mini-batch\ variance \\\\
        \\hat{x_i} &\\gets \\frac{x_i - \\mu_\\beta} {\\sqrt{\\
        \\sigma_{\\beta}^{2} + \\epsilon}} \\qquad &//\ normalize \\\\
        y_i &\\gets \\gamma \\hat{x_i} + \\beta \\qquad &//\ scale\ and\ shift
3062

3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075

    When use_global_stats = True, the :math:`\\mu_{\\beta}`
    and :math:`\\sigma_{\\beta}^{2}` are not the statistics of one mini-batch.
    They are global (or running) statistics. (It usually got from the
    pre-trained model.)
    The training and testing (or inference) have the same behavior:

    ..  math::

        \\hat{x_i} &\\gets \\frac{x_i - \\mu_\\beta} {\\sqrt{\\
        \\sigma_{\\beta}^{2} + \\epsilon}}  \\\\
        y_i &\\gets \\gamma \\hat{x_i} + \\beta

3076
    Args:
Q
qingqing01 已提交
3077
        input(variable): The rank of input variable can be 2, 3, 4, 5.
Q
qiaolongfei 已提交
3078
        act(string, Default None): Activation type, linear|relu|prelu|...
Q
qingqing01 已提交
3079 3080 3081 3082 3083 3084 3085 3086 3087
        is_test (bool, Default False): A flag indicating whether it is in
            test phrase or not.
        momentum(float, Default 0.9): The value used for the moving_mean and
            moving_var computation. The updated formula is:
            :math:`moving\_mean = moving\_mean * momentum + new\_mean * (1. - momentum)`
            :math:`moving\_var = moving\_var * momentum + new\_var * (1. - momentum)`
            Default is 0.9.
        epsilon(float, Default 1e-05): A value added to the denominator for
            numerical stability. Default is 1e-5.
C
chengduo 已提交
3088 3089 3090 3091 3092 3093 3094 3095
        param_attr(ParamAttr|None): The parameter attribute for Parameter `scale`
             of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm
             will create ParamAttr as param_attr. If the Initializer of the param_attr
             is not set, the parameter is initialized with Xavier. Default: None.
        bias_attr(ParamAttr|None): The parameter attribute for the bias of batch_norm.
             If it is set to None or one attribute of ParamAttr, batch_norm
             will create ParamAttr as bias_attr. If the Initializer of the bias_attr
             is not set, the bias is initialized zero. Default: None.
Q
qiaolongfei 已提交
3096
        data_layout(string, default NCHW): NCHW|NHWC
3097
        in_place(bool, Default False): Make the input and output of batch norm reuse memory.
Q
qiaolongfei 已提交
3098 3099 3100 3101
        name(string, Default None): A name for this layer(optional). If set None, the layer
            will be named automatically.
        moving_mean_name(string, Default None): The name of moving_mean which store the global Mean.
        moving_variance_name(string, Default None): The name of the moving_variance which store the global Variance.
Q
qiaolongfei 已提交
3102
        do_model_average_for_mean_and_var(bool, Default False): Do model average for mean and variance or not.
3103
        fuse_with_relu (bool): if True, this OP performs relu after batch norm.
3104 3105 3106 3107 3108
        use_global_stats(bool, Default False): Whether to use global mean and
            variance. In inference or test mode, set use_global_stats to true
            or is_test to true, and the behavior is equivalent.
            In train mode, when setting use_global_stats True, the global mean
            and variance are also used during train period.
3109 3110

    Returns:
Q
qiaolongfei 已提交
3111
        Variable: A tensor variable which is the result after applying batch normalization on the input.
Q
qiaolongfei 已提交
3112 3113 3114 3115 3116

    Examples:

        .. code-block:: python

L
lvmengsi 已提交
3117
            x = fluid.layers.data(name='x', shape=[3, 7, 3, 7], dtype='float32', append_batch_size=False)
Q
qiaolongfei 已提交
3118 3119
            hidden1 = fluid.layers.fc(input=x, size=200, param_attr='fc1.w')
            hidden2 = fluid.layers.batch_norm(input=hidden1)
Y
Yu Yang 已提交
3120
    """
C
chengduo 已提交
3121
    assert bias_attr is not False, "bias_attr should not be False in batch_norm."
Y
Yu Yang 已提交
3122 3123 3124
    helper = LayerHelper('batch_norm', **locals())
    dtype = helper.input_dtype()

W
Wu Yi 已提交
3125 3126 3127 3128
    # use fp32 for bn parameter
    if dtype == core.VarDesc.VarType.FP16:
        dtype = core.VarDesc.VarType.FP32

Y
Yu Yang 已提交
3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146
    input_shape = input.shape
    if data_layout == 'NCHW':
        channel_num = input_shape[1]
    else:
        if data_layout == 'NHWC':
            channel_num = input_shape[-1]
        else:
            raise ValueError("unsupported data layout:" + data_layout)

    param_shape = [channel_num]

    # create parameter
    scale = helper.create_parameter(
        attr=helper.param_attr,
        shape=param_shape,
        dtype=dtype,
        default_initializer=Constant(1.0))
    bias = helper.create_parameter(
3147
        attr=helper.bias_attr, shape=param_shape, dtype=dtype, is_bias=True)
Y
Yu Yang 已提交
3148

3149 3150
    mean = helper.create_parameter(
        attr=ParamAttr(
W
wanghaoshuang 已提交
3151 3152 3153
            name=moving_mean_name,
            initializer=Constant(0.0),
            trainable=False,
W
wanghaoshuang 已提交
3154
            do_model_average=do_model_average_for_mean_and_var),
Q
QI JUN 已提交
3155
        shape=param_shape,
W
Wu Yi 已提交
3156
        dtype=dtype)
3157 3158 3159 3160 3161 3162
    mean.stop_gradient = True

    variance = helper.create_parameter(
        attr=ParamAttr(
            name=moving_variance_name,
            initializer=Constant(1.0),
W
wanghaoshuang 已提交
3163
            trainable=False,
W
wanghaoshuang 已提交
3164
            do_model_average=do_model_average_for_mean_and_var),
Q
QI JUN 已提交
3165
        shape=param_shape,
W
Wu Yi 已提交
3166
        dtype=dtype)
3167
    variance.stop_gradient = True
Y
Yu Yang 已提交
3168 3169 3170 3171 3172 3173

    # create output
    # mean and mean_out share the same memory
    mean_out = mean
    # variance and variance out share the same memory
    variance_out = variance
X
Xin Pan 已提交
3174 3175 3176 3177
    saved_mean = helper.create_variable_for_type_inference(
        dtype=dtype, stop_gradient=True)
    saved_variance = helper.create_variable_for_type_inference(
        dtype=dtype, stop_gradient=True)
Y
Yu Yang 已提交
3178

X
Xin Pan 已提交
3179 3180
    batch_norm_out = input if in_place else helper.create_variable_for_type_inference(
        dtype)
Y
Yu Yang 已提交
3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197

    helper.append_op(
        type="batch_norm",
        inputs={
            "X": input,
            "Scale": scale,
            "Bias": bias,
            "Mean": mean,
            "Variance": variance
        },
        outputs={
            "Y": batch_norm_out,
            "MeanOut": mean_out,
            "VarianceOut": variance_out,
            "SavedMean": saved_mean,
            "SavedVariance": saved_variance
        },
3198 3199 3200 3201
        attrs={
            "momentum": momentum,
            "epsilon": epsilon,
            "is_test": is_test,
3202
            "data_layout": data_layout,
X
Xin Pan 已提交
3203
            "use_mkldnn": False,
3204 3205
            "fuse_with_relu": fuse_with_relu,
            "use_global_stats": use_global_stats
3206
        })
Y
Yu Yang 已提交
3207 3208 3209 3210

    return helper.append_activation(batch_norm_out)


H
heqiaozhi 已提交
3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261
def data_norm(input,
              act=None,
              epsilon=1e-05,
              param_attr=None,
              data_layout='NCHW',
              in_place=False,
              name=None,
              moving_mean_name=None,
              moving_variance_name=None,
              do_model_average_for_mean_and_var=False):
    """
    **Data Normalization Layer**

    Can be used as a normalizer function for conv2d and fully_connected operations.
    The required data format for this layer is one of the following:

    1. NHWC `[batch, in_height, in_width, in_channels]`

    2. NCHW `[batch, in_channels, in_height, in_width]`

    :math:`input` is the input features over a mini-batch.

    ..  math::

        \\mu_{\\beta} &\\gets \\frac{1}{m} \\sum_{i=1}^{m} x_i \\qquad &//\\
        \ mini-batch\ mean \\\\
        \\sigma_{\\beta}^{2} &\\gets \\frac{1}{m} \\sum_{i=1}^{m}(x_i - \\
        \\mu_{\\beta})^2 \\qquad &//\ mini-batch\ variance \\\\
        \\hat{x_i} &\\gets \\frac{x_i - \\mu_\\beta} {\\sqrt{\\
        \\sigma_{\\beta}^{2} + \\epsilon}} \\qquad &//\ normalize \\\\
        y_i &\\gets \\gamma \\hat{x_i} + \\beta \\qquad &//\ scale\ and\ shift

    Args:
        input(variable): The input variable which is a LoDTensor.
        act(string, Default None): Activation type, linear|relu|prelu|...
        epsilon(float, Default 1e-05):
        param_attr(ParamAttr): The parameter attribute for Parameter `scale`.
        data_layout(string, default NCHW): NCHW|NHWC
        in_place(bool, Default False): Make the input and output of batch norm reuse memory.
        name(string, Default None): A name for this layer(optional). If set None, the layer
            will be named automatically.
        moving_mean_name(string, Default None): The name of moving_mean which store the global Mean.
        moving_variance_name(string, Default None): The name of the moving_variance which store the global Variance.
        do_model_average_for_mean_and_var(bool, Default False): Do model average for mean and variance or not.

    Returns:
        Variable: A tensor variable which is the result after applying data normalization on the input.

    Examples:

        .. code-block:: python
3262 3263
            
            import paddle.fluid as fluid
H
heqiaozhi 已提交
3264

3265 3266
            hidden1 = fluid.layers.data(name="hidden1", shape=[200])
            hidden2 = fluid.layers.data_norm(name="hidden2", input=hidden1)
H
heqiaozhi 已提交
3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331
    """
    helper = LayerHelper('data_norm', **locals())
    dtype = helper.input_dtype()

    input_shape = input.shape
    if data_layout == 'NCHW':
        channel_num = input_shape[1]
    else:
        if data_layout == 'NHWC':
            channel_num = input_shape[-1]
        else:
            raise ValueError("unsupported data layout:" + data_layout)

    param_shape = [channel_num]

    batch_size_default = 1e4
    batch_sum_default = 0.0
    batch_square_sum_default = 1e4

    if param_attr and isinstance(param_attr, dict):
        batch_size_default = param_attr.get("batch_size", 1e4)
        batch_sum_default = param_attr.get("batch_sum", 0.0)
        batch_square_sum_default = param_attr.get("batch_square", 1e4)

    # create parameter
    batch_size = helper.create_parameter(
        attr=ParamAttr(
            name=name + '.batch_size',
            initializer=Constant(value=float(batch_size_default)),
            trainable=True),
        shape=param_shape,
        dtype=input.dtype)

    batch_sum = helper.create_parameter(
        attr=ParamAttr(
            name=name + '.batch_sum',
            initializer=Constant(value=float(batch_sum_default)),
            trainable=True),
        shape=param_shape,
        dtype=input.dtype)

    batch_square_sum = helper.create_parameter(
        attr=ParamAttr(
            name=name + '.batch_square_sum',
            initializer=Constant(value=float(batch_square_sum_default)),
            trainable=True),
        shape=param_shape,
        dtype=input.dtype)

    means = helper.create_variable(dtype=dtype, stop_gradient=True)
    scales = helper.create_variable(dtype=dtype, stop_gradient=True)

    data_norm_out = input if in_place else helper.create_variable(dtype=dtype)

    helper.append_op(
        type="data_norm",
        inputs={
            "X": input,
            "BatchSize": batch_size,
            "BatchSum": batch_sum,
            "BatchSquareSum": batch_square_sum
        },
        outputs={"Y": data_norm_out,
                 "Means": means,
                 "Scales": scales},
H
heqiaozhi 已提交
3332
        attrs={"epsilon": epsilon})
H
heqiaozhi 已提交
3333 3334 3335 3336

    return helper.append_activation(data_norm_out)


Y
yuyang18 已提交
3337
@templatedoc()
G
guosheng 已提交
3338 3339 3340 3341 3342 3343 3344 3345 3346 3347
def layer_norm(input,
               scale=True,
               shift=True,
               begin_norm_axis=1,
               epsilon=1e-05,
               param_attr=None,
               bias_attr=None,
               act=None,
               name=None):
    """
Y
yuyang18 已提交
3348
    ${comment}
G
guosheng 已提交
3349 3350 3351

    The formula is as follows:

Y
yuyang18 已提交
3352
    ..  math::
G
guosheng 已提交
3353 3354 3355 3356 3357 3358 3359

        \\mu & = \\frac{1}{H}\\sum_{i=1}^{H} a_i

        \\sigma & = \\sqrt{\\frac{1}{H}\sum_{i=1}^{H}(a_i - \\mu)^2}

        h & = f(\\frac{g}{\\sigma}(a - \\mu) + b)

Y
yuyang18 已提交
3360 3361 3362 3363 3364 3365 3366 3367
    * :math:`a`: the vector representation of the summed inputs to the neurons
    in that layer.

    * :math:`H`: the number of hidden units in a layers

    * :math:`g`: the trainable scale parameter.

    * :math:`b`: the trainable bias parameter.
Y
yuyang18 已提交
3368

G
guosheng 已提交
3369 3370
    Args:
        input(Variable): The input tensor variable.
3371
        scale(bool): Whether to learn the adaptive gain :math:`g` after
S
sneaxiy 已提交
3372
            normalization. Default True.
3373
        shift(bool): Whether to learn the adaptive bias :math:`b` after
S
sneaxiy 已提交
3374 3375
            normalization. Default True.
        begin_norm_axis(int): The normalization will be performed along
G
guosheng 已提交
3376
            dimensions from :attr:`begin_norm_axis` to :attr:`rank(input)`.
S
sneaxiy 已提交
3377
            Default 1.
3378
        epsilon(float): The small value added to the variance to prevent
S
sneaxiy 已提交
3379
            division by zero. Default 1e-05.
G
guosheng 已提交
3380
        param_attr(ParamAttr|None): The parameter attribute for the learnable
S
sneaxiy 已提交
3381 3382
            gain :math:`g`. If :attr:`scale` is False, :attr:`param_attr` is
            omitted. If :attr:`scale` is True and :attr:`param_attr` is None,
3383 3384
            a default :code:`ParamAttr` would be added as scale. The
            :attr:`param_attr` is initialized as 1 if it is added. Default None.
G
guosheng 已提交
3385
        bias_attr(ParamAttr|None): The parameter attribute for the learnable
S
sneaxiy 已提交
3386 3387
            bias :math:`b`. If :attr:`shift` is False, :attr:`bias_attr` is
            omitted. If :attr:`shift` is True and :attr:`param_attr` is None,
3388
            a default :code:`ParamAttr` would be added as bias. The
S
sneaxiy 已提交
3389
            :attr:`bias_attr` is initialized as 0 if it is added. Default None.
G
guosheng 已提交
3390
        act(str): Activation to be applied to the output of layer normalizaiton.
S
sneaxiy 已提交
3391 3392 3393
                  Default None.
        name(str): The name of this layer. It is optional. Default None, and a
                   unique name would be generated automatically.
G
guosheng 已提交
3394 3395

    Returns:
Y
yuyang18 已提交
3396
        ${y_comment}
G
guosheng 已提交
3397 3398 3399

    Examples:

Y
yuyang18 已提交
3400 3401 3402
        >>> data = fluid.layers.data(name='data', shape=[3, 32, 32],
        >>>                          dtype='float32')
        >>> x = fluid.layers.layer_norm(input=data, begin_norm_axis=1)
G
guosheng 已提交
3403
    """
L
lujun 已提交
3404
    assert in_dygraph_mode(
L
lujun 已提交
3405
    ) is not True, "please use FC instead of fc in dygraph mode!"
G
guosheng 已提交
3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419
    helper = LayerHelper('layer_norm', **locals())
    dtype = helper.input_dtype()

    # create intput and parameters
    inputs = {'X': input}
    input_shape = input.shape
    param_shape = [reduce(lambda x, y: x * y, input_shape[begin_norm_axis:])]
    if scale:
        scale = helper.create_parameter(
            attr=helper.param_attr,
            shape=param_shape,
            dtype=dtype,
            default_initializer=Constant(1.0))
        inputs['Scale'] = scale
G
guosheng 已提交
3420
    if shift:
G
guosheng 已提交
3421 3422 3423 3424 3425 3426
        assert bias_attr is not False
        bias = helper.create_parameter(
            attr=helper.bias_attr, shape=param_shape, dtype=dtype, is_bias=True)
        inputs['Bias'] = bias

    # create output
X
Xin Pan 已提交
3427 3428 3429 3430 3431
    mean_out = helper.create_variable_for_type_inference(
        dtype=dtype, stop_gradient=True)
    variance_out = helper.create_variable_for_type_inference(
        dtype=dtype, stop_gradient=True)
    layer_norm_out = helper.create_variable_for_type_inference(dtype)
G
guosheng 已提交
3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446

    helper.append_op(
        type="layer_norm",
        inputs=inputs,
        outputs={
            "Y": layer_norm_out,
            "Mean": mean_out,
            "Variance": variance_out,
        },
        attrs={"epsilon": epsilon,
               "begin_norm_axis": begin_norm_axis})

    return helper.append_activation(layer_norm_out)


D
Dun 已提交
3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458
@templatedoc()
def group_norm(input,
               groups,
               epsilon=1e-05,
               param_attr=None,
               bias_attr=None,
               act=None,
               data_layout='NCHW',
               name=None):
    """
    **Group Normalization Layer**

H
haowang101779990 已提交
3459
    Refer to `Group Normalization <https://arxiv.org/abs/1803.08494>`_ .
D
Dun 已提交
3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506

    Args:
        input(Variable): The input tensor variable.
        groups(int): The number of groups that divided from channels.
        epsilon(float): The small value added to the variance to prevent
            division by zero.
        param_attr(ParamAttr|None): The parameter attribute for the learnable
            scale :math:`g`. If it is set to False, no scale will be added to the output units.
            If it is set to None, the bias is initialized one. Default: None.
        bias_attr(ParamAttr|None): The parameter attribute for the learnable
            bias :math:`b`. If it is set to False, no bias will be added to the output units.
            If it is set to None, the bias is initialized zero. Default: None.
        act(str): Activation to be applied to the output of group normalizaiton.
        data_layout(string|NCHW): Only NCHW is supported.
        name (str): The name of this layer. It is optional.

    Returns:
        Variable: A tensor variable which is the result after applying group normalization on the input.

    Examples:

        >>> data = fluid.layers.data(name='data', shape=[8, 32, 32],
        >>>                          dtype='float32')
        >>> x = fluid.layers.group_norm(input=data, groups=4)
    """
    helper = LayerHelper('group_norm', **locals())
    dtype = helper.input_dtype()

    # create intput and parameters
    inputs = {'X': input}
    input_shape = input.shape
    if data_layout != 'NCHW':
        raise ValueError("unsupported data layout:" + data_layout)
    param_shape = [input_shape[1]]
    if param_attr:
        scale = helper.create_parameter(
            attr=helper.param_attr,
            shape=param_shape,
            dtype=dtype,
            default_initializer=Constant(1.0))
        inputs['Scale'] = scale
    if bias_attr:
        bias = helper.create_parameter(
            attr=helper.bias_attr, shape=param_shape, dtype=dtype, is_bias=True)
        inputs['Bias'] = bias

    # create output
H
heqiaozhi 已提交
3507 3508
    mean_out = helper.create_variable(dtype=dtype, stop_gradient=True)
    variance_out = helper.create_variable(dtype=dtype, stop_gradient=True)
D
dengkaipeng 已提交
3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525
    group_norm_out = helper.create_variable(dtype=dtype)

    helper.append_op(
        type="group_norm",
        inputs=inputs,
        outputs={
            "Y": group_norm_out,
            "Mean": mean_out,
            "Variance": variance_out,
        },
        attrs={"epsilon": epsilon,
               "groups": groups})

    return helper.append_activation(group_norm_out)


@templatedoc()
3526
def spectral_norm(weight, dim=0, power_iters=1, eps=1e-12, name=None):
D
dengkaipeng 已提交
3527 3528 3529
    """
    **Spectral Normalization Layer**

D
dengkaipeng 已提交
3530
    This layer calculates the spectral normalization value of weight parameters of
3531
    fc, conv1d, conv2d, conv3d layers which should be 2-D, 3-D, 4-D, 5-D
D
dengkaipeng 已提交
3532
    Parameters. Calculations are showed as follows.
3533

D
dengkaipeng 已提交
3534 3535 3536
    Step 1:
    Generate vector U in shape of [H], and V in shape of [W].
    While H is the :attr:`dim` th dimension of the input weights,
D
dengkaipeng 已提交
3537
    and W is the product result of remaining dimensions.
D
dengkaipeng 已提交
3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549

    Step 2:
    :attr:`power_iters` shoule be a positive interger, do following
    calculations with U and V for :attr:`power_iters` rounds.

    .. math:: 

        \mathbf{v} := \\frac{\mathbf{W}^{T} \mathbf{u}}{\|\mathbf{W}^{T} \mathbf{u}\|_2}

        \mathbf{u} := \\frac{\mathbf{W}^{T} \mathbf{v}}{\|\mathbf{W}^{T} \mathbf{v}\|_2}

    Step 3:
D
dengkaipeng 已提交
3550
    Calculate :math:`\sigma(\mathbf{W})` and normalize weight values.
D
dengkaipeng 已提交
3551 3552 3553 3554

    .. math::

        \sigma(\mathbf{W}) = \mathbf{u}^{T} \mathbf{W} \mathbf{v}
3555

D
dengkaipeng 已提交
3556
        \mathbf{W} = \\frac{\mathbf{W}}{\sigma(\mathbf{W})}
3557 3558
                

D
dengkaipeng 已提交
3559 3560 3561 3562
    Refer to `Spectral Normalization <https://arxiv.org/abs/1802.05957>`_ .

    Args:
        weight(${weight_type}): ${weight_comment}
D
dengkaipeng 已提交
3563 3564 3565
        dim(int): ${dim_comment}
        power_iters(int): ${power_iters_comment}
        eps(float): ${eps_comment}
D
dengkaipeng 已提交
3566 3567 3568
        name (str): The name of this layer. It is optional.

    Returns:
D
dengkaipeng 已提交
3569
        Variable: A tensor variable of weight parameters after spectral normalization.
D
dengkaipeng 已提交
3570 3571

    Examples:
K
Kaipeng Deng 已提交
3572
       .. code-block:: python
D
dengkaipeng 已提交
3573

K
Kaipeng Deng 已提交
3574 3575 3576 3577 3578
            import paddle.fluid as fluid

            weight = fluid.layers.data(name='weight', shape=[2, 8, 32, 32], 
                                       append_batch_size=False, dtype='float32')
            x = fluid.layers.spectral_norm(weight=weight, dim=1, power_iters=2)
D
dengkaipeng 已提交
3579 3580
    """
    helper = LayerHelper('spectral_norm', **locals())
3581
    dtype = weight.dtype
D
dengkaipeng 已提交
3582 3583 3584

    # create intput and parameters
    inputs = {'Weight': weight}
3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602
    input_shape = weight.shape
    h = input_shape[dim]
    w = np.prod(input_shape) // h

    u = helper.create_parameter(
        attr=ParamAttr(),
        shape=[h],
        dtype=dtype,
        default_initializer=Normal(0., 1.))
    u.stop_gradient = True
    inputs['U'] = u
    v = helper.create_parameter(
        attr=ParamAttr(),
        shape=[w],
        dtype=dtype,
        default_initializer=Normal(0., 1.))
    inputs['V'] = v
    v.stop_gradient = True
D
dengkaipeng 已提交
3603 3604

    # create output
3605
    out = helper.create_variable(dtype=dtype)
D
Dun 已提交
3606 3607

    helper.append_op(
3608
        type="spectral_norm",
D
Dun 已提交
3609
        inputs=inputs,
3610 3611 3612 3613 3614 3615
        outputs={"Out": out, },
        attrs={
            "dim": dim,
            "power_iters": power_iters,
            "eps": eps,
        })
D
Dun 已提交
3616

3617
    return out
D
Dun 已提交
3618 3619


Y
Yu Yang 已提交
3620 3621 3622 3623
def conv2d_transpose(input,
                     num_filters,
                     output_size=None,
                     filter_size=None,
C
chengduoZH 已提交
3624 3625 3626
                     padding=0,
                     stride=1,
                     dilation=1,
3627
                     groups=None,
C
caoying03 已提交
3628
                     param_attr=None,
3629
                     bias_attr=None,
C
chengduoZH 已提交
3630
                     use_cudnn=True,
3631
                     act=None,
C
caoying03 已提交
3632
                     name=None):
Y
Yu Yang 已提交
3633
    """
3634 3635 3636 3637 3638 3639 3640 3641
    **Convlution2D transpose layer**

    The convolution2D transpose layer calculates the output based on the input,
    filter, and dilations, strides, paddings. Input(Input) and output(Output)
    are in NCHW format. Where N is batch size, C is the number of channels,
    H is the height of the feature, and W is the width of the feature.
    Parameters(dilations, strides, paddings) are two elements. These two elements
    represent height and width, respectively. The details of convolution transpose
3642 3643
    layer, please refer to the following explanation and references
    `therein <http://www.matthewzeiler.com/wp-content/uploads/2017/07/cvpr2010.pdf>`_.
3644 3645 3646
    If bias attribution and activation type are provided, bias is added to
    the output of the convolution, and the corresponding activation function
    is applied to the final result.
3647 3648 3649 3650 3651

    For each input :math:`X`, the equation is:

    .. math::

3652
        Out = \sigma (W \\ast X + b)
3653

3654
    Where:
3655 3656 3657

    * :math:`X`: Input value, a tensor with NCHW format.
    * :math:`W`: Filter value, a tensor with MCHW format.
3658 3659 3660 3661
    * :math:`\\ast`: Convolution operation.
    * :math:`b`: Bias value, a 2-D tensor with shape [M, 1].
    * :math:`\\sigma`: Activation function.
    * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.
Y
Yu Yang 已提交
3662

3663 3664 3665 3666
    Example:

        - Input:

3667
          Input shape: :math:`(N, C_{in}, H_{in}, W_{in})`
3668

3669
          Filter shape: :math:`(C_{in}, C_{out}, H_f, W_f)`
3670 3671 3672

        - Output:

3673
          Output shape: :math:`(N, C_{out}, H_{out}, W_{out})`
3674 3675

        Where
Y
Yu Yang 已提交
3676

3677 3678
        .. math::

3679 3680
           H^\prime_{out} &= (H_{in} - 1) * strides[0] - 2 * paddings[0] + dilations[0] * (H_f - 1) + 1 \\\\
           W^\prime_{out} &= (W_{in} - 1) * strides[1] - 2 * paddings[1] + dilations[1] * (W_f - 1) + 1 \\\\
H
haowang101779990 已提交
3681 3682
           H_{out} &\in [ H^\prime_{out}, H^\prime_{out} + strides[0] ) \\\\
           W_{out} &\in [ W^\prime_{out}, W^\prime_{out} + strides[1] )
Y
Yu Yang 已提交
3683 3684

    Args:
3685 3686 3687 3688
        input(Variable): The input image with [N, C, H, W] format.
        num_filters(int): The number of the filter. It is as same as the output
            image channel.
        output_size(int|tuple|None): The output image size. If output size is a
3689 3690 3691 3692
            tuple, it must contain two integers, (image_H, image_W). None if use
            filter_size, padding, and stride to calculate output_size.
            if output_size and filter_size are specified at the same time, They
            should follow the formula above.
3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710
        filter_size(int|tuple|None): The filter size. If filter_size is a tuple,
            it must contain two integers, (filter_size_H, filter_size_W).
            Otherwise, the filter will be a square. None if use output size to
            calculate filter_size.
        padding(int|tuple): The padding size. If padding is a tuple, it must
            contain two integers, (padding_H, padding_W). Otherwise, the
            padding_H = padding_W = padding. Default: padding = 0.
        stride(int|tuple): The stride size. If stride is a tuple, it must
            contain two integers, (stride_H, stride_W). Otherwise, the
            stride_H = stride_W = stride. Default: stride = 1.
        dilation(int|tuple): The dilation size. If dilation is a tuple, it must
            contain two integers, (dilation_H, dilation_W). Otherwise, the
            dilation_H = dilation_W = dilation. Default: dilation = 1.
        groups(int): The groups number of the Conv2d transpose layer. Inspired by
            grouped convolution in Alex Krizhevsky's Deep CNN paper, in which
            when group=2, the first half of the filters is only connected to the
            first half of the input channels, while the second half of the
            filters is only connected to the second half of the input channels.
C
chengduo 已提交
3711 3712 3713 3714 3715 3716 3717 3718 3719 3720
            Default: groups = 1.
        param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights
            of conv2d_transpose. If it is set to None or one attribute of ParamAttr, conv2d_transpose
            will create ParamAttr as param_attr. If the Initializer of the param_attr
            is not set, the parameter is initialized with Xavier. Default: None.
        bias_attr (ParamAttr|bool|None): The parameter attribute for the bias of conv2d_transpose.
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, conv2d_transpose
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
            is not set, the bias is initialized zero. Default: None.
3721
        use_cudnn(bool): Use cudnn kernel or not, it is valid only when the cudnn
C
chengduo 已提交
3722 3723 3724
            library is installed. Default: True.
        act (str): Activation type, if it is set to None, activation is not appended.
            Default: None.
3725
        name(str|None): A name for this layer(optional). If set None, the layer
C
chengduo 已提交
3726
            will be named automatically. Default: True.
Y
Yu Yang 已提交
3727 3728

    Returns:
3729
        Variable: The tensor variable storing the convolution transpose result.
3730 3731

    Raises:
3732 3733
        ValueError: If the shapes of input, filter_size, stride, padding and
                    groups mismatch.
3734 3735 3736 3737

    Examples:
       .. code-block:: python

3738 3739
          data = fluid.layers.data(name='data', shape=[3, 32, 32], dtype='float32')
          conv2d_transpose = fluid.layers.conv2d_transpose(input=data, num_filters=2, filter_size=3)
Y
Yu Yang 已提交
3740
    """
C
chengduo 已提交
3741
    assert param_attr is not False, "param_attr should not be False in conv2d_transpose."
3742 3743 3744 3745 3746 3747 3748 3749
    input_channel = input.shape[1]

    op_type = 'conv2d_transpose'
    if (input_channel == groups and num_filters == input_channel and
            not use_cudnn):
        op_type = 'depthwise_conv2d_transpose'

    helper = LayerHelper(op_type, **locals())
Y
Yu Yang 已提交
3750 3751 3752
    if not isinstance(input, Variable):
        raise TypeError("Input of conv2d_transpose must be Variable")

C
chengduoZH 已提交
3753 3754 3755
    padding = utils.convert_to_list(padding, 2, 'padding')
    stride = utils.convert_to_list(stride, 2, 'stride')
    dilation = utils.convert_to_list(dilation, 2, 'dilation')
G
guosheng 已提交
3756

C
chengduoZH 已提交
3757 3758
    if not isinstance(use_cudnn, bool):
        raise ValueError("use_cudnn should be True or False")
G
guosheng 已提交
3759

Y
Yu Yang 已提交
3760 3761 3762 3763 3764
    if filter_size is None:
        if output_size is None:
            raise ValueError("output_size must be set when filter_size is None")
        if isinstance(output_size, int):
            output_size = [output_size, output_size]
G
guosheng 已提交
3765

Y
Yu Yang 已提交
3766 3767
        h_in = input.shape[2]
        w_in = input.shape[3]
G
guosheng 已提交
3768

C
chengduoZH 已提交
3769
        filter_size_h = (output_size[0] - (h_in - 1) * stride[0] + 2 *
M
minqiyang 已提交
3770
                         padding[0] - 1) // dilation[0] + 1
C
chengduoZH 已提交
3771
        filter_size_w = (output_size[1] - (w_in - 1) * stride[1] + 2 *
M
minqiyang 已提交
3772
                         padding[1] - 1) // dilation[1] + 1
Y
Yu Yang 已提交
3773
        filter_size = [filter_size_h, filter_size_w]
C
chengduoZH 已提交
3774 3775 3776
    else:
        filter_size = utils.convert_to_list(filter_size, 2,
                                            'conv2d_transpose.filter_size')
C
chengduo 已提交
3777

3778 3779 3780 3781 3782 3783 3784
    if output_size is None:
        output_size = []
    elif isinstance(output_size, list) or isinstance(output_size, int):
        output_size = utils.convert_to_list(output_size, 2, 'output_size')
    else:
        raise ValueError("output_size should be list or int")
    padding = utils.convert_to_list(padding, 2, 'padding')
3785
    groups = 1 if groups is None else groups
M
minqiyang 已提交
3786
    filter_shape = [input_channel, num_filters // groups] + filter_size
C
chengduo 已提交
3787

Y
Yu Yang 已提交
3788 3789 3790
    img_filter = helper.create_parameter(
        dtype=input.dtype, shape=filter_shape, attr=helper.param_attr)

X
Xin Pan 已提交
3791
    pre_bias = helper.create_variable_for_type_inference(dtype=input.dtype)
Y
Yu Yang 已提交
3792
    helper.append_op(
3793
        type=op_type,
Y
Yu Yang 已提交
3794 3795
        inputs={'Input': [input],
                'Filter': [img_filter]},
3796
        outputs={'Output': pre_bias},
C
chengduoZH 已提交
3797
        attrs={
3798
            'output_size': output_size,
3799 3800 3801 3802 3803
            'strides': stride,
            'paddings': padding,
            'dilations': dilation,
            'groups': groups,
            'use_cudnn': use_cudnn
Y
Yu Yang 已提交
3804 3805
        })

3806 3807 3808
    pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2)
    out = helper.append_activation(pre_act)
    return out
Y
Yu Yang 已提交
3809 3810


3811
def conv3d_transpose(input,
Y
Yu Yang 已提交
3812 3813 3814
                     num_filters,
                     output_size=None,
                     filter_size=None,
C
chengduoZH 已提交
3815 3816 3817
                     padding=0,
                     stride=1,
                     dilation=1,
3818
                     groups=None,
C
caoying03 已提交
3819
                     param_attr=None,
3820
                     bias_attr=None,
C
chengduoZH 已提交
3821
                     use_cudnn=True,
3822
                     act=None,
C
caoying03 已提交
3823
                     name=None):
Y
Yu Yang 已提交
3824
    """
3825
    **Convlution3D transpose layer**
3826

3827
    The convolution3D transpose layer calculates the output based on the input,
3828
    filter, and dilations, strides, paddings. Input(Input) and output(Output)
3829 3830 3831 3832 3833 3834
    are in NCDHW format. Where N is batch size, C is the number of channels,
    D is the depth of the feature, H is the height of the feature, and W
    is the width of the feature. Parameters(dilations, strides, paddings) are
    two elements. These two elements represent height and width, respectively.
    The details of convolution transpose layer, please refer to the following
    explanation and references `therein <http://www.matthewzeiler.com/wp-content/uploads/2017/07/cvpr2010.pdf>`_.
3835 3836 3837
    If bias attribution and activation type are provided, bias is added to
    the output of the convolution, and the corresponding activation function
    is applied to the final result.
3838 3839 3840 3841 3842

    For each input :math:`X`, the equation is:

    .. math::

3843
        Out = \sigma (W \\ast X + b)
3844 3845 3846

    In the above equation:

3847 3848
    * :math:`X`: Input value, a tensor with NCDHW format.
    * :math:`W`: Filter value, a tensor with MCDHW format.
3849 3850 3851 3852
    * :math:`\\ast`: Convolution operation.
    * :math:`b`: Bias value, a 2-D tensor with shape [M, 1].
    * :math:`\\sigma`: Activation function.
    * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.
Y
Yu Yang 已提交
3853

3854 3855 3856 3857
    Example:

        - Input:

3858
          Input shape: :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})`
3859

3860
          Filter shape: :math:`(C_{in}, C_{out}, D_f, H_f, W_f)`
3861 3862 3863

        - Output:

3864
          Output shape: :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})`
3865 3866

        Where
Y
Yu Yang 已提交
3867

3868 3869
        .. math::

3870 3871 3872
           D_{out} &= (D_{in} - 1) * strides[0] - 2 * paddings[0] + dilations[0] * (D_f - 1) + 1 \\\\
           H_{out} &= (H_{in} - 1) * strides[1] - 2 * paddings[1] + dilations[1] * (H_f - 1) + 1 \\\\
           W_{out} &= (W_{in} - 1) * strides[2] - 2 * paddings[2] + dilations[2] * (W_f - 1) + 1
Y
Yu Yang 已提交
3873 3874

    Args:
3875
        input(Variable): The input image with [N, C, D, H, W] format.
3876 3877 3878
        num_filters(int): The number of the filter. It is as same as the output
            image channel.
        output_size(int|tuple|None): The output image size. If output size is a
3879
            tuple, it must contain three integers, (image_D, image_H, image_W). This
3880 3881
            parameter only works when filter_size is None.
        filter_size(int|tuple|None): The filter size. If filter_size is a tuple,
3882
            it must contain three integers, (filter_size_D, filter_size_H, filter_size_W).
3883 3884 3885
            Otherwise, the filter will be a square. None if use output size to
            calculate filter_size.
        padding(int|tuple): The padding size. If padding is a tuple, it must
3886 3887
            contain three integers, (padding_D, padding_H, padding_W). Otherwise, the
            padding_D = padding_H = padding_W = padding. Default: padding = 0.
3888
        stride(int|tuple): The stride size. If stride is a tuple, it must
3889 3890
            contain three integers, (stride_D, stride_H, stride_W). Otherwise, the
            stride_D = stride_H = stride_W = stride. Default: stride = 1.
3891
        dilation(int|tuple): The dilation size. If dilation is a tuple, it must
3892 3893 3894
            contain three integers, (dilation_D, dilation_H, dilation_W). Otherwise, the
            dilation_D = dilation_H = dilation_W = dilation. Default: dilation = 1.
        groups(int): The groups number of the Conv3d transpose layer. Inspired by
3895 3896 3897 3898 3899
            grouped convolution in Alex Krizhevsky's Deep CNN paper, in which
            when group=2, the first half of the filters is only connected to the
            first half of the input channels, while the second half of the
            filters is only connected to the second half of the input channels.
            Default: groups=1
C
chengduo 已提交
3900 3901 3902 3903 3904 3905 3906 3907 3908
        param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights
            of conv3d_transpose. If it is set to None or one attribute of ParamAttr, conv3d_transpose
            will create ParamAttr as param_attr. If the Initializer of the param_attr
            is not set, the parameter is initialized with Xavier. Default: None.
        bias_attr (ParamAttr|bool|None): The parameter attribute for the bias of conv3d_transpose.
            If it is set to False, no bias will be added to the output units.
            If it is set to None or one attribute of ParamAttr, conv3d_transpose
            will create ParamAttr as bias_attr. If the Initializer of the bias_attr
            is not set, the bias is initialized zero. Default: None.
3909 3910
        use_cudnn(bool): Use cudnn kernel or not, it is valid only when the cudnn
            library is installed. Default: True
C
chengduo 已提交
3911 3912
        act (str): Activation type, if it is set to None, activation is not appended.
            Default: None.
3913 3914
        name(str|None): A name for this layer(optional). If set None, the layer
            will be named automatically.
Y
Yu Yang 已提交
3915 3916

    Returns:
3917
        Variable: The tensor variable storing the convolution transpose result.
3918 3919

    Raises:
3920 3921
        ValueError: If the shapes of input, filter_size, stride, padding and
                    groups mismatch.
3922 3923 3924 3925

    Examples:
       .. code-block:: python

3926 3927
          data = fluid.layers.data(name='data', shape=[3, 12, 32, 32], dtype='float32')
          conv3d_transpose = fluid.layers.conv3d_transpose(input=data, num_filters=2, filter_size=3)
Y
Yu Yang 已提交
3928
    """
C
chengduo 已提交
3929
    assert param_attr is not False, "param_attr should not be False in conv3d_transpose."
3930 3931
    l_type = "conv3d_transpose"
    helper = LayerHelper(l_type, **locals())
Y
Yu Yang 已提交
3932
    if not isinstance(input, Variable):
3933
        raise TypeError("Input of conv3d_transpose must be Variable")
Y
Yu Yang 已提交
3934 3935
    input_channel = input.shape[1]

3936 3937 3938
    padding = utils.convert_to_list(padding, 3, 'padding')
    stride = utils.convert_to_list(stride, 3, 'stride')
    dilation = utils.convert_to_list(dilation, 3, 'dilation')
C
chengduoZH 已提交
3939

C
chengduoZH 已提交
3940 3941 3942
    if not isinstance(use_cudnn, bool):
        raise ValueError("use_cudnn should be True or False")

Y
Yu Yang 已提交
3943 3944 3945 3946 3947 3948
    if filter_size is None:
        if output_size is None:
            raise ValueError("output_size must be set when filter_size is None")
        if isinstance(output_size, int):
            output_size = [output_size, output_size]

3949 3950 3951
        d_in = input.shape[2]
        h_in = input.shape[3]
        w_in = input.shape[4]
C
chengduoZH 已提交
3952

3953
        filter_size_d = (output_size[0] - (d_in - 1) * stride[0] + 2 *
M
minqiyang 已提交
3954
                         padding[0] - 1) // dilation[0] + 1
3955
        filter_size_h = (output_size[1] - (h_in - 1) * stride[1] + 2 *
M
minqiyang 已提交
3956
                         padding[1] - 1) // dilation[1] + 1
3957
        filter_size_w = (output_size[2] - (w_in - 1) * stride[2] + 2 *
M
minqiyang 已提交
3958
                         padding[2] - 1) // dilation[2] + 1
3959
        filter_size = [filter_size_d, filter_size_h, filter_size_w]
C
chengduoZH 已提交
3960
    else:
3961 3962
        filter_size = utils.convert_to_list(filter_size, 3,
                                            'conv3d_transpose.filter_size')
Y
Yu Yang 已提交
3963

3964
    groups = 1 if groups is None else groups
M
minqiyang 已提交
3965
    filter_shape = [input_channel, num_filters // groups] + filter_size
Y
Yu Yang 已提交
3966 3967 3968
    img_filter = helper.create_parameter(
        dtype=input.dtype, shape=filter_shape, attr=helper.param_attr)

X
Xin Pan 已提交
3969
    pre_bias = helper.create_variable_for_type_inference(dtype=input.dtype)
Y
Yu Yang 已提交
3970
    helper.append_op(
3971
        type=l_type,
Y
Yu Yang 已提交
3972 3973
        inputs={'Input': [input],
                'Filter': [img_filter]},
3974
        outputs={'Output': pre_bias},
C
chengduoZH 已提交
3975 3976 3977 3978
        attrs={
            'strides': stride,
            'paddings': padding,
            'dilations': dilation,
3979
            'groups': groups,
C
chengduoZH 已提交
3980 3981
            'use_cudnn': use_cudnn
        })
Y
Yu Yang 已提交
3982

3983 3984
    pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2)
    out = helper.append_activation(pre_act)
Y
Yu Yang 已提交
3985
    return out
Y
yangyaming 已提交
3986 3987


Y
yangyaming 已提交
3988
def sequence_expand(x, y, ref_level=-1, name=None):
3989
    """Sequence Expand Layer. This layer will expand the input variable **x**
Y
yangyaming 已提交
3990 3991 3992 3993
    according to specified level lod of **y**. Please note that lod level of
    **x** is at most 1 and rank of **x** is at least 2. When rank of **x**
    is greater than 2, then it would be viewed as a 2-D tensor.
    Following examples will explain how sequence_expand works:
3994 3995 3996 3997 3998

    .. code-block:: text

        * Case 1
            x is a LoDTensor:
3999
                x.lod  = [[2,        2]]
Y
yangyaming 已提交
4000
                x.data = [[a], [b], [c], [d]]
4001 4002 4003
                x.dims = [4, 1]

            y is a LoDTensor:
4004 4005
                y.lod = [[2,    2],
                         [3, 3, 1, 1]]
4006

Y
yangyaming 已提交
4007
            ref_level: 0
4008

Y
yangyaming 已提交
4009
            then output is a 1-level LoDTensor:
4010
                out.lod =  [[2,        2,        2,        2]]
Y
yangyaming 已提交
4011
                out.data = [[a], [b], [a], [b], [c], [d], [c], [d]]
4012 4013 4014 4015
                out.dims = [8, 1]

        * Case 2
            x is a Tensor:
Y
yangyaming 已提交
4016
                x.data = [[a], [b], [c]]
4017 4018 4019
                x.dims = [3, 1]

            y is a LoDTensor:
4020
                y.lod = [[2, 0, 3]]
4021

Y
yangyaming 已提交
4022
            ref_level: -1
4023

Y
yangyaming 已提交
4024 4025 4026
            then output is a Tensor:
                out.data = [[a], [a], [c], [c], [c]]
                out.dims = [5, 1]
4027 4028 4029
    Args:
        x (Variable): The input variable which is a Tensor or LoDTensor.
        y (Variable): The input variable which is a LoDTensor.
Y
yangyaming 已提交
4030 4031
        ref_level (int): Lod level of `y` to be referred by `x`. If set to -1,
                         refer the last level of lod.
C
caoying03 已提交
4032
        name(str|None): A name for this layer(optional). If set None, the layer
Y
yangyaming 已提交
4033
                        will be named automatically.
4034 4035 4036 4037 4038 4039

    Returns:
        Variable: The expanded variable which is a LoDTensor.

    Examples:
        .. code-block:: python
4040 4041
	
            import paddle.fluid.layers as layers
4042 4043 4044
            x = fluid.layers.data(name='x', shape=[10], dtype='float32')
            y = fluid.layers.data(name='y', shape=[10, 20],
                             dtype='float32', lod_level=1)
Y
yangyaming 已提交
4045
            out = layers.sequence_expand(x=x, y=y, ref_level=0)
4046
    """
L
lujun 已提交
4047
    assert not in_dygraph_mode(), (
4048
        "sequence layer is not supported in dygraph mode yet.")
Y
yangyaming 已提交
4049
    helper = LayerHelper('sequence_expand', input=x, **locals())
4050
    dtype = helper.input_dtype()
X
Xin Pan 已提交
4051
    tmp = helper.create_variable_for_type_inference(dtype)
4052
    helper.append_op(
Y
yangyaming 已提交
4053 4054 4055 4056 4057
        type='sequence_expand',
        inputs={'X': x,
                'Y': y},
        outputs={'Out': tmp},
        attrs={'ref_level': ref_level})
4058
    return tmp
4059 4060


C
chengduo 已提交
4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108
def sequence_expand_as(x, y, name=None):
    """Sequence Expand As Layer. This layer will expand the input variable **x**
    according to the zeroth level lod of **y**. Current implementation requires
    the level number of Input(Y)'s lod must be 1, and the first dimension of
    Input(X) should be equal to the size of Input(Y)'s zeroth level lod, and
    lod of Input(X) is not considered.

    Following examples will explain how sequence_expand_as works:

    .. code-block:: text

        * Case 1:

            Given a 1-level LoDTensor input(X)
                X.data = [[a], [b], [c], [d]]
                X.dims = [4, 1]
            and input(Y)
                Y.lod = [[0, 3, 6, 7, 8]]
            ref_level: 0
            then we get 1-level LoDTensor
                Out.lod =  [[0,            3,              6,  7,  8]]
                Out.data = [[a], [a], [a], [b], [b], [b], [c], [d]]
                Out.dims = [8, 1]

        * Case 2:

            Given a common Tensor input(X)
                X.data = [[a, b], [c, d], [e, f]]
                X.dims = [3, 2]
            and input(Y)
                Y.lod = [[0, 2, 3, 6]]
            ref_level: 0
            then we get a common LoDTensor
                Out.lod =  [[0,             2,     3,                    6]]
                Out.data = [[a, b], [a, b] [c, d], [e, f], [e, f], [e, f]]
                Out.dims = [6, 2]

    Args:
        x (Variable): The input variable which is a Tensor or LoDTensor.
        y (Variable): The input variable which is a LoDTensor.
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.

    Returns:
        Variable: The expanded variable which is a LoDTensor.

    Examples:
        .. code-block:: python
4109
            import paddle.fluid.layers as layers
C
chengduo 已提交
4110 4111 4112 4113 4114 4115

            x = fluid.layers.data(name='x', shape=[10], dtype='float32')
            y = fluid.layers.data(name='y', shape=[10, 20],
                             dtype='float32', lod_level=1)
            out = layers.sequence_expand_as(x=x, y=y)
    """
L
lujun 已提交
4116
    assert not in_dygraph_mode(), (
4117
        "sequence layer is not supported in dygraph mode yet.")
C
chengduo 已提交
4118 4119
    helper = LayerHelper('sequence_expand_as', input=x, **locals())
    dtype = helper.input_dtype()
X
Xin Pan 已提交
4120
    tmp = helper.create_variable_for_type_inference(dtype)
C
chengduo 已提交
4121 4122 4123 4124 4125 4126 4127 4128
    helper.append_op(
        type='sequence_expand_as',
        inputs={'X': x,
                'Y': y},
        outputs={'Out': tmp})
    return tmp


F
fengjiayi 已提交
4129
@templatedoc()
4130
def sequence_pad(x, pad_value, maxlen=None, name=None):
F
fengjiayi 已提交
4131 4132 4133 4134 4135
    """
    ${comment}

    Args:
        x(Variable): Input variable which should contain lod information.
M
minqiyang 已提交
4136 4137 4138
        pad_value(Variable): The Variable that holds values that will be fill
            into padded steps. It can be a scalar or a tensor whose shape
            equals to time steps in sequences. If it's a scalar, it will be
F
fengjiayi 已提交
4139
            automatically broadcasted to the shape of time step.
M
minqiyang 已提交
4140 4141 4142 4143
        maxlen(int, default None): The length of padded sequences. It can be
            None or any positive int. When it is None, all sequences will be
            padded up to the length of the longest one among them; when it a
            certain positive value, it must be greater than the length of the
4144 4145 4146
            longest original sequence.
        name(str|None): A name for this layer(optional). If set None, the layer
            will be named automatically.
M
minqiyang 已提交
4147

F
fengjiayi 已提交
4148
    Returns:
M
minqiyang 已提交
4149
        Variable: The padded sequence batch and the original lengths before
4150
                  padding. All sequences has the same length.
M
minqiyang 已提交
4151

F
fengjiayi 已提交
4152 4153 4154 4155 4156 4157 4158
    Examples:
        .. code-block:: python

            import numpy

            x = fluid.layers.data(name='y', shape=[10, 5],
                             dtype='float32', lod_level=1)
G
gmcather 已提交
4159
            pad_value = fluid.layers.assign(
D
dongzhihong 已提交
4160
                input=numpy.array([0.0], dtype=numpy.float32))
F
fengjiayi 已提交
4161 4162 4163
            out = fluid.layers.sequence_pad(x=x, pad_value=pad_value)
    """

L
lujun 已提交
4164
    assert not in_dygraph_mode(), (
4165
        "sequence layer is not supported in dygraph mode yet.")
F
fengjiayi 已提交
4166 4167
    helper = LayerHelper('sequence_pad', input=x, **locals())
    dtype = helper.input_dtype()
X
Xin Pan 已提交
4168 4169
    out = helper.create_variable_for_type_inference(dtype)
    length = helper.create_variable_for_type_inference(dtype)
4170 4171 4172 4173

    pad_value.stop_gradient = True
    length.stop_gradient = True

F
fengjiayi 已提交
4174 4175 4176 4177 4178 4179
    if maxlen is None:
        maxlen = -1
    helper.append_op(
        type='sequence_pad',
        inputs={'X': x,
                'PadValue': pad_value},
4180 4181
        outputs={'Out': out,
                 'Length': length},
F
fengjiayi 已提交
4182
        attrs={'padded_length': maxlen})
4183
    return out, length
F
fengjiayi 已提交
4184 4185


4186
def sequence_unpad(x, length, name=None):
Y
Yibing Liu 已提交
4187
    """
4188
    **Sequence Unpad Layer**
Y
Yibing Liu 已提交
4189

4190 4191
    This layer removes the padding data in the input sequences and convert
    them into sequences with actual length as output, identitied by lod
Y
Yibing Liu 已提交
4192 4193 4194 4195 4196 4197 4198 4199 4200
    information.

    .. code-block:: text

	Example:

	Given input Variable **x**:
	    x.data = [[ 1.0,  2.0,  3.0,  4.0,  5.0],
		      [ 6.0,  7.0,  8.0,  9.0, 10.0],
4201 4202 4203
		      [11.0, 12.0, 13.0, 14.0, 15.0]],

	in which there are 3 sequences padded to length 5, and the acutal length
4204
	specified by input Variable **length**:
Y
Yibing Liu 已提交
4205 4206 4207 4208 4209 4210

	    length.data = [[2], [3], [4]],

	after unpadding, the output Variable will be:

	    out.data = [[1.0, 2.0, 6.0, 7.0, 8.0, 11.0, 12.0, 13.0, 14.0]]
4211
	    out.lod = [[2, 3, 4]]
Y
Yibing Liu 已提交
4212 4213 4214 4215 4216 4217

    Args:
        x(Variable): Input Variable which contains the padded sequences with
            equal length.
        length(Variable): The Variable that specifies the actual ength of
            sequences after unpadding.
4218 4219
        name(str|None): A name for this layer(optional). If set None, the layer
            will be named automatically.
Y
Yibing Liu 已提交
4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231

    Returns:
        Variable: The Variable contains the unpadded sequences.

    Examples:
        .. code-block:: python

            x = fluid.layers.data(name='x', shape=[10, 5], dtype='float32')
            len = fluid.layers.data(name='length', shape=[1], dtype='int64')
            out = fluid.layers.sequence_unpad(x=x, length=len)
    """

L
lujun 已提交
4232
    assert not in_dygraph_mode(), (
4233
        "sequence layer is not supported in dygraph mode yet.")
Y
Yibing Liu 已提交
4234 4235
    helper = LayerHelper('sequence_unpad', input=x, **locals())
    dtype = helper.input_dtype()
X
Xin Pan 已提交
4236
    out = helper.create_variable_for_type_inference(dtype)
Y
Yibing Liu 已提交
4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247

    length.stop_gradient = True

    helper.append_op(
        type='sequence_unpad',
        inputs={'X': x,
                'Length': length},
        outputs={'Out': out})
    return out


4248 4249 4250 4251 4252 4253 4254
def beam_search(pre_ids,
                pre_scores,
                ids,
                scores,
                beam_size,
                end_id,
                level=0,
4255
                is_accumulated=True,
4256 4257
                name=None,
                return_parent_idx=False):
4258
    """
4259 4260
    Beam search is a classical algorithm for selecting candidate words in a
    machine translation task.
Y
Yan Chunwei 已提交
4261 4262 4263

    Refer to `Beam search <https://en.wikipedia.org/wiki/Beam_search>`_
    for more details.
M
minqiyang 已提交
4264 4265

    This layer does the search in beams for one time step. Specifically, it
4266 4267 4268
    selects the top-K candidate word ids of current step from :attr:`ids`
    according to their :attr:`scores` for all source sentences, where K is
    :attr:`beam_size` and :attr:`ids, scores` are predicted results from the
4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279
    computation cell. If :attr:`ids` is not set, it will be calculated out
    according to :attr:`scores`. Additionally, :attr:`pre_ids` and
    :attr:`pre_scores` are the output of beam_search at previous step, they
    are needed for special use to handle ended candidate translations.

    Note that if :attr:`is_accumulated` is :attr:`True`, the :attr:`scores`
    passed in should be accumulated scores. Else, the :attr:`scores` are
    considered as the straightforward scores and will be transformed to the
    log field and accumulated the :attr:`pre_scores` in this operator.
    Length penalty should be done with extra operators before calculating the
    accumulated scores if needed.
4280 4281 4282 4283

    Please see the following demo for a fully beam search usage example:

        fluid/tests/book/test_machine_translation.py
Y
Yan Chunwei 已提交
4284

4285
    Args:
4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308
        pre_ids(Variable): The LodTensor variable which is the output of
            beam_search at previous step. It should be a LodTensor with shape
            :math:`(batch_size, 1)` and lod
            :math:`[[0, 1, ... , batch_size], [0, 1, ..., batch_size]]` at the
            first step.
        pre_scores(Variable): The LodTensor variable which is the output of
            beam_search at previous step.
        ids(Variable): The LodTensor variable containing the candidates ids.
            Its shape should be :math:`(batch_size \\times beam_size, K)`,
            where :math:`K` supposed to be :attr:`beam_size`.
        scores(Variable): The LodTensor variable containing the accumulated
            scores corresponding to :attr:`ids` and its shape is the same as
            the shape of :attr:`ids`.
        beam_size(int): The beam width used in beam search.
        end_id(int): The id of end token.
        level(int, default 0): It can be ignored and mustn't change currently.
            It means the source level of lod, which is explained as following.
            The lod level of :attr:`ids` should be 2. The first level is source
            level which describes how many prefixes (branchs) for each source
            sentece (beam), and the second level is sentence level which
            describes how these candidates belong to the prefix. The paths
            linking prefixes and selected candidates are organized and reserved
            in lod.
4309 4310
        is_accumulated(bool, default True): Whether the input :attr:`score` is
             accumulated scores.
4311 4312
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.
4313 4314 4315 4316
        return_parent_idx(bool): Whether to return an extra Tensor variable 
                        preserving the selected_ids' parent indice in pre_ids
                        in output, which can be used to gather cell states at
                        the next time step.
F
fengjiayi 已提交
4317

4318
    Returns:
4319 4320 4321 4322
        Variable: The LodTensor tuple containing the selected ids and the \
            corresponding scores. If :attr:`return_parent_idx` is :attr:`True`, \
            an extra Tensor variable preserving the selected_ids' parent indice \
            is included.
Y
Yan Chunwei 已提交
4323 4324 4325 4326

    Examples:
        .. code-block:: python

4327 4328
            import paddle.fluid as fluid

4329 4330 4331
            # Suppose `probs` contains predicted results from the computation
            # cell and `pre_ids` and `pre_scores` is the output of beam_search
            # at previous step.
4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343
            beam_size = 4
            end_id = 1
            pre_ids = fluid.layers.data(
                name='pre_id', shape=[1], lod_level=2, dtype='int64')
            pre_scores = fluid.layers.data(
                name='pre_scores', shape=[1], lod_level=2, dtype='float32')
            probs = fluid.layers.data(
                name='probs', shape=[10000], dtype='float32')
            topk_scores, topk_indices = fluid.layers.topk(probs, k=beam_size)
            accu_scores = fluid.layers.elementwise_add(
                x=fluid.layers.log(x=topk_scores),
                y=fluid.layers.reshape(pre_scores, shape=[-1]),
4344
                axis=0)
4345
            selected_ids, selected_scores = fluid.layers.beam_search(
4346 4347 4348 4349 4350 4351 4352
                pre_ids=pre_ids,
                pre_scores=pre_scores,
                ids=topk_indices,
                scores=accu_scores,
                beam_size=beam_size,
                end_id=end_id)
    """
Q
Qiao Longfei 已提交
4353
    helper = LayerHelper('beam_search', **locals())
4354 4355 4356 4357 4358 4359
    score_type = pre_scores.dtype
    id_type = pre_ids.dtype

    inputs = {"pre_ids": pre_ids, "pre_scores": pre_scores, "scores": scores}
    if ids is not None:
        inputs["ids"] = ids
Q
Qiao Longfei 已提交
4360

X
Xin Pan 已提交
4361 4362 4363
    selected_scores = helper.create_variable_for_type_inference(
        dtype=score_type)
    selected_ids = helper.create_variable_for_type_inference(dtype=id_type)
4364 4365 4366 4367 4368
    # parent_idx is a tensor used to gather cell states at the next time
    # step. Though lod in selected_ids can also be used to gather by
    # sequence_expand, it is not efficient.
    # gather_op's index input only supports int32 dtype currently
    parent_idx = helper.create_variable_for_type_inference(dtype="int32")
Q
Qiao Longfei 已提交
4369 4370 4371

    helper.append_op(
        type='beam_search',
4372
        inputs=inputs,
Q
Qiao Longfei 已提交
4373 4374 4375
        outputs={
            'selected_ids': selected_ids,
            'selected_scores': selected_scores,
4376
            'parent_idx': parent_idx
Q
Qiao Longfei 已提交
4377 4378 4379 4380 4381 4382
        },
        attrs={
            # TODO(ChunweiYan) to assure other value support
            'level': level,
            'beam_size': beam_size,
            'end_id': end_id,
4383
            'is_accumulated': is_accumulated,
Q
Qiao Longfei 已提交
4384
        })
4385 4386 4387 4388
    if return_parent_idx:
        return selected_ids, selected_scores, parent_idx
    else:
        return selected_ids, selected_scores
Q
Qiao Longfei 已提交
4389 4390


4391 4392 4393 4394 4395 4396 4397
def beam_search_decode(ids, scores, beam_size, end_id, name=None):
    """
    Beam Search Decode Layer. This layer constructs the full hypotheses for
    each source sentence by walking back along the LoDTensorArray :attr:`ids`
    whose lods can be used to restore the path in the beam search tree.
    Please see the following demo for a fully beam search usage example:
        fluid/tests/book/test_machine_translation.py
G
guosheng 已提交
4398

4399 4400 4401 4402 4403 4404 4405 4406 4407
    Args:
        ids(Variable): The LodTensorArray variable containing the selected ids
            of all steps.
        scores(Variable): The LodTensorArray variable containing the selected
            scores of all steps.
        beam_size(int): The beam width used in beam search.
        end_id(int): The id of end token.
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.
G
guosheng 已提交
4408

4409 4410 4411 4412 4413 4414
    Returns:
        Variable: The LodTensor pair containing the generated id sequences \
            and the corresponding scores. The shapes and lods of the two \
            LodTensor are same. The lod level is 2 and the two levels \
            separately indicate how many hypotheses each source sentence has \
            and how many ids each hypothesis has.
G
guosheng 已提交
4415

4416 4417
    Examples:
        .. code-block:: python
T
Tink_Y 已提交
4418

4419 4420
            import paddle.fluid as fluid

4421 4422
            # Suppose `ids` and `scores` are LodTensorArray variables reserving
            # the selected ids and scores of all steps
4423 4424 4425
            ids = fluid.layers.create_array(dtype='int64')
            scores = fluid.layers.create_array(dtype='float32')
            finished_ids, finished_scores = fluid.layers.beam_search_decode(
4426 4427 4428
                ids, scores, beam_size=5, end_id=0)
    """
    helper = LayerHelper('beam_search_decode', **locals())
X
Xin Pan 已提交
4429 4430
    sentence_ids = helper.create_variable_for_type_inference(dtype=ids.dtype)
    sentence_scores = helper.create_variable_for_type_inference(dtype=ids.dtype)
4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445

    helper.append_op(
        type="beam_search_decode",
        inputs={"Ids": ids,
                "Scores": scores},
        outputs={
            "SentenceIds": sentence_ids,
            "SentenceScores": sentence_scores
        },
        attrs={"beam_size": beam_size,
               "end_id": end_id})

    return sentence_ids, sentence_scores


Y
yangyaming 已提交
4446 4447 4448 4449
def lstm_unit(x_t,
              hidden_t_prev,
              cell_t_prev,
              forget_bias=0.0,
Y
yangyaming 已提交
4450
              param_attr=None,
C
caoying03 已提交
4451 4452
              bias_attr=None,
              name=None):
Y
yangyaming 已提交
4453 4454 4455 4456
    """Lstm unit layer. The equation of a lstm step is:

        .. math::

4457
            i_t & = \sigma(W_{x_i}x_{t} + W_{h_i}h_{t-1} + b_i)
Y
yangyaming 已提交
4458

4459
            f_t & = \sigma(W_{x_f}x_{t} + W_{h_f}h_{t-1} + b_f)
Y
yangyaming 已提交
4460

4461
            c_t & = f_tc_{t-1} + i_t tanh (W_{x_c}x_t + W_{h_c}h_{t-1} + b_c)
Y
yangyaming 已提交
4462

4463
            o_t & = \sigma(W_{x_o}x_{t} + W_{h_o}h_{t-1} + b_o)
Y
yangyaming 已提交
4464 4465 4466

            h_t & = o_t tanh(c_t)

4467 4468 4469 4470 4471 4472
    The inputs of lstm unit include :math:`x_t`, :math:`h_{t-1}` and
    :math:`c_{t-1}`. The 2nd dimensions of :math:`h_{t-1}` and :math:`c_{t-1}`
    should be same. The implementation separates the linear transformation and
    non-linear transformation apart. Here, we take :math:`i_t` as an example.
    The linear transformation is applied by calling a `fc` layer and the
    equation is:
Y
yangyaming 已提交
4473 4474 4475

        .. math::

4476
            L_{i_t} = W_{x_i}x_{t} + W_{h_i}h_{t-1} + b_i
Y
yangyaming 已提交
4477 4478 4479 4480 4481 4482 4483 4484

    The non-linear transformation is applied by calling `lstm_unit_op` and the
    equation is:

        .. math::

            i_t = \sigma(L_{i_t})

Y
yangyaming 已提交
4485
    This layer has two outputs including :math:`h_t` and :math:`o_t`.
Y
yangyaming 已提交
4486 4487

    Args:
Y
yangyaming 已提交
4488 4489 4490 4491 4492 4493
        x_t (Variable): The input value of current step, a 2-D tensor with shape
            M x N, M for batch size and N for input size.
        hidden_t_prev (Variable): The hidden value of lstm unit, a 2-D tensor
            with shape M x S, M for batch size and S for size of lstm unit.
        cell_t_prev (Variable): The cell value of lstm unit, a 2-D tensor with
            shape M x S, M for batch size and S for size of lstm unit.
Y
yangyaming 已提交
4494
        forget_bias (float): The forget bias of lstm unit.
C
chengduo 已提交
4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506
        param_attr(ParamAttr|None): The parameter attribute for the learnable
                               hidden-hidden weights.
                               If it is set to None or one attribute of ParamAttr,
                               lstm_unit will create ParamAttr as param_attr.
                               If the Initializer of the param_attr is not set, the
                               parameter is initialized with Xavier. Default: None.
        bias_attr (ParamAttr|None): The bias attribute for the learnable bias
                              weights. If it is set to False, no bias will be added
                              to the output units. If it is set to None or one attribute of ParamAttr,
                              lstm_unit will create ParamAttr as bias_attr.
                              If the Initializer of the bias_attr is not set,
                              the bias is initialized zero. Default: None.
C
caoying03 已提交
4507 4508
        name(str|None): A name for this layer(optional). If set None, the layer
                       will be named automatically.
Y
yangyaming 已提交
4509 4510

    Returns:
Y
yangyaming 已提交
4511
        tuple: The hidden value and cell value of lstm unit.
Y
yangyaming 已提交
4512 4513

    Raises:
4514 4515 4516 4517
        ValueError: The ranks of **x_t**, **hidden_t_prev** and **cell_t_prev**
                    not be 2 or the 1st dimensions of **x_t**, **hidden_t_prev**
                    and **cell_t_prev** not be the same or the 2nd dimensions of
                    **hidden_t_prev** and **cell_t_prev** not be the same.
Y
yangyaming 已提交
4518 4519 4520 4521 4522

    Examples:

        .. code-block:: python

4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535
            import paddle.fluid as fluid

            dict_dim, emb_dim, hidden_dim = 128, 64, 512
            data = fluid.layers.data(name='step_data', shape=[1], dtype='int32')
            x = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim])
            pre_hidden = fluid.layers.data(
                name='pre_hidden', shape=[hidden_dim], dtype='float32')
            pre_cell = fluid.layers.data(
                name='pre_cell', shape=[hidden_dim], dtype='float32')
            hidden = fluid.layers.lstm_unit(
                x_t=x,
                hidden_t_prev=pre_hidden,
                cell_t_prev=pre_cell)
Y
yangyaming 已提交
4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549
    """
    helper = LayerHelper('lstm_unit', **locals())

    if len(x_t.shape) != 2:
        raise ValueError("Rank of x_t must be 2.")

    if len(hidden_t_prev.shape) != 2:
        raise ValueError("Rank of hidden_t_prev must be 2.")

    if len(cell_t_prev.shape) != 2:
        raise ValueError("Rank of cell_t_prev must be 2.")

    if x_t.shape[0] != hidden_t_prev.shape[0] or x_t.shape[
            0] != cell_t_prev.shape[0]:
Y
yangyaming 已提交
4550
        raise ValueError("The 1st dimensions of x_t, hidden_t_prev and "
4551 4552 4553 4554
                         "cell_t_prev must be the same.")

    if hidden_t_prev.shape[1] != cell_t_prev.shape[1]:
        raise ValueError("The 2nd dimensions of hidden_t_prev and "
Y
yangyaming 已提交
4555 4556
                         "cell_t_prev must be the same.")

Y
yangyaming 已提交
4557 4558 4559
    if bias_attr is None:
        bias_attr = ParamAttr()

Y
yangyaming 已提交
4560
    size = cell_t_prev.shape[1]
4561
    concat_out = concat(input=[x_t, hidden_t_prev], axis=1)
Y
yangyaming 已提交
4562 4563
    fc_out = fc(input=concat_out,
                size=4 * size,
Y
yangyaming 已提交
4564
                param_attr=param_attr,
4565
                bias_attr=bias_attr)
Y
yangyaming 已提交
4566
    dtype = x_t.dtype
X
Xin Pan 已提交
4567 4568
    c = helper.create_variable_for_type_inference(dtype)
    h = helper.create_variable_for_type_inference(dtype)
Y
yangyaming 已提交
4569 4570 4571 4572 4573 4574 4575 4576 4577

    helper.append_op(
        type='lstm_unit',
        inputs={"X": fc_out,
                "C_prev": cell_t_prev},
        outputs={"C": c,
                 "H": h},
        attrs={"forget_bias": forget_bias})

Y
yangyaming 已提交
4578
    return h, c
G
guosheng 已提交
4579 4580


C
caoying03 已提交
4581
def reduce_sum(input, dim=None, keep_dim=False, name=None):
G
guosheng 已提交
4582
    """
Y
yangyaming 已提交
4583
    Computes the sum of tensor elements over the given dimension.
G
guosheng 已提交
4584 4585 4586

    Args:
        input (Variable): The input variable which is a Tensor or LoDTensor.
W
whs 已提交
4587
        dim (list|int|None): The dimensions along which the sum is performed. If
Y
yangyaming 已提交
4588 4589
            :attr:`None`, sum all elements of :attr:`input` and return a
            Tensor variable with a single element, otherwise must be in the
W
whs 已提交
4590 4591
            range :math:`[-rank(input), rank(input))`. If :math:`dim[i] < 0`,
            the dimension to reduce is :math:`rank + dim[i]`.
4592
        keep_dim (bool|False): Whether to reserve the reduced dimension in the
Y
yangyaming 已提交
4593
            output Tensor. The result tensor will have one fewer dimension
G
guosheng 已提交
4594
            than the :attr:`input` unless :attr:`keep_dim` is true.
C
caoying03 已提交
4595 4596
        name(str|None): A name for this layer(optional). If set None, the layer
                       will be named automatically.
G
guosheng 已提交
4597 4598 4599

    Returns:
        Variable: The reduced Tensor variable.
F
fengjiayi 已提交
4600

G
guosheng 已提交
4601 4602 4603 4604 4605 4606
    Examples:
        .. code-block:: python

            # x is a Tensor variable with following elements:
            #    [[0.2, 0.3, 0.5, 0.9]
            #     [0.1, 0.2, 0.6, 0.7]]
Q
qiaolongfei 已提交
4607
            # Each example is followed by the corresponding output tensor.
G
guosheng 已提交
4608 4609 4610 4611
            fluid.layers.reduce_sum(x)  # [3.5]
            fluid.layers.reduce_sum(x, dim=0)  # [0.3, 0.5, 1.1, 1.6]
            fluid.layers.reduce_sum(x, dim=-1)  # [1.9, 1.6]
            fluid.layers.reduce_sum(x, dim=1, keep_dim=True)  # [[1.9], [1.6]]
W
whs 已提交
4612 4613 4614 4615

            # x is a Tensor variable with shape [2, 2, 2] and elements as below:
            #      [[[1, 2], [3, 4]],
            #      [[5, 6], [7, 8]]]
Q
qiaolongfei 已提交
4616
            # Each example is followed by the corresponding output tensor.
W
whs 已提交
4617 4618 4619
            fluid.layers.reduce_sum(x, dim=[1, 2]) # [10, 26]
            fluid.layers.reduce_sum(x, dim=[0, 1]) # [16, 20]

G
guosheng 已提交
4620 4621
    """
    helper = LayerHelper('reduce_sum', **locals())
X
Xin Pan 已提交
4622
    out = helper.create_variable_for_type_inference(dtype=helper.input_dtype())
W
whs 已提交
4623 4624
    if dim is not None and not isinstance(dim, list):
        dim = [dim]
G
guosheng 已提交
4625 4626 4627 4628 4629
    helper.append_op(
        type='reduce_sum',
        inputs={'X': input},
        outputs={'Out': out},
        attrs={
W
whs 已提交
4630
            'dim': dim if dim != None else [0],
G
guosheng 已提交
4631 4632 4633 4634
            'keep_dim': keep_dim,
            'reduce_all': True if dim == None else False
        })
    return out
G
guosheng 已提交
4635 4636


C
caoying03 已提交
4637
def reduce_mean(input, dim=None, keep_dim=False, name=None):
G
guosheng 已提交
4638
    """
Y
Yibing Liu 已提交
4639
    Computes the mean of the input tensor's elements along the given dimension.
G
guosheng 已提交
4640 4641 4642

    Args:
        input (Variable): The input variable which is a Tensor or LoDTensor.
Y
Yibing Liu 已提交
4643 4644 4645
        dim (list|int|None): The dimension along which the mean is computed. If
            `None`, compute the mean over all elements of :attr:`input`
            and return a variable with a single element, otherwise it
Y
yangyaming 已提交
4646
            must be in the range :math:`[-rank(input), rank(input))`. If
4647
            :math:`dim[i] < 0`, the dimension to reduce is
Y
Yibing Liu 已提交
4648
            :math:`rank(input) + dim[i]`.
Y
yangyaming 已提交
4649 4650
        keep_dim (bool): Whether to reserve the reduced dimension in the
            output Tensor. The result tensor will have one fewer dimension
G
guosheng 已提交
4651
            than the :attr:`input` unless :attr:`keep_dim` is true.
Y
Yibing Liu 已提交
4652
        name(str|None): A name for this layer(optional). If set `None`, the layer
C
caoying03 已提交
4653
                       will be named automatically.
G
guosheng 已提交
4654 4655

    Returns:
Y
Yibing Liu 已提交
4656
        Variable: The reduced mean Variable.
F
fengjiayi 已提交
4657

G
guosheng 已提交
4658 4659 4660 4661 4662 4663 4664 4665 4666 4667
    Examples:
        .. code-block:: python

            # x is a Tensor variable with following elements:
            #    [[0.2, 0.3, 0.5, 0.9]
            #     [0.1, 0.2, 0.6, 0.7]]
            # Each example is followed by the correspending output tensor.
            fluid.layers.reduce_mean(x)  # [0.4375]
            fluid.layers.reduce_mean(x, dim=0)  # [0.15, 0.25, 0.55, 0.8]
            fluid.layers.reduce_mean(x, dim=-1)  # [0.475, 0.4]
F
stash  
fengjiayi 已提交
4668 4669
            fluid.layers.reduce_mean(
                x, dim=1, keep_dim=True)  # [[0.475], [0.4]]
W
whs 已提交
4670 4671 4672 4673 4674 4675 4676

            # x is a Tensor variable with shape [2, 2, 2] and elements as below:
            #      [[[1.0, 2.0], [3.0, 4.0]],
            #      [[5.0, 6.0], [7.0, 8.0]]]
            # Each example is followed by the correspending output tensor.
            fluid.layers.reduce_mean(x, dim=[1, 2]) # [2.5, 6.5]
            fluid.layers.reduce_mean(x, dim=[0, 1]) # [4.0, 5.0]
G
guosheng 已提交
4677 4678
    """
    helper = LayerHelper('reduce_mean', **locals())
X
Xin Pan 已提交
4679
    out = helper.create_variable_for_type_inference(dtype=helper.input_dtype())
W
whs 已提交
4680 4681
    if dim is not None and not isinstance(dim, list):
        dim = [dim]
G
guosheng 已提交
4682 4683 4684 4685 4686
    helper.append_op(
        type='reduce_mean',
        inputs={'X': input},
        outputs={'Out': out},
        attrs={
W
whs 已提交
4687
            'dim': dim if dim != None else [0],
G
guosheng 已提交
4688 4689 4690 4691
            'keep_dim': keep_dim,
            'reduce_all': True if dim == None else False
        })
    return out
4692 4693


C
caoying03 已提交
4694
def reduce_max(input, dim=None, keep_dim=False, name=None):
4695
    """
Y
yangyaming 已提交
4696
    Computes the maximum of tensor elements over the given dimension.
4697 4698 4699

    Args:
        input (Variable): The input variable which is a Tensor or LoDTensor.
W
whs 已提交
4700
        dim (list|int|None): The dimension along which the maximum is computed.
Y
yangyaming 已提交
4701 4702 4703
            If :attr:`None`, compute the maximum over all elements of
            :attr:`input` and return a Tensor variable with a single element,
            otherwise must be in the range :math:`[-rank(input), rank(input))`.
W
whs 已提交
4704
            If :math:`dim[i] < 0`, the dimension to reduce is :math:`rank + dim[i]`.
Y
yangyaming 已提交
4705 4706
        keep_dim (bool): Whether to reserve the reduced dimension in the
            output Tensor. The result tensor will have one fewer dimension
4707
            than the :attr:`input` unless :attr:`keep_dim` is true.
C
caoying03 已提交
4708 4709
        name(str|None): A name for this layer(optional). If set None, the layer
                       will be named automatically.
4710 4711 4712

    Returns:
        Variable: The reduced Tensor variable.
Y
yangyaming 已提交
4713

4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724
    Examples:
        .. code-block:: python

            # x is a Tensor variable with following elements:
            #    [[0.2, 0.3, 0.5, 0.9]
            #     [0.1, 0.2, 0.6, 0.7]]
            # Each example is followed by the correspending output tensor.
            fluid.layers.reduce_max(x)  # [0.9]
            fluid.layers.reduce_max(x, dim=0)  # [0.2, 0.3, 0.6, 0.9]
            fluid.layers.reduce_max(x, dim=-1)  # [0.9, 0.7]
            fluid.layers.reduce_max(x, dim=1, keep_dim=True)  # [[0.9], [0.7]]
W
whs 已提交
4725 4726 4727 4728 4729 4730 4731

            # x is a Tensor variable with shape [2, 2, 2] and elements as below:
            #      [[[1.0, 2.0], [3.0, 4.0]],
            #      [[5.0, 6.0], [7.0, 8.0]]]
            # Each example is followed by the correspending output tensor.
            fluid.layers.reduce_max(x, dim=[1, 2]) # [4.0, 8.0]
            fluid.layers.reduce_max(x, dim=[0, 1]) # [7.0, 8.0]
4732 4733
    """
    helper = LayerHelper('reduce_max', **locals())
X
Xin Pan 已提交
4734
    out = helper.create_variable_for_type_inference(dtype=helper.input_dtype())
W
whs 已提交
4735 4736
    if dim is not None and not isinstance(dim, list):
        dim = [dim]
4737 4738 4739 4740 4741
    helper.append_op(
        type='reduce_max',
        inputs={'X': input},
        outputs={'Out': out},
        attrs={
W
whs 已提交
4742
            'dim': dim if dim != None else [0],
4743 4744 4745 4746 4747 4748
            'keep_dim': keep_dim,
            'reduce_all': True if dim == None else False
        })
    return out


C
caoying03 已提交
4749
def reduce_min(input, dim=None, keep_dim=False, name=None):
4750
    """
Y
yangyaming 已提交
4751
    Computes the minimum of tensor elements over the given dimension.
4752 4753 4754

    Args:
        input (Variable): The input variable which is a Tensor or LoDTensor.
W
whs 已提交
4755
        dim (list|int|None): The dimensions along which the minimum is computed.
Y
yangyaming 已提交
4756 4757 4758
            If :attr:`None`, compute the minimum over all elements of
            :attr:`input` and return a Tensor variable with a single element,
            otherwise must be in the range :math:`[-rank(input), rank(input))`.
W
whs 已提交
4759
            If :math:`dim[i] < 0`, the dimension to reduce is :math:`rank + dim[i]`.
Y
yangyaming 已提交
4760 4761
        keep_dim (bool): Whether to reserve the reduced dimension in the
            output Tensor. The result tensor will have one fewer dimension
4762
            than the :attr:`input` unless :attr:`keep_dim` is true.
C
caoying03 已提交
4763 4764
        name(str|None): A name for this layer(optional). If set None, the layer
                       will be named automatically.
4765 4766 4767

    Returns:
        Variable: The reduced Tensor variable.
Y
yangyaming 已提交
4768

4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779
    Examples:
        .. code-block:: python

            # x is a Tensor variable with following elements:
            #    [[0.2, 0.3, 0.5, 0.9]
            #     [0.1, 0.2, 0.6, 0.7]]
            # Each example is followed by the correspending output tensor.
            fluid.layers.reduce_min(x)  # [0.1]
            fluid.layers.reduce_min(x, dim=0)  # [0.1, 0.2, 0.5, 0.7]
            fluid.layers.reduce_min(x, dim=-1)  # [0.2, 0.1]
            fluid.layers.reduce_min(x, dim=1, keep_dim=True)  # [[0.2], [0.1]]
W
whs 已提交
4780 4781 4782 4783 4784 4785 4786

            # x is a Tensor variable with shape [2, 2, 2] and elements as below:
            #      [[[1.0, 2.0], [3.0, 4.0]],
            #      [[5.0, 6.0], [7.0, 8.0]]]
            # Each example is followed by the correspending output tensor.
            fluid.layers.reduce_min(x, dim=[1, 2]) # [1.0, 5.0]
            fluid.layers.reduce_min(x, dim=[0, 1]) # [1.0, 2.0]
4787 4788
    """
    helper = LayerHelper('reduce_min', **locals())
X
Xin Pan 已提交
4789
    out = helper.create_variable_for_type_inference(dtype=helper.input_dtype())
W
whs 已提交
4790 4791
    if dim is not None and not isinstance(dim, list):
        dim = [dim]
4792 4793 4794 4795 4796
    helper.append_op(
        type='reduce_min',
        inputs={'X': input},
        outputs={'Out': out},
        attrs={
W
whs 已提交
4797
            'dim': dim if dim != None else [0],
4798 4799 4800 4801
            'keep_dim': keep_dim,
            'reduce_all': True if dim == None else False
        })
    return out
G
guosheng 已提交
4802 4803


4804 4805 4806 4807 4808 4809
def reduce_prod(input, dim=None, keep_dim=False, name=None):
    """
    Computes the product of tensor elements over the given dimension.

    Args:
        input (Variable): The input variable which is a Tensor or LoDTensor.
W
whs 已提交
4810
        dim (list|int|None): The dimensions along which the product is performed. If
4811 4812
            :attr:`None`, multipy all elements of :attr:`input` and return a
            Tensor variable with a single element, otherwise must be in the
W
whs 已提交
4813 4814
            range :math:`[-rank(input), rank(input))`. If :math:`dim[i] < 0`,
            the dimension to reduce is :math:`rank + dim[i]`.
4815 4816 4817
        keep_dim (bool|False): Whether to reserve the reduced dimension in the
            output Tensor. The result tensor will have one fewer dimension
            than the :attr:`input` unless :attr:`keep_dim` is true.
Y
yangyaming 已提交
4818
        name(str|None): A name for this layer(optional). If set None, the
Z
zhouhanqing 已提交
4819
            layer will be named automatically.
4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833

    Returns:
        Variable: The reduced Tensor variable.

    Examples:
        .. code-block:: python

            # x is a Tensor variable with following elements:
            #    [[0.2, 0.3, 0.5, 0.9]
            #     [0.1, 0.2, 0.6, 0.7]]
            # Each example is followed by the correspending output tensor.
            fluid.layers.reduce_prod(x)  # [0.0002268]
            fluid.layers.reduce_prod(x, dim=0)  # [0.02, 0.06, 0.3, 0.63]
            fluid.layers.reduce_prod(x, dim=-1)  # [0.027, 0.0084]
Y
yangyaming 已提交
4834
            fluid.layers.reduce_prod(x, dim=1,
Z
zhouhanqing 已提交
4835
                                     keep_dim=True)  # [[0.027], [0.0084]]
W
whs 已提交
4836 4837 4838 4839 4840 4841 4842

            # x is a Tensor variable with shape [2, 2, 2] and elements as below:
            #      [[[1.0, 2.0], [3.0, 4.0]],
            #      [[5.0, 6.0], [7.0, 8.0]]]
            # Each example is followed by the correspending output tensor.
            fluid.layers.reduce_prod(x, dim=[1, 2]) # [24.0, 1680.0]
            fluid.layers.reduce_prod(x, dim=[0, 1]) # [105.0, 384.0]
4843 4844
    """
    helper = LayerHelper('reduce_prod', **locals())
X
Xin Pan 已提交
4845
    out = helper.create_variable_for_type_inference(dtype=helper.input_dtype())
W
whs 已提交
4846 4847
    if dim is not None and not isinstance(dim, list):
        dim = [dim]
4848 4849 4850 4851 4852
    helper.append_op(
        type='reduce_prod',
        inputs={'X': input},
        outputs={'Out': out},
        attrs={
W
whs 已提交
4853
            'dim': dim if dim != None else [0],
4854 4855 4856 4857 4858 4859
            'keep_dim': keep_dim,
            'reduce_all': True if dim == None else False
        })
    return out


Z
zhoukunsheng 已提交
4860 4861
def reduce_all(input, dim=None, keep_dim=False, name=None):
    """
Z
zhoukunsheng 已提交
4862
    Computes the ``logical and`` of tensor elements over the given dimension.
Z
zhoukunsheng 已提交
4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881

    Args:
        input (Variable): The input variable which is a Tensor or LoDTensor.
        dim (list|int|None): The dimension along which the logical and is computed.
            If :attr:`None`, compute the logical and over all elements of
            :attr:`input` and return a Tensor variable with a single element,
            otherwise must be in the range :math:`[-rank(input), rank(input))`.
            If :math:`dim[i] < 0`, the dimension to reduce is :math:`rank + dim[i]`.
        keep_dim (bool): Whether to reserve the reduced dimension in the
            output Tensor. The result tensor will have one fewer dimension
            than the :attr:`input` unless :attr:`keep_dim` is true.
        name(str|None): A name for this layer(optional). If set None, the layer
                       will be named automatically.

    Returns:
        Variable: The reduced Tensor variable.

    Examples:
        .. code-block:: python
Z
zhoukunsheng 已提交
4882
        
Z
zhoukunsheng 已提交
4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911
            # x is a bool Tensor variable with following elements:
            #    [[True, False]
            #     [True, True]]
            # Each example is followed by the correspending output tensor.
            fluid.layers.reduce_all(x)  # False 
            fluid.layers.reduce_all(x, dim=0)  # [True, False]
            fluid.layers.reduce_all(x, dim=-1)  # [False, True]
            fluid.layers.reduce_all(x, dim=1,
                                     keep_dim=True)  # [[False], [True]]

    """
    helper = LayerHelper('reduce_all', **locals())
    out = helper.create_variable_for_type_inference(dtype=helper.input_dtype())
    if dim is not None and not isinstance(dim, list):
        dim = [dim]
    helper.append_op(
        type='reduce_all',
        inputs={'X': input},
        outputs={'Out': out},
        attrs={
            'dim': dim if dim != None else [0],
            'keep_dim': keep_dim,
            'reduce_all': True if dim == None else False
        })
    return out


def reduce_any(input, dim=None, keep_dim=False, name=None):
    """
Z
zhoukunsheng 已提交
4912
    Computes the ``logical or`` of tensor elements over the given dimension.
Z
zhoukunsheng 已提交
4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931

    Args:
        input (Variable): The input variable which is a Tensor or LoDTensor.
        dim (list|int|None): The dimension along which the logical or is computed.
            If :attr:`None`, compute the logical or over all elements of
            :attr:`input` and return a Tensor variable with a single element,
            otherwise must be in the range :math:`[-rank(input), rank(input))`.
            If :math:`dim[i] < 0`, the dimension to reduce is :math:`rank + dim[i]`.
        keep_dim (bool): Whether to reserve the reduced dimension in the
            output Tensor. The result tensor will have one fewer dimension
            than the :attr:`input` unless :attr:`keep_dim` is true.
        name(str|None): A name for this layer(optional). If set None, the layer
                       will be named automatically.

    Returns:
        Variable: The reduced Tensor variable.

    Examples:
        .. code-block:: python
Z
zhoukunsheng 已提交
4932

Z
zhoukunsheng 已提交
4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954
            # x is a bool Tensor variable with following elements:
            #    [[True, False]
            #     [False, False]]
            # Each example is followed by the correspending output tensor.
            fluid.layers.reduce_any(x)  # True
            fluid.layers.reduce_any(x, dim=0)  # [True, False]
            fluid.layers.reduce_any(x, dim=-1)  # [True, False]
            fluid.layers.reduce_any(x, dim=1,
                                     keep_dim=True)  # [[True], [False]]

    """
    helper = LayerHelper('reduce_any', **locals())
    out = helper.create_variable_for_type_inference(dtype=helper.input_dtype())
    if dim is not None and not isinstance(dim, list):
        dim = [dim]
    helper.append_op(
        type='reduce_any',
        inputs={'X': input},
        outputs={'Out': out},
        attrs={
            'dim': dim if dim != None else [0],
            'keep_dim': keep_dim,
4955 4956 4957 4958 4959
            'reduce_all': True if dim == None else False
        })
    return out


C
caoying03 已提交
4960
def split(input, num_or_sections, dim=-1, name=None):
G
guosheng 已提交
4961
    """
C
caoying03 已提交
4962
    Split the input tensor into multiple sub-tensors.
G
guosheng 已提交
4963 4964 4965

    Args:
        input (Variable): The input variable which is a Tensor or LoDTensor.
C
caoying03 已提交
4966 4967 4968 4969 4970
        num_or_sections (int|list): If :attr:`num_or_sections` is an integer,
            then the integer indicates the number of equal sized sub-tensors
            that the tensor will be divided into. If :attr:`num_or_sections`
            is a list of integers, the length of list indicates the number of
            sub-tensors and the integers indicate the sizes of sub-tensors'
G
guosheng 已提交
4971
            :attr:`dim` dimension orderly.
C
caoying03 已提交
4972
        dim (int): The dimension along which to split. If :math:`dim < 0`, the
G
guosheng 已提交
4973
            dimension to split along is :math:`rank(input) + dim`.
C
caoying03 已提交
4974 4975
        name(str|None): A name for this layer(optional). If set None, the layer
                       will be named automatically.
G
guosheng 已提交
4976 4977

    Returns:
D
dzhwinter 已提交
4978
        list(Variable): The list of segmented tensor variables.
G
guosheng 已提交
4979 4980 4981 4982 4983 4984 4985 4986 4987

    Examples:
        .. code-block:: python

            # x is a Tensor variable with shape [3, 9, 5]:
            x0, x1, x2 = fluid.layers.split(x, num_or_sections=3, dim=1)
            x0.shape  # [3, 3, 5]
            x1.shape  # [3, 3, 5]
            x2.shape  # [3, 3, 5]
F
stash  
fengjiayi 已提交
4988 4989
            x0, x1, x2 = fluid.layers.split(
                x, num_or_sections=[2, 3, 4], dim=1)
G
guosheng 已提交
4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000
            x0.shape  # [3, 2, 5]
            x1.shape  # [3, 3, 5]
            x2.shape  # [3, 4, 5]
    """
    helper = LayerHelper('split', **locals())
    input_shape = input.shape
    dim = (len(input_shape) + dim) if dim < 0 else dim
    if isinstance(num_or_sections, int):
        assert num_or_sections > 1, 'num_or_sections must be more than 1.'
        num = num_or_sections
    else:
T
tink2123 已提交
5001
        assert len(num_or_sections) <= input_shape[
G
guosheng 已提交
5002 5003 5004
            dim], 'len(num_or_sections) must not be more than input.shape[dim].'
        num = len(num_or_sections)
    outs = [
X
Xin Pan 已提交
5005
        helper.create_variable_for_type_inference(dtype=helper.input_dtype())
G
guosheng 已提交
5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018
        for i in range(num)
    ]
    helper.append_op(
        type='split',
        inputs={'X': input},
        outputs={'Out': outs},
        attrs={
            'num': num_or_sections if isinstance(num_or_sections, int) else 0,
            'sections': num_or_sections
            if isinstance(num_or_sections, list) else [],
            'axis': dim
        })
    return outs
C
caoying03 已提交
5019 5020 5021 5022 5023 5024 5025 5026 5027


def l2_normalize(x, axis, epsilon=1e-12, name=None):
    """
    **L2 normalize Layer**

    The l2 normalize layer normalizes `x` along dimension `axis` using an L2
    norm. For a 1-D tensor (`dim` is fixed to 0), this layer computes

5028
    .. math::
5029 5030

        y = \\frac{x}{ \sqrt{\sum {x^2} + epsion }}
C
caoying03 已提交
5031 5032 5033 5034 5035

    For `x` with more dimensions, this layer independently normalizes each 1-D
    slice along dimension `axis`.

    Args:
5036
        x(Variable|list): The input tensor to l2_normalize layer.
5037
        axis(int): The axis on which to apply normalization. If `axis < 0`, \
5038 5039
            the dimension to normalization is rank(X) + axis. -1 is the
            last dimension.
5040
        epsilon(float): The epsilon value is used to avoid division by zero, \
5041
            the defalut value is 1e-12.
5042
        name(str|None): A name for this layer(optional). If set None, the layer \
5043
            will be named automatically.
C
caoying03 已提交
5044 5045

    Returns:
5046
        Variable: The output tensor variable is the same shape with `x`.
C
caoying03 已提交
5047 5048

    Examples:
5049

C
caoying03 已提交
5050 5051
        .. code-block:: python

5052 5053 5054 5055
            data = fluid.layers.data(name="data",
                                     shape=(3, 17, 13),
                                     dtype="float32")
            normed = fluid.layers.l2_normalize(x=data, axis=1)
C
caoying03 已提交
5056 5057
    """

F
fengjiayi 已提交
5058 5059
    if len(x.shape) == 1:
        axis = 0
C
caoying03 已提交
5060 5061
    helper = LayerHelper("l2_normalize", **locals())

X
Xin Pan 已提交
5062 5063
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
    norm = helper.create_variable_for_type_inference(dtype=x.dtype)
C
caoying03 已提交
5064
    helper.append_op(
5065 5066 5067 5068
        type="norm",
        inputs={"X": x},
        outputs={"Out": out,
                 "Norm": norm},
C
caoying03 已提交
5069
        attrs={
5070 5071
            "axis": 1 if axis is None else axis,
            "epsilon": epsilon,
C
caoying03 已提交
5072 5073
        })
    return out
5074 5075


S
sneaxiy 已提交
5076
def matmul(x, y, transpose_x=False, transpose_y=False, alpha=1.0, name=None):
G
guosheng 已提交
5077
    """
Y
ying 已提交
5078 5079 5080 5081
    Applies matrix multiplication to two tensors.

    Currently, the input tensors' rank can be any, but when the rank of any
    inputs is bigger than 3, this two inputs' rank should be equal.
G
guosheng 已提交
5082

C
chengduoZH 已提交
5083
    The actual behavior depends on the shapes of :math:`x`, :math:`y` and the
5084
    flag values of :attr:`transpose_x`, :attr:`transpose_y`. Specifically:
G
guosheng 已提交
5085

5086 5087 5088 5089 5090
    - If a transpose flag is specified, the last two dimensions of the tensor
      are transposed. If the tensor is rank-1 of shape :math:`[D]`, then for
      :math:`x` it is treated as :math:`[1, D]` in nontransposed form and as
      :math:`[D, 1]` in transposed form, whereas for :math:`y` it is the
      opposite: It is treated as :math:`[D, 1]` in nontransposed form and as
5091
      :math:`[1, D]` in transposed form.
G
guosheng 已提交
5092

C
chengduoZH 已提交
5093
    - After transpose, the two tensors are 2-D or n-D and matrix multiplication
5094
      performs in the following way.
G
guosheng 已提交
5095

5096
      - If both are 2-D, they are multiplied like conventional matrices.
C
chengduoZH 已提交
5097
      - If either is n-D, it is treated as a stack of matrices residing in the
Y
ying 已提交
5098
        last two dimensions and a batched matrix multiply supporting broadcast
5099
        applies on the two tensors.
G
guosheng 已提交
5100

Y
ying 已提交
5101 5102
    Also note that if the raw tensor :math:`x` or :math:`y` is rank-1 and
    nontransposed, the prepended or appended dimension :math:`1` will be
C
chengduoZH 已提交
5103
    removed after matrix multiplication.
G
guosheng 已提交
5104 5105 5106

    Args:
        x (Variable): The input variable which is a Tensor or LoDTensor.
5107 5108 5109
        y (Variable): The input variable which is a Tensor or LoDTensor.
        transpose_x (bool): Whether to transpose :math:`x` before multiplication.
        transpose_y (bool): Whether to transpose :math:`y` before multiplication.
S
sneaxiy 已提交
5110
        alpha (float): The scale of output. Default 1.0.
5111
        name(str|None): A name for this layer(optional). If set None, the layer
5112
            will be named automatically.
G
guosheng 已提交
5113 5114

    Returns:
5115
        Variable: The product Tensor variable.
G
guosheng 已提交
5116

G
guosheng 已提交
5117 5118 5119
    Examples:
        .. code-block:: python

5120
            # Examples to clarify shapes of the inputs and output
C
chengduoZH 已提交
5121 5122
            # x: [B, ..., M, K], y: [B, ..., K, N]
            fluid.layers.matmul(x, y)  # out: [B, ..., M, N]
Y
ying 已提交
5123

5124 5125
            # x: [B, M, K], y: [B, K, N]
            fluid.layers.matmul(x, y)  # out: [B, M, N]
Y
ying 已提交
5126

5127 5128
            # x: [B, M, K], y: [K, N]
            fluid.layers.matmul(x, y)  # out: [B, M, N]
Y
ying 已提交
5129

5130 5131
            # x: [M, K], y: [K, N]
            fluid.layers.matmul(x, y)  # out: [M, N]
Y
ying 已提交
5132 5133 5134 5135

            # x: [B, M, K], y: [K]
            fluid.layers.matmul(x, y)  # out: [B, M]

5136 5137
            # x: [K], y: [K]
            fluid.layers.matmul(x, y)  # out: [1]
5138

Y
ying 已提交
5139
            # x: [M], y: [N]
5140
            fluid.layers.matmul(x, y, True, True)  # out: [M, N]
G
guosheng 已提交
5141
    """
Y
ying 已提交
5142 5143 5144 5145 5146 5147 5148

    def __check_input(x, y):
        x_shape = list(x.shape)
        y_shape = list(y.shape)
        if len(x_shape) == 1:
            x_shape = [1] + x_shape
        if len(y_shape) == 1:
Y
ying 已提交
5149
            y_shape = y_shape + [1]
Y
ying 已提交
5150 5151 5152 5153 5154 5155 5156

        # check the inner 2 dimensions
        if transpose_x:
            x_shape[-2], x_shape[-1] = x_shape[-1], x_shape[-2]
        if transpose_y:
            y_shape[-2], y_shape[-1] = y_shape[-1], y_shape[-2]
        if x_shape[-1] != y_shape[-2]:
5157 5158
            raise ValueError("Invalid inputs for matmul. x: %s, y: %s\n" %
                             (x_shape, y_shape))
Y
ying 已提交
5159

C
chengduo 已提交
5160
        if len(y_shape) > 2 and len(x_shape) > 2:
Y
ying 已提交
5161
            for i, dim_x in enumerate(x_shape[:-2]):
P
phlrain 已提交
5162 5163 5164
                # don't check neg shape
                if dim_x < 0 or y_shape[i] < 0:
                    continue
Y
ying 已提交
5165
                if dim_x != y_shape[i]:
C
chengduo 已提交
5166 5167
                    raise ValueError("Invalid inputs for matmul. x(%s), y(%s)" %
                                     (x.shape, y.shape))
Y
ying 已提交
5168 5169 5170

    __check_input(x, y)

5171
    helper = LayerHelper('matmul', **locals())
X
Xin Pan 已提交
5172
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
G
guosheng 已提交
5173
    helper.append_op(
5174 5175 5176 5177
        type='matmul',
        inputs={'X': x,
                'Y': y},
        outputs={'Out': out},
S
sneaxiy 已提交
5178 5179 5180
        attrs={
            'transpose_X': transpose_x,
            'transpose_Y': transpose_y,
S
sneaxiy 已提交
5181
            'alpha': float(alpha),
S
sneaxiy 已提交
5182
        })
5183
    return out
5184 5185


5186
def topk(input, k, name=None):
Q
qingqing01 已提交
5187 5188 5189 5190
    """
    This operator is used to find values and indices of the k largest entries
    for the last dimension.

F
fengjiayi 已提交
5191
    If the input is a vector (1-D Tensor), finds the k largest entries in the vector
Q
qingqing01 已提交
5192 5193 5194 5195 5196 5197
    and outputs their values and indices as vectors. Thus values[j] is the j-th
    largest entry in input, and its index is indices[j].

    If the input is a Tensor with higher rank, this operator computes the top k
    entries along the last dimension.

F
fengjiayi 已提交
5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218
    For example:

    .. code-block:: text

        If:
            input = [[5, 4, 2, 3],
                     [9, 7, 10, 25],
                     [6, 2, 10, 1]]
            k = 2

        Then:
            The first output:
            values = [[5, 4],
                      [10, 25],
                      [6, 10]]

            The second output:
            indices = [[0, 1],
                       [2, 3],
                       [0, 2]]

Q
qingqing01 已提交
5219 5220 5221
    Args:
        input(Variable): The input variable which can be a vector or Tensor with
            higher rank.
W
whs 已提交
5222
        k(int | Variable):  The number of top elements to look for along the last dimension
F
fengjiayi 已提交
5223
                 of input.
5224
        name(str|None): A name for this layer(optional). If set None, the layer
5225
                       will be named automatically.
F
fengjiayi 已提交
5226
                       Default: None
Q
qingqing01 已提交
5227 5228

    Returns:
5229 5230 5231
        Tuple[Variable]: A tuple with two elements. Each element is a Variable.
        The first one is k largest elements along each last
        dimensional slice. The second one is indices of values
F
fengjiayi 已提交
5232
        within the last dimension of input.
Q
qingqing01 已提交
5233

F
fengjiayi 已提交
5234 5235
    Raises:
        ValueError: If k < 1 or k is not less than the last dimension of input
Q
qingqing01 已提交
5236 5237 5238 5239

    Examples:
        .. code-block:: python

5240 5241
            import paddle.fluid.layers as layers
            input = layers.data(name="input", shape=[13, 11], dtype='float32')
Q
qingqing01 已提交
5242 5243 5244
            top5_values, top5_indices = layers.topk(input, k=5)
    """
    helper = LayerHelper("top_k", **locals())
X
Xin Pan 已提交
5245 5246
    values = helper.create_variable_for_type_inference(dtype=input.dtype)
    indices = helper.create_variable_for_type_inference(dtype="int64")
W
whs 已提交
5247 5248 5249 5250 5251 5252
    inputs = {"X": [input]}
    attrs = None
    if isinstance(k, Variable):
        inputs['K'] = k
    else:
        attrs = {'k': k}
Q
qingqing01 已提交
5253 5254
    helper.append_op(
        type="top_k",
W
whs 已提交
5255
        inputs=inputs,
Q
qingqing01 已提交
5256 5257
        outputs={"Out": [values],
                 "Indices": [indices]},
W
whs 已提交
5258
        attrs=attrs)
Q
qingqing01 已提交
5259 5260 5261 5262 5263
    values.stop_gradient = True
    indices.stop_gradient = True
    return values, indices


5264
def edit_distance(input, label, normalized=True, ignored_tokens=None):
5265
    """
Y
ying 已提交
5266 5267 5268 5269 5270 5271 5272 5273 5274
    EditDistance operator computes the edit distances between a batch of
    hypothesis strings and their references. Edit distance, also called
    Levenshtein distance, measures how dissimilar two strings are by counting
    the minimum number of operations to transform one string into anthor.
    Here the operations include insertion, deletion, and substitution.

    For example, given hypothesis string A = "kitten" and reference
    B = "sitting", the edit distance is 3 for A will be transformed into B
    at least after two substitutions and one insertion:
W
wanghaoshuang 已提交
5275

Y
ying 已提交
5276
    "kitten" -> "sitten" -> "sittin" -> "sitting"
W
wanghaoshuang 已提交
5277

5278
    The input is a LoDTensor consisting of all the hypothesis strings with
Y
ying 已提交
5279 5280
    the total number denoted by `batch_size`, and the separation is specified
    by the LoD information. And the `batch_size` reference strings are arranged
5281
    in order in the same way in the input LoDTensor.
W
wanghaoshuang 已提交
5282

5283
    The output contains the `batch_size` results and each stands for the edit
Y
ying 已提交
5284 5285
    distance for a pair of strings respectively. If Attr(normalized) is true,
    the edit distance will be divided by the length of reference string.
W
wanghaoshuang 已提交
5286

5287 5288 5289
    Args:
        input(Variable): The indices for hypothesis strings.
        label(Variable): The indices for reference strings.
5290
        normalized(bool, default True): Indicated whether to normalize the edit distance by
Y
ying 已提交
5291
                          the length of reference string.
5292
        ignored_tokens(list<int>, default None): Tokens that should be removed before
Y
ying 已提交
5293
                                     calculating edit distance.
5294
        name (str): The name of this layer. It is optional.
5295

W
wanghaoshuang 已提交
5296
    Returns:
W
wanghaoshuang 已提交
5297
        Variable: sequence-to-sequence edit distance in shape [batch_size, 1].
W
wanghaoshuang 已提交
5298 5299 5300 5301

    Examples:
        .. code-block:: python

T
tink2123 已提交
5302 5303
            x = fluid.layers.data(name='x', shape=[1], dtype='float32')
            y = fluid.layers.data(name='y', shape=[1], dtype='float32')
5304
            cost = fluid.layers.edit_distance(input=x,label=y)
5305
    """
5306
    helper = LayerHelper("edit_distance", **locals())
5307

5308
    # remove some tokens from input and labels
W
wanghaoshuang 已提交
5309
    if ignored_tokens is not None and len(ignored_tokens) > 0:
X
Xin Pan 已提交
5310 5311
        erased_input = helper.create_variable_for_type_inference(dtype="int64")
        erased_label = helper.create_variable_for_type_inference(dtype="int64")
5312 5313 5314 5315 5316

        helper.append_op(
            type="sequence_erase",
            inputs={"X": [input]},
            outputs={"Out": [erased_input]},
W
wanghaoshuang 已提交
5317
            attrs={"tokens": ignored_tokens})
5318 5319 5320 5321 5322
        input = erased_input

        helper.append_op(
            type="sequence_erase",
            inputs={"X": [label]},
W
whs 已提交
5323
            outputs={"Out": [erased_label]},
W
wanghaoshuang 已提交
5324
            attrs={"tokens": ignored_tokens})
5325 5326
        label = erased_label

5327
    # edit distance op
X
Xin Pan 已提交
5328 5329
    edit_distance_out = helper.create_variable_for_type_inference(dtype="int64")
    sequence_num = helper.create_variable_for_type_inference(dtype="int64")
5330 5331 5332 5333
    helper.append_op(
        type="edit_distance",
        inputs={"Hyps": [input],
                "Refs": [label]},
5334 5335
        outputs={"Out": [edit_distance_out],
                 "SequenceNum": [sequence_num]},
5336 5337
        attrs={"normalized": normalized})

5338
    return edit_distance_out, sequence_num
5339 5340 5341 5342 5343


def ctc_greedy_decoder(input, blank, name=None):
    """
    This op is used to decode sequences by greedy policy by below steps:
Y
yi.wu 已提交
5344

Y
ying 已提交
5345 5346 5347 5348
    1. Get the indexes of max value for each row in input. a.k.a.
       numpy.argmax(input, axis=0).
    2. For each sequence in result of step1, merge repeated tokens between two
       blanks and delete all blanks.
5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365

    A simple example as below:

    .. code-block:: text

        Given:

        input.data = [[0.6, 0.1, 0.3, 0.1],
                      [0.3, 0.2, 0.4, 0.1],
                      [0.1, 0.5, 0.1, 0.3],
                      [0.5, 0.1, 0.3, 0.1],

                      [0.5, 0.1, 0.3, 0.1],
                      [0.2, 0.2, 0.2, 0.4],
                      [0.2, 0.2, 0.1, 0.5],
                      [0.5, 0.1, 0.3, 0.1]]

5366
        input.lod = [[4, 4]]
M
minqiyang 已提交
5367

W
whs 已提交
5368
        Computation:
5369

W
whs 已提交
5370 5371 5372 5373 5374 5375
        step1: Apply argmax to first input sequence which is input.data[0:4]. Then we get:
               [[0], [2], [1], [0]]
        step2: merge repeated tokens and remove blank which is 0. Then we get first output sequence:
               [[2], [1]]

        Finally:
5376 5377 5378 5379 5380

        output.data = [[2],
                       [1],
                       [3]]

5381
        output.lod = [[2, 1]]
5382

W
whs 已提交
5383

5384 5385
    Args:

Y
ying 已提交
5386 5387 5388 5389 5390 5391 5392 5393 5394
        input(Variable): (LoDTensor<float>), the probabilities of
                         variable-length sequences, which is a 2-D Tensor with
                         LoD information. It's shape is [Lp, num_classes + 1],
                         where Lp is the sum of all input sequences' length and
                         num_classes is the true number of classes. (not
                         including the blank label).
        blank(int): the blank label index of Connectionist Temporal
                    Classification (CTC) loss, which is in thehalf-opened
                    interval [0, num_classes + 1).
5395
        name (str): The name of this layer. It is optional.
5396 5397

    Returns:
H
haowang101779990 已提交
5398 5399 5400
        Variable: CTC greedy decode result which is a 2-D tensor with shape [Lp, 1]. \
                  'Lp' is the sum if all output sequences' length. If all the sequences \
                  in result were empty, the result LoDTensor will be [-1] with  \
M
minqiyang 已提交
5401
                  LoD [[]] and dims [1, 1].
5402 5403 5404 5405

    Examples:
        .. code-block:: python

S
SunGaofeng 已提交
5406
            import paddle.fluid as fluid
5407 5408
            x = fluid.layers.data(name='x', shape=[8], dtype='float32')
            cost = fluid.layers.ctc_greedy_decoder(input=x, blank=0)
W
wanghaoshuang 已提交
5409
    """
5410
    helper = LayerHelper("ctc_greedy_decoder", **locals())
Q
qingqing01 已提交
5411
    _, topk_indices = topk(input, k=1)
5412 5413

    # ctc align op
X
Xin Pan 已提交
5414
    ctc_out = helper.create_variable_for_type_inference(dtype="int64")
5415 5416 5417
    helper.append_op(
        type="ctc_align",
        inputs={"Input": [topk_indices]},
W
wanghaoshuang 已提交
5418
        outputs={"Output": [ctc_out]},
5419 5420
        attrs={"merge_repeated": True,
               "blank": blank})
5421
    return ctc_out
5422 5423


W
Wu Yi 已提交
5424
def warpctc(input, label, blank=0, norm_by_times=False, use_cudnn=False):
W
wanghaoshuang 已提交
5425
    """
5426 5427
    An operator integrating the open source Warp-CTC library
    (https://github.com/baidu-research/warp-ctc)
W
wanghaoshuang 已提交
5428
    to compute Connectionist Temporal Classification (CTC) loss.
5429 5430
    It can be aliased as softmax with CTC, since a native softmax activation is
    interated to the Warp-CTC library, to to normlize values for each row of the
W
wanghaoshuang 已提交
5431 5432 5433
    input tensor.

    Args:
5434
       input (Variable): The unscaled probabilities of variable-length sequences,
W
wanghaoshuang 已提交
5435 5436 5437 5438
         which is a 2-D Tensor with LoD information.
         It's shape is [Lp, num_classes + 1], where Lp is the sum of all input
         sequences' length and num_classes is the true number of classes.
         (not including the blank label).
5439
       label (Variable): The ground truth of variable-length sequence,
5440 5441 5442
         which is a 2-D Tensor with LoD information. It is of the shape [Lg, 1],
         where Lg is th sum of all labels' length.
       blank (int, default 0): The blank label index of Connectionist
W
wanghaoshuang 已提交
5443 5444
         Temporal Classification (CTC) loss, which is in the
         half-opened interval [0, num_classes + 1).
5445 5446 5447
       norm_by_times(bool, default false): Whether to normalize the gradients
         by the number of time-step, which is also the sequence's length.
         There is no need to normalize the gradients if warpctc layer was
5448
         follewed by a mean_op.
W
Wu Yi 已提交
5449
       use_cudnn (bool, default false): Whether to use cudnn.
W
wanghaoshuang 已提交
5450 5451

    Returns:
5452 5453
        Variable: The Connectionist Temporal Classification (CTC) loss,
        which is a 2-D Tensor of the shape [batch_size, 1].
W
wanghaoshuang 已提交
5454 5455

    Examples:
5456

W
wanghaoshuang 已提交
5457
        .. code-block:: python
5458

5459 5460 5461
            label = fluid.layers.data(shape=[11, 8], dtype='float32', lod_level=1)
            predict = fluid.layers.data(shape=[11, 1], dtype='float32')
            cost = fluid.layers.warpctc(input=predict, label=label)
W
wanghaoshuang 已提交
5462 5463

    """
F
fengjiayi 已提交
5464
    helper = LayerHelper('warpctc', **locals())
X
Xin Pan 已提交
5465 5466
    loss_out = helper.create_variable_for_type_inference(dtype=input.dtype)
    grad_out = helper.create_variable_for_type_inference(dtype=input.dtype)
W
wanghaoshuang 已提交
5467 5468 5469 5470 5471 5472
    helper.append_op(
        type='warpctc',
        inputs={'Logits': [input],
                'Label': [label]},
        outputs={'WarpCTCGrad': [grad_out],
                 'Loss': [loss_out]},
W
Wu Yi 已提交
5473 5474 5475 5476 5477
        attrs={
            'blank': blank,
            'norm_by_times': norm_by_times,
            'use_cudnn': use_cudnn
        })
W
wanghaoshuang 已提交
5478
    return loss_out
5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493


def sequence_reshape(input, new_dim):
    """
    **Sequence Reshape Layer**

    This layer will rearrange the input sequences. The new dimension is set by
    user. Length of each sequence is computed according to original length,
    original dimension and new dimension. The following example will help to
    illustrate the function of this layer:

    .. code-block:: text

        x is a LoDTensor:
            x.lod  = [[0, 2, 6]]
5494 5495 5496
            x.data = [[1,  2], [3,  4],
                      [5,  6], [7,  8],
                      [9, 10], [11, 12]]
5497 5498 5499 5500 5501
            x.dims = [6, 2]

        set new_dim = 4

        then out is a LoDTensor:
5502

5503
            out.lod  = [[0, 1, 3]]
5504 5505 5506 5507

            out.data = [[1,  2,  3,  4],
                        [5,  6,  7,  8],
                        [9, 10, 11, 12]]
5508 5509 5510 5511 5512 5513 5514
            out.dims = [3, 4]

    Currently, only 1-level LoDTensor is supported and please make sure
    (original length * original dimension) can be divided by new dimension with
    no remainder for each sequence.

    Args:
5515 5516 5517

       input (Variable): A 2-D LoDTensor with shape being [N, M] where M for dimension.
       new_dim (int): New dimension that the input LoDTensor is reshaped to.
5518 5519

    Returns:
5520

5521 5522 5523 5524 5525
        Variable: Reshaped LoDTensor according to new dimension.

    Examples:
        .. code-block:: python

B
bdzhuxiaoning 已提交
5526 5527 5528
            import paddle.fluid as fluid
            x = fluid.layers.data(name='x', shape=[2, 6], append_batch_size=False, dtype='float32', lod_level=1)
            x_reshaped = fluid.layers.sequence_reshape(input=x, new_dim=4)
5529
    """
L
lujun 已提交
5530
    assert not in_dygraph_mode(), (
5531
        "sequence layer is not supported in dygraph mode yet.")
5532
    helper = LayerHelper('sequence_reshape', **locals())
X
Xin Pan 已提交
5533
    out = helper.create_variable_for_type_inference(helper.input_dtype())
5534 5535 5536 5537 5538 5539
    helper.append_op(
        type='sequence_reshape',
        inputs={'X': [input]},
        outputs={'Out': [out]},
        attrs={'new_dim': new_dim})
    return out
Y
ying 已提交
5540 5541


5542 5543 5544 5545
# FIXME(wuyi): let docstring_checker.py understand @autodoc.
# For now, the comments in c++ use types like Tensor, but in python side
# the type is often "Variable", and arguments may vary.
@templatedoc(op_type="nce")
Y
Yang Yu 已提交
5546 5547 5548 5549 5550 5551
def nce(input,
        label,
        num_total_classes,
        sample_weight=None,
        param_attr=None,
        bias_attr=None,
C
chengduo 已提交
5552
        num_neg_samples=None,
5553 5554 5555
        name=None,
        sampler="uniform",
        custom_dist=None,
5556 5557
        seed=0,
        is_sparse=False):
5558 5559 5560 5561 5562 5563 5564
    """
    ${comment}

    Args:
        input (Variable): input variable.
        label (Variable): label.
        num_total_classes (int):${num_total_classes_comment}
5565 5566
        sample_weight (Variable|None): A Variable of shape [batch_size, 1]
            storing a weight for each sample. The default weight for each
5567
            sample is 1.0.
C
chengduo 已提交
5568 5569 5570 5571 5572 5573 5574 5575 5576
        param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights
             of nce. If it is set to None or one attribute of ParamAttr, nce
             will create ParamAttr as param_attr. If the Initializer of the param_attr
             is not set, the parameter is initialized with Xavier. Default: None.
        bias_attr (ParamAttr|bool|None): The parameter attribute for the bias of nce.
             If it is set to False, no bias will be added to the output units.
             If it is set to None or one attribute of ParamAttr, nce
             will create ParamAttr as bias_attr. If the Initializer of the bias_attr
             is not set, the bias is initialized zero. Default: None.
5577
        num_neg_samples (int): ${num_neg_samples_comment}
C
chengduo 已提交
5578 5579
        name (str|None): A name for this layer(optional). If set None, the layer
             will be named automatically. Default: None.
5580 5581 5582
        sampler (str): The sampler used to sample class from negtive classes.
                       It can be 'uniform', 'log_uniform' or 'custom_dist'.
                       default: 'uniform'.
5583
        custom_dist (float[]): A float[] with size=num_total_classes.
5584 5585 5586 5587
                       It is used when sampler is set to 'custom_dist'.
                       custom_dist[i] is the probsbility of i-th class to be sampled.
                       default: None.
        seed (int): The seed used in sampler. default: 0.
5588
        is_sparse(bool): The flag indicating whether to use sparse update, the weight@GRAD and bias@GRAD will be changed to SelectedRows.
F
fengjiayi 已提交
5589

5590
    Returns:
Y
Yibing Liu 已提交
5591 5592 5593 5594 5595 5596
        Variable: The output nce loss.

    Examples:
        .. code-block:: python


Y
Yibing Liu 已提交
5597
	    import numpy as np
Y
Yibing Liu 已提交
5598

Y
Yibing Liu 已提交
5599 5600 5601 5602 5603 5604 5605 5606
	    window_size = 5
	    words = []
	    for i in xrange(window_size):
		words.append(fluid.layers.data(
		    name='word_{0}'.format(i), shape=[1], dtype='int64'))

	    dict_size = 10000
	    label_word = int(window_size / 2) + 1
Y
Yibing Liu 已提交
5607

Y
Yibing Liu 已提交
5608 5609 5610 5611
	    embs = []
	    for i in xrange(window_size):
		if i == label_word:
		    continue
Y
Yibing Liu 已提交
5612

Y
Yibing Liu 已提交
5613 5614 5615
		emb = fluid.layers.embedding(input=words[i], size=[dict_size, 32],
				   param_attr='embed', is_sparse=True)
		embs.append(emb)
5616

Y
Yibing Liu 已提交
5617 5618 5619 5620
	    embs = fluid.layers.concat(input=embs, axis=1)
	    loss = fluid.layers.nce(input=embs, label=words[label_word],
		      num_total_classes=dict_size, param_attr='nce.w_0',
		      bias_attr='nce.b_0')
5621

Y
Yibing Liu 已提交
5622 5623 5624 5625 5626 5627 5628 5629
	    #or use custom distribution
	    dist = np.array([0.05,0.5,0.1,0.3,0.05])
	    loss = fluid.layers.nce(input=embs, label=words[label_word],
		      num_total_classes=5, param_attr='nce.w_1',
		      bias_attr='nce.b_1',
		      num_neg_samples=3,
		      sampler="custom_dist",
		      custom_dist=dist)
5630
    """
Y
Yang Yu 已提交
5631 5632 5633
    helper = LayerHelper('nce', **locals())
    assert isinstance(input, Variable)
    assert isinstance(label, Variable)
C
chengduo 已提交
5634 5635

    dim = input.shape[1]
Y
Yang Yu 已提交
5636 5637 5638 5639 5640 5641
    num_true_class = label.shape[1]
    w = helper.create_parameter(
        attr=helper.param_attr,
        shape=[num_total_classes, dim],
        is_bias=False,
        dtype=input.dtype)
5642
    inputs = {}
C
chengduo 已提交
5643 5644 5645 5646 5647 5648 5649
    if helper.bias_attr:
        b = helper.create_parameter(
            attr=helper.bias_attr,
            shape=[num_total_classes, 1],
            is_bias=True,
            dtype=input.dtype)
        inputs['Bias'] = b
X
Xin Pan 已提交
5650 5651 5652
    cost = helper.create_variable_for_type_inference(dtype=input.dtype)
    sample_logits = helper.create_variable_for_type_inference(dtype=input.dtype)
    sample_labels = helper.create_variable_for_type_inference(dtype=label.dtype)
Y
Yang Yu 已提交
5653

5654 5655 5656 5657
    inputs['Input'] = input
    inputs['Label'] = label
    inputs['Weight'] = w
    inputs['SampleWeight'] = sample_weight if sample_weight is not None else []
5658 5659 5660 5661 5662 5663 5664

    if sampler == "uniform":
        sampler = 0
    elif sampler == "log_uniform":
        sampler = 1
    elif sampler == "custom_dist":
        assert custom_dist is not None
5665 5666
        # assert isinstance(custom_dist, Variable)

Y
Yibing Liu 已提交
5667
        custom_dist_len = num_total_classes
5668 5669 5670 5671 5672 5673
        alias_probs_ = [0] * custom_dist_len
        alias_ = [0] * custom_dist_len
        bigs = []
        littles = []
        for i in range(custom_dist_len):
            normal_prob = custom_dist[i] * custom_dist_len
5674
            if normal_prob - 1.0 > 0:
5675
                bigs.append((i, normal_prob))
5676
            elif 1.0 - normal_prob > 0:
5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691
                littles.append((i, normal_prob))
            else:
                alias_probs_[i] = normal_prob
                alias_[i] = -1

        while len(bigs) and len(littles):
            big = bigs.pop(0)
            little = littles.pop(0)

            big_idx = big[0]
            big_prob = big[1]

            alias_probs_[little[0]] = little[1]
            alias_[little[0]] = big_idx
            big_left = big[1] + little[1] - 1
5692
            if big_left - 1.0 > 0:
5693
                bigs.append((big_idx, big_left))
5694
            elif 1.0 - big_left > 0:
5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708
                littles.append((big_idx, big_left))
            else:
                alias_probs_[big_idx] = big_left
                alias_[big_idx] = -1

        if len(bigs):
            big = bigs.pop(0)
            alias_probs_[big[0]] = 1.0
            alias_[big[0]] = -1
        if len(littles):
            little = littles.pop(0)
            alias_probs_[little[0]] = 1.0
            alias_[little[0]] = -1

5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723
        def _init_by_numpy_array(numpy_array):
            ret = helper.create_parameter(
                attr=ParamAttr(),
                shape=numpy_array.shape,
                dtype=numpy_array.dtype,
                default_initializer=NumpyArrayInitializer(numpy_array))
            ret.stop_gradient = True
            return ret

        inputs['CustomDistProbs'] = _init_by_numpy_array(
            np.array(custom_dist).astype('float32'))
        inputs['CustomDistAlias'] = _init_by_numpy_array(
            np.array(alias_).astype('int32'))
        inputs['CustomDistAliasProbs'] = _init_by_numpy_array(
            np.array(alias_probs_).astype('float32'))
5724 5725 5726 5727
        sampler = 2
    else:
        raise Exception("Unsupported sampler type.")

5728 5729 5730 5731 5732
    if num_neg_samples is None:
        num_neg_samples = 10
    else:
        num_neg_samples = int(num_neg_samples)

5733 5734 5735 5736
    remote_prefetch = is_sparse
    print(
        "With sparse mode, if your models has only small parameter prefetch may cause speed down"
    )
5737

Y
Yang Yu 已提交
5738 5739
    attrs = {
        'num_total_classes': int(num_total_classes),
5740 5741
        'num_neg_samples': num_neg_samples,
        'seed': seed,
5742
        'sampler': sampler,
5743 5744
        'is_sparse': is_sparse,
        'remote_prefetch': remote_prefetch
Y
Yang Yu 已提交
5745
    }
Y
Yang Yu 已提交
5746 5747 5748

    helper.append_op(
        type='nce',
C
chengduo 已提交
5749
        inputs=inputs,
Y
Yang Yu 已提交
5750 5751 5752 5753 5754 5755
        outputs={
            'Cost': cost,
            'SampleLogits': sample_logits,
            'SampleLabels': sample_labels
        },
        attrs=attrs)
Y
Yang Yu 已提交
5756
    return cost / (num_neg_samples + 1)
5757 5758


C
chengduo 已提交
5759 5760
def hsigmoid(input,
             label,
5761
             num_classes,
C
chengduo 已提交
5762 5763
             param_attr=None,
             bias_attr=None,
J
JiabinYang 已提交
5764
             name=None,
5765 5766 5767
             path_table=None,
             path_code=None,
             is_custom=False,
J
JiabinYang 已提交
5768
             is_sparse=False):
W
weixing02 已提交
5769 5770
    """
    The hierarchical sigmoid operator is used to accelerate the training
M
minqiyang 已提交
5771
    process of language model. This operator organizes the classes into a
M
minqiyang 已提交
5772
    complete binary tree, or you can use is_custom to pass your own tree to
5773
    implement hierarchical. Each leaf node represents a class(a word) and each
G
guosheng 已提交
5774 5775 5776 5777 5778 5779
    internal node acts as a binary classifier. For each word there's a unique
    path from root to it's leaf node, hsigmoid calculate the cost for each
    internal node on the path, and sum them to get a total cost. hsigmoid can
    achive a acceleration from :math:`O(N)` to :math:`O(logN)`, where :math:`N`
    represents the size of word dict.

5780
    Using default tree you can Refer to `Hierarchical Probabilistic Neural Network Language Model
G
guosheng 已提交
5781
    <http://www.iro.umontreal.ca/~lisa/pointeurs/hierarchical-nnlm-aistats05.pdf>`_
M
minqiyang 已提交
5782

5783 5784
    And if you want to use the costumed tree by set 'is_custom' as true you may need to do following things first:

H
haowang101779990 已提交
5785 5786 5787 5788
    1. using your word dict to build a binary tree, each leaf node should be an word of your word dict
    2. build a dict to store word_id -> word's leaf to root path, we call it path_table.
    3. build a dict to store word_id -> code of word's leaf to root path, we call it path_code. Code
       means label of each binary classification, using 1 indicate true, 0 indicate false.
M
minqiyang 已提交
5789
    4. now, each word should has its path and code along the path, you can pass a batch of path and code
H
haowang101779990 已提交
5790
       related to the same batch of inputs.
5791

W
weixing02 已提交
5792
    Args:
M
minqiyang 已提交
5793
        input (Variable): The input tensor variable with shape
G
guosheng 已提交
5794 5795 5796 5797
            :math:`[N \\times D]`, where :math:`N` is the size of mini-batch,
            and :math:`D` is the feature size.
        label (Variable): The tensor variable contains labels of training data.
            It's a tensor with shape is :math:`[N \\times 1]`.
M
minqiyang 已提交
5798 5799
        num_classes: (int), The number of classes, must not be less than 2. with default tree this has to be set,
            it should never be None under is_custom=False, but while is_custom is true, it should be non leaf num
5800
            which indicates the num of classes using by binary classify.
C
chengduo 已提交
5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811
        param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights
             of hsigmoid. If it is set to None or one attribute of ParamAttr, hsigmoid
             will create ParamAttr as param_attr. If the Initializer of the param_attr
             is not set, the parameter is initialized with Xavier. Default: None.
        bias_attr (ParamAttr|bool|None): The parameter attribute for the bias of hsigmoid.
             If it is set to False, no bias will be added to the output units.
             If it is set to None or one attribute of ParamAttr, hsigmoid
             will create ParamAttr as bias_attr. If the Initializer of the bias_attr
             is not set, the bias is initialized zero. Default: None.
        name (str|None): A name for this layer(optional). If set None, the layer
             will be named automatically. Default: None.
M
minqiyang 已提交
5812
        path_table: (Variable|None) this variable can store each batch of samples' path to root,
5813
            it should be in leaf -> root order
M
minqiyang 已提交
5814 5815 5816
            path_table should have the same shape with path_code, and for each sample i path_table[i] indicates a np.array like
            structure and each element in this array is indexes in parent nodes' Weight Matrix.
        path_code:  (Variable|None) this variable can store each batch of samples' code,
5817
            each code consist with every code of parent nodes. it should be in leaf -> root order
M
minqiyang 已提交
5818
        is_custom: (bool|False)using user defined binary tree instead of default complete binary tree, if costum is
5819
             set you need to set path_table/path_code/num_classes, otherwise num_classes should be set
M
minqiyang 已提交
5820
        is_sparse: (bool|False)using sparse update instead of dense update, if set, the gradient
5821
             of W and input will be sparse.
W
weixing02 已提交
5822 5823

    Returns:
J
JiabinYang 已提交
5824
        Out: (LodTensor) The cost of hierarchical sigmoid operator. the shape is [N, 1]
W
weixing02 已提交
5825 5826 5827 5828 5829

    Examples:

        .. code-block:: python

G
guosheng 已提交
5830 5831 5832
            x = fluid.layers.data(name='x', shape=[2], dtype='float32')
            y = fluid.layers.data(name='y', shape=[1], dtype='int64')
            out = fluid.layers.hsigmoid(input=x, label=y, num_classes=6)
W
weixing02 已提交
5833 5834 5835 5836
    """

    helper = LayerHelper('hierarchical_sigmoid', **locals())
    dtype = helper.input_dtype()
X
Xin Pan 已提交
5837 5838
    out = helper.create_variable_for_type_inference(dtype)
    pre_out = helper.create_variable_for_type_inference(dtype)
W
weixing02 已提交
5839
    dim = input.shape[1]
5840
    if ((num_classes is None) or (num_classes < 2)) and (not is_custom):
J
JiabinYang 已提交
5841 5842 5843
        raise ValueError(
            "num_classes must not be less than 2 with default tree")

5844 5845 5846 5847 5848 5849 5850 5851 5852
    if (not is_custom) and (is_sparse):
        print("Sparse mode should not be used without custom tree")
        is_sparse = False

    if (not is_custom) and ((path_table is not None) or
                            (path_code is not None)):
        raise ValueError(
            "only num_classes should be passed without custom tree")

5853
    if (is_custom) and (path_code is None):
5854
        raise ValueError("path_code should not be None with custom tree")
5855
    elif (is_custom) and (path_table is None):
5856
        raise ValueError("path_table should not be None with custom tree")
5857
    elif (is_custom) and (num_classes is None):
5858
        raise ValueError("num_classes should not be None with custom tree")
5859 5860 5861
    else:
        pass

J
JiabinYang 已提交
5862
    weights = None
5863 5864 5865 5866
    remote_prefetch = is_sparse
    print(
        "With sparse mode, if your models has only small parameter prefetch may cause speed down"
    )
5867
    if not is_custom:
J
JiabinYang 已提交
5868 5869 5870 5871 5872 5873 5874 5875
        weights = helper.create_parameter(
            attr=helper.param_attr,
            shape=[num_classes - 1, dim],
            is_bias=False,
            dtype=input.dtype)
    else:
        weights = helper.create_parameter(
            attr=helper.param_attr,
5876
            shape=[num_classes, dim],
J
JiabinYang 已提交
5877 5878
            is_bias=False,
            dtype=input.dtype)
5879 5880 5881
    inputs = {
        "X": input,
        "W": weights,
5882
        "PathTable": path_table,
5883
        "PathCode": path_code,
5884 5885
        "Label": label
    }
W
weixing02 已提交
5886
    if helper.bias_attr:
5887
        if not is_custom:
J
JiabinYang 已提交
5888 5889
            bias = helper.create_parameter(
                attr=helper.bias_attr,
J
JiabinYang 已提交
5890
                shape=[num_classes - 1, 1],
J
JiabinYang 已提交
5891 5892 5893 5894 5895 5896
                is_bias=True,
                dtype=input.dtype)
            inputs['Bias'] = bias
        else:
            bias = helper.create_parameter(
                attr=helper.bias_attr,
5897
                shape=[num_classes, 1],
J
JiabinYang 已提交
5898 5899 5900
                is_bias=True,
                dtype=input.dtype)
            inputs['Bias'] = bias
W
weixing02 已提交
5901 5902
    helper.append_op(
        type="hierarchical_sigmoid",
W
weixing02 已提交
5903
        inputs=inputs,
W
weixing02 已提交
5904
        outputs={"Out": out,
5905 5906 5907 5908 5909 5910 5911
                 "PreOut": pre_out,
                 "W_Out": weights},
        attrs={
            "num_classes": num_classes,
            "is_sparse": is_sparse,
            "remote_prefetch": remote_prefetch
        })
W
weixing02 已提交
5912 5913 5914
    return out


Y
fix ci.  
ying 已提交
5915
def transpose(x, perm, name=None):
Y
ying 已提交
5916 5917 5918 5919 5920 5921 5922
    """
    Permute the dimensions of `input` according to `perm`.

    The `i`-th dimension  of the returned tensor will correspond to the
    perm[i]-th dimension of `input`.

    Args:
5923 5924 5925
        x (Variable): The input Tensor.
        perm (list): A permutation of the dimensions of `input`.
        name (str): The name of this layer. It is optional.
Y
ying 已提交
5926 5927 5928 5929 5930 5931 5932

    Returns:
        Variable: A transposed Tensor.

    Examples:
        .. code-block:: python

5933
            # use append_batch_size=False to avoid prepending extra
5934
            # batch size in shape
5935
            x = fluid.layers.data(name='x', shape=[5, 10, 15],
5936
                            dtype='float32', append_batch_size=False)
Y
fix ci.  
ying 已提交
5937
            x_transposed = layers.transpose(x, perm=[1, 0, 2])
Y
ying 已提交
5938 5939
    """

Y
fix ci.  
ying 已提交
5940
    if len(perm) != len(x.shape):
Y
ying 已提交
5941 5942 5943
        raise ValueError(
            "Input(perm) is the permutation of dimensions of Input(input). "
            "It's length shoud be equal to Input(input)'s rank.")
Y
ying 已提交
5944 5945 5946 5947 5948 5949
    for idx, dim in enumerate(perm):
        if dim >= len(x.shape):
            raise ValueError(
                "Each element in perm should be less than x's rank. "
                "%d-th element in perm is %d which accesses x's rank %d." %
                (idx, perm[idx], len(x.shape)))
Y
ying 已提交
5950 5951

    helper = LayerHelper('transpose', **locals())
X
Xin Pan 已提交
5952 5953
    out = helper.create_variable_for_type_inference(x.dtype)
    x_shape = helper.create_variable_for_type_inference(x.dtype)
Y
ying 已提交
5954
    helper.append_op(
5955
        type='transpose2',
Y
fix ci.  
ying 已提交
5956
        inputs={'X': [x]},
5957 5958
        outputs={'Out': [out],
                 'XShape': [x_shape]},
Y
ying 已提交
5959 5960
        attrs={'axis': perm})
    return out
5961 5962


5963 5964 5965 5966 5967 5968 5969
def im2sequence(input,
                filter_size=1,
                stride=1,
                padding=0,
                input_image_size=None,
                out_stride=1,
                name=None):
5970
    """
5971 5972 5973 5974 5975 5976 5977
    Extracts image patches from the input tensor to form a tensor of shape
    {input.batch_size * output_height * output_width, filter_size_H *
    filter_size_W * input.channels} which is similar with im2col.
    This op use filter / kernel to scan images and convert these images to
    sequences. After expanding, the number of time step are
    output_height * output_width for an image, in which output_height and
    output_width are calculated by below equation:
5978 5979 5980 5981 5982 5983 5984 5985 5986 5987

    .. math::

        output\_size = 1 + \
            (2 * padding + img\_size - block\_size + stride - 1) / stride

    And the dimension of each time step is block_y * block_x * input.channels.

    Args:
        input (Variable): The input should be a tensor in NCHW format.
W
wanghaoshuang 已提交
5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005

        filter_size(int|tuple|None): The filter size. If filter_size is a tuple,
            it must contain two integers, (filter_size_H, filter_size_W).
            Otherwise, the filter will be a square.

        stride(int|tuple): The stride size. If stride is a tuple, it must
            contain two integers, (stride_H, stride_W). Otherwise, the
            stride_H = stride_W = stride. Default: stride = 1.

        padding(int|tuple): The padding size. If padding is a tuple, it can
            contain two integers like (padding_H, padding_W) which means
            padding_up = padding_down = padding_H and
            padding_left = padding_right = padding_W. Or it can use
            (padding_up, padding_left, padding_down, padding_right) to indicate
            paddings of four direction. Otherwise, a scalar padding means
            padding_up = padding_down = padding_left = padding_right = padding
            Default: padding = 0.

6006 6007 6008 6009 6010 6011 6012 6013 6014
        input_image_size(Variable): the input contains image real size.It's dim
            is [batchsize, 2]. It is dispensable.It is just for batch inference.

        out_stride(int|tuple): The scaling of image through CNN. It is
            dispensable. It is valid only when input_image_size is not null.
            If out_stride is tuple,  it must contain two intergers,
            (out_stride_H, out_stride_W). Otherwise,
            the out_stride_H = out_stride_W = out_stride.

6015 6016 6017
        name (int): The name of this layer. It is optional.

    Returns:
W
wanghaoshuang 已提交
6018 6019 6020 6021 6022
        output: The output is a LoDTensor with shape
        {input.batch_size * output_height * output_width,
        filter_size_H * filter_size_W * input.channels}.
        If we regard output as a matrix, each row of this matrix is
        a step of a sequence.
6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049

    Examples:

        .. code-block:: text

            Given:

            x = [[[[ 6.  2.  1.]
                   [ 8.  3.  5.]
                   [ 0.  2.  6.]]

                  [[ 2.  4.  4.]
                   [ 6.  3.  0.]
                   [ 6.  4.  7.]]]

                 [[[ 6.  7.  1.]
                   [ 5.  7.  9.]
                   [ 2.  4.  8.]]

                  [[ 1.  2.  1.]
                   [ 1.  3.  5.]
                   [ 9.  0.  8.]]]]

            x.dims = {2, 2, 3, 3}

            And:

W
wanghaoshuang 已提交
6050 6051 6052
            filter = [2, 2]
            stride = [1, 1]
            padding = [0, 0]
6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064

            Then:

            output.data = [[ 6.  2.  8.  3.  2.  4.  6.  3.]
                           [ 2.  1.  3.  5.  4.  4.  3.  0.]
                           [ 8.  3.  0.  2.  6.  3.  6.  4.]
                           [ 3.  5.  2.  6.  3.  0.  4.  7.]
                           [ 6.  7.  5.  7.  1.  2.  1.  3.]
                           [ 7.  1.  7.  9.  2.  1.  3.  5.]
                           [ 5.  7.  2.  4.  1.  3.  9.  0.]
                           [ 7.  9.  4.  8.  3.  5.  0.  8.]]

6065
            output.dims = {8, 8}
6066

6067
            output.lod = [[4, 4]]
6068

T
Tink_Y 已提交
6069
    Examples:
6070 6071 6072

        .. code-block:: python

6073 6074
            output = fluid.layers.im2sequence(
                input=layer, stride=[1, 1], filter_size=[2, 2])
6075 6076

    """
L
lujun 已提交
6077
    assert not in_dygraph_mode(), (
6078
        "sequence layer is not supported in dygraph mode yet.")
W
wanghaoshuang 已提交
6079 6080 6081 6082 6083 6084 6085 6086 6087 6088

    if isinstance(filter_size, int):
        filter_size = [filter_size, filter_size]
    if isinstance(stride, int):
        stride = [stride, stride]
    if isinstance(padding, int):
        padding = [padding, padding]
    if len(padding) == 2:
        padding.append(padding[0])
        padding.append(padding[1])
6089 6090 6091 6092 6093 6094 6095
    inputs = {"X": input}
    attrs = {"kernels": filter_size, "strides": stride, "padding": padding}
    if input_image_size:
        if isinstance(out_stride, int):
            out_stride = [out_stride, out_stride]
        inputs["Y"] = input_image_size
        attrs["out_stride"] = out_stride
6096
    helper = LayerHelper('im2sequence', **locals())
X
Xin Pan 已提交
6097
    out = helper.create_variable_for_type_inference(dtype=helper.input_dtype())
6098
    helper.append_op(
6099
        type='im2sequence', inputs=inputs, outputs={'Out': out}, attrs=attrs)
6100
    return out
6101 6102


Y
yuyang18 已提交
6103
@templatedoc()
6104
def row_conv(input, future_context_size, param_attr=None, act=None):
Y
yuyang18 已提交
6105 6106
    """
    ${comment}
6107 6108

    Args:
Y
yuyang18 已提交
6109
        input (${x_type}): ${x_comment}.
Y
yangyaming 已提交
6110 6111
        future_context_size (int): Future context size. Please note, the shape
            of convolution kernel is [future_context_size + 1, D].
6112 6113 6114 6115 6116
        param_attr (ParamAttr): Attributes of parameters, including
            name, initializer etc.
        act (str): Non-linear activation to be applied to output variable.

    Returns:
Y
yuyang18 已提交
6117
        ${out_comment}.
6118 6119

    Examples:
Y
yuyang18 已提交
6120 6121 6122 6123
        >>> import paddle.fluid as fluid
        >>> x = fluid.layers.data(name='x', shape=[16],
        >>>                        dtype='float32', lod_level=1)
        >>> out = fluid.layers.row_conv(input=x, future_context_size=2)
6124 6125 6126 6127 6128 6129
    """
    helper = LayerHelper('row_conv', **locals())
    dtype = helper.input_dtype()
    filter_shape = [future_context_size + 1, input.shape[1]]
    filter_param = helper.create_parameter(
        attr=helper.param_attr, shape=filter_shape, dtype=dtype)
X
Xin Pan 已提交
6130
    out = helper.create_variable_for_type_inference(dtype)
6131 6132 6133 6134 6135
    helper.append_op(
        type='row_conv',
        inputs={'X': [input],
                'Filter': [filter_param]},
        outputs={'Out': [out]})
Y
yangyaming 已提交
6136
    return helper.append_activation(out)
6137 6138


Y
yuyang18 已提交
6139
@templatedoc()
6140 6141
def multiplex(inputs, index):
    """
Y
yuyang18 已提交
6142 6143
    ${comment}

L
lujun 已提交
6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186
    For Example:

    .. code-block:: text

        case 1:

        Given:

        X = [[[0,0,3,4], [0,1,3,4], [0,2,4,4], [0,3,3,4]],
             [[1,0,3,4], [1,1,7,8], [1,2,4,2], [1,3,3,4]],
             [[2,0,3,4], [2,1,7,8], [2,2,4,2], [2,3,3,4]],
             [[3,0,3,4], [3,1,7,8], [3,2,4,2], [3,3,3,4]]]

        index = [3,0,1,2]

        out:[[3 0 3 4]    // X[3,0] (3 = index[i], 0 = i); i=0
             [0 1 3 4]    // X[0,1] (0 = index[i], 1 = i); i=1
             [1 2 4 2]    // X[1,2] (0 = index[i], 2 = i); i=2
             [2 3 3 4]]   // X[2,3] (0 = index[i], 3 = i); i=3

        case 2:

        Given:

        X = [[[0,0,3,4], [0,1,3,4], [0,2,4,4], [0,3,3,4]],
             [[1,0,3,4], [1,1,7,8], [1,2,4,2], [1,3,3,4]]]

        index = [1,0]

        out:[[1 0 3 4]    // X[1,0] (3 = index[0], 0 = i); i=1
             [0 1 3 4]    // X[0,1] (0 = index[1], 1 = i); i=2
             [0 2 4 4]    // X[0,2] (0 = 0, 2 = i); i=3
             [0 3 3 4]]   // X[0,3] (0 = 0, 3 = i); i=4

    Examples:

    .. code-block:: python

        import paddle.fluid as fluid
        x1 = fluid.layers.data(name='x1', shape=[4], dtype='float32')
        x2 = fluid.layers.data(name='x2', shape=[4], dtype='float32')
        index = fluid.layers.data(name='index', shape=[1], dtype='int32')
        out = fluid.layers.multiplex(inputs=[x1, x2], index=index)
6187 6188

    Args:
Y
yuyang18 已提交
6189 6190
       inputs (list): ${x_comment}.
       index (${ids_type}): ${ids_comment}.
6191 6192

    Returns:
Y
yuyang18 已提交
6193
        ${out_comment}.
6194 6195
    """
    helper = LayerHelper('multiplex', **locals())
Y
yangyaming 已提交
6196 6197 6198 6199 6200

    if not isinstance(inputs, list) and len(inputs) < 2:
        raise ValueError("inputs should be a list object and contains at least "
                         "2 elements.")

X
Xin Pan 已提交
6201
    out = helper.create_variable_for_type_inference(inputs[0].dtype)
6202 6203 6204 6205 6206 6207
    helper.append_op(
        type='multiplex',
        inputs={'X': inputs,
                'Ids': index},
        outputs={'Out': [out]})
    return out
6208 6209


6210 6211 6212
def softmax_with_cross_entropy(logits,
                               label,
                               soft_label=False,
J
jerrywgz 已提交
6213
                               ignore_index=kIgnoreIndex,
6214
                               numeric_stable_mode=True,
6215 6216
                               return_softmax=False,
                               axis=-1):
6217 6218
    """
    **Softmax With Cross Entropy Operator.**
6219

6220
    Cross entropy loss with softmax is used as the output layer extensively. This
6221 6222 6223
    operator computes the softmax normalized values for dimension :attr:`axis` of 
    the input tensor, after which cross-entropy loss is computed. This provides 
    a more numerically stable gradient.
6224

6225 6226 6227
    Because this operator performs a softmax on logits internally, it expects
    unscaled logits. This operator should not be used with the output of
    softmax operator since that would produce incorrect results.
6228

6229 6230 6231 6232
    When the attribute :attr:`soft_label` is set :attr:`False`, this operators 
    expects mutually exclusive hard labels, each sample in a batch is in exactly 
    one class with a probability of 1.0. Each sample in the batch will have a 
    single label.
6233

6234
    The equation is as follows:
6235

6236
    1) Hard label (one-hot label, so every sample has exactly one class)
6237

6238 6239 6240 6241
    .. math::

        loss_j =  -\\text{logit}_{label_j} +
        \\log\\left(\\sum_{i=0}^{K}\\exp(\\text{logit}_i)\\right), j = 1,..., K
6242

6243 6244 6245
    2) Soft label (each sample can have a distribution over all classes)

    .. math::
6246

6247 6248 6249 6250
        loss_j =  -\\sum_{i=0}^{K}\\text{label}_i
        \\left(\\text{logit}_i - \\log\\left(\\sum_{i=0}^{K}
        \\exp(\\text{logit}_i)\\right)\\right), j = 1,...,K

6251 6252
    3) If :attr:`numeric_stable_mode` is :attr:`True`, softmax is calculated 
    first by:
S
sneaxiy 已提交
6253 6254

    .. math::
6255

H
haowang101779990 已提交
6256
        max_j &= \\max_{i=0}^{K}{\\text{logit}_i}
S
sneaxiy 已提交
6257

H
haowang101779990 已提交
6258
        log\\_max\\_sum_j &= \\log\\sum_{i=0}^{K}\\exp(logit_i - max_j)
S
sneaxiy 已提交
6259

H
haowang101779990 已提交
6260
        softmax_j &= \\exp(logit_j - max_j - {log\\_max\\_sum}_j)
S
sneaxiy 已提交
6261 6262 6263

    and then cross entropy loss is calculated by softmax and label.

6264
    Args:
6265 6266 6267 6268 6269 6270
        logits (Variable): The input tensor of unscaled log probabilities.
        label (Variable): The ground truth  tensor. If :attr:`soft_label`
            is set to :attr:`True`, Label is a Tensor<float/double> in the 
            same shape with :attr:`logits`. If :attr:`soft_label` is set to 
            :attr:`True`, Label is a Tensor<int64> in the same shape with 
            :attr:`logits` expect shape in dimension :attr:`axis` as 1.
6271
        soft_label (bool): A flag to indicate whether to interpretate the given
6272
            labels as soft labels. Default False.
M
minqiyang 已提交
6273 6274
        ignore_index (int): Specifies a target value that is ignored and does
                            not contribute to the input gradient. Only valid
6275 6276
                            if :attr:`soft_label` is set to :attr:`False`. 
                            Default: kIgnoreIndex
S
sneaxiy 已提交
6277 6278
        numeric_stable_mode (bool): A flag to indicate whether to use a more
                                    numerically stable algorithm. Only valid
6279 6280 6281 6282
                                    when :attr:`soft_label` is :attr:`False` 
                                    and GPU is used. When :attr:`soft_label` 
                                    is :attr:`True` or CPU is used, the 
                                    algorithm is always numerically stable.
6283
                                    Note that the speed may be slower when use
6284
                                    stable algorithm. Default: True
6285
        return_softmax (bool): A flag indicating whether to return the softmax
6286
                               along with the cross entropy loss. Default: False
6287 6288 6289
        axis (int): The index of dimension to perform softmax calculations. It 
                    should be in range :math:`[-1, rank - 1]`, while :math:`rank`
                    is the rank of input :attr:`logits`. Default: -1.
6290

6291
    Returns:
H
haowang101779990 已提交
6292 6293
        Variable or Tuple of two Variables: Return the cross entropy loss if \
                                            `return_softmax` is False, otherwise the tuple \
6294 6295 6296 6297
                                            (loss, softmax), softmax is in the same shape \
                                            with input logits and cross entropy loss is in \
                                            the same shape with input logits except shape \
                                            in dimension :attr:`axis` as 1.
6298 6299 6300 6301 6302 6303 6304

    Examples:
        .. code-block:: python

            data = fluid.layers.data(name='data', shape=[128], dtype='float32')
            label = fluid.layers.data(name='label', shape=[1], dtype='int64')
            fc = fluid.layers.fc(input=data, size=100)
F
stash  
fengjiayi 已提交
6305 6306
            out = fluid.layers.softmax_with_cross_entropy(
                logits=fc, label=label)
6307 6308
    """
    helper = LayerHelper('softmax_with_cross_entropy', **locals())
X
Xin Pan 已提交
6309 6310
    softmax = helper.create_variable_for_type_inference(dtype=logits.dtype)
    loss = helper.create_variable_for_type_inference(dtype=logits.dtype)
6311 6312 6313 6314 6315 6316
    helper.append_op(
        type='softmax_with_cross_entropy',
        inputs={'Logits': logits,
                'Label': label},
        outputs={'Softmax': softmax,
                 'Loss': loss},
S
sneaxiy 已提交
6317 6318 6319
        attrs={
            'soft_label': soft_label,
            'ignore_index': ignore_index,
6320 6321
            'numeric_stable_mode': numeric_stable_mode,
            'axis': axis
S
sneaxiy 已提交
6322
        })
6323 6324 6325 6326

    if return_softmax:
        return loss, softmax

6327 6328 6329
    return loss


6330 6331 6332
def sampled_softmax_with_cross_entropy(logits,
                                       label,
                                       num_samples,
X
xuezhong 已提交
6333
                                       num_true=1,
6334
                                       remove_accidental_hits=True,
X
xuezhong 已提交
6335 6336 6337
                                       use_customized_samples=False,
                                       customized_samples=None,
                                       customized_probabilities=None,
6338
                                       seed=0):
X
xuezhong 已提交
6339 6340 6341 6342 6343
    """
    **Sampled Softmax With Cross Entropy Operator.**

    Cross entropy loss with sampled softmax is used as the output layer for 
    larger output classes extensively. This operator samples a number of samples
6344
    for all examples, and computes the softmax normalized values for each 
X
xuezhong 已提交
6345 6346 6347 6348 6349 6350 6351 6352
    row of the sampled tensor, after which cross-entropy loss is computed. 

    Because this operator performs a softmax on logits internally, it expects
    unscaled logits. This operator should not be used with the output of
    softmax operator since that would produce incorrect results.
    
    For examples with T true labels (T >= 1), we assume that each true label has
    a probability of 1/T. For each sample, S samples are generated using a
X
xuezhong 已提交
6353
    log uniform distribution. True labels are concatenated with these samples to
X
xuezhong 已提交
6354 6355 6356 6357 6358 6359 6360 6361
    form T + S samples for each example. So, assume the shape of logits is
    [N x K], the shape for samples is [N x (T+S)]. For each sampled label, a 
    probability is calculated, which corresponds to the Q(y|x) in 
    [Jean et al., 2014](http://arxiv.org/abs/1412.2007).
    
    Logits are sampled according to the sampled labels. Then if 
    remove_accidental_hits is True, if a sample[i, j] accidentally hits true 
    labels, then the corresponding sampled_logits[i, j] is minus by 1e20 to 
X
xuezhong 已提交
6362
    make its softmax result close to zero. Then sampled logits are subtracted by
X
xuezhong 已提交
6363 6364 6365 6366 6367 6368 6369 6370 6371 6372 6373
    logQ(y|x), these sampled logits and re-indexed labels are used to compute 
    a softmax with cross entropy.

    Args:
        logits (Variable): The unscaled log probabilities, which is a 2-D tensor
            with shape [N x K]. N is the batch_size, and K is the class number.
        label (Variable): The ground truth which is a 2-D tensor. Label is a 
            Tensor<int64> with shape [N x T], where T is the number of true 
            labels per example. 
        num_samples (int): The number for each example, num_samples should be 
            less than the number of class.
6374
        num_true(int): The number of target classes per training example.
X
xuezhong 已提交
6375 6376 6377 6378 6379
        remove_accidental_hits (bool): A flag indicating whether to remove 
            accidental hits when sampling. If True and if a sample[i, j] 
            accidentally hits true labels, then the corresponding 
            sampled_logits[i, j] is minus by 1e20 to make its softmax result 
            close to zero. Default is True.
X
xuezhong 已提交
6380
        use_customized_samples (bool): Whether to use custom samples and probabities to sample
6381
            logits.
X
xuezhong 已提交
6382 6383 6384 6385 6386
        customized_samples (Variable): User defined samples, which is a 2-D tensor
            with shape [N, T + S]. S is the num_samples, and T is the number of true 
            labels per example. 
        customized_probabilities (Variable): User defined probabilities of samples, 
            a 2-D tensor which has the same shape with customized_samples.
6387 6388 6389
        seed (int): The random seed for generating random number, which is used
            in the process of sampling. Default is 0.

X
xuezhong 已提交
6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409
    Returns:
        Variable: Return the cross entropy loss which is a 2-D tensor with shape
                  [N x 1].

    Examples:
        .. code-block:: python

            logits = fluid.layers.data(name='data', shape=[256], dtype='float32')
            label = fluid.layers.data(name='label', shape=[5], dtype='int64')
            fc = fluid.layers.fc(input=data, size=100)
            out = fluid.layers.sampled_softmax_with_cross_entropy(
                logits=fc, label=label, num_samples=25)
    """
    helper = LayerHelper('sample_logits', **locals())
    samples = helper.create_variable_for_type_inference(dtype='int64')
    probabilities = helper.create_variable_for_type_inference(
        dtype=logits.dtype)
    sampled_logits \
        = helper.create_variable_for_type_inference(dtype=logits.dtype)
    sampled_label = helper.create_variable_for_type_inference(dtype='int64')
X
xuezhong 已提交
6410 6411
    sampled_softlabel = helper.create_variable_for_type_inference(
        dtype=logits.dtype)
6412 6413
    logits_dim = helper.create_variable_for_type_inference(dtype=logits.dtype)
    labels_dim = helper.create_variable_for_type_inference(dtype=label.type)
X
xuezhong 已提交
6414 6415 6416 6417 6418

    helper.append_op(
        type='sample_logits',
        inputs={
            'Logits': logits,
X
xuezhong 已提交
6419
            'Labels': label,
X
xuezhong 已提交
6420 6421
            'CustomizedSamples': customized_samples,
            'CustomizedProbabilities': customized_probabilities
X
xuezhong 已提交
6422 6423 6424 6425
        },
        outputs={
            'Samples': samples,
            'Probabilities': probabilities,
X
xuezhong 已提交
6426
            'SampledLabels': sampled_label,
6427 6428 6429
            'SampledLogits': sampled_logits,
            'LogitsDim': logits_dim,
            'LabelsDim': labels_dim
X
xuezhong 已提交
6430 6431
        },
        attrs={
X
xuezhong 已提交
6432
            'use_customized_samples': use_customized_samples,
6433
            'uniq': True,
X
xuezhong 已提交
6434 6435 6436 6437
            'remove_accidental_hits': remove_accidental_hits,
            'num_samples': num_samples,
            'seed': seed
        })
X
xuezhong 已提交
6438 6439
    loss = helper.create_variable_for_type_inference(dtype=logits.dtype)
    softmax = helper.create_variable_for_type_inference(dtype=logits.dtype)
X
xuezhong 已提交
6440 6441 6442 6443 6444 6445
    helper.append_op(
        type='one_hot',
        inputs={'X': sampled_label},
        attrs={'depth': num_samples + 1},
        outputs={'Out': sampled_softlabel})

6446 6447
    helper.append_op(
        type='softmax_with_cross_entropy',
X
xuezhong 已提交
6448
        inputs={'Logits': sampled_logits,
X
xuezhong 已提交
6449
                'Label': sampled_softlabel},
X
xuezhong 已提交
6450 6451 6452
        outputs={'Softmax': softmax,
                 'Loss': loss},
        attrs={
X
xuezhong 已提交
6453
            'soft_label': True,
X
xuezhong 已提交
6454 6455 6456
            'ignore_index': False,
            'numeric_stable_mode': False
        })
X
xuezhong 已提交
6457
    return loss / num_true
X
xuezhong 已提交
6458 6459


6460 6461
def smooth_l1(x, y, inside_weight=None, outside_weight=None, sigma=None):
    """
Y
Yibing Liu 已提交
6462 6463
    This layer computes the smooth L1 loss for Variable :attr:`x` and :attr:`y`.
    It takes the first dimension of :attr:`x` and :attr:`y` as batch size.
Q
qingqing01 已提交
6464
    For each instance, it computes the smooth L1 loss element by element first
6465
    and then sums all the losses. So the shape of ouput Variable is
6466
    [batch_size, 1].
6467

6468 6469
    Args:
        x (Variable): A tensor with rank at least 2. The input value of smooth
Q
qingqing01 已提交
6470
            L1 loss op with shape [batch_size, dim1, ..., dimN].
6471
        y (Variable): A tensor with rank at least 2. The target value of smooth
Y
Yibing Liu 已提交
6472
            L1 loss op with same shape as :attr:`x`.
6473
        inside_weight (Variable|None):  A tensor with rank at least 2. This
6474 6475
            input is optional and should have same shape with :attr:`x`. If
            provided, the result of (:attr:`x` - :attr:`y`) will be multiplied
Y
Yibing Liu 已提交
6476
            by this tensor element by element.
6477
        outside_weight (Variable|None): A tensor with rank at least 2. This
6478 6479
            input is optional and should have same shape with :attr:`x`. If
            provided, the out smooth L1 loss will be multiplied by this tensor
Y
Yibing Liu 已提交
6480
            element by element.
6481
        sigma (float|None): Hyper parameter of smooth L1 loss layer. A float
6482 6483
           scalar with default value 1.0.

6484
    Returns:
6485
        Variable: The output smooth L1 loss with shape [batch_size, 1].
6486 6487 6488 6489 6490

    Examples:
        .. code-block:: python

            data = fluid.layers.data(name='data', shape=[128], dtype='float32')
F
stash  
fengjiayi 已提交
6491 6492
            label = fluid.layers.data(
                name='label', shape=[100], dtype='float32')
6493
            fc = fluid.layers.fc(input=data, size=100)
F
fengjiayi 已提交
6494
            out = fluid.layers.smooth_l1(x=fc, y=label)
6495
    """
6496

6497
    helper = LayerHelper('smooth_l1_loss', **locals())
X
Xin Pan 已提交
6498 6499
    diff = helper.create_variable_for_type_inference(dtype=x.dtype)
    loss = helper.create_variable_for_type_inference(dtype=x.dtype)
6500 6501 6502 6503 6504 6505 6506 6507 6508 6509
    helper.append_op(
        type='smooth_l1_loss',
        inputs={
            'X': x,
            'Y': y,
            'InsideWeight': inside_weight,
            'OutsideWeight': outside_weight
        },
        outputs={'Diff': diff,
                 'Out': loss},
6510
        attrs={'sigma': sigma if sigma is not None else 1.0})
6511
    return loss
6512 6513 6514 6515


def one_hot(input, depth):
    """
Y
Yibing Liu 已提交
6516
    This layer creates the one-hot representations for input indices.
6517 6518

    Args:
Y
Yibing Liu 已提交
6519 6520
        input(Variable): Input indices, last dimension must be 1.
        depth(scalar): An interger defining the depth of the one-hot dimension.
6521 6522

    Returns:
Y
Yibing Liu 已提交
6523
        Variable: The one-hot representations of input.
6524 6525

    Examples:
C
caoying03 已提交
6526
        .. code-block:: python
6527

Y
Yibing Liu 已提交
6528 6529
            label = fluid.layers.data(name="label", shape=[1], dtype="int64")
            one_hot_label = fluid.layers.one_hot(input=label, depth=10)
6530 6531
    """
    helper = LayerHelper("one_hot", **locals())
X
Xin Pan 已提交
6532
    one_hot_out = helper.create_variable_for_type_inference(dtype='float32')
6533 6534 6535 6536
    helper.append_op(
        type="one_hot",
        inputs={'X': input},
        attrs={'depth': depth},
6537 6538
        outputs={'Out': one_hot_out},
        stop_gradient=True)
6539
    return one_hot_out
Y
Yu Yang 已提交
6540 6541


Y
Yu Yang 已提交
6542
def autoincreased_step_counter(counter_name=None, begin=1, step=1):
Y
Yu Yang 已提交
6543
    """
Y
yi.wu 已提交
6544 6545 6546
    Create an auto-increase variable
    which will be automatically increased by 1 every mini-batch
    Return the run counter of the main program, default is started from 1.
Y
Yu Yang 已提交
6547 6548 6549 6550 6551 6552

    Args:
        counter_name(str): The counter name, default is '@STEP_COUNTER@'.
        begin(int): The first value of this counter.
        step(int): The increment step between each execution.

6553 6554
    Returns:
        Variable: The global run counter.
Y
yi.wu 已提交
6555 6556 6557 6558 6559

    Examples:
        .. code-block:: python

           global_step = fluid.layers.autoincreased_step_counter(
Y
Yibing Liu 已提交
6560
               counter_name='@LR_DECAY_COUNTER@', begin=0, step=1)
Y
Yu Yang 已提交
6561 6562
    """
    helper = LayerHelper('global_step_counter')
Y
Yu Yang 已提交
6563 6564
    if counter_name is None:
        counter_name = '@STEP_COUNTER@'
Y
Yu Yang 已提交
6565 6566 6567 6568 6569
    counter, is_new_var = helper.create_or_get_global_variable(
        name=counter_name, dtype='int64', shape=[1], persistable=True)
    if is_new_var:
        helper.set_variable_initializer(
            counter, initializer=Constant(
Y
Yu Yang 已提交
6570
                value=begin - 1, force_cpu=True))
W
Wu Yi 已提交
6571
        helper.main_program.global_block()._prepend_op(
Y
Yu Yang 已提交
6572 6573
            type='increment',
            inputs={'X': [counter]},
Y
Yu Yang 已提交
6574
            outputs={'Out': [counter]},
M
minqiyang 已提交
6575 6576
            attrs={'step': float(step)},
            stop_gradient=True)
Y
Yu Yang 已提交
6577 6578 6579
        counter.stop_gradient = True

    return counter
Y
yangyaming 已提交
6580 6581


6582
def reshape(x, shape, actual_shape=None, act=None, inplace=False, name=None):
C
caoying03 已提交
6583
    """
C
caoying03 已提交
6584 6585
    Gives a new shape to the input Tensor without changing its data.

6586 6587 6588 6589 6590
    The target shape can be given by :attr:`shape` or :attr:`actual_shape`.
    :attr:`shape` is a list of integer while :attr:`actual_shape` is a tensor
    variable. :attr:`actual_shape` has a higher priority than :attr:`shape`
    if it is provided, while :attr:`shape` still should be set correctly to
    gurantee shape inference in compile-time.
C
caoying03 已提交
6591

6592
    Some tricks exist when specifying the target shape.
C
caoying03 已提交
6593

6594 6595 6596 6597
    1. -1 means the value of this dimension is inferred from the total element
    number of x and remaining dimensions. Thus one and only one dimension can
    be set -1.

6598
    2. 0 means the actual dimension value is going to be copied from the
6599 6600 6601 6602
    corresponding dimension of x. The indice of 0s in shape can not exceed
    Rank(X).

    Here are some examples to explain it.
C
caoying03 已提交
6603 6604

    1. Given a 3-D tensor x with a shape [2, 4, 6], and the target shape
W
wanghaoshuang 已提交
6605
    is [6, 8], the reshape operator will transform x into a 2-D tensor with
6606
    shape [6, 8] and leaving x's data unchanged.
C
caoying03 已提交
6607

6608
    2. Given a 3-D tensor x with a shape [2, 4, 6], and the target shape
6609 6610
    specified is [2, 3, -1, 2], the reshape operator will transform x into a
    4-D tensor with shape [2, 3, 4, 2] and leaving x's data unchanged. In this
W
wanghaoshuang 已提交
6611 6612
    case, one dimension of the target shape is set to -1, the value of this
    dimension is inferred from the total element number of x and remaining
6613
    dimensions.
C
caoying03 已提交
6614

6615
    3. Given a 3-D tensor x with a shape [2, 4, 6], and the target shape
6616 6617 6618 6619
    is [-1, 0, 3, 2], the reshape operator will transform x into a 4-D tensor
    with shape [2, 4, 3, 2] and leaving x's data unchanged. In this case,
    besides -1, 0 means the actual dimension value is going to be copied from
    the corresponding dimension of x.
C
caoying03 已提交
6620 6621

    Args:
6622
        x(variable): The input tensor.
C
caoying03 已提交
6623 6624
        shape(list): The new shape. At most one dimension of the new shape can
                     be -1.
6625 6626 6627 6628 6629
        actual_shape(variable): An optional input. If provided, reshape
                                according to this given shape rather than
                                :attr:`shape` specifying shape. That is to
                                say :attr:`actual_shape` has a higher priority
                                than :attr:`shape`.
6630 6631
        act (str): The non-linear activation to be applied to the reshaped tensor
                   variable.
C
chengduozh 已提交
6632 6633 6634
        inplace(bool): If ``inplace`` is `True`, the input and output of ``layers.reshape``
                       are the same variable, otherwise, the input and output of
                       ``layers.reshape`` are different variables. Note that if :attr:`x`
C
chengduozh 已提交
6635
                       is more than one layer's input, ``inplace`` must be :attr:`False`.
6636
        name (str): The name of this layer. It is optional.
C
caoying03 已提交
6637

6638
    Returns:
G
guosheng 已提交
6639 6640 6641 6642
        Variable: The reshaped tensor variable if :attr:`act` is None. It is a \
                  new tensor variable if :attr:`inplace` is :attr:`False`, \
                  otherwise it is :attr:`x`. If :attr:`act` is not None, return \
                  the activated tensor variable.
C
caoying03 已提交
6643

X
Xin Pan 已提交
6644 6645 6646
    Raises:
        TypeError: if actual_shape is neither Variable nor None.

C
caoying03 已提交
6647 6648
    Examples:
        .. code-block:: python
G
guosheng 已提交
6649

6650
            data = fluid.layers.data(
6651
                name='data', shape=[2, 4, 6], dtype='float32')
C
caoying03 已提交
6652
            reshaped = fluid.layers.reshape(
G
guosheng 已提交
6653
                x=data, shape=[-1, 0, 3, 2], inplace=True)
C
caoying03 已提交
6654 6655 6656
    """

    if not (isinstance(shape, list) or isinstance(shape, tuple)):
L
luotao1 已提交
6657
        raise ValueError("Input shape must be a python list or tuple.")
X
Xin Pan 已提交
6658 6659 6660 6661 6662
    inputs = {"X": x}
    if isinstance(actual_shape, Variable):
        inputs["Shape"] = actual_shape
    elif actual_shape is not None:
        raise TypeError("actual_shape should either be Variable or None")
C
caoying03 已提交
6663

6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678
    # Validate the shape
    unk_dim_idx = -1
    for dim_idx, dim_size in enumerate(shape):
        if dim_size == -1:
            assert unk_dim_idx == -1, (
                "Only one dimension in shape can be unknown.")
            unk_dim_idx = dim_idx
        elif dim_size == 0:
            assert dim_idx < len(x.shape), (
                "The indice of 0s in shape can not exceed Rank(X).")
        else:
            assert dim_size > 0, (
                "Each dimension size given in shape must not be negtive "
                "except one unknown dimension.")

6679
    helper = LayerHelper("reshape2", **locals())
6680 6681
    out = x if inplace else helper.create_variable_for_type_inference(
        dtype=x.dtype)
X
Xin Pan 已提交
6682
    x_shape = helper.create_variable_for_type_inference(dtype=x.dtype)
C
caoying03 已提交
6683
    helper.append_op(
6684
        type="reshape2",
X
Xin Pan 已提交
6685
        inputs=inputs,
D
dzhwinter 已提交
6686
        attrs={"shape": shape},
6687 6688
        outputs={"Out": out,
                 "XShape": x_shape})
C
caoying03 已提交
6689

D
dzhwinter 已提交
6690
    return helper.append_activation(out)
6691

6692

6693
def squeeze(input, axes, name=None):
Y
Yibing Liu 已提交
6694
    """
M
minqiyang 已提交
6695 6696 6697
    Remove single-dimensional entries from the shape of a tensor. Takes a
    parameter axes with a list of axes to squeeze. If axes is not provided, all
    the single dimensions will be removed from the shape. If an axis is
Y
Yibing Liu 已提交
6698
    selected with shape entry not equal to one, an error is raised.
M
minqiyang 已提交
6699

H
haowang101779990 已提交
6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720
    For example:

    .. code-block:: text

        Case 1:

          Given
            X.shape = (1, 3, 1, 5)
          and
            axes = [0]
          we get:
            Out.shape = (3, 1, 5)

        Case 2:

          Given
            X.shape = (1, 3, 1, 5)
          and
            axes = []
          we get:
            Out.shape = (3, 5)
M
minqiyang 已提交
6721

Y
Yibing Liu 已提交
6722
    Args:
6723
        input (Variable): The input variable to be squeezed.
Y
Yibing Liu 已提交
6724
        axes (list): List of integers, indicating the dimensions to be squeezed.
6725
        name (str|None): Name for this layer.
Y
Yibing Liu 已提交
6726 6727 6728 6729 6730 6731 6732

    Returns:
        Variable: Output squeezed variable.

    Examples:
        .. code-block:: python

6733
            import paddle.fluid.layers as layers
Y
Yibing Liu 已提交
6734
            x = layers.data(name='x', shape=[5, 1, 10])
6735
            y = layers.squeeze(input=x, axes=[1])
Y
Yibing Liu 已提交
6736
    """
L
lujun 已提交
6737
    assert not in_dygraph_mode(), (
L
lujun 已提交
6738
        "squeeze layer is not supported in dygraph mode yet.")
Y
Yibing Liu 已提交
6739
    helper = LayerHelper("squeeze", **locals())
X
Xin Pan 已提交
6740 6741
    out = helper.create_variable_for_type_inference(dtype=input.dtype)
    x_shape = helper.create_variable_for_type_inference(dtype=input.dtype)
Y
Yibing Liu 已提交
6742
    helper.append_op(
6743
        type="squeeze2",
6744
        inputs={"X": input},
Y
Yibing Liu 已提交
6745
        attrs={"axes": axes},
6746 6747
        outputs={"Out": out,
                 "XShape": x_shape})
Y
Yibing Liu 已提交
6748

6749 6750 6751
    return out


6752
def unsqueeze(input, axes, name=None):
Y
Yibing Liu 已提交
6753
    """
M
minqiyang 已提交
6754 6755 6756
    Insert single-dimensional entries to the shape of a tensor. Takes one
    required argument axes, a list of dimensions that will be inserted.
    Dimension indices in axes are as seen in the output tensor.
Y
Yibing Liu 已提交
6757

M
minqiyang 已提交
6758
    For example:
H
haowang101779990 已提交
6759 6760 6761

    .. code-block:: text

M
minqiyang 已提交
6762
      Given a tensor such that tensor with shape [3, 4, 5],
Y
Yibing Liu 已提交
6763
      then Unsqueezed tensor with axes=[0, 4] has shape [1, 3, 4, 5, 1].
M
minqiyang 已提交
6764

Y
Yibing Liu 已提交
6765
    Args:
6766
        input (Variable): The input variable to be unsqueezed.
Y
Yibing Liu 已提交
6767
        axes (list): List of integers, indicating the dimensions to be inserted.
6768
        name (str|None): Name for this layer.
Y
Yibing Liu 已提交
6769 6770 6771 6772 6773 6774 6775 6776

    Returns:
        Variable: Output unsqueezed variable.

    Examples:
        .. code-block:: python

            x = layers.data(name='x', shape=[5, 10])
6777
            y = layers.unsequeeze(input=x, axes=[1])
Y
Yibing Liu 已提交
6778 6779
    """
    helper = LayerHelper("unsqueeze", **locals())
X
Xin Pan 已提交
6780 6781
    out = helper.create_variable_for_type_inference(dtype=input.dtype)
    x_shape = helper.create_variable_for_type_inference(dtype=input.dtype)
Y
Yibing Liu 已提交
6782
    helper.append_op(
6783
        type="unsqueeze2",
6784
        inputs={"X": input},
Y
Yibing Liu 已提交
6785
        attrs={"axes": axes},
6786 6787
        outputs={"Out": out,
                 "XShape": x_shape})
Y
Yibing Liu 已提交
6788

6789 6790
    return out

6791

Y
yangyaming 已提交
6792
def lod_reset(x, y=None, target_lod=None):
Y
yangyaming 已提交
6793
    """
Y
Yibing Liu 已提交
6794
    Set LoD of :attr:`x` to a new one specified by :attr:`y` or
6795 6796 6797 6798
    :attr:`target_lod`. When :attr:`y` provided, :attr:`y.lod` would be
    considered as target LoD first, otherwise :attr:`y.data` would be
    considered as target LoD. If :attr:`y` is not provided, target LoD should
    be specified by :attr:`target_lod`. If target LoD is specified by
Y
Yibing Liu 已提交
6799
    :attr:`Y.data` or :attr:`target_lod`, only one level LoD is supported.
Y
yangyaming 已提交
6800 6801 6802 6803 6804 6805

    .. code-block:: text

        * Example 1:

            Given a 1-level LoDTensor x:
6806
                x.lod =  [[ 2,           3,                   1 ]]
Y
yangyaming 已提交
6807 6808 6809
                x.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]]
                x.dims = [6, 1]

6810
            target_lod: [4, 2]
Y
yangyaming 已提交
6811 6812

            then we get a 1-level LoDTensor:
6813
                out.lod =  [[4,                          2]]
Y
yangyaming 已提交
6814 6815 6816 6817 6818 6819
                out.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]]
                out.dims = [6, 1]

        * Example 2:

            Given a 1-level LoDTensor x:
6820
                x.lod =  [[2,            3,                   1]]
Y
yangyaming 已提交
6821 6822 6823 6824
                x.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]]
                x.dims = [6, 1]

            y is a Tensor:
6825
                y.data = [[2, 4]]
Y
yangyaming 已提交
6826 6827 6828
                y.dims = [1, 3]

            then we get a 1-level LoDTensor:
6829
                out.lod =  [[2,            4]]
Y
yangyaming 已提交
6830 6831 6832 6833 6834 6835
                out.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]]
                out.dims = [6, 1]

        * Example 3:

            Given a 1-level LoDTensor x:
6836
                x.lod =  [[2,            3,                   1]]
Y
yangyaming 已提交
6837 6838 6839 6840
                x.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]]
                x.dims = [6, 1]

            y is a 2-level LoDTensor:
6841
                y.lod =  [[2, 2], [2, 2, 1, 1]]
Y
yangyaming 已提交
6842 6843 6844 6845
                y.data = [[1.1], [2.1], [3.1], [4.1], [5.1], [6.1]]
                y.dims = [6, 1]

            then we get a 2-level LoDTensor:
6846
                out.lod =  [[2, 2], [2, 2, 1, 1]]
Y
yangyaming 已提交
6847 6848 6849 6850 6851
                out.data = [[1.0], [2.0], [3.0], [4.0], [5.0], [6.0]]
                out.dims = [6, 1]

    Args:
        x (Variable): Input variable which could be a Tensor or LodTensor.
6852
        y (Variable|None): If provided, output's LoD would be derived
Y
Yibing Liu 已提交
6853
                           from :attr:`y`.
Y
yangyaming 已提交
6854
        target_lod (list|tuple|None): One level LoD which should be considered
Y
Yibing Liu 已提交
6855
                                      as target LoD when :attr:`y` not provided.
Y
yangyaming 已提交
6856 6857

    Returns:
Y
Yibing Liu 已提交
6858
        Variable: Output variable with LoD specified by this layer.
Y
yangyaming 已提交
6859 6860

    Raises:
Y
Yibing Liu 已提交
6861
        ValueError: If :attr:`y` and :attr:`target_lod` are both None.
Y
yangyaming 已提交
6862 6863 6864 6865 6866 6867 6868 6869 6870

    Examples:
        .. code-block:: python

            x = layers.data(name='x', shape=[10])
            y = layers.data(name='y', shape=[10, 20], lod_level=2)
            out = layers.lod_reset(x=x, y=y)
    """
    helper = LayerHelper("lod_reset", **locals())
X
Xin Pan 已提交
6871
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
Y
yangyaming 已提交
6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885
    if y is not None:
        helper.append_op(
            type="lod_reset", inputs={'X': x,
                                      'Y': y}, outputs={'Out': out})
    elif target_lod is not None:
        helper.append_op(
            type="lod_reset",
            inputs={'X': x},
            attrs={'target_lod': target_lod},
            outputs={'Out': out})
    else:
        raise ValueError("y and target_lod should not be both None.")

    return out
D
dragonwarrior 已提交
6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896


def lrn(input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None):
    """
    Local Response Normalization Layer. This layer performs a type of
    "lateral inhibition" by normalizing over local input regions.

    The formula is as follows:

    .. math::

X
xiaoting 已提交
6897
      Output(i, x, y) = Input(i, x, y) / \\left(k + \\alpha \\sum\\limits^{\\min(C-1, i + n/2)}_{j = \\max(0, i - n/2)}(Input(j, x, y))^2\\right)^{\\beta}
D
dragonwarrior 已提交
6898 6899 6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925

    In the above equation:

    * :math:`n`: The number of channels to sum over.
    * :math:`k`: The offset (avoid being divided by 0).
    * :math:`alpha`: The scaling parameter.
    * :math:`beta`: The exponent parameter.

    Refer to `ImageNet Classification with Deep Convolutional Neural Networks
    <https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf>`_

    Args:
        input (Variable): The input tensor of this layer, and the dimension of input tensor must be 4.
        n (int, default 5): The number of channels to sum over.
        k (float, default 1.0): An offset (usually positive to avoid dividing by 0).
        alpha (float, default 1e-4): The scaling parameter.
        beta (float, default 0.75): The exponent.
        name (str, default None): A name for this operation.

    Raises:
        ValueError: If rank of the input tensor is not 4.

    Returns:
        A tensor variable storing the transformation result.

    Examples:
        .. code-block:: python

F
stash  
fengjiayi 已提交
6926 6927
          data = fluid.layers.data(
              name="data", shape=[3, 112, 112], dtype="float32")
D
dragonwarrior 已提交
6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939
          lrn = fluid.layers.lrn(input=data)
    """
    helper = LayerHelper('lrn', **locals())
    dtype = helper.input_dtype()
    input_shape = input.shape
    dims = len(input_shape)

    if dims != 4:
        raise ValueError(
            "dims of input must be 4(not %d), and it's order must be NCHW" %
            (dims))

X
Xin Pan 已提交
6940 6941 6942
    mid_out = helper.create_variable_for_type_inference(
        dtype=dtype, stop_gradient=True)
    lrn_out = helper.create_variable_for_type_inference(dtype)
D
dragonwarrior 已提交
6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955
    helper.append_op(
        type="lrn",
        inputs={"X": input},
        outputs={
            "Out": lrn_out,
            "MidOut": mid_out,
        },
        attrs={"n": n,
               "k": k,
               "alpha": alpha,
               "beta": beta})

    return lrn_out
G
guosheng 已提交
6956 6957 6958 6959


def pad(x, paddings, pad_value=0., name=None):
    """
G
guosheng 已提交
6960
    Pads a tensor with a constant value given by :attr:`pad_value`, and the
W
wanghaoshuang 已提交
6961
    padded width is specified by :attr:`paddings`.
G
guosheng 已提交
6962

G
guosheng 已提交
6963 6964 6965 6966
    Specifically, the number of values padded before the contents of :attr:`x`
    in dimension :attr:`i` is indicated by :attr:`paddings[i]`, and the number
    of values padded after the contents of :attr:`x` in dimension :attr:`i` is
    indicated by :attr:`paddings[i+1]`.
G
guosheng 已提交
6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987 6988

    See below for an example.

    .. code-block:: text

        Given:
            x = [[1, 2], [3, 4]]

            paddings = [0, 1, 1, 2]

            pad_value = 0

        Return:

            out = [[0, 1, 2, 0, 0]
                   [0, 3, 4, 0, 0]
                   [0, 0, 0, 0, 0]]

    Args:
        x (Variable): The input tensor variable.
        paddings (list): A list of integers. Its elements specify the padded
                         width before and after for each dimension in turn.
W
wanghaoshuang 已提交
6989
                         The length of :attr:paddings must be
G
guosheng 已提交
6990 6991 6992 6993 6994 6995 6996 6997 6998 6999
                         :math:`rank(x) \\times 2`.
        pad_value (float): The constant value used to pad.
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.

    Returns:
        Variable: The padded tensor variable.

    Examples:
        .. code-block:: python
G
guosheng 已提交
7000

G
guosheng 已提交
7001
            # x is a rank 2 tensor variable.
S
SunGaofeng 已提交
7002 7003
            import paddle.fluid as fluid
            x = fluid.layers.data(name='data', shape=[224], dtype='float32')
G
guosheng 已提交
7004 7005 7006 7007 7008
            out = fluid.layers.pad(
                x=x, paddings=[0, 1, 1, 2], pad_value=0.)
    """
    helper = LayerHelper('pad', input=x, **locals())
    dtype = helper.input_dtype()
X
Xin Pan 已提交
7009
    out = helper.create_variable_for_type_inference(dtype)
G
guosheng 已提交
7010 7011 7012 7013 7014 7015 7016
    helper.append_op(
        type='pad',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={'paddings': paddings,
               'pad_value': float(pad_value)})
    return out
7017 7018


C
chengduo 已提交
7019 7020 7021 7022 7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049
def pad_constant_like(x, y, pad_value=0., name=None):
    """
    Pad input(Y) with :attr:`pad_value`, the number of values padded to
    the edges of each axis is specified by the difference of the shape
    of X and Y. ((0, shape_x_0 - shape_y_0), ... (0, shape_x_n - shape_y_n))
    unique pad widths for each axis. The input should be a k-D
    tensor(k > 0 and k < 7).

    See below for an example.

    .. code-block:: text

        Given:
            X = [[[[ 0,  1,  2],
                   [ 3,  4,  5]],
                  [[ 6,  7,  8],
                   [ 9, 10, 11]],
                  [[12, 13, 14],
                   [15, 16, 17]]],
                 [[[18, 19, 20],
                   [21, 22, 23]],
                  [[24, 25, 26],
                   [27, 28, 29]],
                  [[30, 31, 32],
                   [33, 34, 35]]]]
            X.shape = (2, 3, 2, 3)

            Y = [[[[35, 36, 37]],
                  [[38, 39, 40]],
                  [[41, 42, 43]]]]
            Y.shape = (1, 3, 1, 3)
T
Tink_Y 已提交
7050 7051
		And
            pad_value = -1,
C
chengduo 已提交
7052

T
Tink_Y 已提交
7053 7054 7055 7056 7057 7058 7059 7060 7061 7062 7063 7064 7065 7066
        Return:
            Out = [[[[35, 36, 37],
                     [-1, -1, -1]],
                    [[38, 39, 40],
                     [-1, -1, -1]],
                    [[41, 42, 43],
                     [-1, -1, -1]]],
                  [[[-1, -1, -1],
                    [-1, -1, -1]],
                   [[-1, -1, -1],
                    [-1, -1, -1]],
                   [[-1, -1, -1],
                    [-1, -1, -1]]]]
            Out.shape = (2, 3, 2, 3)
C
chengduo 已提交
7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082

    Args:
        x (Variable): The input tensor variable.
        y (Variable): The input tensor variable.
        pad_value (float): The constant value used to pad.
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.

    Returns:
        Variable: The padded tensor variable.

    Examples:
        .. code-block:: python

            # x is a rank 4 tensor variable, x.shape = (2, 3, 2, 3)
            # y is a rank 4 tensor variable, y.shape = (1, 3, 1, 3)
S
SunGaofeng 已提交
7083 7084 7085
            import paddle.fluid as fluid
            x = fluid.layers.data(name='x', shape=[2,3,2,3], dtype='float32')
            y = fluid.layers.data(name='y', shape=[1,3,1,3], dtype='float32')
C
chengduo 已提交
7086 7087 7088 7089 7090
            out = fluid.layers.pad_constant_like(x=x, y=y, pad_value=0.)
            # out is a rank 4 tensor variable, and out.shape = [2, 3 ,2 , 3]
    """
    helper = LayerHelper('pad_constant_like', input=x, **locals())
    dtype = helper.input_dtype()
X
Xin Pan 已提交
7091
    out = helper.create_variable_for_type_inference(dtype)
C
chengduo 已提交
7092 7093 7094 7095 7096 7097 7098 7099 7100
    helper.append_op(
        type='pad_constant_like',
        inputs={'X': x,
                'Y': y},
        outputs={'Out': out},
        attrs={'pad_value': float(pad_value)})
    return out


7101 7102 7103 7104 7105 7106 7107
def label_smooth(label,
                 prior_dist=None,
                 epsilon=0.1,
                 dtype="float32",
                 name=None):
    """
    Label smoothing is a mechanism to regularize the classifier layer and is
7108 7109
    called label-smoothing regularization (LSR).

7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132
    Label smoothing is proposed to encourage the model to be less confident,
    since optimizing the log-likelihood of the correct label directly may
    cause overfitting and reduce the ability of the model to adapt. Label
    smoothing replaces the ground-truth label :math:`y` with the weighted sum
    of itself and some fixed distribution :math:`\mu`. For class :math:`k`,
    i.e.

    .. math::

        \\tilde{y_k} = (1 - \epsilon) * y_k + \epsilon * \mu_k,

    where :math:`1 - \epsilon` and :math:`\epsilon` are the weights
    respectively, and :math:`\\tilde{y}_k` is the smoothed label. Usually
    uniform distribution is used for :math:`\mu`.

    See more details about label smoothing in https://arxiv.org/abs/1512.00567.

    Args:
        label(Variable): The input variable containing the label data. The
                          label data should use one-hot representation.
        prior_dist(Variable): The prior distribution to be used to smooth
                              labels. If not provided, an uniform distribution
                              is used. The shape of :attr:`prior_dist` should
7133
                              be :math:`(1, class\_num)`.
7134 7135
        epsilon(float): The weight used to mix up the original ground-truth
                        distribution and the fixed distribution.
7136
        dtype(np.dtype|core.VarDesc.VarType|str): The type of data : float32,
7137 7138 7139 7140 7141 7142 7143 7144 7145
                                                  float_64, int etc.
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.

    Returns:
        Variable: The tensor variable containing the smoothed labels.

    Examples:
        .. code-block:: python
7146 7147
            
            import paddle.fluid.layers as layers
7148 7149 7150 7151 7152 7153 7154 7155 7156 7157

            label = layers.data(name="label", shape=[1], dtype="float32")
            one_hot_label = layers.one_hot(input=label, depth=10)
            smooth_label = layers.label_smooth(
                label=one_hot_label, epsilon=0.1, dtype="float32")
    """
    if epsilon > 1. or epsilon < 0.:
        raise ValueError("The value of epsilon must be between 0 and 1.")
    helper = LayerHelper("label_smooth", **locals())
    label.stop_gradient = True
X
Xin Pan 已提交
7158
    smooth_label = helper.create_variable_for_type_inference(dtype)
7159 7160 7161 7162 7163 7164 7165
    helper.append_op(
        type="label_smooth",
        inputs={"X": label,
                "PriorDist": prior_dist} if prior_dist else {"X": label},
        outputs={"Out": smooth_label},
        attrs={"epsilon": float(epsilon)})
    return smooth_label
7166 7167


W
wopeizl 已提交
7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185
@templatedoc()
def roi_pool(input, rois, pooled_height=1, pooled_width=1, spatial_scale=1.0):
    """
    ${comment}

    Args:
        input (Variable): ${x_comment}
        rois (Variable): ROIs (Regions of Interest) to pool over.
        pooled_height (integer): ${pooled_height_comment} Default: 1
        pooled_width (integer): ${pooled_width_comment} Default: 1
        spatial_scale (float): ${spatial_scale_comment} Default: 1.0

    Returns:
        Variable: ${out_comment}.

    Examples:
        .. code-block:: python

7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198
            import paddle.fluid as fluid

            x = fluid.layers.data(
                name='x', shape=[8, 112, 112], dtype='float32')
            rois = fluid.layers.data(
                name='roi', shape=[4], lod_level=1, dtype='float32')
            pool_out = fluid.layers.roi_pool(
                input=x,
                rois=rois,
                pooled_height=7,
                pooled_width=7,
                spatial_scale=1.0)

W
wopeizl 已提交
7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215
    """
    helper = LayerHelper('roi_pool', **locals())
    dtype = helper.input_dtype()
    pool_out = helper.create_variable_for_type_inference(dtype)
    argmaxes = helper.create_variable_for_type_inference(dtype='int32')
    helper.append_op(
        type="roi_pool",
        inputs={"X": input,
                "ROIs": rois},
        outputs={"Out": pool_out,
                 "Argmax": argmaxes},
        attrs={
            "pooled_height": pooled_height,
            "pooled_width": pooled_width,
            "spatial_scale": spatial_scale
        })
    return pool_out
W
whs 已提交
7216 7217


J
jerrywgz 已提交
7218 7219 7220 7221 7222 7223
@templatedoc()
def roi_align(input,
              rois,
              pooled_height=1,
              pooled_width=1,
              spatial_scale=1.0,
J
jerrywgz 已提交
7224 7225
              sampling_ratio=-1,
              name=None):
J
jerrywgz 已提交
7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241
    """
    ${comment}

    Args:
        input (Variable): ${x_comment}
        rois (Variable): ROIs (Regions of Interest) to pool over.
        pooled_height (integer): ${pooled_height_comment} Default: 1
        pooled_width (integer): ${pooled_width_comment} Default: 1
        spatial_scale (float): ${spatial_scale_comment} Default: 1.0
        sampling_ratio(intger): ${sampling_ratio_comment} Default: -1

    Returns:
        Variable: ${out_comment}.
    Examples:
        .. code-block:: python

J
jerrywgz 已提交
7242 7243 7244 7245
            x = fluid.layers.data(
                name='data', shape=[256, 32, 32], dtype='float32')
            rois = fluid.layers.data(
                name='rois', shape=[4], dtype='float32')
7246 7247 7248
            align_out = fluid.layers.roi_align(input=x,
                                               rois=rois,
                                               pooled_height=7,
J
jerrywgz 已提交
7249 7250 7251 7252 7253 7254
                                               pooled_width=7,
                                               spatial_scale=0.5,
                                               sampling_ratio=-1)
    """
    helper = LayerHelper('roi_align', **locals())
    dtype = helper.input_dtype()
X
Xin Pan 已提交
7255
    align_out = helper.create_variable_for_type_inference(dtype)
J
jerrywgz 已提交
7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269
    helper.append_op(
        type="roi_align",
        inputs={"X": input,
                "ROIs": rois},
        outputs={"Out": align_out},
        attrs={
            "pooled_height": pooled_height,
            "pooled_width": pooled_width,
            "spatial_scale": spatial_scale,
            "sampling_ratio": sampling_ratio
        })
    return align_out


W
whs 已提交
7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281 7282 7283 7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295
def dice_loss(input, label, epsilon=0.00001):
    """
    Dice loss for comparing the similarity of two batch of data,
    usually is used for binary image segmentation i.e. labels are binary.
    The dice loss can be defined as below equation:

    .. math::

        dice\_loss &= 1 - \\frac{2 * intersection\_area}{total\_area} \\\\
                  &= \\frac{(total\_area - intersection\_area) - intersection\_area}{total\_area} \\\\
                  &= \\frac{(union\_area - intersection\_area)}{total\_area}


    Args:
        input (Variable): The predictions with rank>=2. The first dimension is batch size,
                          and the last dimension is class number.
        label (Variable): The groud truth with the same rank with input. The first dimension
                          is batch size, and the last dimension is 1.
        epsilon (float): The epsilon will be added to the numerator and denominator.
                         If both input and label are empty, it makes sure dice is 1.
                         Default: 0.00001

    Returns:
        dice_loss (Variable): The dice loss with shape [1].

    Examples:
7296 7297
        .. code-block:: python

S
SunGaofeng 已提交
7298 7299 7300
            import paddle.fluid as fluid
            x = fluid.layers.data(name='data', shape = [3, 224, 224, 2], dtype='float32')
            label = fluid.layers.data(name='label', shape=[3, 224, 224, 1], dtype='float32')
W
whs 已提交
7301
            predictions = fluid.layers.softmax(x)
S
SunGaofeng 已提交
7302
            loss = fluid.layers.dice_loss(input=predictions, label=label)
W
whs 已提交
7303 7304
    """
    label = one_hot(label, depth=input.shape[-1])
7305
    reduce_dim = list(range(1, len(input.shape)))
W
whs 已提交
7306 7307 7308 7309 7310 7311
    inse = reduce_sum(input * label, dim=reduce_dim)
    dice_denominator = reduce_sum(
        input, dim=reduce_dim) + reduce_sum(
            label, dim=reduce_dim)
    dice_score = 1 - inse * 2 / (dice_denominator + epsilon)
    return reduce_mean(dice_score)
7312 7313


7314 7315 7316 7317
def image_resize(input,
                 out_shape=None,
                 scale=None,
                 name=None,
7318
                 resample='BILINEAR',
7319 7320
                 actual_shape=None,
                 align_corners=True,
T
tink2123 已提交
7321
                 align_mode=1):
7322
    """
Q
qiaolongfei 已提交
7323
    **Resize a Batch of Images**
F
stash  
fengjiayi 已提交
7324

7325
    The input must be a tensor of the shape (num_batches, channels, in_h, in_w),
7326 7327 7328
    and the resizing only applies on the last two dimensions(hight and width).

    Supporting resample methods:
Q
update  
qiaolongfei 已提交
7329

7330
        'BILINEAR' : Bilinear interpolation
T
Tink_Y 已提交
7331

7332
        'NEAREST' : Nearest neighbor interpolation
F
stash  
fengjiayi 已提交
7333

7334 7335 7336 7337 7338 7339 7340 7341 7342 7343
    Nearest neighbor interpolation is to perform nearest neighbor interpolation
    in both the 3rd dimention(in height direction) and the 4th dimention(in width 
    direction) on input tensor.
            
    Bilinear interpolation is an extension of linear interpolation for 
    interpolating functions of two variables (e.g. H-direction and 
    W-direction in this op) on a rectilinear 2D grid. The key idea is 
    to perform linear interpolation first in one direction, and then 
    again in the other direction.

T
tink2123 已提交
7344
    Align_corners and align_mode are optinal parameters,the calculation method 
7345 7346 7347 7348
    of interpolation can be selected by them.

    Example:

T
Tink_Y 已提交
7349
    .. code-block:: text
7350

T
Tink_Y 已提交
7351
        For scale:
7352
          
T
Tink_Y 已提交
7353
            if align_corners = True && out_size > 1 :
7354

T
Tink_Y 已提交
7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365
              scale_factor = (in_size-1.0)/(out_size-1.0)
            
            else:
              
              scale_factor = float(in_size/out_size)
            
          
        Nearest neighbor interpolation:
          
          if:
              align_corners = False
7366

T
Tink_Y 已提交
7367 7368
              input : (N,C,H_in,W_in)
              output: (N,C,H_out,W_out) where:
7369

T
Tink_Y 已提交
7370 7371
              H_out = floor (H_{in} * scale_{factor})
              W_out = floor (W_{in} * scale_{factor})
7372

T
Tink_Y 已提交
7373 7374
          else:
              align_corners = True
7375

T
Tink_Y 已提交
7376 7377
              input : (N,C,H_in,W_in)
              output: (N,C,H_out,W_out) where:
7378

T
Tink_Y 已提交
7379 7380
              H_out = round(H_{in} * scale_{factor})
              W_out = round(W_{in} * scale_{factor})
7381

T
Tink_Y 已提交
7382 7383 7384 7385 7386 7387 7388 7389 7390 7391
        Bilinear interpolation:

          if:
              align_corners = False , align_mode = 0
              
              input : (N,C,H_in,W_in)
              output: (N,C,H_out,W_out) where:
              
              H_out = (H_{in}+0.5) * scale_{factor} - 0.5
              W_out = (W_{in}+0.5) * scale_{factor} - 0.5
7392

T
Tink_Y 已提交
7393 7394 7395 7396
          else:
           
              input : (N,C,H_in,W_in)
              output: (N,C,H_out,W_out) where:
7397

T
Tink_Y 已提交
7398 7399
              H_out = H_{in} * scale_{factor}
              W_out = W_{in} * scale_{factor}
7400 7401 7402 7403 7404 7405 7406 7407 7408

    For details of nearest neighbor interpolation, please refer to Wikipedia: 
    https://en.wikipedia.org/wiki/Nearest-neighbor_interpolation.

    For details of bilinear interpolation, please refer to Wikipedia: 
    https://en.wikipedia.org/wiki/Bilinear_interpolation.



7409
    Args:
7410
        input (Variable): The input tensor of image resize layer,
7411 7412
                          This is a 4-D tensor of the shape
                          (num_batches, channels, in_h, in_w).
7413
        out_shape(list|tuple|Variable|None): Output shape of image resize
7414 7415
                                    layer, the shape is (out_h, out_w).
                                    Default: None
D
dengkaipeng 已提交
7416
        scale(float|None): The multiplier for the input height or width. At
D
dengkaipeng 已提交
7417
             least one of :attr:`out_shape` or :attr:`scale` must be set. 
D
dengkaipeng 已提交
7418
             And :attr:`out_shape` has a higher priority than :attr:`scale`. 
D
dengkaipeng 已提交
7419
             Default: None.
7420 7421
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.
7422
        resample(str): The resample method. It supports 'BILINEAR' and 'NEAREST'
7423
                       currently.
7424
                       Default: 'BILINEAR'
7425 7426 7427
        actual_shape(Variable): An optional input to specify output shape
                                dynamically. If provided, image resize
                                according to this given shape rather than
7428
                                :attr:`out_shape` and :attr:`scale` specifying
7429 7430 7431 7432 7433 7434 7435
                                shape. That is to say actual_shape has the
                                highest priority. It is recommended to use
                                actual_shape instead of :attr:`out_shape` if you
                                want to specify output shape dynamically. When
                                using actual_shape to specify output shape, one of
                                :attr:`out_shape` and :attr:`scale` should also be
                                set, otherwise errors would be occured in graph
7436 7437
                                constructing stage.
                                Default: None
7438 7439 7440 7441
        align_corners(bool) :  An optional bool, If True, the centers of the 4 corner pixels of the 
                               input and output tensors are aligned, preserving the values at the 
                               corner pixels.
                               Default: True
T
tink2123 已提交
7442
        align_mode(int)  :  An optional for bilinear interpolation. can be \'0\' 
T
tink2123 已提交
7443 7444
                            for src_idx = scale*(dst_indx+0.5)-0.5 , can be \'1\' for 
                            src_idx = scale*dst_index .
7445 7446

    Returns:
Q
update  
qiaolongfei 已提交
7447 7448
        Variable: The output is a 4-D tensor of the shape
        (num_batches, channls, out_h, out_w).
F
stash  
fengjiayi 已提交
7449

7450 7451 7452
    Raises:
        TypeError: out_shape should be a list or tuple or Variable.
        TypeError: actual_shape should either be Variable or None.
7453
        ValueError: The 'resample' of image_resize can only be 'BILINEAR'
7454 7455 7456
                    or 'NEAREST' currently.
        ValueError: One of out_shape and scale must not be None.
        ValueError: out_shape length should be 2.
D
dengkaipeng 已提交
7457
        ValueError: scale should be greater than zero.
7458 7459
        TypeError: align_corners shoule be a bool value
        ValueError: align_mode can only be '0' or '1'
7460

7461 7462 7463
    Examples:
        .. code-block:: python

R
ruri 已提交
7464
            input = fluid.layers.data(name="input", shape=[3,6,9], dtype="float32")
7465
            out = fluid.layers.image_resize(input, out_shape=[12, 12], resample="NEAREST")
7466
    """
7467 7468 7469 7470
    resample_methods = {
        'BILINEAR': 'bilinear',
        'NEAREST': 'nearest',
    }
7471 7472
    if resample not in resample_methods:
        raise ValueError(
7473
            "The 'resample' of image_resize can only be 'BILINEAR' or 'NEAREST' currently."
7474
        )
7475
    resample_type = resample_methods[resample]
7476 7477 7478 7479 7480 7481

    if not isinstance(align_corners, bool):
        raise TypeError("Attr align_corners should be a bool value")
    if align_mode != 0 and align_mode != 1:
        raise ValueError("align_mode can only be 0 or 1")

7482
    if out_shape is None and scale is None:
7483
        raise ValueError("One of out_shape and scale must not be None.")
7484
    helper = LayerHelper('{}_interp'.format(resample_type), **locals())
7485
    dtype = helper.input_dtype()
7486 7487 7488 7489

    def _is_list_or_turple_(data):
        return (isinstance(data, list) or isinstance(data, tuple))

7490
    inputs = {"X": input}
D
dengkaipeng 已提交
7491
    attrs = {
D
dengkaipeng 已提交
7492 7493
        "out_h": 0,
        "out_w": 0,
D
dengkaipeng 已提交
7494 7495 7496 7497 7498
        "interp_method": resample_type,
        "align_corners": align_corners,
        "align_mode": align_mode
    }

7499
    if out_shape is not None:
7500 7501 7502 7503
        if isinstance(out_shape, Variable):
            warnings.warn("out_shape as Variable type is deprecated, \
                    it is recommended to use actual_shape instead of \
                    out_shape to specify output shape dynamically.")
7504
            inputs['OutSize'] = out_shape
7505 7506
        else:
            if not (_is_list_or_turple_(out_shape)):
D
dengkaipeng 已提交
7507 7508
                raise TypeError(
                    "out_shape should be a list or tuple or Variable.")
7509 7510 7511 7512 7513 7514 7515
            if len(out_shape) != 2:
                raise ValueError("out_shape length should be 2.")

            out_shape = list(map(int, out_shape))
            attrs['out_h'] = out_shape[0]
            attrs['out_w'] = out_shape[1]

7516
    else:
D
dengkaipeng 已提交
7517 7518
        if scale <= 0:
            raise ValueError("scale should be greater than zero.")
D
dengkaipeng 已提交
7519
        attrs['scale'] = float(scale)
7520

7521 7522 7523 7524 7525
    if isinstance(actual_shape, Variable):
        inputs["OutSize"] = actual_shape
    elif actual_shape is not None:
        raise TypeError("actual_shape should either be Variable or None.")

X
Xin Pan 已提交
7526
    out = helper.create_variable_for_type_inference(dtype)
7527
    helper.append_op(
7528
        type='{}_interp'.format(resample_type),
7529
        inputs=inputs,
7530
        outputs={"Out": out},
D
dengkaipeng 已提交
7531
        attrs=attrs)
7532
    return out
F
stash  
fengjiayi 已提交
7533 7534


7535
@templatedoc(op_type="bilinear_interp")
7536 7537 7538 7539
def resize_bilinear(input,
                    out_shape=None,
                    scale=None,
                    name=None,
7540 7541
                    actual_shape=None,
                    align_corners=True,
T
tink2123 已提交
7542
                    align_mode=1):
7543
    """
7544 7545
    Resize input by performing bilinear interpolation based on given
    output shape which specified by actual_shape, out_shape and scale
7546 7547
    in priority order.

7548 7549 7550 7551
    Bilinear interpolation is an extension of linear interpolation for
    interpolating functions of two variables (e.g. H-direction and
    W-direction in this op) on a rectilinear 2D grid. The key idea is
    to perform linear interpolation first in one direction, and then
7552 7553
    again in the other direction.

7554
    For details of bilinear interpolation, please refer to Wikipedia:
7555
    https://en.wikipedia.org/wiki/Bilinear_interpolation
Y
yuyang18 已提交
7556

T
tink2123 已提交
7557
    Align_corners and align_mode are optinal parameters,the calculation 
7558 7559 7560 7561
    method of interpolation can be selected by them.

    Example:

T
Tink_Y 已提交
7562
    .. code-block:: text
7563

T
Tink_Y 已提交
7564
        For scale:
7565
          
T
Tink_Y 已提交
7566
            if align_corners = True && out_size > 1 :
7567

T
Tink_Y 已提交
7568 7569 7570 7571 7572
              scale_factor = (in_size-1.0)/(out_size-1.0)
            
            else:
              
              scale_factor = float(in_size/out_size)     
7573

T
Tink_Y 已提交
7574 7575 7576 7577 7578 7579 7580 7581 7582 7583
        Bilinear interpolation:

          if:
              align_corners = False , align_mode = 0
              
              input : (N,C,H_in,W_in)
              output: (N,C,H_out,W_out) where:
              
              H_out = (H_{in}+0.5) * scale_{factor} - 0.5
              W_out = (W_{in}+0.5) * scale_{factor} - 0.5
7584 7585


T
Tink_Y 已提交
7586
          else:
T
tink2123 已提交
7587

T
Tink_Y 已提交
7588 7589
              input : (N,C,H_in,W_in)
              output: (N,C,H_out,W_out) where:
7590

T
Tink_Y 已提交
7591 7592
              H_out = H_{in} * scale_{factor}
              W_out = W_{in} * scale_{factor}
7593 7594 7595



Y
yuyang18 已提交
7596 7597 7598
    Args:
        input(${x_type}): ${x_comment}.

D
dengkaipeng 已提交
7599 7600 7601
        out_shape(list|tuple|Variable|None): Output shape of resize bilinear
                                    layer, the shape is (out_h, out_w).
                                    Default: None
7602

Y
yuyang18 已提交
7603
        scale(float|None): The multiplier for the input height or width. At
D
dengkaipeng 已提交
7604
             least one of :attr:`out_shape` or :attr:`scale` must be set. 
D
dengkaipeng 已提交
7605
             And :attr:`out_shape` has a higher priority than :attr:`scale`. 
D
dengkaipeng 已提交
7606
             Default: None.
Y
yuyang18 已提交
7607 7608

        name(str|None): The output variable name.
7609 7610 7611
        actual_shape(Variable): An optional input to specify output shape
                                dynamically. If provided, image resize
                                according to this given shape rather than
7612
                                :attr:`out_shape` and :attr:`scale` specifying
7613 7614 7615 7616 7617 7618 7619
                                shape. That is to say actual_shape has the
                                highest priority. It is recommended to use
                                actual_shape instead of :attr:`out_shape` if you
                                want to specify output shape dynamically. When
                                using actual_shape to specify output shape, one of
                                :attr:`out_shape` and :attr:`scale` should also be
                                set, otherwise errors would be occured in graph
7620 7621
                                constructing stage.
                                Default: None
7622 7623
        align_corners(bool): ${align_corners_comment}
        align_mode(bool): ${align_mode_comment}
Y
yuyang18 已提交
7624 7625 7626

    Returns:
        ${out_comment}.
7627 7628 7629 7630

    Examples:
        .. code-block:: python

R
ruri 已提交
7631
            input = fluid.layers.data(name="input", shape=[3,6,9], dtype="float32")
7632
            out = fluid.layers.resize_bilinear(input, out_shape=[12, 12])
7633 7634
    """

7635 7636
    return image_resize(input, out_shape, scale, name, 'BILINEAR', actual_shape,
                        align_corners, align_mode)
7637 7638


7639
@templatedoc(op_type="nearest_interp")
7640 7641 7642 7643
def resize_nearest(input,
                   out_shape=None,
                   scale=None,
                   name=None,
7644 7645
                   actual_shape=None,
                   align_corners=True):
7646
    """
7647
    Resize input by performing nearest neighbor interpolation in both the
T
Tink_Y 已提交
7648 7649
    3rd dimension(in height direction) and the 4th dimension(in width
    direction) based on given output shape which is specified by actual_shape,
7650 7651
    out_shape and scale in priority order.

7652 7653
    Example:

T
Tink_Y 已提交
7654 7655 7656 7657 7658
    .. code-block:: text

        For scale:
          
            if align_corners = True && out_size > 1 :
7659

T
Tink_Y 已提交
7660 7661 7662 7663 7664 7665 7666 7667
              scale_factor = (in_size-1.0)/(out_size-1.0)
            
            else:
              
              scale_factor = float(in_size/out_size)
            
          
        Nearest neighbor interpolation:
7668
          
T
Tink_Y 已提交
7669 7670
          if:
              align_corners = False
7671

T
Tink_Y 已提交
7672 7673
              input : (N,C,H_in,W_in)
              output: (N,C,H_out,W_out) where:
7674

T
Tink_Y 已提交
7675 7676
              H_out = floor(H_{in} * scale_{factor})
              W_out = floor(W_{in} * scale_{factor})
7677

T
Tink_Y 已提交
7678 7679
          else:
              align_corners = True
7680

T
Tink_Y 已提交
7681 7682
              input : (N,C,H_in,W_in)
              output: (N,C,H_out,W_out) where:
7683

T
Tink_Y 已提交
7684 7685
              H_out = round(H_{in} * scale_{factor})
              W_out = round(W_{in} * scale_{factor})
7686 7687


7688
    For details of nearest neighbor interpolation, please refer to Wikipedia:
7689
    https://en.wikipedia.org/wiki/Nearest-neighbor_interpolation
Y
yuyang18 已提交
7690 7691 7692 7693

    Args:
        input(${x_type}): ${x_comment}.

D
dengkaipeng 已提交
7694 7695 7696
        out_shape(list|tuple|Variable|None): Output shape of resize nearest
                                    layer, the shape is (out_h, out_w).
                                    Default: None
7697

Y
yuyang18 已提交
7698
        scale(float|None): The multiplier for the input height or width. At
D
dengkaipeng 已提交
7699
             least one of :attr:`out_shape` or :attr:`scale` must be set. 
D
dengkaipeng 已提交
7700
             And :attr:`out_shape` has a higher priority than :attr:`scale`. 
D
dengkaipeng 已提交
7701
             Default: None.
Y
yuyang18 已提交
7702 7703

        name(str|None): The output variable name.
7704 7705 7706
        actual_shape(Variable): An optional input to specify output shape
                                dynamically. If provided, image resize
                                according to this given shape rather than
7707
                                :attr:`out_shape` and :attr:`scale` specifying
7708 7709 7710 7711 7712 7713 7714
                                shape. That is to say actual_shape has the
                                highest priority. It is recommended to use
                                actual_shape instead of :attr:`out_shape` if you
                                want to specify output shape dynamically. When
                                using actual_shape to specify output shape, one of
                                :attr:`out_shape` and :attr:`scale` should also be
                                set, otherwise errors would be occured in graph
7715 7716
                                constructing stage.
                                Default: None
7717
        align_corners(bool): ${align_corners_comment}
Y
yuyang18 已提交
7718 7719 7720

    Returns:
        ${out_comment}.
7721 7722 7723 7724

    Examples:
        .. code-block:: python

R
ruri 已提交
7725
            input = fluid.layers.data(name="input", shape=[3,6,9], dtype="float32")
7726
            out = fluid.layers.resize_nearest(input, out_shape=[12, 12])
7727 7728
    """

7729 7730
    return image_resize(input, out_shape, scale, name, 'NEAREST', actual_shape,
                        align_corners)
7731 7732 7733 7734


def image_resize_short(input, out_short_len, resample='BILINEAR'):
    """
7735 7736 7737
    Resize a batch of images. The short edge of input images will be
    resized to the given 'out_short_len'. The long edge of input images
    will be resized proportionately to make images' length-width ratio
7738 7739 7740 7741 7742 7743 7744
    constant.

    Args:
        input (Variable): The input tensor of image resize layer,
                          This is a 4-D tensor of the shape
                          (num_batches, channels, in_h, in_w).
        out_short_len(int): The length of output images' short edge.
7745
        resample (str): resample method, default: BILINEAR.
F
fengjiayi 已提交
7746

7747
    Returns:
Q
update  
qiaolongfei 已提交
7748
        Variable: The output is a 4-D tensor of the shape
7749
        (num_batches, channls, out_h, out_w).
R
ruri 已提交
7750 7751 7752 7753 7754 7755

    Examples:
        .. code-block:: python

            input = fluid.layers.data(name="input", shape=[3,6,9], dtype="float32")
            out = fluid.layers.image_resize_short(input, out_short_len=3)
7756 7757 7758 7759 7760 7761 7762 7763 7764 7765
    """
    in_shape = input.shape
    if len(in_shape) != 4:
        raise ValueError(
            "The rank of input must be 4 (num_batches, channels, in_h, in_w).")
    hw = in_shape[2:4]
    short_idx = hw.index(min(hw))
    long_idx = 1 - short_idx
    out_shape = list(hw)
    out_shape[short_idx] = out_short_len
F
fengjiayi 已提交
7766 7767 7768
    out_shape[long_idx] = int(
        float(out_shape[long_idx]) * (float(out_short_len) / float(hw[
            short_idx])) + 0.5)
7769 7770 7771
    return image_resize(input=input, out_shape=out_shape, resample=resample)


W
whs 已提交
7772 7773
def gather(input, index):
    """
Q
qiaolongfei 已提交
7774 7775
    **Gather Layer**

7776
    Output is obtained by gathering entries of the outer-most dimension
W
whs 已提交
7777 7778 7779 7780
    of X indexed by `index` and concatenate them together.

    .. math::

7781
        Out = X[Index]
W
whs 已提交
7782 7783 7784 7785 7786 7787 7788


    .. code-block:: text


                Given:

7789 7790
                X = [[1, 2],
                     [3, 4],
W
whs 已提交
7791 7792 7793 7794 7795 7796 7797 7798 7799 7800
                     [5, 6]]

                Index = [1, 2]

                Then:

                Out = [[3, 4],
                       [5, 6]]

    Args:
7801
        input (Variable): The source input with rank>=1.
W
whs 已提交
7802 7803 7804 7805 7806 7807
        index (Variable): The index input with rank=1.

    Returns:
        output (Variable): The output is a tensor with the same rank as input.

    Examples:
W
whs 已提交
7808

W
whs 已提交
7809 7810
        .. code-block:: python

Y
Yibing Liu 已提交
7811 7812
            x = fluid.layers.data(name='x', shape=[-1, 5], dtype='float32')
            index = fluid.layers.data(name='index', shape=[-1, 1], dtype='int32')
W
whs 已提交
7813 7814 7815 7816
            output = fluid.layers.gather(x, index)
    """
    helper = LayerHelper('gather', **locals())
    dtype = helper.input_dtype()
X
Xin Pan 已提交
7817
    out = helper.create_variable_for_type_inference(dtype)
W
whs 已提交
7818 7819 7820 7821 7822 7823 7824 7825
    helper.append_op(
        type="gather",
        inputs={"X": input,
                "Index": index},
        outputs={"Out": out})
    return out


7826 7827 7828 7829 7830 7831 7832 7833 7834 7835 7836 7837 7838 7839 7840 7841 7842 7843 7844 7845 7846 7847 7848 7849 7850 7851 7852 7853 7854 7855 7856
def scatter(input, index, updates, name=None):
    """
    **Scatter Layer**

    Output is obtained by updating the input on selected indices on the first
    axis.

    .. math::

        Out = X
        Out[Ids] = Updates

    Args:
        input (Variable): The source input with rank>=1.
        index (Variable): The index input with rank=1. Its dtype should be
                          int32 or int64 as it is used as indexes.
        updates (Variable): The updated value of scatter op.
        name (str|None): The output variable name. Default None.

    Returns:
        output (Variable): The output is a tensor with the same shape as input.

    Examples:

        .. code-block:: python

            output = fluid.layers.scatter(input, index, updates)

    """
    helper = LayerHelper('scatter', **locals())
    dtype = helper.input_dtype()
X
Xin Pan 已提交
7857
    out = helper.create_variable_for_type_inference(dtype)
7858 7859 7860 7861 7862 7863 7864 7865 7866
    helper.append_op(
        type="scatter",
        inputs={"X": input,
                "Ids": index,
                "Updates": updates},
        outputs={"Out": out})
    return out


Q
Qingsheng Li 已提交
7867 7868 7869 7870 7871 7872 7873 7874 7875
def sequence_scatter(input, index, updates, name=None):
    """
    **Sequence Scatter Layer**

    This operator scatters the Updates tensor to the input X. It uses the LoD
    information of Ids to select the rows to update, and use the values in Ids as
    the columns to update in each row of X.

    Here is an example:
H
haowang101779990 已提交
7876

Q
Qingsheng Li 已提交
7877
    Given the following input:
H
haowang101779990 已提交
7878

Q
Qingsheng Li 已提交
7879
    .. code-block:: text
H
haowang101779990 已提交
7880

Q
Qingsheng Li 已提交
7881 7882 7883 7884 7885 7886 7887 7888 7889 7890 7891 7892
        input.data = [[1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
                      [1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
                      [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]]
        input.dims = [3, 6]

        index.data = [[0], [1], [2], [5], [4], [3], [2], [1], [3], [2], [5], [4]]
        index.lod =  [[0,        3,                       8,                 12]]

        updates.data = [[0.3], [0.3], [0.4], [0.1], [0.2], [0.3], [0.4], [0.0], [0.2], [0.3], [0.1], [0.4]]
        updates.lod =  [[  0,            3,                                 8,                         12]]

    Then we have the output:
H
haowang101779990 已提交
7893

Q
Qingsheng Li 已提交
7894
    .. code-block:: text
H
haowang101779990 已提交
7895

Q
Qingsheng Li 已提交
7896 7897 7898 7899 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910
        out.data = [[1.3, 1.3, 1.4, 1.0, 1.0, 1.0],
                    [1.0, 1.0, 1.4, 1.3, 1.2, 1.1],
                    [1.0, 1.0, 1.3, 1.2, 1.4, 1.1]]
        out.dims = X.dims = [3, 6]

    Args:
        input (Variable): The source input with rank>=1.
        index (Variable): A LoD Tensor. The index input of sequence scatter op
            where input will be  updated. The index input with rank=1. Its dtype
            should be int32 or int64 as it is used as indexes.
        updates (Variable): A LoD Tensor. The values to scatter to the input
            tensor X, must be a LoDTensor with the same LoD information as index.
        name (str|None): The output variable name. Default None.

    Returns:
H
haowang101779990 已提交
7911
        Variable: The output is a tensor with the same shape as input.
Q
Qingsheng Li 已提交
7912 7913 7914 7915

    Examples:

        .. code-block:: python
7916 7917
	
            import paddle.fluid.layers as layers
Q
Qingsheng Li 已提交
7918

7919 7920 7921
            input = layers.data( name="x", shape=[3, 6], append_batch_size=False, dtype='float32' )
            index = layers.data( name='index', shape=[1], dtype='int32')
            updates = layers.data( name='updates', shape=[1], dtype='float32')
Q
Qingsheng Li 已提交
7922 7923 7924
            output = fluid.layers.sequence_scatter(input, index, updates)

    """
L
lujun 已提交
7925
    assert not in_dygraph_mode(), (
7926
        "sequence layer is not supported in dygraph mode yet.")
Q
Qingsheng Li 已提交
7927 7928
    helper = LayerHelper('sequence_scatter', **locals())
    dtype = helper.input_dtype()
X
Xin Pan 已提交
7929
    out = helper.create_variable_for_type_inference(dtype)
Q
Qingsheng Li 已提交
7930 7931 7932 7933 7934 7935 7936 7937 7938
    helper.append_op(
        type="sequence_scatter",
        inputs={"X": input,
                "Ids": index,
                "Updates": updates},
        outputs={"Out": out})
    return out


Y
yuyang18 已提交
7939 7940 7941 7942 7943 7944 7945 7946 7947 7948 7949 7950 7951
@templatedoc()
def random_crop(x, shape, seed=None):
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        shape(${shape_type}): ${shape_comment}
        seed(int|${seed_type}|None): ${seed_comment} By default, the seed will
            get from `random.randint(-65536, 65535)`.

    Returns:
        ${out_comment}
7952

7953 7954 7955
    Examples:
        >>> img = fluid.layers.data("img", [3, 256, 256])
        >>> cropped_img = fluid.layers.random_crop(img, shape=[3, 224, 224])
Y
yuyang18 已提交
7956
    """
F
stash  
fengjiayi 已提交
7957
    helper = LayerHelper("random_crop", **locals())
F
fengjiayi 已提交
7958
    dtype = x.dtype
X
Xin Pan 已提交
7959
    out = helper.create_variable_for_type_inference(dtype)
Y
yuyang18 已提交
7960
    if seed is None:
7961
        seed = np.random.randint(-65536, 65536)
F
fengjiayi 已提交
7962
    op_attrs = {"shape": shape}
F
stash  
fengjiayi 已提交
7963
    if isinstance(seed, int):
F
fengjiayi 已提交
7964 7965 7966 7967 7968
        op_attrs["startup_seed"] = seed
        seed = helper.create_variable(
            name=unique_name.generate("random_crop_seed"),
            dtype="int64",
            persistable=True)
F
stash  
fengjiayi 已提交
7969 7970 7971 7972
    elif not isinstance(seed, Variable):
        raise ValueError("'seed' must be a Variable or an int.")
    helper.append_op(
        type="random_crop",
F
fix  
fengjiayi 已提交
7973
        inputs={"X": x,
F
stash  
fengjiayi 已提交
7974 7975
                "Seed": seed},
        outputs={"Out": out,
F
fengjiayi 已提交
7976 7977
                 "SeedOut": seed},
        attrs=op_attrs)
F
stash  
fengjiayi 已提交
7978
    return out
W
whs 已提交
7979 7980


7981
def log(x, name=None):
W
wanghaoshuang 已提交
7982 7983 7984 7985 7986
    """
    Calculates the natural log of the given input tensor, element-wise.

    .. math::

7987
        Out = \\ln(x)
W
wanghaoshuang 已提交
7988 7989

    Args:
7990
        x (Variable): Input tensor.
7991 7992
        name (str|None, default None): A name for this layer If set None,
            the layer will be named automatically.
W
wanghaoshuang 已提交
7993 7994 7995 7996 7997 7998 7999 8000

    Returns:
        Variable: The natural log of the input tensor computed element-wise.

    Examples:

        .. code-block:: python

8001
            output = fluid.layers.log(x)
W
wanghaoshuang 已提交
8002 8003
    """
    helper = LayerHelper('log', **locals())
W
wanghaoshuang 已提交
8004
    dtype = helper.input_dtype(input_param_name='x')
X
Xin Pan 已提交
8005
    out = helper.create_variable_for_type_inference(dtype)
W
wanghaoshuang 已提交
8006
    helper.append_op(type="log", inputs={"X": x}, outputs={"Out": out})
W
wanghaoshuang 已提交
8007 8008 8009
    return out


8010
def relu(x, name=None):
W
wanghaoshuang 已提交
8011 8012
    """
    Relu takes one input data (Tensor) and produces one output data (Tensor)
8013
    where the rectified linear function, y = max(0, x), is applied to
W
wanghaoshuang 已提交
8014 8015 8016 8017
    the tensor elementwise.

    .. math::

8018
        Out = \\max(0, x)
W
wanghaoshuang 已提交
8019 8020

    Args:
8021
        x (Variable): The input tensor.
8022 8023
        name (str|None, default None): A name for this layer If set None,
            the layer will be named automatically.
W
wanghaoshuang 已提交
8024 8025 8026 8027 8028 8029 8030 8031

    Returns:
        Variable: The output tensor with the same shape as input.

    Examples:

        .. code-block:: python

8032
            x = fluid.layers.data(name="x", shape=[3, 4], dtype="float32")
8033
            output = fluid.layers.relu(x)
W
wanghaoshuang 已提交
8034 8035
    """
    helper = LayerHelper('relu', **locals())
W
wanghaoshuang 已提交
8036
    dtype = helper.input_dtype(input_param_name='x')
X
Xin Pan 已提交
8037
    out = helper.create_variable_for_type_inference(dtype)
X
Xin Pan 已提交
8038 8039
    helper.append_op(
        type="relu", inputs={"X": helper.input('x')}, outputs={"Out": out})
W
wanghaoshuang 已提交
8040
    return out
8041 8042


C
chengduo 已提交
8043 8044 8045 8046 8047 8048 8049 8050 8051 8052 8053 8054 8055 8056 8057 8058 8059 8060 8061 8062 8063 8064 8065 8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081 8082 8083
@templatedoc()
def selu(x, scale=None, alpha=None, name=None):
    """
    ${comment}

    Args:
        x (Variable): The input tensor.
        scale(float, None): If the scale is not set,
            the default value is 1.0507009873554804934193349852946.
            For more information about this value, please refer
            to: https://arxiv.org/abs/1706.02515.
        alpha(float, None): If the alpha is not set,
            the default value is 1.6732632423543772848170429916717.
            For more information about this value, please refer
            to: https://arxiv.org/abs/1706.02515.
        name (str|None, default None): A name for this layer If set None,
            the layer will be named automatically.

    Returns:
        Variable: The output tensor with the same shape as input.

    Examples:

        .. code-block:: python

            output = fluid.layers.selu(x)
    """
    helper = LayerHelper('selu', **locals())
    dtype = helper.input_dtype(input_param_name='x')
    out = helper.create_variable_for_type_inference(dtype)
    attrs = {}
    if scale is not None:
        attrs["scale"] = scale
    if alpha is not None:
        attrs["alpha"] = alpha

    helper.append_op(
        type="selu", inputs={"X": x}, outputs={"Out": out}, attrs=attrs)
    return out


W
whs 已提交
8084 8085 8086
def mean_iou(input, label, num_classes):
    """
    Mean Intersection-Over-Union is a common evaluation metric for
8087 8088 8089 8090
    semantic image segmentation, which first computes the IOU for each
    semantic class and then computes the average over classes.
    IOU is defined as follows:

W
whs 已提交
8091
    .. math::
8092

H
haowang101779990 已提交
8093
        IOU = \\frac{true\_positive}{(true\_positive + false\_positive + false\_negative)}.
W
whs 已提交
8094

8095
    The predictions are accumulated in a confusion matrix and mean-IOU
W
whs 已提交
8096 8097 8098 8099 8100
    is then calculated from it.


    Args:
        input (Variable): A Tensor of prediction results for semantic labels with type int32 or int64.
8101
        label (Variable): A Tensor of ground truth labels with type int32 or int64.
W
whs 已提交
8102
                           Its shape should be the same as input.
8103
        num_classes (int): The possible number of labels.
W
whs 已提交
8104 8105

    Returns:
M
minqiyang 已提交
8106 8107
        mean_iou (Variable),out_wrong(Variable),out_correct(Variable):

H
haowang101779990 已提交
8108
                     Three variables:
M
minqiyang 已提交
8109

H
haowang101779990 已提交
8110 8111 8112
                     - mean_iou : A Tensor representing the mean intersection-over-union with shape [1].
                     - out_wrong: A Tensor with shape [num_classes]. The wrong numbers of each class.
                     - out_correct: A Tensor with shape [num_classes]. The correct numbers of each class.
W
whs 已提交
8113 8114 8115 8116

    Examples:

        .. code-block:: python
8117

W
whs 已提交
8118 8119 8120 8121
            iou, wrongs, corrects = fluid.layers.mean_iou(predict, label, num_classes)
    """
    helper = LayerHelper('mean_iou', **locals())
    dtype = helper.input_dtype()
X
Xin Pan 已提交
8122 8123 8124
    out_mean_iou = helper.create_variable_for_type_inference(dtype='float32')
    out_wrong = helper.create_variable_for_type_inference(dtype='int32')
    out_correct = helper.create_variable_for_type_inference(dtype='int32')
W
whs 已提交
8125 8126
    helper.append_op(
        type="mean_iou",
W
whs 已提交
8127 8128
        inputs={"Predictions": input,
                "Labels": label},
W
whs 已提交
8129
        outputs={
W
whs 已提交
8130 8131 8132
            "OutMeanIou": out_mean_iou,
            "OutWrong": out_wrong,
            "OutCorrect": out_correct
W
whs 已提交
8133 8134 8135
        },
        attrs={"num_classes": num_classes})
    return out_mean_iou, out_wrong, out_correct
8136 8137 8138 8139 8140 8141 8142 8143 8144 8145 8146 8147 8148 8149 8150 8151 8152 8153 8154 8155 8156 8157 8158 8159 8160 8161 8162 8163 8164 8165 8166 8167 8168 8169 8170 8171 8172 8173 8174 8175 8176 8177


def crop(x, shape=None, offsets=None, name=None):
    """
    Crop input into output, as specified by offsets and shape.

    .. code-block:: text

        * Case 1:
            Given
                X = [[0, 1, 2, 0, 0]
                     [0, 3, 4, 0, 0]
                     [0, 0, 0, 0, 0]],
            and
                shape = [2, 2],
                offsets = [0, 1],
            output is:
                Out = [[1, 2],
                       [3, 4]].
        * Case 2:
            Given
                X = [[0, 1, 2, 5, 0]
                     [0, 3, 4, 6, 0]
                     [0, 0, 0, 0, 0]],
            and shape is tensor
                shape = [[0, 0, 0]
                         [0, 0, 0]]
            and
                offsets = [0, 1],

            output is:
                Out = [[1, 2, 5],
                       [3, 4, 6]].

    Args:
        x (Variable): The input tensor variable.
        shape (Variable|list/tuple of integer): The output shape is specified
            by `shape`, which can a Variable or a list/tupe of integer.
            If a tensor Variable, it's rank must be the same as `x`. This way
            is suitable for the case that the output shape may be changed each
            iteration. If a list/tupe of integer, it's length must be the same
            as the rank of `x`
S
SunGaofeng 已提交
8178
        offsets (Variable|list/tuple of integer|None): Specifies the cropping
8179
            offsets at each dimension. It can be a Variable or or a list/tupe
S
SunGaofeng 已提交
8180
            of integers. If a tensor Variable, it's rank must be the same as `x`.
8181 8182 8183 8184 8185 8186 8187 8188 8189 8190 8191 8192 8193 8194 8195 8196 8197
            This way is suitable for the case that the offsets may be changed
            each iteration. If a list/tupe of integer, it's length must be the
            same as the rank of `x`. If None, the offsets are 0 at each
            dimension.
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.

    Returns:
        Variable: The cropped tensor variable.

    Raises:
        ValueError: If shape is not a list, tuple or Variable.

    Examples:

        .. code-block:: python

S
SunGaofeng 已提交
8198
            import paddle.fluid as fluid
8199 8200 8201 8202 8203 8204
            x = fluid.layers.data(name="x", shape=[3, 5], dtype="float32")
            y = fluid.layers.data(name="y", shape=[2, 3], dtype="float32")
            crop = fluid.layers.crop(x, shape=y)

            # or
            z = fluid.layers.data(name="z", shape=[3, 5], dtype="float32")
T
Tink_Y 已提交
8205
            crop = fluid.layers.crop(z, shape=[-1, 2, 3])
8206 8207 8208 8209 8210

    """
    helper = LayerHelper('crop', **locals())

    if not (isinstance(shape, list) or isinstance(shape, tuple) or \
8211
            isinstance(shape, Variable)):
8212 8213 8214 8215 8216
        raise ValueError("The shape should be a list, tuple or Variable.")

    if offsets is None:
        offsets = [0] * len(x.shape)

X
Xin Pan 已提交
8217
    out = helper.create_variable_for_type_inference(x.dtype)
8218 8219 8220 8221 8222 8223 8224 8225 8226 8227 8228 8229 8230 8231 8232 8233 8234
    ipts = {'X': x}
    attrs = {}
    if isinstance(shape, Variable):
        ipts['Y'] = shape
    else:
        attrs['shape'] = shape
    if isinstance(offsets, Variable):
        ipts['Offsets'] = offsets
    else:
        attrs['offsets'] = offsets

    helper.append_op(
        type='crop',
        inputs=ipts,
        outputs={'Out': out},
        attrs=None if len(attrs) == 0 else attrs)
    return out
8235 8236


W
whs 已提交
8237 8238 8239 8240 8241 8242 8243 8244 8245 8246 8247 8248 8249 8250 8251 8252 8253
def affine_grid(theta, out_shape, name=None):
    """
    It generates a grid of (x,y) coordinates using the parameters of
    the affine transformation that correspond to a set of points where
    the input feature map should be sampled to produce the transformed
    output feature map.

    .. code-block:: text

        * Case 1:

          Given:

              theta = [[[x_11, x_12, x_13]
                        [x_14, x_15, x_16]]
                       [[x_21, x_22, x_23]
                        [x_24, x_25, x_26]]]
8254

W
whs 已提交
8255
              out_shape = [2, 3, 5, 5]
8256

W
whs 已提交
8257
          Step 1:
8258

W
whs 已提交
8259 8260 8261
              Generate normalized coordinates according to out_shape.
              The values of the normalized coordinates are in the interval between -1 and 1.
              The shape of the normalized coordinates is [2, H, W] as below:
8262

W
whs 已提交
8263 8264 8265 8266 8267 8268 8269 8270 8271 8272 8273 8274 8275 8276 8277 8278 8279 8280 8281 8282 8283 8284 8285 8286 8287 8288 8289 8290 8291 8292 8293 8294 8295 8296 8297 8298 8299 8300 8301 8302 8303 8304 8305 8306 8307
              C = [[[-1.  -1.  -1.  -1.  -1. ]
                    [-0.5 -0.5 -0.5 -0.5 -0.5]
                    [ 0.   0.   0.   0.   0. ]
                    [ 0.5  0.5  0.5  0.5  0.5]
                    [ 1.   1.   1.   1.   1. ]]
                   [[-1.  -0.5  0.   0.5  1. ]
                    [-1.  -0.5  0.   0.5  1. ]
                    [-1.  -0.5  0.   0.5  1. ]
                    [-1.  -0.5  0.   0.5  1. ]
                    [-1.  -0.5  0.   0.5  1. ]]]
              C[0] is the coordinates in height axis and  C[1] is the coordinates in width axis.

          Step2:

              Tanspose and reshape C to shape [H * W, 2] and append ones to last dimension. The we get:
              C_ = [[-1.  -1.   1. ]
                    [-0.5 -1.   1. ]
                    [ 0.  -1.   1. ]
                    [ 0.5 -1.   1. ]
                    [ 1.  -1.   1. ]
                    [-1.  -0.5  1. ]
                    [-0.5 -0.5  1. ]
                    [ 0.  -0.5  1. ]
                    [ 0.5 -0.5  1. ]
                    [ 1.  -0.5  1. ]
                    [-1.   0.   1. ]
                    [-0.5  0.   1. ]
                    [ 0.   0.   1. ]
                    [ 0.5  0.   1. ]
                    [ 1.   0.   1. ]
                    [-1.   0.5  1. ]
                    [-0.5  0.5  1. ]
                    [ 0.   0.5  1. ]
                    [ 0.5  0.5  1. ]
                    [ 1.   0.5  1. ]
                    [-1.   1.   1. ]
                    [-0.5  1.   1. ]
                    [ 0.   1.   1. ]
                    [ 0.5  1.   1. ]
                    [ 1.   1.   1. ]]
          Step3:
              Compute output by equation $$Output[i] = C_ * Theta[i]^T$$

    Args:
        theta (Variable): A batch of affine transform parameters with shape [N, 2, 3].
M
minqiyang 已提交
8308
        out_shape (Variable | list | tuple): The shape of target output with format [N, C, H, W].
H
haowang101779990 已提交
8309
                                             ``out_shape`` can be a Variable or a list or tuple.
W
whs 已提交
8310 8311 8312 8313 8314 8315 8316 8317 8318 8319 8320 8321
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.

    Returns:
        Variable: The output with shape [N, H, W, 2].

    Raises:
        ValueError: If the type of arguments is not supported.

    Examples:

        .. code-block:: python
H
haowang101779990 已提交
8322

S
SunGaofeng 已提交
8323
            import paddle.fluid as fluid
W
whs 已提交
8324 8325 8326 8327 8328 8329 8330 8331 8332 8333 8334
            theta = fluid.layers.data(name="x", shape=[2, 3], dtype="float32")
            out_shape = fluid.layers.data(name="y", shape=[-1], dtype="float32")
            data = fluid.layers.affine_grid(theta, out_shape)

            # or
            data = fluid.layers.affine_grid(theta, [5, 3, 28, 28])

    """
    helper = LayerHelper('affine_grid')

    if not (isinstance(out_shape, list) or isinstance(out_shape, tuple) or \
8335
            isinstance(out_shape, Variable)):
W
whs 已提交
8336 8337 8338 8339 8340 8341 8342 8343 8344 8345 8346 8347 8348 8349 8350 8351 8352 8353 8354 8355 8356
        raise ValueError("The out_shape should be a list, tuple or Variable.")

    if not isinstance(theta, Variable):
        raise ValueError("The theta should be a Variable.")

    out = helper.create_variable_for_type_inference(theta.dtype)
    ipts = {'Theta': theta}
    attrs = {}
    if isinstance(out_shape, Variable):
        ipts['OutputShape'] = out_shape
    else:
        attrs['output_shape'] = out_shape

    helper.append_op(
        type='affine_grid',
        inputs=ipts,
        outputs={'Output': out},
        attrs=None if len(attrs) == 0 else attrs)
    return out


8357 8358
def rank_loss(label, left, right, name=None):
    """
H
haowang101779990 已提交
8359

8360 8361
    **Rank loss layer for RankNet**

H
haowang101779990 已提交
8362
    `RankNet <http://icml.cc/2015/wp-content/uploads/2015/06/icml_ranking.pdf>`_
8363 8364 8365
    is a pairwise ranking model with a training sample consisting of a pair
    of documents, A and B. Label P indicates whether A is ranked higher than B
    or not:
M
minqiyang 已提交
8366

8367 8368
    P = {0, 1} or {0, 0.5, 1}, where 0.5 means that there is no information
    about the rank of the input pair.
M
minqiyang 已提交
8369

H
haowang101779990 已提交
8370 8371
    Rank loss layer takes three inputs: left ( :math:`o_i` ), right ( :math:`o_j` ) and
    label ( :math:`P_{i,j}` ). The inputs respectively represent RankNet's output scores
8372 8373
    for documents A and B and the value of label P. The following equation
    computes rank loss C_{i,j} from the inputs:
M
minqiyang 已提交
8374

H
haowang101779990 已提交
8375 8376 8377 8378 8379 8380 8381 8382
    .. math::

      C_{i,j} &= -\\tilde{P_{ij}} * o_{i,j} + \log(1 + e^{o_{i,j}}) \\\\

      o_{i,j} &=  o_i - o_j  \\\\

      \\tilde{P_{i,j}} &= \\left \{0, 0.5, 1 \\right \} \ or \ \\left \{0, 1 \\right \}

M
minqiyang 已提交
8383 8384 8385

    Rank loss layer takes batch inputs with size batch_size (batch_size >= 1).

8386 8387 8388 8389 8390 8391 8392 8393 8394 8395 8396 8397 8398 8399 8400 8401 8402
    Args:
        label (Variable): Indicats whether A ranked higher than B or not.
        left (Variable): RankNet's output score for doc A.
        right (Variable): RankNet's output score for doc B.
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.

    Returns:
        list: The value of rank loss.

    Raises:
        ValueError: Any of label, left, and right is not a variable.

    Examples:

        .. code-block:: python

8403 8404 8405
            label = fluid.layers.data(name="label", shape=[-1, 1], dtype="float32")
            left = fluid.layers.data(name="left", shape=[-1, 1], dtype="float32")
            right = fluid.layers.data(name="right", shape=[-1, 1], dtype="float32")
8406 8407 8408 8409 8410 8411 8412 8413 8414 8415 8416 8417 8418 8419
            out = fluid.layers.rank_loss(label, left, right)

    """
    helper = LayerHelper('rank_loss', **locals())

    if not (isinstance(label, Variable)):
        raise ValueError("The label should be a Variable")

    if not (isinstance(left, Variable)):
        raise ValueError("The left should be a Variable")

    if not (isinstance(right, Variable)):
        raise ValueError("The right should be a Variable")

X
Xin Pan 已提交
8420
    out = helper.create_variable_for_type_inference("float32")
8421 8422 8423 8424 8425 8426 8427 8428

    helper.append_op(
        type='rank_loss',
        inputs={"Label": label,
                "Left": left,
                "Right": right},
        outputs={'Out': out})
    return out
8429 8430


M
minqiyang 已提交
8431 8432
def margin_rank_loss(label, left, right, margin=0.1, name=None):
    """
M
minqiyang 已提交
8433
    Margin Ranking Loss Layer for ranking problem,
M
minqiyang 已提交
8434
    which compares left score and right score passed in.
M
minqiyang 已提交
8435
    The ranking loss can be defined as following equation:
M
minqiyang 已提交
8436 8437 8438

    .. math::

H
haowang101779990 已提交
8439
        rank\_loss = max(0, -label * (left - right) + margin)
M
minqiyang 已提交
8440 8441

    Args:
M
minqiyang 已提交
8442
       label (Variable): Indicates whether the left is ranked higher than the right or not.
M
minqiyang 已提交
8443 8444
       left (Variable): Ranking score for left.
       right (Variable): Ranking score for right.
M
minqiyang 已提交
8445
       margin (float): Indicates the given margin.
M
minqiyang 已提交
8446 8447
       name (str|None): A name for this layer (optional). If set None, the layer
                       will be named automatically.
H
haowang101779990 已提交
8448

M
minqiyang 已提交
8449
    Returns:
M
minqiyang 已提交
8450
       Variable: The ranking loss.
H
haowang101779990 已提交
8451

M
minqiyang 已提交
8452
    Raises:
M
minqiyang 已提交
8453
       ValueError: Any of label, left, and right is not a Variable.
H
haowang101779990 已提交
8454

M
minqiyang 已提交
8455
    Examples:
H
haowang101779990 已提交
8456

M
minqiyang 已提交
8457
        .. code-block:: python
H
haowang101779990 已提交
8458

Y
Yibing Liu 已提交
8459 8460 8461
           label = fluid.layers.data(name="label", shape=[-1, 1], dtype="float32")
           left = fluid.layers.data(name="left", shape=[-1, 1], dtype="float32")
           right = fluid.layers.data(name="right", shape=[-1, 1], dtype="float32")
M
minqiyang 已提交
8462 8463
           out = fluid.layers.margin_rank_loss(label, left, right)
    """
M
minqiyang 已提交
8464
    helper = LayerHelper('margin_rank_loss', **locals())
M
minqiyang 已提交
8465 8466 8467 8468 8469 8470
    if not isinstance(label, Variable):
        raise ValueError("The label should be a Variable.")
    if not isinstance(left, Variable):
        raise ValueError("The left should be a Variable.")
    if not isinstance(right, Variable):
        raise ValueError("The right should be a Variable.")
X
Xin Pan 已提交
8471 8472
    out = helper.create_variable_for_type_inference(left.dtype)
    act = helper.create_variable_for_type_inference(left.dtype)
M
minqiyang 已提交
8473 8474 8475 8476 8477 8478 8479 8480 8481 8482 8483
    helper.append_op(
        type='margin_rank_loss',
        inputs={"Label": label,
                "X1": left,
                "X2": right},
        outputs={'Out': out,
                 'Activated': act},
        attrs={'margin': margin})
    return out


W
whs 已提交
8484 8485 8486 8487 8488 8489 8490 8491 8492 8493 8494 8495
def pad2d(input,
          paddings=[0, 0, 0, 0],
          mode='constant',
          pad_value=0.0,
          data_format="NCHW",
          name=None):
    """
    Pad 2-d images accordding to 'paddings' and 'mode'.
    If mode is 'reflect', paddings[0] and paddings[1] must be no greater
    than height-1. And the width dimension has the same condition.

    Example:
T
Tink_Y 已提交
8496
        .. code-block:: text
W
whs 已提交
8497

T
Tink_Y 已提交
8498
	      Given that X is a channel of image from input:
M
minqiyang 已提交
8499

T
Tink_Y 已提交
8500 8501
	      X = [[1, 2, 3],
		   [4, 5, 6]]
M
minqiyang 已提交
8502

T
Tink_Y 已提交
8503
	      Case 0:
M
minqiyang 已提交
8504

T
Tink_Y 已提交
8505 8506 8507
		paddings = [0, 1, 2, 3],
		mode = 'constant'
		pad_value = 0
M
minqiyang 已提交
8508

T
Tink_Y 已提交
8509 8510 8511
		Out = [[0, 0, 1, 2, 3, 0, 0, 0]
		       [0, 0, 4, 5, 6, 0, 0, 0]
		       [0, 0, 0, 0, 0, 0, 0, 0]]
M
minqiyang 已提交
8512

T
Tink_Y 已提交
8513
	      Case 1:
M
minqiyang 已提交
8514

T
Tink_Y 已提交
8515 8516
		paddings = [0, 1, 2, 1],
		mode = 'reflect'
M
minqiyang 已提交
8517

T
Tink_Y 已提交
8518 8519 8520
		Out = [[3, 2, 1, 2, 3, 2]
		       [6, 5, 4, 5, 6, 5]
		       [3, 2, 1, 2, 3, 2]]
M
minqiyang 已提交
8521

T
Tink_Y 已提交
8522
	      Case 2:
M
minqiyang 已提交
8523

T
Tink_Y 已提交
8524 8525
		paddings = [0, 1, 2, 1],
		mode = 'edge'
M
minqiyang 已提交
8526

T
Tink_Y 已提交
8527 8528 8529
		Out = [[1, 1, 1, 2, 3, 3]
		       [4, 4, 4, 5, 6, 6]
		       [4, 4, 4, 5, 6, 6]]
M
minqiyang 已提交
8530 8531


W
whs 已提交
8532 8533
    Args:
        input (Variable): The input image with [N, C, H, W] format or [N, H, W, C] format.
8534
        paddings (tuple|list|Variable): The padding size. If padding is a tuple, it must
W
whs 已提交
8535 8536 8537 8538 8539 8540 8541 8542 8543 8544 8545 8546 8547 8548 8549 8550 8551 8552 8553 8554 8555 8556 8557
            contain four integers, (padding_top, padding_bottom, padding_left, padding_right).
            Default: padding = [0, 0, 0, 0].
        mode (str): Three modes: constant(default), reflect, edge. Default: constant
        pad_value (float32): The value to fill the padded areas in constant mode. Default: 0
        data_format (str): An optional string from: "NHWC", "NCHW". Specify the data format of
                           the input data.
                           Default: "NCHW"
        name (str|None): A name for this layer(optional). If set None, the layer
            will be named automatically.

    Returns:
        Variable: The tensor variable padded accordding to paddings and mode.


    Examples:
        .. code-block:: python

          data = fluid.layers.data(name='data', shape=[3, 32, 32], dtype='float32')
          result = fluid.layers.pad2d(input=data, padding=[1,2,3,4], mode='reflect')
    """

    helper = LayerHelper('pad2d', **locals())
    dtype = helper.input_dtype(input_param_name='input')
X
Xin Pan 已提交
8558
    out = helper.create_variable_for_type_inference(dtype)
8559 8560 8561 8562 8563 8564 8565 8566 8567
    inputs = {'X': input}
    attrs = {'mode': mode, 'pad_value': pad_value, 'data_format': data_format}

    if isinstance(paddings, Variable):
        inputs['Paddings'] = paddings
        attrs['paddings'] = []
    else:
        attrs['paddings'] = paddings

W
whs 已提交
8568
    helper.append_op(
8569
        type='pad2d', inputs=inputs, outputs={"Out": out}, attrs=attrs)
W
whs 已提交
8570 8571 8572 8573

    return out


8574 8575 8576 8577 8578 8579 8580 8581 8582 8583 8584 8585
@templatedoc()
def elu(x, alpha=1.0, name=None):
    """
    ${comment}
    Args:
        x(${x_type}): ${x_comment}
        alpha(${alpha_type}|1.0): ${alpha_comment}
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.

    Returns:
        output(${out_type}): ${out_comment}
Z
ZhenWang 已提交
8586 8587 8588 8589 8590

    Examples:

        .. code-block:: python

Z
ZhenWang 已提交
8591 8592
            x = fluid.layers.data(name="x", shape=[3,10,32,32], dtype="float32")
            y = fluid.layers.elu(x, alpha=0.2)
8593 8594
    """
    helper = LayerHelper('elu', **locals())
X
Xin Pan 已提交
8595
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
8596 8597 8598 8599 8600 8601 8602 8603 8604 8605 8606 8607 8608 8609 8610 8611 8612 8613 8614 8615
    helper.append_op(
        type='elu',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={'alpha': alpha})
    return out


@templatedoc()
def relu6(x, threshold=6.0, name=None):
    """
    ${comment}
    Args:
        x(${x_type}): ${x_comment}
        threshold(${threshold_type}|6.0): ${threshold_comment}
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.

    Returns:
        output(${out_type}): ${out_comment}
Z
ZhenWang 已提交
8616 8617 8618 8619 8620

    Examples:

        .. code-block:: python

Z
ZhenWang 已提交
8621 8622
            x = fluid.layers.data(name="x", shape=[3,10,32,32], dtype="float32")
            y = fluid.layers.relu6(x, threshold=6.0)
8623 8624
    """
    helper = LayerHelper('relu6', **locals())
X
Xin Pan 已提交
8625
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
8626 8627 8628 8629 8630 8631 8632 8633 8634 8635 8636 8637 8638 8639 8640 8641 8642 8643 8644 8645
    helper.append_op(
        type='relu6',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={'threshold': threshold})
    return out


@templatedoc()
def pow(x, factor=1.0, name=None):
    """
    ${comment}
    Args:
        x(${x_type}): ${x_comment}
        factor(${factor_type}|1.0): ${factor_comment}
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.

    Returns:
        output(${out_type}): ${out_comment}
Z
ZhenWang 已提交
8646 8647 8648 8649 8650

    Examples:

        .. code-block:: python

Z
ZhenWang 已提交
8651 8652
            x = fluid.layers.data(name="x", shape=[3,10,32,32], dtype="float32")
            y = fluid.layers.pow(x, factor=2.0)
8653 8654
    """
    helper = LayerHelper('pow', **locals())
X
Xin Pan 已提交
8655
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
8656 8657 8658 8659 8660 8661 8662 8663 8664 8665 8666 8667 8668 8669 8670 8671 8672 8673 8674 8675 8676
    helper.append_op(
        type='pow',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={'factor': factor})
    return out


@templatedoc()
def stanh(x, scale_a=2.0 / 3.0, scale_b=1.7159, name=None):
    """
    ${comment}
    Args:
        x(${x_type}): ${x_comment}
        scale_a(${scale_a_type}|2.0 / 3.0): ${scale_a_comment}
        scale_b(${scale_b_type}|1.7159): ${scale_b_comment}
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.

    Returns:
        output(${out_type}): ${out_comment}
Z
ZhenWang 已提交
8677 8678 8679 8680 8681

    Examples:

        .. code-block:: python

Z
ZhenWang 已提交
8682
            x = fluid.layers.data(name="x", shape=[3,10,32,32], dtype="float32")
Z
ZhenWang 已提交
8683
            y = fluid.layers.stanh(x, scale_a=0.67, scale_b=1.72)
8684 8685
    """
    helper = LayerHelper('stanh', **locals())
X
Xin Pan 已提交
8686
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
8687 8688 8689 8690 8691 8692 8693 8694 8695 8696 8697 8698 8699 8700 8701 8702 8703 8704 8705 8706 8707 8708
    helper.append_op(
        type='stanh',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={'scale_a': scale_a,
               'scale_b': scale_b})
    return out


@templatedoc()
def hard_sigmoid(x, slope=0.2, offset=0.5, name=None):
    """
    ${comment}
    Args:
        x(${x_type}): ${x_comment}
        slope(${slope_type}|0.2): ${slope_comment}
        offset(${offset_type}|0.5): ${offset_comment}
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.

    Returns:
        output(${out_type}): ${out_comment}
Z
ZhenWang 已提交
8709 8710 8711 8712 8713

    Examples:

        .. code-block:: python

Z
ZhenWang 已提交
8714 8715
            x = fluid.layers.data(name="x", shape=[3,10,32,32], dtype="float32")
            y = fluid.layers.hard_sigmoid(x, slope=0.3, offset=0.8)
8716 8717
    """
    helper = LayerHelper('hard_sigmoid', **locals())
X
Xin Pan 已提交
8718
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
8719 8720 8721 8722 8723 8724 8725 8726 8727 8728 8729 8730 8731 8732 8733 8734 8735 8736 8737 8738 8739
    helper.append_op(
        type='hard_sigmoid',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={'slope': slope,
               'offset': offset})
    return out


@templatedoc()
def swish(x, beta=1.0, name=None):
    """
    ${comment}
    Args:
        x(${x_type}): ${x_comment}
        beta(${beta_type}|1.0): ${beta_comment}
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.

    Returns:
        output(${out_type}): ${out_comment}
Z
ZhenWang 已提交
8740 8741 8742 8743 8744

    Examples:

        .. code-block:: python

Z
ZhenWang 已提交
8745 8746
            x = fluid.layers.data(name="x", shape=[3,10,32,32], dtype="float32")
            y = fluid.layers.swish(x, beta=2.0)
8747 8748
    """
    helper = LayerHelper('swish', **locals())
X
Xin Pan 已提交
8749
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
8750 8751 8752 8753 8754 8755 8756 8757
    helper.append_op(
        type='swish',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={'slope': beta})
    return out


J
jerrywgz 已提交
8758 8759 8760 8761
def prelu(x, mode, param_attr=None, name=None):
    """
    Equation:

H
haowang101779990 已提交
8762 8763
    .. math::
        y = \max(0, x) + \\alpha * \min(0, x)
J
jerrywgz 已提交
8764 8765 8766

    Args:
        x (Variable): The input tensor.
J
jerrywgz 已提交
8767
        param_attr(ParamAttr|None): The parameter attribute for the learnable
T
Tink_Y 已提交
8768
          weight (alpha).
J
jerrywgz 已提交
8769
        mode (string): The mode for weight sharing. It supports all, channel
T
Tink_Y 已提交
8770 8771 8772
          and element. all: all elements share same weight
          channel:elements in a channel share same weight
          element:each element has a weight
J
jerrywgz 已提交
8773
        name(str|None): A name for this layer(optional). If set None, the layer
T
Tink_Y 已提交
8774
          will be named automatically.
J
jerrywgz 已提交
8775 8776 8777 8778 8779 8780 8781 8782

    Returns:
        Variable: The output tensor with the same shape as input.

    Examples:

        .. code-block:: python

J
jerrywgz 已提交
8783
            x = fluid.layers.data(name="x", shape=[10,10], dtype="float32")
J
jerrywgz 已提交
8784 8785 8786 8787 8788 8789 8790 8791 8792 8793 8794 8795 8796
            mode = 'channel'
            output = fluid.layers.prelu(x,mode)
    """
    helper = LayerHelper('prelu', **locals())
    if mode not in ['all', 'channel', 'element']:
        raise ValueError('mode should be one of all, channel, element.')
    alpha_shape = [1]
    if mode == 'channel':
        alpha_shape = [1, x.shape[1], 1, 1]
    elif mode == 'element':
        alpha_shape = x.shape
    dtype = helper.input_dtype(input_param_name='x')
    alpha = helper.create_parameter(
Q
Qiao Longfei 已提交
8797
        attr=helper.param_attr,
J
jerrywgz 已提交
8798 8799 8800 8801
        shape=alpha_shape,
        dtype='float32',
        is_bias=False,
        default_initializer=Constant(1.0))
X
Xin Pan 已提交
8802
    out = helper.create_variable_for_type_inference(dtype)
J
jerrywgz 已提交
8803 8804 8805 8806 8807 8808 8809 8810 8811
    helper.append_op(
        type="prelu",
        inputs={"X": x,
                'Alpha': alpha},
        attrs={"mode": mode},
        outputs={"Out": out})
    return out


8812 8813 8814 8815 8816 8817 8818 8819 8820 8821
@templatedoc()
def brelu(x, t_min=0.0, t_max=24.0, name=None):
    """
    ${comment}
    Args:
        x(${x_type}): ${x_comment}
        t_min(${t_min_type}|0.0): ${t_min_comment}
        t_max(${t_max_type}|24.0): ${t_max_comment}
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.
8822
    Returns:
8823
        output(${out_type}): ${out_comment}
8824 8825 8826

    Examples:

8827
    .. code-block:: python
8828

H
haowang101779990 已提交
8829 8830
            x = fluid.layers.data(name="x", shape=[2,3,16,16], dtype="float32")
            y = fluid.layers.brelu(x, t_min=1.0, t_max=20.0)
8831 8832
    """
    helper = LayerHelper('brelu', **locals())
X
Xin Pan 已提交
8833
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
8834 8835 8836 8837 8838 8839 8840 8841 8842 8843 8844 8845 8846 8847 8848 8849 8850 8851
    helper.append_op(
        type='brelu',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={'t_min': t_min,
               't_max': t_max})
    return out


@templatedoc()
def leaky_relu(x, alpha=0.02, name=None):
    """
    ${comment}
    Args:
        x(${x_type}): ${x_comment}
        alpha(${alpha_type}|0.02): ${alpha_comment}
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.
8852
    Returns:
8853
        output(${out_type}): ${out_comment}
8854 8855 8856 8857 8858

    Examples:

        .. code-block:: python

H
haowang101779990 已提交
8859 8860
            x = fluid.layers.data(name="x", shape=[2,3,16,16], dtype="float32")
            y = fluid.layers.leaky_relu(x, alpha=0.01)
8861 8862
    """
    helper = LayerHelper('leaky_relu', **locals())
X
Xin Pan 已提交
8863
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
8864 8865 8866 8867 8868 8869 8870 8871 8872 8873 8874 8875 8876 8877 8878 8879 8880
    helper.append_op(
        type='leaky_relu',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={'alpha': alpha})
    return out


@templatedoc()
def soft_relu(x, threshold=40.0, name=None):
    """
    ${comment}
    Args:
        x(${x_type}): ${x_comment}
        threshold(${threshold_type}|40.0): ${threshold_comment}
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.
8881
    Returns:
8882
        output(${out_type}): ${out_comment}
8883 8884 8885 8886 8887

    Examples:

        .. code-block:: python

H
haowang101779990 已提交
8888 8889
            x = fluid.layers.data(name="x", shape=[2,3,16,16], dtype="float32")
            y = fluid.layers.soft_relu(x, threshold=20.0)
8890 8891
    """
    helper = LayerHelper('soft_relu', **locals())
X
Xin Pan 已提交
8892
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
8893 8894 8895 8896 8897 8898 8899 8900
    helper.append_op(
        type='soft_relu',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={'threshold': threshold})
    return out


8901 8902 8903 8904
def flatten(x, axis=1, name=None):
    """
    **Flatten layer**
    Flattens the input tensor into a 2D matrix.
M
minqiyang 已提交
8905

H
haowang101779990 已提交
8906
    For Example:
M
minqiyang 已提交
8907

H
haowang101779990 已提交
8908
    .. code-block:: text
8909

H
haowang101779990 已提交
8910 8911 8912 8913 8914 8915 8916 8917 8918 8919 8920 8921 8922 8923 8924 8925 8926 8927 8928 8929 8930
        Case 1:

          Given
            X.shape = (3, 100, 100, 4)

          and
            axis = 2

          We get:
            Out.shape = (3 * 100, 4 * 100)

        Case 2:

          Given
            X.shape = (3, 100, 100, 4)

          and
            axis = 0

          We get:
            Out.shape = (1, 3 * 100 * 100 * 4)
8931 8932 8933

    Args:
        x (Variable): A tensor of rank >= axis.
8934 8935
        axis (int): Indicate up to which input dimensions (exclusive) should
                    be flattened to the outer dimension of the output.
8936 8937 8938 8939 8940 8941 8942 8943
                    The value for axis must be in the range [0, R], where R
                    is the rank of the input tensor. When axis = 0, the shape
                    of the output tensor is (1, (d_0 X d_1 ... d_n), where the
                    shape of the input tensor is (d_0, d_1, ... d_n).
        name(str|None): A name for this layer(optional). If set None, the layer
                        will be named automatically.

    Returns:
H
haowang101779990 已提交
8944 8945 8946
        Variable: A 2D tensor with the contents of the input tensor, with input \
                  dimensions up to axis flattened to the outer dimension of \
                  the output and remaining input dimensions flattened into the \
8947 8948 8949 8950
                  inner dimension of the output.

    Raises:
        ValueError: If x is not a variable.
8951
        ValueError: If axis is not in range [0, rank(x)].
8952 8953 8954 8955 8956 8957 8958 8959 8960 8961 8962 8963 8964 8965 8966 8967

    Examples:

        .. code-block:: python

            x = fluid.layers.data(name="x", shape=[4, 4, 3], dtype="float32")
            out = fluid.layers.flatten(x=x, axis=2)
    """
    helper = LayerHelper('flatten', **locals())

    if not (isinstance(x, Variable)):
        raise ValueError("The input x should be a Variable")

    if not (isinstance(axis, int)) or axis > len(x.shape) or axis < 0:
        raise ValueError("The axis should be a int, and in range [0, rank(x)]")

X
Xin Pan 已提交
8968 8969
    out = helper.create_variable_for_type_inference(x.dtype)
    x_shape = helper.create_variable_for_type_inference(x.dtype)
8970
    helper.append_op(
8971
        type='flatten2',
8972
        inputs={"X": x},
8973 8974
        outputs={'Out': out,
                 'XShape': x_shape},
8975 8976
        attrs={"axis": axis})
    return out
X
Xin Pan 已提交
8977 8978


C
chenweihang 已提交
8979
def sequence_enumerate(input, win_size, pad_value=0, name=None):
C
chenweihang 已提交
8980
    """
C
chenweihang 已提交
8981
    Generate a new sequence for the input index sequence, which enumerates all the
M
minqiyang 已提交
8982
    sub-sequences with length `win_size` of the input.
C
chenweihang 已提交
8983 8984
    The enumerated sequence has the same 1st dimension with variable `input`, and
    the 2nd dimension is `win_size`, padded by `pad_value` if necessary in generation.
M
minqiyang 已提交
8985

H
haowang101779990 已提交
8986 8987 8988 8989 8990 8991 8992 8993 8994 8995 8996 8997 8998 8999 9000 9001 9002
    .. code-block:: text

        Case 1:

          Input:
            X.lod = [[0, 3, 5]]
            X.data = [[1], [2], [3], [4], [5]]
            X.dims = [5, 1]

          Attrs:
            win_size = 2
            pad_value = 0

          Output:
            Out.lod = [[0, 3, 5]]
            Out.data = [[1, 2], [2, 3], [3, 0], [4, 5], [5, 0]]
            Out.dims = [5, 2]
C
chenweihang 已提交
9003 9004

    Args:
C
chenweihang 已提交
9005 9006 9007
        input (Variable): The input variable which is a index sequence.
        win_size (int): The window size for enumerating all sub-sequences.
        pad_value (int): The padding value, default 0.
C
chenweihang 已提交
9008 9009 9010 9011 9012 9013 9014

    Returns:
        Variable: The enumerate sequence variable which is a LoDTensor.

    Examples:
        .. code-block:: python

9015
            x = fluid.layers.data(shape[-1, 1], dtype='int32', lod_level=1)
C
chenweihang 已提交
9016 9017
            out = fluid.layers.sequence_enumerate(input=x, win_size=3, pad_value=0)
    """
L
lujun 已提交
9018
    assert not in_dygraph_mode(), (
9019
        "sequence layer is not supported in dygraph mode yet.")
C
chenweihang 已提交
9020
    helper = LayerHelper('sequence_enumerate', **locals())
X
Xin Pan 已提交
9021 9022
    out = helper.create_variable_for_type_inference(
        helper.input_dtype(), stop_gradient=True)
C
chenweihang 已提交
9023 9024 9025 9026 9027 9028
    helper.append_op(
        type='sequence_enumerate',
        inputs={'X': input},
        outputs={'Out': out},
        attrs={'win_size': win_size,
               'pad_value': pad_value})
M
minqiyang 已提交
9029
    return out
9030

9031

S
sneaxiy 已提交
9032 9033 9034 9035 9036 9037 9038 9039 9040
def sequence_mask(x, maxlen=None, dtype='int64', name=None):
    """
    **SequenceMask Layer**

    This layer outputs a mask according to the input :code:`x` and
    :code:`maxlen` with data type of :code:`dtype`.

    Supposing :code:`x` is a Tensor with shape [d_1, d_2, ..., d_n], the
    :code:`y` is a mask with shape [d_1, d_2, ..., d_n, maxlen], where:
9041

S
sneaxiy 已提交
9042
    .. math::
9043

S
sneaxiy 已提交
9044 9045 9046
        y(i_1, i_2,..., i_n, j) = (j < x(i_1, i_2,..., i_n))

    Args:
9047
        x (Variable): Input tensor of sequence_mask layer,
S
sneaxiy 已提交
9048 9049 9050 9051
                      whose elements are integers less than :code:`maxlen`.
        maxlen (int|None): Maximum length of the sequence. If :code:`maxlen`
                           is None, it would be replace with :math:`max(x)`.
        dtype (np.dtype|core.VarDesc.VarType|str): Data type of the output.
9052 9053 9054
        name (str|None): A name for this layer(optional). If set None, the
                         layer will be named automatically.

S
sneaxiy 已提交
9055 9056
    Returns:
        Variable: The output sequence mask.
9057

9058 9059 9060 9061 9062 9063 9064 9065
    Examples:
        .. code-block:: python
	
            import paddle.fluid.layers as layers

            x = fluid.layers.data(name='x', shape=[10], dtype='float32', lod_level=1)
            mask = layers.sequence_mask(x=x)

S
sneaxiy 已提交
9066
    """
L
lujun 已提交
9067
    assert not in_dygraph_mode(), (
9068
        "sequence layer is not supported in dygraph mode yet.")
S
sneaxiy 已提交
9069

Q
qingqing01 已提交
9070
    helper = LayerHelper('sequence_mask', **locals())
S
sneaxiy 已提交
9071
    if name is None:
X
Xin Pan 已提交
9072
        out = helper.create_variable_for_type_inference(dtype=dtype)
S
sneaxiy 已提交
9073
    else:
X
Xin Pan 已提交
9074
        out = helper.create_variable_for_type_inference(dtype=dtype, name=name)
S
sneaxiy 已提交
9075

Q
qingqing01 已提交
9076 9077 9078
    helper.append_op(
        type='sequence_mask',
        inputs={'X': [x]},
S
sneaxiy 已提交
9079 9080
        outputs={'Y': out},
        attrs={
9081
            'maxlen': maxlen if maxlen is not None else -1,
S
sneaxiy 已提交
9082 9083 9084
            'out_dtype': out.dtype
        })
    return out
S
sneaxiy 已提交
9085 9086


X
Xin Pan 已提交
9087
def stack(x, axis=0):
S
sneaxiy 已提交
9088 9089 9090 9091
    """
    **Stack Layer**

    This layer stacks all of the input :code:`x` along axis.
9092 9093 9094 9095 9096 9097 9098

    Input :code:`x` can be a single variable, a :code:`list` of variables,
    or a :code:`tuple` of variables. If :code:`x` is a :code:`list` or
    :code:`tuple`, the shapes of all these variables must be the same.
    Supposing the shape of each input is :math:`[d_0, d_1, ..., d_{n-1}]`,
    the shape of the output variable would be
    :math:`[d_0, d_1, ..., d_{axis}=len(x), ..., d_{n-1}]`.
S
sneaxiy 已提交
9099
    If :code:`axis` < 0, it would be replaced with :code:`axis+rank(x[0])+1`.
9100
    If :code:`axis` is None, it would be replaced with 0.
S
sneaxiy 已提交
9101

C
chengduozh 已提交
9102 9103
    For Example:

C
chengduozh 已提交
9104 9105 9106 9107 9108 9109 9110 9111 9112 9113 9114 9115 9116 9117 9118 9119 9120 9121 9122 9123 9124 9125 9126 9127 9128 9129 9130 9131 9132 9133 9134 9135 9136 9137 9138 9139 9140 9141
    .. code-block:: text

        Case 1:
          Input:
            x[0].data = [ [1.0 , 2.0 ] ]
            x[0].dims = [1, 2]
            x[1].data = [ [3.0 , 4.0 ] ]
            x[1].dims = [1, 2]
            x[2].data = [ [5.0 , 6.0 ] ]
            x[2].dims = [1, 2]

          Attrs:
            axis = 0

          Output:
            Out.data =[ [ [1.0, 2.0] ],
                        [ [3.0, 4.0] ],
                        [ [5.0, 6.0] ] ]
            Out.dims = [3, 1, 2]

        Case 2:
          Given
            x[0].data = [ [1.0 , 2.0 ] ]
            x[0].dims = [1, 2]
            x[1].data = [ [3.0 , 4.0 ] ]
            x[1].dims = [1, 2]
            x[2].data = [ [5.0 , 6.0 ] ]
            x[2].dims = [1, 2]

          Attrs:
            axis = 1 or axis = -2

          Output:
            Out.data =[ [ [1.0, 2.0]
                          [3.0, 4.0]
                          [5.0, 6.0] ] ]
            Out.dims = [1, 3, 2]

S
sneaxiy 已提交
9142
    Args:
9143
        x (Variable|list(Variable)|tuple(Variable)): Input variables.
S
sneaxiy 已提交
9144
        axis (int|None): The axis along which all inputs are stacked.
9145

S
sneaxiy 已提交
9146 9147
    Returns:
        Variable: The stacked variable.
9148

9149 9150 9151 9152 9153 9154 9155 9156
    Examples:
        .. code-block:: python

            import paddle.fluid.layers as layers
            x1 = layers.data(name='x1', shape[1, 2], dtype='int32')
            x2 = layers.data(name='x2', shape[1, 2], dtype='int32')
            data = layers.stack([x1,x2])

S
sneaxiy 已提交
9157 9158
    """

X
Xin Pan 已提交
9159 9160 9161 9162 9163 9164
    helper = LayerHelper('stack', **locals())
    axis = 0 if axis is None else axis

    if not isinstance(x, list) and not isinstance(x, tuple):
        x = [x]

X
Xin Pan 已提交
9165
    out = helper.create_variable_for_type_inference(x[0].dtype)
X
Xin Pan 已提交
9166
    helper.append_op(
S
sneaxiy 已提交
9167 9168
        type='stack', inputs={'X': x}, outputs={'Y': out},
        attrs={'axis': axis})
9169

X
Xin Pan 已提交
9170
    return out
D
dzhwinter 已提交
9171 9172 9173 9174 9175 9176 9177


def unstack(x, axis=0, num=None):
    """
    **UnStack Layer**

    This layer unstacks input :code:`x` into several tensors along axis.
M
minqiyang 已提交
9178

D
dzhwinter 已提交
9179 9180 9181
    If :code:`axis` < 0, it would be replaced with :code:`axis+rank(x)`.
    If :code:`num` is None, it would be inferred from :code:`x.shape[axis]`,
    and if :code:`x.shape[axis]` <= 0 or is unknown, :code:`ValueError` is
M
minqiyang 已提交
9182
    raised.
D
dzhwinter 已提交
9183 9184

    Args:
M
minqiyang 已提交
9185
        x (Variable): Input variable.
D
dzhwinter 已提交
9186 9187
        axis (int): The axis along which the input is unstacked.
        num (int|None): The number of output variables.
M
minqiyang 已提交
9188

D
dzhwinter 已提交
9189 9190
    Returns:
        list(Variable): The unstacked variables.
M
minqiyang 已提交
9191

D
dzhwinter 已提交
9192 9193 9194 9195 9196 9197 9198 9199 9200 9201
    """

    helper = LayerHelper('unstack', **locals())
    if num is None:
        if axis is None or x.shape[axis] <= 0:
            raise ValueError('unknown unstack number')
        else:
            num = x.shape[axis]

    outs = []
Y
Yibing Liu 已提交
9202
    for _ in range(num):
X
Xin Pan 已提交
9203
        outs.append(helper.create_variable_for_type_inference(x.dtype))
D
dzhwinter 已提交
9204 9205 9206 9207 9208 9209 9210 9211

    helper.append_op(
        type='unstack',
        inputs={'X': [x]},
        outputs={'Y': outs},
        attrs={'axis': axis,
               'num': num})
    return outs
W
whs 已提交
9212 9213 9214 9215 9216 9217 9218 9219 9220 9221 9222 9223


def expand(x, expand_times, name=None):
    """Expand operator tiles the input by given times number. You should set times
    number for each dimension by providing attribute 'expand_times'. The rank of X
    should be in [1, 6]. Please note that size of 'expand_times' must be the same
    with X's rank. Following is a using case:


    .. code-block:: text

        Input(X) is a 3-D tensor with shape [2, 3, 1]:
M
minqiyang 已提交
9224

W
whs 已提交
9225 9226 9227 9228
                [
                   [[1], [2], [3]],
                   [[4], [5], [6]]
                ]
M
minqiyang 已提交
9229

W
whs 已提交
9230
        Attr(expand_times):  [1, 2, 2]
M
minqiyang 已提交
9231

W
whs 已提交
9232
        Output(Out) is a 3-D tensor with shape [2, 6, 2]:
M
minqiyang 已提交
9233

W
whs 已提交
9234 9235 9236 9237
                [
                    [[1, 1], [2, 2], [3, 3], [1, 1], [2, 2], [3, 3]],
                    [[4, 4], [5, 5], [6, 6], [4, 4], [5, 5], [6, 6]]
                ]
M
minqiyang 已提交
9238

W
whs 已提交
9239 9240 9241 9242 9243 9244 9245 9246 9247 9248 9249 9250 9251 9252 9253 9254
    Args:
        x (Variable): A tensor with rank in [1, 6].
        expand_times (list|tuple): Expand times number for each dimension.

    Returns:
        Variable: The expanded variable which is a LoDTensor. After expanding, size of each dimension of Output(Out) is equal to ithe size of the corresponding dimension of Input(X) multiplying the corresponding value given by expand_times.


    Examples:
        .. code-block:: python

            x = fluid.layers.data(name='x', shape=[10], dtype='float32')
            out = fluid.layers.expand(x=x, expand_times=[1, 2, 2])
    """
    helper = LayerHelper('expand', input=x, **locals())
    dtype = helper.input_dtype(input_param_name='x')
X
Xin Pan 已提交
9255
    out = helper.create_variable_for_type_inference(dtype)
W
whs 已提交
9256 9257 9258 9259 9260 9261
    helper.append_op(
        type='expand',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={'expand_times': expand_times})
    return out
S
sneaxiy 已提交
9262 9263


G
fix  
gongweibao 已提交
9264 9265 9266
from paddle.fluid.framework import convert_np_dtype_to_dtype_


G
gongweibao 已提交
9267
@templatedoc()
G
fix  
gongweibao 已提交
9268 9269 9270 9271 9272 9273 9274 9275 9276
def uniform_random_batch_size_like(input,
                                   shape,
                                   dtype='float32',
                                   input_dim_idx=0,
                                   output_dim_idx=0,
                                   min=-1.0,
                                   max=1.0,
                                   seed=0):
    """
G
gongweibao 已提交
9277
    ${comment}
G
fix  
gongweibao 已提交
9278 9279

    Args:
G
gongweibao 已提交
9280 9281 9282
        input (Variable): ${input_comment}
        shape (tuple|list): ${shape_comment}
        input_dim_idx (Int): ${input_dim_idx_comment}
G
gongweibao 已提交
9283
        output_dim_idx (Int): ${output_dim_idx_comment}
G
gongweibao 已提交
9284 9285 9286
        min (Float): ${min_comment}
        max (Float): ${max_comment}
        seed (Int): ${seed_comment}
G
fix  
gongweibao 已提交
9287 9288
        dtype(np.dtype|core.VarDesc.VarType|str): The type of data : float32, float_16, int etc
    Returns:
G
gongweibao 已提交
9289
        out (Variable): ${out_comment}
G
fix  
gongweibao 已提交
9290

9291 9292 9293
    Examples:
        .. code-block:: python

9294 9295
            import paddle.fluid.layers as layers 

9296 9297
            input = layers.data(name="input", shape=[13, 11], dtype='float32')
            out = layers.uniform_random_batch_size_like(input, [-1, 11])
G
fix  
gongweibao 已提交
9298 9299 9300
    """

    helper = LayerHelper('uniform_random_batch_size_like', **locals())
X
Xin Pan 已提交
9301
    out = helper.create_variable_for_type_inference(dtype)
G
fix  
gongweibao 已提交
9302 9303 9304 9305 9306 9307 9308 9309 9310 9311 9312 9313 9314 9315 9316 9317
    c_dtype = convert_np_dtype_to_dtype_(dtype)
    helper.append_op(
        type='uniform_random_batch_size_like',
        inputs={'Input': input},
        outputs={'Out': out},
        attrs={
            'shape': shape,
            'input_dim_idx': input_dim_idx,
            'output_dim_idx': output_dim_idx,
            'min': min,
            'max': max,
            'seed': seed,
            'dtype': c_dtype
        })

    return out
G
fix  
gongweibao 已提交
9318 9319


G
gongweibao 已提交
9320
@templatedoc()
X
Xin Pan 已提交
9321
def gaussian_random(shape, mean=0.0, std=1.0, seed=0, dtype='float32'):
G
fix  
gongweibao 已提交
9322
    """
G
gongweibao 已提交
9323
    ${comment}
G
fix  
gongweibao 已提交
9324 9325

    Args:
G
gongweibao 已提交
9326 9327 9328 9329
        shape (tuple|list): ${shape_comment}
        mean (Float): ${mean_comment}
        std (Float): ${std_comment}
        seed (Int): ${seed_comment}
G
fix  
gongweibao 已提交
9330 9331 9332
        dtype(np.dtype|core.VarDesc.VarType|str): Output data type.

    Returns:
G
gongweibao 已提交
9333
        out (Variable): ${out_comment}
G
fix  
gongweibao 已提交
9334

9335 9336 9337
    Examples:
        .. code-block:: python

J
JesseyXujin 已提交
9338
            import paddle.fluid.layers as layers
9339
            out = layers.gaussian_random(shape=[20, 30])
G
fix  
gongweibao 已提交
9340 9341 9342
    """

    helper = LayerHelper('gaussian_random', **locals())
X
Xin Pan 已提交
9343
    out = helper.create_variable_for_type_inference(dtype)
G
fix  
gongweibao 已提交
9344 9345 9346 9347 9348 9349 9350 9351 9352 9353
    c_dtype = convert_np_dtype_to_dtype_(dtype)
    helper.append_op(
        type='gaussian_random',
        outputs={'Out': out},
        attrs={
            'shape': shape,
            'mean': mean,
            'std': std,
            'seed': seed,
            'dtype': c_dtype,
X
Xin Pan 已提交
9354
            'use_mkldnn': False
G
fix  
gongweibao 已提交
9355 9356 9357 9358 9359
        })

    return out


G
gongweibao 已提交
9360
@templatedoc()
G
fix  
gongweibao 已提交
9361
def sampling_id(x, min=0.0, max=1.0, seed=0, dtype='float32'):
G
fix  
gongweibao 已提交
9362
    """
G
gongweibao 已提交
9363
    ${comment}
G
fix  
gongweibao 已提交
9364 9365

    Args:
G
gongweibao 已提交
9366 9367 9368 9369
        x (Variable): ${x_comment}
        min (Float): ${min_comment}
        max (Float): ${max_comment}
        seed (Float): ${seed_comment}
G
fix  
gongweibao 已提交
9370
        dtype(np.dtype|core.VarDesc.VarType|str): The type of output data : float32, float_16, int etc
G
fix  
gongweibao 已提交
9371 9372

    Returns:
G
gongweibao 已提交
9373
        out (Variable): ${out_comment}
G
fix  
gongweibao 已提交
9374

9375 9376 9377
    Examples:
        .. code-block:: python

Y
Yibing Liu 已提交
9378
            x = fluid.layers.data(
9379 9380 9381 9382 9383
                name="X",
                shape=[13, 11],
                dtype='float32',
                append_batch_size=False)

Y
Yibing Liu 已提交
9384
            out = fluid.layers.sampling_id(x)
G
fix  
gongweibao 已提交
9385 9386 9387
    """

    helper = LayerHelper('sampling_id', **locals())
X
Xin Pan 已提交
9388
    out = helper.create_variable_for_type_inference(dtype)
G
fix  
gongweibao 已提交
9389 9390 9391 9392 9393 9394 9395 9396 9397 9398 9399
    helper.append_op(
        type='sampling_id',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={'min': min,
               'max': max,
               'seed': seed})

    return out


G
gongweibao 已提交
9400
@templatedoc()
G
fix  
gongweibao 已提交
9401 9402 9403 9404 9405 9406 9407 9408 9409
def gaussian_random_batch_size_like(input,
                                    shape,
                                    input_dim_idx=0,
                                    output_dim_idx=0,
                                    mean=0.0,
                                    std=1.0,
                                    seed=0,
                                    dtype='float32'):
    """
G
gongweibao 已提交
9410
    ${comment}
G
fix  
gongweibao 已提交
9411 9412

    Args:
G
gongweibao 已提交
9413 9414
        input (Variable): ${input_comment}
        shape (tuple|list): ${shape_comment}
G
gongweibao 已提交
9415
        input_dim_idx (Int): ${input_dim_idx_comment}
G
gongweibao 已提交
9416 9417 9418 9419
        output_dim_idx (Int): ${output_dim_idx_comment}
        mean (Float): ${mean_comment}
        std (Float): ${std_comment}
        seed (Int): ${seed_comment}
G
fix  
gongweibao 已提交
9420
        dtype(np.dtype|core.VarDesc.VarType|str): The type of output data : float32, float_16, int etc
G
fix  
gongweibao 已提交
9421 9422

    Returns:
G
gongweibao 已提交
9423
        out (Variable): ${out_comment}
9424 9425 9426 9427

    Examples:
        .. code-block:: python

Y
Yibing Liu 已提交
9428
            input = fluid.layers.data(name="input", shape=[13, 11], dtype='float32')
9429

Y
Yibing Liu 已提交
9430
            out = fluid.layers.gaussian_random_batch_size_like(
9431
                input, shape=[-1, 11], mean=1.0, std=2.0)
G
fix  
gongweibao 已提交
9432 9433 9434
    """

    helper = LayerHelper('gaussian_random_batch_size_like', **locals())
X
Xin Pan 已提交
9435
    out = helper.create_variable_for_type_inference(dtype)
G
fix  
gongweibao 已提交
9436 9437 9438 9439 9440 9441 9442 9443 9444 9445 9446 9447 9448 9449 9450 9451 9452 9453
    c_dtype = convert_np_dtype_to_dtype_(dtype)
    helper.append_op(
        type='gaussian_random_batch_size_like',
        inputs={'Input': input},
        outputs={'Out': out},
        attrs={
            'shape': shape,
            'input_dim_idx': input_dim_idx,
            'output_dim_idx': output_dim_idx,
            'mean': mean,
            'std': std,
            'seed': seed,
            'dtype': c_dtype
        })

    return out


G
gongweibao 已提交
9454
@templatedoc()
X
Xin Pan 已提交
9455
def sum(x):
G
fix  
gongweibao 已提交
9456
    """
G
gongweibao 已提交
9457
    ${comment}
G
fix  
gongweibao 已提交
9458 9459

    Args:
G
gongweibao 已提交
9460
        x (Variable): ${x_comment}
G
fix  
gongweibao 已提交
9461 9462

    Returns:
G
gongweibao 已提交
9463
        out (Variable): ${out_comment}
9464 9465 9466 9467 9468 9469

    Examples:
        .. code-block:: python

            input = layers.data(name="input", shape=[13, 11], dtype='float32')
            out = layers.sum(input)
G
fix  
gongweibao 已提交
9470 9471 9472
    """

    helper = LayerHelper('sum', **locals())
X
Xin Pan 已提交
9473 9474
    out = helper.create_variable_for_type_inference(
        dtype=helper.input_dtype('x'))
G
fix  
gongweibao 已提交
9475 9476 9477 9478
    helper.append_op(
        type='sum',
        inputs={'X': x},
        outputs={'Out': out},
X
Xin Pan 已提交
9479
        attrs={'use_mkldnn': False})
G
fix  
gongweibao 已提交
9480 9481 9482 9483

    return out


G
gongweibao 已提交
9484
@templatedoc()
G
fix  
gongweibao 已提交
9485 9486
def slice(input, axes, starts, ends):
    """
G
gongweibao 已提交
9487
    ${comment}
G
fix  
gongweibao 已提交
9488 9489

    Args:
G
gongweibao 已提交
9490 9491 9492 9493
        input (Variable): ${input_comment}.
        axes (List): ${axes_comment}
        starts (List): ${starts_comment}
        ends (List): ${ends_comment}
G
fix  
gongweibao 已提交
9494 9495

    Returns:
G
gongweibao 已提交
9496
        out (Variable): ${out_comment}
G
fix  
gongweibao 已提交
9497

9498 9499 9500 9501 9502 9503 9504 9505 9506 9507 9508
    Examples:
        .. code-block:: python

            starts = [1, 0, 2]
            ends = [3, 3, 4]
            axes = [0, 1, 2]

            input = layers.data(
                name="input", shape=[3, 4, 5, 6], dtype='float32')

            out = layers.slice(input, axes=axes, starts=starts, ends=ends)
G
fix  
gongweibao 已提交
9509 9510 9511
    """

    helper = LayerHelper('slice', **locals())
X
Xin Pan 已提交
9512 9513
    out = helper.create_variable_for_type_inference(
        dtype=helper.input_dtype('input'))
G
fix  
gongweibao 已提交
9514 9515 9516 9517 9518 9519 9520 9521 9522 9523 9524 9525 9526
    helper.append_op(
        type='slice',
        inputs={'Input': input},
        outputs={'Out': out},
        attrs={'axes': axes,
               'starts': starts,
               'ends': ends})

    return out


def shape(input):
    """
C
chengduozh 已提交
9527 9528
    **Shape Layer**

C
fix doc  
chengduozh 已提交
9529
    Get the shape of the input.
G
fix  
gongweibao 已提交
9530 9531

    Args:
C
chengduozh 已提交
9532
        input (Variable): The input variable.
G
fix  
gongweibao 已提交
9533 9534

    Returns:
C
fix doc  
chengduozh 已提交
9535
        Variable: The shape of the input variable.
G
fix  
gongweibao 已提交
9536

9537 9538 9539 9540 9541 9542
    Examples:
        .. code-block:: python

            input = layers.data(
                name="input", shape=[3, 100, 100], dtype="float32")
            out = layers.shape(input)
G
fix  
gongweibao 已提交
9543 9544 9545
    """

    helper = LayerHelper('shape', **locals())
9546
    out = helper.create_variable_for_type_inference(dtype='int32')
G
fix  
gongweibao 已提交
9547
    helper.append_op(
G
fix  
gongweibao 已提交
9548
        type='shape', inputs={'Input': input}, outputs={'Out': out})
G
fix  
gongweibao 已提交
9549 9550

    return out
G
merge  
gongweibao 已提交
9551 9552


Z
zhoukunsheng 已提交
9553 9554 9555 9556
def rank(input):
    """
    **Rank Layer**

Z
zhoukunsheng 已提交
9557
    Returns the number of dimensions for a tensor, which is a 0-D int32 Tensor.
Z
zhoukunsheng 已提交
9558 9559 9560 9561 9562 9563 9564 9565 9566 9567 9568 9569 9570 9571 9572 9573 9574 9575 9576 9577 9578

    Args:
        input (Variable): The input variable.

    Returns:
        Variable: The rank of the input variable.

    Examples:
        .. code-block:: python

            input = layers.data(
                name="input", shape=[3, 100, 100], dtype="float32")
            rank = layers.rank(input) # 4
    """

    ndims = len(input.shape)
    out = assign(np.array(ndims, 'int32'))

    return out


S
sneaxiy 已提交
9579 9580 9581 9582
def _elementwise_op(helper):
    op_type = helper.layer_type
    x = helper.kwargs.get('x', None)
    y = helper.kwargs.get('y', None)
L
lujun 已提交
9583
    if in_dygraph_mode():
X
Xin Pan 已提交
9584 9585 9586
        x = base.to_variable(x)
        y = base.to_variable(y)

S
sneaxiy 已提交
9587 9588 9589 9590
    assert x is not None, 'x cannot be None in {}'.format(op_type)
    assert y is not None, 'y cannot be None in {}'.format(op_type)
    axis = helper.kwargs.get('axis', -1)
    use_mkldnn = helper.kwargs.get('use_mkldnn', False)
S
sneaxiy 已提交
9591 9592
    name = helper.kwargs.get('name', None)
    if name is None:
X
Xin Pan 已提交
9593
        out = helper.create_variable_for_type_inference(dtype=x.dtype)
S
sneaxiy 已提交
9594 9595 9596
    else:
        out = helper.create_variable(
            name=name, dtype=x.dtype, persistable=False)
S
sneaxiy 已提交
9597

S
sneaxiy 已提交
9598 9599 9600 9601 9602 9603 9604 9605 9606 9607 9608
    helper.append_op(
        type=op_type,
        inputs={'X': x,
                'Y': y},
        outputs={'Out': out},
        attrs={'axis': axis,
               'use_mkldnn': use_mkldnn})
    return helper.append_activation(out)


@templatedoc()
S
sneaxiy 已提交
9609
def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None):
S
sneaxiy 已提交
9610 9611 9612 9613 9614 9615 9616 9617
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        scale(${scale_type}): ${scale_comment}
        bias(${bias_type}): ${bias_comment}
        bias_after_scale(${bias_after_scale_type}): ${bias_after_scale_comment}
S
sneaxiy 已提交
9618
        act(basestring|None): Activation applied to the output.
M
minqiyang 已提交
9619
        name(basestring|None): Name of the output.
S
sneaxiy 已提交
9620 9621 9622 9623 9624 9625

    Returns:
        out(${out_type}): ${out_comment}
    """

    helper = LayerHelper('scale', **locals())
S
sneaxiy 已提交
9626
    if name is None:
X
Xin Pan 已提交
9627
        out = helper.create_variable_for_type_inference(dtype=x.dtype)
S
sneaxiy 已提交
9628 9629 9630
    else:
        out = helper.create_variable(
            name=name, dtype=x.dtype, persistable=False)
S
sneaxiy 已提交
9631 9632 9633 9634 9635 9636 9637 9638 9639 9640

    helper.append_op(
        type='scale',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={
            'scale': float(scale),
            'bias': float(bias),
            'bias_after_scale': bias_after_scale
        })
S
sneaxiy 已提交
9641
    return helper.append_activation(out)
S
sneaxiy 已提交
9642 9643


X
Xin Pan 已提交
9644
def elementwise_add(x, y, axis=-1, act=None, name=None):
S
sneaxiy 已提交
9645 9646 9647
    return _elementwise_op(LayerHelper('elementwise_add', **locals()))


X
Xin Pan 已提交
9648
def elementwise_div(x, y, axis=-1, act=None, name=None):
S
sneaxiy 已提交
9649 9650 9651
    return _elementwise_op(LayerHelper('elementwise_div', **locals()))


X
Xin Pan 已提交
9652
def elementwise_sub(x, y, axis=-1, act=None, name=None):
S
sneaxiy 已提交
9653 9654 9655
    return _elementwise_op(LayerHelper('elementwise_sub', **locals()))


X
Xin Pan 已提交
9656
def elementwise_mul(x, y, axis=-1, act=None, name=None):
S
sneaxiy 已提交
9657 9658 9659
    return _elementwise_op(LayerHelper('elementwise_mul', **locals()))


X
Xin Pan 已提交
9660
def elementwise_max(x, y, axis=-1, act=None, name=None):
S
sneaxiy 已提交
9661 9662 9663
    return _elementwise_op(LayerHelper('elementwise_max', **locals()))


X
Xin Pan 已提交
9664
def elementwise_min(x, y, axis=-1, act=None, name=None):
S
sneaxiy 已提交
9665 9666 9667
    return _elementwise_op(LayerHelper('elementwise_min', **locals()))


X
Xin Pan 已提交
9668
def elementwise_pow(x, y, axis=-1, act=None, name=None):
S
sneaxiy 已提交
9669 9670 9671
    return _elementwise_op(LayerHelper('elementwise_pow', **locals()))


9672 9673 9674 9675 9676 9677 9678 9679
def elementwise_mod(x, y, axis=-1, act=None, name=None):
    return _elementwise_op(LayerHelper('elementwise_mod', **locals()))


def elementwise_floordiv(x, y, axis=-1, act=None, name=None):
    return _elementwise_op(LayerHelper('elementwise_floordiv', **locals()))


S
sneaxiy 已提交
9680
for func in [
9681 9682 9683 9684 9685 9686 9687 9688 9689
        elementwise_add,
        elementwise_div,
        elementwise_sub,
        elementwise_mul,
        elementwise_max,
        elementwise_min,
        elementwise_pow,
        elementwise_mod,
        elementwise_floordiv,
S
sneaxiy 已提交
9690 9691 9692 9693 9694
]:
    op_proto = OpProtoHolder.instance().get_op_proto(func.__name__)
    func.__doc__ = _generate_doc_string_(
        op_proto,
        additional_args_lines=[
S
sneaxiy 已提交
9695 9696
            "act (basestring|None): Activation applied to the output.",
            "name (basestring|None): Name of the output."
S
sneaxiy 已提交
9697
        ])
M
minqiyang 已提交
9698 9699


9700
def _logical_op(op_name, x, y, out=None, name=None, binary_op=True):
M
minqiyang 已提交
9701 9702
    helper = LayerHelper(op_name, **locals())

M
minqiyang 已提交
9703 9704
    if binary_op:
        assert x.dtype == y.dtype
M
minqiyang 已提交
9705 9706 9707

    if out is None:
        if name is None:
X
Xin Pan 已提交
9708
            out = helper.create_variable_for_type_inference(dtype=x.dtype)
M
minqiyang 已提交
9709 9710 9711 9712 9713 9714 9715 9716 9717 9718 9719 9720 9721 9722 9723
        else:
            out = helper.create_variable(
                name=name, dtype=x.dtype, persistable=False)

    if binary_op:
        helper.append_op(
            type=op_name, inputs={"X": x,
                                  "Y": y}, outputs={"Out": out})
    else:
        helper.append_op(type=op_name, inputs={"X": x}, outputs={"Out": out})

    return out


@templatedoc()
9724
def logical_and(x, y, out=None, name=None):
M
minqiyang 已提交
9725 9726 9727 9728 9729 9730 9731 9732 9733 9734 9735
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        y(${y_type}): ${y_comment}
        out(Tensor): Output tensor of logical operation.
        name(basestring|None): Name of the output.

    Returns:
        out(${out_type}): ${out_comment}
9736 9737 9738 9739 9740 9741 9742 9743 9744

    Examples:
        .. code-block:: python

            left = fluid.layers.data(
                name='left', shape=[1], dtype='int32')
            right = fluid.layers.data(
                name='right', shape=[1], dtype='int32')
            result = fluid.layers.logical_and(x=left, y=right)
M
minqiyang 已提交
9745 9746 9747 9748 9749 9750 9751
    """

    return _logical_op(
        op_name="logical_and", x=x, y=y, name=name, out=out, binary_op=True)


@templatedoc()
9752
def logical_or(x, y, out=None, name=None):
M
minqiyang 已提交
9753 9754 9755 9756 9757 9758 9759 9760 9761 9762 9763
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        y(${y_type}): ${y_comment}
        out(Tensor): Output tensor of logical operation.
        name(basestring|None): Name of the output.

    Returns:
        out(${out_type}): ${out_comment}
9764 9765 9766 9767 9768 9769 9770 9771 9772

    Examples:
        .. code-block:: python

            left = fluid.layers.data(
                name='left', shape=[1], dtype='int32')
            right = fluid.layers.data(
                name='right', shape=[1], dtype='int32')
            result = fluid.layers.logical_or(x=left, y=right)
M
minqiyang 已提交
9773 9774 9775 9776 9777 9778 9779
    """

    return _logical_op(
        op_name="logical_or", x=x, y=y, name=name, out=out, binary_op=True)


@templatedoc()
9780
def logical_xor(x, y, out=None, name=None):
M
minqiyang 已提交
9781 9782 9783 9784 9785 9786 9787 9788 9789 9790 9791
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        y(${y_type}): ${y_comment}
        out(Tensor): Output tensor of logical operation.
        name(basestring|None): Name of the output.

    Returns:
        out(${out_type}): ${out_comment}
9792 9793 9794 9795 9796 9797 9798 9799 9800

    Examples:
        .. code-block:: python

            left = fluid.layers.data(
                name='left', shape=[1], dtype='int32')
            right = fluid.layers.data(
                name='right', shape=[1], dtype='int32')
            result = fluid.layers.logical_xor(x=left, y=right)
M
minqiyang 已提交
9801 9802 9803 9804 9805 9806 9807
    """

    return _logical_op(
        op_name="logical_xor", x=x, y=y, name=name, out=out, binary_op=True)


@templatedoc()
9808
def logical_not(x, out=None, name=None):
M
minqiyang 已提交
9809 9810 9811 9812 9813 9814 9815 9816 9817 9818
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        out(Tensor): Output tensor of logical operation.
        name(basestring|None): Name of the output.

    Returns:
        out(${out_type}): ${out_comment}
9819 9820 9821 9822 9823 9824 9825

    Examples:
        .. code-block:: python

            left = fluid.layers.data(
                name='left', shape=[1], dtype='int32')
            result = fluid.layers.logical_not(x=left)
M
minqiyang 已提交
9826 9827 9828 9829
    """

    return _logical_op(
        op_name="logical_not", x=x, y=None, name=name, out=out, binary_op=False)
9830 9831 9832 9833 9834 9835 9836 9837 9838 9839 9840 9841 9842 9843 9844


@templatedoc()
def clip(x, min, max, name=None):
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        min(${min_type}): ${min_comment}
        max(${max_type}): ${max_comment}
        name(basestring|None): Name of the output.

    Returns:
        out(${out_type}): ${out_comment}
9845 9846 9847 9848

    Examples:
        .. code-block:: python

S
SunGaofeng 已提交
9849
            import paddle.fluid as fluid
9850 9851 9852
            input = fluid.layers.data(
                name='data', shape=[1], dtype='float32')
            reward = fluid.layers.clip(x=input, min=-1.0, max=1.0)
9853 9854 9855 9856 9857
    """

    helper = LayerHelper("clip", **locals())

    if name is None:
9858 9859
        name = unique_name.generate_with_ignorable_key(".".join(
            [helper.name, 'tmp']))
S
sneaxiy 已提交
9860 9861 9862

    out = helper.create_variable(
        type=x.type, name=name, dtype=x.dtype, persistable=False)
9863 9864 9865 9866 9867 9868 9869 9870 9871 9872 9873 9874 9875 9876 9877 9878 9879 9880 9881 9882 9883 9884 9885

    helper.append_op(
        type="clip",
        inputs={"X": x},
        attrs={"min": min,
               "max": max},
        outputs={"Out": out})

    return out


@templatedoc()
def clip_by_norm(x, max_norm, name=None):
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        max_norm(${max_norm_type}): ${max_norm_comment}
        name(basestring|None): Name of the output.

    Returns:
        out(${out_type}): ${out_comment}
9886 9887 9888 9889 9890 9891 9892

    Examples:
        .. code-block:: python

            input = fluid.layers.data(
                name='data', shape=[1], dtype='float32')
            reward = fluid.layers.clip_by_norm(x=input, max_norm=1.0)
9893 9894 9895 9896 9897
    """

    helper = LayerHelper("clip_by_norm", **locals())

    if name is None:
9898 9899
        name = unique_name.generate_with_ignorable_key(".".join(
            [helper.name, 'tmp']))
S
sneaxiy 已提交
9900 9901 9902

    out = helper.create_variable(
        type=x.type, name=name, dtype=x.dtype, persistable=False)
9903 9904 9905 9906 9907 9908 9909 9910

    helper.append_op(
        type="clip_by_norm",
        inputs={"X": x},
        attrs={"max_norm": max_norm},
        outputs={"Out": out})

    return out
X
Xin Pan 已提交
9911 9912 9913 9914 9915 9916 9917 9918 9919 9920 9921 9922 9923 9924 9925 9926 9927 9928


@templatedoc()
def mean(x, name=None):
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        name(basestring|None): Name of the output.

    Returns:
        out(${out_type}): ${out_comment}
    """

    helper = LayerHelper("mean", **locals())

    if name is None:
X
Xin Pan 已提交
9929
        out = helper.create_variable_for_type_inference(dtype=x.dtype)
X
Xin Pan 已提交
9930 9931 9932 9933 9934 9935 9936 9937 9938 9939
    else:
        out = helper.create_variable(
            name=name, dtype=x.dtype, persistable=False)

    helper.append_op(
        type="mean", inputs={"X": x}, attrs={}, outputs={"Out": out})

    return out


C
chengduo 已提交
9940 9941 9942 9943 9944 9945 9946 9947 9948 9949 9950 9951 9952 9953 9954 9955 9956 9957 9958 9959 9960 9961 9962
@templatedoc()
def merge_selected_rows(x, name=None):
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        name(basestring|None): Name of the output.

    Returns:
        out(${out_type}): ${out_comment}
    """

    helper = LayerHelper("merge_selected_rows", **locals())
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
    helper.append_op(
        type="merge_selected_rows",
        inputs={"X": x},
        attrs={},
        outputs={"Out": out})
    return out


X
Xin Pan 已提交
9963 9964 9965 9966 9967 9968 9969 9970 9971 9972 9973 9974 9975 9976
@templatedoc()
def mul(x, y, x_num_col_dims=1, y_num_col_dims=1, name=None):
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        y(${y_type}): ${y_comment}
        x_num_col_dims(${x_num_col_dims_type}): ${x_num_col_dims_comment}
        y_num_col_dims(${y_num_col_dims_type}): ${y_num_col_dims_comment}
        name(basestring|None): Name of the output.

    Returns:
        out(${out_type}): ${out_comment}
9977 9978 9979 9980 9981 9982 9983 9984 9985 9986 9987 9988

    Examples:
        .. code-block:: python
            
            import paddle.fluid as fluid
            dataX = fluid.layers.data(name="dataX", append_batch_size = False, shape=[2, 5], dtype="float32")
            dataY = fluid.layers.data(name="dataY", append_batch_size = False, shape=[5, 3], dtype="float32")
            output = fluid.layers.mul(dataX, dataY,
                                      x_num_col_dims = 1,
                                      y_num_col_dims = 1)
            

X
Xin Pan 已提交
9989 9990 9991 9992 9993
    """

    helper = LayerHelper("mul", **locals())

    if name is None:
X
Xin Pan 已提交
9994
        out = helper.create_variable_for_type_inference(dtype=x.dtype)
X
Xin Pan 已提交
9995 9996 9997 9998 9999 10000 10001 10002 10003
    else:
        out = helper.create_variable(
            name=name, dtype=x.dtype, persistable=False)

    helper.append_op(
        type="mul",
        inputs={"X": x,
                "Y": y},
        attrs={
X
fix  
Xin Pan 已提交
10004 10005
            "x_num_col_dims": x_num_col_dims,
            "y_num_col_dims": y_num_col_dims
X
Xin Pan 已提交
10006 10007 10008 10009 10010 10011
        },
        outputs={"Out": out})
    return out


@templatedoc()
J
jerrywgz 已提交
10012 10013 10014
def sigmoid_cross_entropy_with_logits(x,
                                      label,
                                      ignore_index=kIgnoreIndex,
10015 10016
                                      name=None,
                                      normalize=False):
X
Xin Pan 已提交
10017 10018 10019 10020 10021 10022
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        label(${label_type}): ${label_comment}
10023
        ignore_index(&{ignore_index}): ${ignore_index_comment}
X
Xin Pan 已提交
10024
        name(basestring|None): Name of the output.
10025 10026
        normalize(bool): If true, divide the output by the number of
            targets != ignore_index.
X
Xin Pan 已提交
10027 10028 10029

    Returns:
        out(${out_type}): ${out_comment}
10030 10031 10032 10033 10034 10035 10036 10037 10038 10039 10040 10041 10042 10043

    Examples:
        .. code-block:: python

            input = fluid.layers.data(
                name='data', shape=[10], dtype='float32')
            label = fluid.layers.data(
                name='data', shape=[10], dtype='float32')
            loss = fluid.layers.sigmoid_cross_entropy_with_logits(
                x=input,
                label=label,
                ignore_index=-1,
                normalize=True) # or False
            # loss = fluid.layers.reduce_sum(loss) # summation of loss
X
Xin Pan 已提交
10044 10045 10046 10047 10048
    """

    helper = LayerHelper("sigmoid_cross_entropy_with_logits", **locals())

    if name is None:
X
Xin Pan 已提交
10049
        out = helper.create_variable_for_type_inference(dtype=x.dtype)
X
Xin Pan 已提交
10050 10051 10052 10053 10054 10055 10056 10057
    else:
        out = helper.create_variable(
            name=name, dtype=x.dtype, persistable=False)

    helper.append_op(
        type="sigmoid_cross_entropy_with_logits",
        inputs={"X": x,
                "Label": label},
10058 10059
        attrs={"ignore_index": ignore_index,
               'normalize': normalize},
X
Xin Pan 已提交
10060 10061 10062 10063 10064 10065 10066 10067 10068 10069 10070 10071 10072 10073 10074 10075
        outputs={"Out": out})
    return out


@templatedoc()
def maxout(x, groups, name=None):
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        groups(${groups_type}): ${groups_comment}
        name(basestring|None): Name of the output.

    Returns:
        out(${out_type}): ${out_comment}
J
jerrywgz 已提交
10076 10077 10078 10079 10080 10081 10082 10083 10084

    Examples:
        .. code-block:: python

            input = fluid.layers.data(
                name='data', 
                shape=[256, 32, 32], 
                dtype='float32')
            out = fluid.layers.maxout(input, groups=2)
X
Xin Pan 已提交
10085 10086 10087 10088
    """
    helper = LayerHelper("maxout", **locals())

    if name is None:
X
Xin Pan 已提交
10089
        out = helper.create_variable_for_type_inference(dtype=x.dtype)
X
Xin Pan 已提交
10090 10091 10092 10093 10094 10095 10096 10097 10098 10099
    else:
        out = helper.create_variable(
            name=name, dtype=x.dtype, persistable=False)

    helper.append_op(
        type="maxout",
        inputs={"X": x},
        attrs={"groups": groups},
        outputs={"Out": out})
    return out
10100 10101


J
JiabinYang 已提交
10102
def space_to_depth(x, blocksize, name=None):
J
JiabinYang 已提交
10103
    """
J
JiabinYang 已提交
10104
    Gives a blocksize to space_to_depth the input LoDtensor with Layout: [batch, channel, height, width]
10105 10106 10107

    This op rearranges blocks of spatial data, into depth. More specifically, this op outputs a copy of the
    input LoDtensor where values from the height and width dimensions are moved to the channel dimension.
J
JiabinYang 已提交
10108
    The attr blocksize indicates the input block size.
10109 10110

    space_to_depth will reorgnize the elements of input with shape[batch, channel, height, width] according
J
JiabinYang 已提交
10111
    to blocksize to construct output with shape [batch, channel * blocksize * blocksize, height/blocksize, width/blocksize]:
10112 10113

    space_to_depth is used to This operation is useful for resizing the activations between convolutions
J
JiabinYang 已提交
10114
    (but keeping all data)
J
JiabinYang 已提交
10115

J
JiabinYang 已提交
10116
    - Non-overlapping blocks of size block_size x block size are rearranged into depth at each location.
10117
    - The depth of the output tensor is block_size * block_size * input channel
J
JiabinYang 已提交
10118 10119 10120 10121 10122
    - The Y, X coordinates within each block of the input become the high order component of the output channel index
    - channel should be divisible by square of blocksize
    - height, width should be divsible by blocksize


J
JiabinYang 已提交
10123
    Args:
J
JiabinYang 已提交
10124
        x(variable): The input LoDtensor.
J
JiabinYang 已提交
10125
        blocksize(variable): The blocksize to select the element on each feature map should be > 2
J
JiabinYang 已提交
10126 10127

    Returns:
J
JiabinYang 已提交
10128
        Variable: The output LoDtensor.
J
JiabinYang 已提交
10129 10130

    Raises:
J
JiabinYang 已提交
10131
        TypeError: blocksize type must be a long.
J
JiabinYang 已提交
10132 10133 10134

    Examples:
        .. code-block:: python
10135 10136 10137
	
            import paddle.fluid as fluid
            import numpy as np
J
JiabinYang 已提交
10138 10139

            data = fluid.layers.data(
10140
                name='data', shape=[1, 4, 2, 2], dtype='float32', append_batch_size=False)
J
JiabinYang 已提交
10141
            space_to_depthed = fluid.layers.space_to_depth(
J
JiabinYang 已提交
10142
                x=data, blocksize=2)
10143 10144 10145 10146 10147 10148

            exe = fluid.Executor(fluid.CUDAPlace(0))
            data_np = np.arange(0,16).reshape((1,4,2,2)).astype('float32')
            out_main = exe.run(fluid.default_main_program(),
                          feed={'data': data_np},
                          fetch_list=[space_to_depthed])
10149

J
JiabinYang 已提交
10150 10151
    """

J
JiabinYang 已提交
10152
    helper = LayerHelper("space_to_depth", **locals())
J
JiabinYang 已提交
10153

J
JiabinYang 已提交
10154 10155
    if not (isinstance(blocksize, int)):
        raise ValueError("blocksize must be a python Int")
J
JiabinYang 已提交
10156 10157

    if name is None:
J
JiabinYang 已提交
10158 10159
        out = helper.create_variable_for_type_inference(
            dtype=x.dtype)  #fix create
J
JiabinYang 已提交
10160 10161 10162 10163 10164
    else:
        out = helper.create_variable(
            name=name, dtype=x.dtype, persistable=False)

    helper.append_op(
J
JiabinYang 已提交
10165
        type="space_to_depth",
J
JiabinYang 已提交
10166
        inputs={"X": x},
J
JiabinYang 已提交
10167
        attrs={"blocksize": blocksize},
J
JiabinYang 已提交
10168
        outputs={"Out": out})
J
JiabinYang 已提交
10169 10170
    return out

J
JiabinYang 已提交
10171

S
sneaxiy 已提交
10172 10173
@templatedoc()
def sequence_reverse(x, name=None):
10174
    """
S
sneaxiy 已提交
10175 10176 10177 10178 10179 10180 10181 10182
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        name(basestring|None): Name of the output.

    Returns:
        out(${y_type}): ${y_comment}
B
bdzhuxiaoning 已提交
10183 10184 10185 10186 10187 10188 10189

    Examples:
        .. code-block:: python

            import paddle.fluid as fluid
            x = fluid.layers.data(name='x', shape=[2, 6], dtype='float32')
            x_reversed = fluid.layers.sequence_reverse(x)
S
sneaxiy 已提交
10190
    """
L
lujun 已提交
10191
    assert not in_dygraph_mode(), (
10192
        "sequence layer is not supported in dygraph mode yet.")
S
sneaxiy 已提交
10193 10194
    helper = LayerHelper("sequence_reverse", **locals())
    if name is None:
S
sneaxiy 已提交
10195
        out = helper.create_variable_for_type_inference(dtype=x.dtype)
S
sneaxiy 已提交
10196 10197 10198 10199 10200 10201 10202 10203 10204 10205
    else:
        out = helper.create_variable(
            name=name, dtype=x.dtype, persistable=False)

    helper.append_op(
        type="sequence_reverse",
        inputs={"X": x},
        outputs={"Y": out},
        attrs=dict())
    return out
S
sneaxiy 已提交
10206 10207


10208 10209 10210 10211 10212 10213
def affine_channel(x,
                   scale=None,
                   bias=None,
                   data_layout='NCHW',
                   name=None,
                   act=None):
10214 10215 10216 10217 10218
    """
    Applies a separate affine transformation to each channel of the input.
    Useful for replacing spatial batch norm with its equivalent fixed
    transformation. The input also can be 2D tensor and applies a affine
    transformation in second dimension.
10219

10220 10221 10222 10223 10224 10225 10226 10227 10228 10229 10230 10231
    Args:
        x (Variable): Feature map input can be a 4D tensor with order NCHW
            or NHWC. It also can be a 2D tensor and the affine transformation
            is applied in the second dimension.
        scale (Variable): 1D input of shape (C), the c-th element is the scale
            factor of the affine transformation for the c-th channel of
            the input.
        bias (Variable): 1D input of shape (C), the c-th element is the bias
            of the affine transformation for the c-th channel of the input.
        data_layout (string, default NCHW): NCHW or NHWC. If input is 2D
            tensor, you can ignore data_layout.
        name (str, default None): The name of this layer.
10232
        act (str, default None): Activation to be applied to the output of this layer.
10233 10234 10235 10236 10237 10238 10239

    Returns:
        out (Variable): A tensor of the same shape and data layout with x.
    """
    helper = LayerHelper("affine_channel", **locals())

    if name is None:
X
Xin Pan 已提交
10240
        out = helper.create_variable_for_type_inference(dtype=x.dtype)
10241 10242 10243 10244 10245 10246 10247 10248 10249 10250 10251
    else:
        out = helper.create_variable(
            name=name, dtype=x.dtype, persistable=False)

    helper.append_op(
        type="affine_channel",
        inputs={"X": x,
                'Scale': scale,
                'Bias': bias},
        attrs={"data_layout": data_layout},
        outputs={"Out": out})
10252
    return helper.append_activation(out)
10253 10254


B
barrierye 已提交
10255
def similarity_focus(input, axis, indexes, name=None):
10256
    """
B
barrierye 已提交
10257
    SimilarityFocus Operator
B
barrierye 已提交
10258 10259

    Generate a similarity focus mask with the same shape of input using the following method:
M
minqiyang 已提交
10260

10261 10262 10263
    1. Extract the 3-D tensor(here the first dimension is BatchSize) corresponding
       to the axis according to the indexes. For example, if axis=1 and indexes=[a],
       it will get the matrix T=X[:, a, :, :]. In this case, if the shape of input X
B
barrierye 已提交
10264
       is (BatchSize, A, B, C), the shape of tensor T is (BatchSize, B, C).
10265 10266 10267 10268 10269 10270 10271
    2. For each index, find the largest numbers in the tensor T, so that the same
       row and same column has at most one number(what it means is that if the
       largest number has been found in the i-th row and the j-th column, then
       the numbers in the i-th row or j-th column will be skipped. And then the
       next largest number will be selected from the remaining numbers. Obviously
       there will be min(B, C) numbers), and mark the corresponding position of the
       3-D similarity focus mask as 1, otherwise as 0. Do elementwise-or for
B
barrierye 已提交
10272
       each index.
B
barrierye 已提交
10273 10274 10275 10276
    3. Broadcast the 3-D similarity focus mask to the same shape of input X.

    Refer to `Similarity Focus Layer <http://www.aclweb.org/anthology/N16-1108>`_

B
barrierye 已提交
10277 10278 10279 10280 10281 10282 10283 10284 10285 10286 10287 10288 10289 10290 10291 10292 10293 10294 10295 10296 10297 10298 10299 10300 10301 10302 10303 10304 10305 10306 10307 10308 10309 10310 10311 10312 10313 10314 10315 10316 10317 10318 10319 10320 10321 10322 10323 10324 10325
    .. code-block:: text

        * Example :

            Given a 4-D tensor x with the shape (BatchSize, C, A, B), where C is
            the number of channels and the shape of feature map is (A, B):
                x.shape = (2, 3, 2, 2)
                x.data = [[[[0.8, 0.1],
                            [0.4, 0.5]],

                           [[0.9, 0.7],
                            [0.9, 0.9]],

                           [[0.8, 0.9],
                            [0.1, 0.2]]],


                          [[[0.2, 0.5],
                            [0.3, 0.4]],

                           [[0.9, 0.7],
                            [0.8, 0.4]],

                           [[0.0, 0.2],
                            [0.4, 0.7]]]]

            Given axis: 1 (the axis of the channel)
            Given indexes: [0]

            then we get a 4-D tensor out with the same shape of input x:
                out.shape = (2, 3, 2, 2)
                out.data = [[[[1.0, 0.0],
                              [0.0, 1.0]],

                             [[1.0, 0.0],
                              [0.0, 1.0]],

                             [[1.0, 0.0],
                              [0.0, 1.0]]],

                            [[[0.0, 1.0],
                              [1.0, 0.0]],

                             [[0.0, 1.0],
                              [1.0, 0.0]],

                             [[0.0, 1.0],
                              [1.0, 0.0]]]]

B
barrierye 已提交
10326
    Args:
10327
        input(Variable): The input tensor variable(default float). It should
B
barrierye 已提交
10328
            be a 4-D tensor with shape [BatchSize, A, B, C].
B
barrierye 已提交
10329
        axis(int): Indicating the dimension to be selected. It can only be
B
barrierye 已提交
10330
            1, 2 or 3.
B
barrierye 已提交
10331
        indexes(list): Indicating the indexes of the selected dimension.
B
barrierye 已提交
10332 10333

    Returns:
H
haowang101779990 已提交
10334 10335
        Variable: A tensor variable with the same shape and same type \
                  as the input.
10336

B
barrierye 已提交
10337 10338
    Examples:
        .. code-block:: python
H
haowang101779990 已提交
10339

B
barrierye 已提交
10340
            data = fluid.layers.data(
Y
Yibing Liu 已提交
10341 10342
                name='data', shape=[-1, 3, 2, 2], dtype='float32')
            fluid.layers.similarity_focus(input=data, axis=1, indexes=[0])
B
barrierye 已提交
10343 10344 10345 10346 10347 10348 10349 10350 10351 10352 10353 10354
    """
    helper = LayerHelper('similarity_focus', **locals())
    # check attrs
    if isinstance(axis, int) is False:
        raise TypeError("axis must be int type.")
    if isinstance(indexes, list) is False:
        raise TypeError("indexes must be list type.")
    if axis != 1 and axis != 2 and axis != 3:
        raise ValueError("axis must be 1, 2 or 3.")
    if len(indexes) == 0:
        raise ValueError("indexes can not be empty.")

B
barrierye 已提交
10355 10356 10357 10358 10359
    if name is None:
        out = helper.create_variable_for_type_inference(dtype=input.dtype)
    else:
        out = helper.create_variable(
            name=name, dtype=input.dtype, persistable=False)
B
barrierye 已提交
10360 10361 10362 10363 10364 10365 10366
    helper.append_op(
        type='similarity_focus',
        inputs={'X': input},
        outputs={'Out': out},
        attrs={"axis": axis,
               "indexes": indexes})
    return out
B
barrierye 已提交
10367 10368


M
minqiyang 已提交
10369 10370
def hash(input, hash_size, num_hash=1, name=None):
    """
M
minqiyang 已提交
10371 10372
    Hash the input to an integer whose value is less than the given hash size.

M
minqiyang 已提交
10373 10374
    The hash algorithm we used was xxHash - Extremely fast hash algorithm
    (https://github.com/Cyan4973/xxHash/tree/v0.6.5)
M
minqiyang 已提交
10375 10376 10377 10378 10379 10380 10381 10382 10383 10384 10385 10386 10387 10388 10389 10390 10391 10392 10393 10394 10395 10396 10397 10398 10399 10400 10401 10402 10403 10404 10405 10406 10407 10408 10409 10410 10411 10412

    A simple example as below:

    .. code-block:: text

        Given:

        # shape [2, 2]
        input.data = [
            [[1], [2]],
            [[3], [4]],
        ]

        input.lod = [[0, 2]]

        hash_size = 10000

        num_hash = 4

        Then:

        Hash op will take all number in input's 2nd dimension as hash algorithm's
        input for each time. Each input will be hashed for 4 times, and get an
        array whose length is 4. Each value in the array ranges from 0 to 9999.

        # shape [2, 4]
        output.data = [
            [[9662], [9217], [1129], [8487]],
            [[8310], [1327], [1654], [4567]],
        ]

        output.lod = [[0, 2]]

    Args:
        input (Variable): The input variable which is a one-hot word. The
            dimensions of the input variable must be 2.
        hash_size (int): The space size for hash algorithm. The output value
            will keep in the range:math:`[0, hash_size - 1]`.
M
minqiyang 已提交
10413
        num_hash (int): The times of hash, default 1.
M
minqiyang 已提交
10414
        name (str, default None): The name of this layer.
M
minqiyang 已提交
10415 10416 10417 10418 10419 10420

    Returns:
       Variable: The hash result variable which is a LoDTensor.

    Examples:
       .. code-block:: python
H
haowang101779990 已提交
10421

10422
           x = fluid.layers.data(name="x", shape=[1], dtype='int32', lod_level=1)
M
minqiyang 已提交
10423
           out = fluid.layers.hash(input=x, num_hash=4, hash_size=1000)
M
minqiyang 已提交
10424 10425
    """
    helper = LayerHelper('hash', **locals())
M
minqiyang 已提交
10426 10427
    out = helper.create_variable_for_type_inference(
        helper.input_dtype(), stop_gradient=True)
M
minqiyang 已提交
10428 10429 10430 10431 10432 10433 10434
    helper.append_op(
        type='hash',
        inputs={'X': input},
        outputs={'Out': out},
        attrs={'num_hash': num_hash,
               'mod_by': hash_size})
    return out
G
gmcather 已提交
10435 10436


D
dengkaipeng 已提交
10437
@templatedoc()
10438 10439
def grid_sampler(x, grid, name=None):
    """
10440
    This operation samples input X by using bilinear interpolation based on
H
haowang101779990 已提交
10441
    flow field grid, which is usually gennerated by :code:`affine_grid` . The grid of
10442 10443 10444 10445
    shape [N, H, W, 2] is the concatenation of (grid_x, grid_y) coordinates
    with shape [N, H, W] each, where grid_x is indexing the 4th dimension
    (in width dimension) of input data x and grid_y is indexng the 3rd
    dimention (in height dimension), finally results is the bilinear
10446
    interpolation value of 4 nearest corner points.
10447

H
haowang101779990 已提交
10448
    .. code-block:: text
10449

H
haowang101779990 已提交
10450 10451
        Step 1:
        Get (x, y) grid coordinates and scale to [0, H-1/W-1].
10452

H
haowang101779990 已提交
10453 10454
        grid_x = 0.5 * (grid[:, :, :, 0] + 1) * (W - 1)
        grid_y = 0.5 * (grid[:, :, :, 1] + 1) * (H - 1)
10455

H
haowang101779990 已提交
10456 10457 10458
        Step 2:
        Indices input data X with grid (x, y) in each [H, W] area, and bilinear
        interpolate point value by 4 nearest points.
10459

H
haowang101779990 已提交
10460 10461 10462 10463 10464 10465 10466 10467 10468
          wn ------- y_n ------- en
          |           |           |
          |          d_n          |
          |           |           |
         x_w --d_w-- grid--d_e-- x_e
          |           |           |
          |          d_s          |
          |           |           |
          ws ------- y_s ------- wn
10469

H
haowang101779990 已提交
10470 10471 10472 10473
        x_w = floor(x)              // west side x coord
        x_e = x_w + 1               // east side x coord
        y_n = floor(y)              // north side y coord
        y_s = y_s + 1               // south side y coord
10474

H
haowang101779990 已提交
10475 10476 10477 10478
        d_w = grid_x - x_w          // distance to west side
        d_e = x_e - grid_x          // distance to east side
        d_n = grid_y - y_n          // distance to north side
        d_s = y_s - grid_y          // distance to south side
10479

H
haowang101779990 已提交
10480 10481 10482 10483
        wn = X[:, :, y_n, x_w]      // north-west point value
        en = X[:, :, y_n, x_e]      // north-east point value
        ws = X[:, :, y_s, x_w]      // south-east point value
        es = X[:, :, y_s, x_w]      // north-east point value
10484

H
haowang101779990 已提交
10485 10486
        output = wn * d_e * d_s + en * d_w * d_s
               + ws * d_e * d_n + es * d_w * d_n
D
dengkaipeng 已提交
10487 10488

    Args:
10489 10490 10491
        x(Variable): Input data of shape [N, C, H, W].
        grid(Variable): Input grid tensor of shape [N, H, W, 2].
        name (str, default None): The name of this layer.
D
dengkaipeng 已提交
10492 10493

    Returns:
H
haowang101779990 已提交
10494
        Variable: Output of shape [N, C, H, W] data samples input X
10495 10496
        using bilnear interpolation based on input grid.

H
haowang101779990 已提交
10497 10498 10499 10500
    Examples:

        .. code-block:: python

K
Kaipeng Deng 已提交
10501 10502 10503 10504 10505
            import paddle.fluid as fluid

            x = fluid.layers.data(name='x', shape=[10, 32, 32], dtype='float32')
            theta = fluid.layers.data(name='theta', shape=[2, 3], dtype='float32')
            grid = fluid.layers.affine_grid(theta=theta, out_shape=[3, 10, 32, 32])
H
haowang101779990 已提交
10506
            out = fluid.layers.grid_sampler(x=x, grid=grid)
10507

D
dengkaipeng 已提交
10508 10509 10510 10511 10512 10513 10514 10515 10516
    """
    helper = LayerHelper("grid_sampler", **locals())

    if not isinstance(x, Variable):
        return ValueError("The x should be a Variable")

    if not isinstance(grid, Variable):
        return ValueError("The grid should be a Variable")

10517
    out = helper.create_variable_for_type_inference(x.dtype)
D
dengkaipeng 已提交
10518 10519
    ipts = {'X': x, 'Grid': grid}

10520
    helper.append_op(type='grid_sampler', inputs=ipts, outputs={'Output': out})
10521 10522 10523
    return out


G
gmcather 已提交
10524 10525 10526 10527 10528 10529 10530 10531 10532 10533 10534 10535 10536 10537 10538 10539 10540 10541 10542 10543 10544 10545 10546 10547 10548 10549 10550
def log_loss(input, label, epsilon=1e-4, name=None):
    """
    **Negative Log Loss Layer**

    This layer accepts input predictions and target label and returns the
    negative log loss.

    .. math::

        Out = -label * \\log{(input + \\epsilon)}
              - (1 - label) * \\log{(1 - input + \\epsilon)}

    Args:
        input (Variable|list):  a 2-D tensor with shape [N x 1], where N is the
                                batch size. This input is a probability computed
                                by the previous operator.
        label (Variable|list):  the ground truth which is a 2-D tensor with
                                shape [N x 1], where N is the batch size.
        epsilon (float): epsilon
        name (string): the name of log_loss

    Returns:
        Variable: A 2-D tensor with shape [N x 1], the negative log loss.

    Examples:
        .. code-block:: python

Y
Yibing Liu 已提交
10551 10552
          label = fluid.layers.data(name='label', shape=[1], dtype='int64')
          prob = fluid.layers.data(name='prob', shape=[10], dtype='float32')
G
gmcather 已提交
10553 10554 10555 10556 10557 10558 10559 10560 10561 10562 10563 10564 10565 10566 10567 10568 10569 10570 10571
          cost = fluid.layers.log_loss(input=prob, label=label)
    """
    helper = LayerHelper('log_loss', **locals())

    if name is None:
        loss = helper.create_variable_for_type_inference(dtype=input.dtype)
    else:
        loss = helper.create_variable(
            name=name, dtype=input.dtype, persistable=False)

    helper.append_op(
        type='log_loss',
        inputs={'Predicted': [input],
                'Labels': [label]},
        outputs={'Loss': [loss]},
        attrs={'epsilon': epsilon})
    return loss


H
heqiaozhi 已提交
10572 10573 10574 10575 10576 10577 10578 10579 10580 10581 10582 10583 10584 10585 10586 10587 10588 10589 10590
def teacher_student_sigmoid_loss(input,
                                 label,
                                 soft_max_up_bound=15.0,
                                 soft_max_lower_bound=-15.0):
    """
    **Teacher Student Log Loss Layer**

    This layer accepts input predictions and target label and returns the
    teacher_student loss.

    .. math::
        loss = max(x, 0) - x * z + log(1 + exp(-abs(x))) + max(x, 0) - x * z' + log(1 + exp(-abs(x)))

    Args:
        input (Variable|list):  a 2-D tensor with shape [N x 1], where N is the
                                batch size. This input is a probability computed
                                by the previous operator.
        label (Variable|list):  the ground truth which is a 2-D tensor with
                                shape [N x 1], where N is the batch size.
M
minqiyang 已提交
10591
        soft_max_up_bound  (float):  if input > soft_max_up_bound, will be bound
H
heqiaozhi 已提交
10592 10593 10594 10595 10596 10597 10598
        soft_max_lower_bound (float): if input < soft_max_lower_bound, will be bound

    Returns:
        Variable: A 2-D tensor with shape [N x 1], the teacher_student_sigmoid_loss.

    Examples:
        .. code-block:: python
10599 10600
          
          import paddle.fluid as fluid
H
heqiaozhi 已提交
10601

10602 10603 10604 10605 10606
          batch_size = 64
          label = fluid.layers.data(
                    name="label", shape=[batch_size, 1], dtype="int64", append_batch_size=False)
          similarity = fluid.layers.data(
                    name="similarity", shape=[batch_size, 1], dtype="float32", append_batch_size=False)
H
heqiaozhi 已提交
10607
          cost = fluid.layers.teacher_student_sigmoid_loss(input=similarity, label=label)
10608

H
heqiaozhi 已提交
10609 10610 10611 10612 10613 10614 10615 10616 10617 10618 10619 10620 10621
    """
    helper = LayerHelper('teacher_student_sigmoid_loss', **locals())
    out = helper.create_variable(dtype=input.dtype)
    helper.append_op(
        type='teacher_student_sigmoid_loss',
        inputs={'X': [input],
                'Label': [label]},
        outputs={'Y': [out]},
        attrs={"soft_max_lower_bound": float(soft_max_lower_bound), \
                "soft_max_up_bound": float(soft_max_up_bound)})
    return out


G
gmcather 已提交
10622 10623 10624 10625
def add_position_encoding(input, alpha, beta, name=None):
    """
    **Add Position Encoding Layer**

H
haowang101779990 已提交
10626
    This layer accepts an input 3D-Tensor of shape [N x M x P], and returns an
G
gmcather 已提交
10627 10628
    output Tensor of shape [N x M x P] with positional encoding value.

H
haowang101779990 已提交
10629
    Refer to `Attention Is All You Need <http://arxiv.org/pdf/1706.03762.pdf>`_ .
G
gmcather 已提交
10630 10631

    .. math::
H
haowang101779990 已提交
10632 10633 10634
        PE(pos, 2i) &= \\sin{(pos / 10000^{2i / P})}   \\\\
        PE(pos, 2i + 1) &= \\cos{(pos / 10000^{2i / P})}  \\\\
        Out(:, pos, i) &= \\alpha * input(:, pos, i) + \\beta * PE(pos, i)
G
gmcather 已提交
10635 10636

    Where:
H
haowang101779990 已提交
10637 10638
      - :math:`PE(pos, 2i)` : the increment for the number at even position
      - :math:`PE(pos, 2i + 1)` : the increment for the number at odd position
G
gmcather 已提交
10639 10640 10641 10642 10643 10644 10645 10646 10647 10648 10649 10650 10651

    Args:
        input (Variable): 3-D input tensor with shape [N x M x P]
        alpha (float): multiple of Input Tensor
        beta (float): multiple of Positional Encoding Tensor
        name (string): the name of position encoding layer

    Returns:
        Variable: A 3-D Tensor of shape [N x M x P] with positional encoding.

    Examples:
        .. code-block:: python

10652 10653 10654 10655 10656 10657 10658 10659 10660
          import paddle.fluid as fluid

          tensor = fluid.layers.data(
              name='tensor',
              shape=[32, 64, 512],
              dtype='float32',
              append_batch_size=False)
          position_tensor = fluid.layers.add_position_encoding(
              input=tensor, alpha=1.0, beta=1.0)
H
haowang101779990 已提交
10661

G
gmcather 已提交
10662 10663 10664 10665 10666 10667 10668 10669 10670 10671 10672 10673 10674 10675 10676 10677
    """
    helper = LayerHelper('add_position_encoding', **locals())
    dtype = helper.input_dtype()

    if name is None:
        out = helper.create_variable_for_type_inference(dtype=dtype)
    else:
        out = helper.create_variable(name=name, dtype=dtype, persistable=False)

    helper.append_op(
        type="add_position_encoding",
        inputs={"X": input},
        outputs={"Out": out},
        attrs={"alpha": alpha,
               "beta": beta})
    return out
Q
Qiao Longfei 已提交
10678 10679 10680 10681 10682 10683 10684 10685 10686 10687


def bilinear_tensor_product(x,
                            y,
                            size,
                            act=None,
                            name=None,
                            param_attr=None,
                            bias_attr=None):
    """
Q
Qiao Longfei 已提交
10688
    **Add Bilinear Tensor Product Layer**
Q
Qiao Longfei 已提交
10689

Q
Qiao Longfei 已提交
10690
    This layer performs bilinear tensor product on two inputs.
Q
Qiao Longfei 已提交
10691 10692 10693
    For example:

    .. math::
H
haowang101779990 已提交
10694
       out_{i} = x * W_{i} * {y^\mathrm{T}}, i=0,1,...,size-1
Q
Qiao Longfei 已提交
10695

Q
Qiao Longfei 已提交
10696
    In this formula:
10697 10698
      - :math:`x`: the first input contains M elements, shape is [batch_size, M].
      - :math:`y`: the second input contains N elements, shape is [batch_size, N].
Q
Qiao Longfei 已提交
10699
      - :math:`W_{i}`: the i-th learned weight, shape is [M, N]
H
haowang101779990 已提交
10700
      - :math:`out_{i}`: the i-th element of out, shape is [batch_size, size].
Q
Qiao Longfei 已提交
10701 10702 10703
      - :math:`y^\mathrm{T}`: the transpose of :math:`y_{2}`.

    Args:
10704 10705
        x (Variable): 2-D input tensor with shape [batch_size, M]
        y (Variable): 2-D input tensor with shape [batch_size, N]
Q
Qiao Longfei 已提交
10706 10707 10708
        size (int): The dimension of this layer.
        act (str, default None): Activation to be applied to the output of this layer.
        name (str, default None): The name of this layer.
Q
Qiao Longfei 已提交
10709
        param_attr (ParamAttr, default None): The parameter attribute for the learnable w.
Q
Qiao Longfei 已提交
10710
            parameters/weights of this layer.
Q
Qiao Longfei 已提交
10711
        bias_attr (ParamAttr, default None): The parameter attribute for the bias
Q
Qiao Longfei 已提交
10712 10713 10714 10715
            of this layer. If it is set to False, no bias will be added to the output units.
            If it is set to None, the bias is initialized zero. Default: None.

    Returns:
Q
Qiao Longfei 已提交
10716
        Variable: A 2-D Tensor of shape [batch_size, size].
Q
Qiao Longfei 已提交
10717 10718 10719 10720

    Examples:
        .. code-block:: python

Y
Yibing Liu 已提交
10721 10722 10723
          layer1 = fluid.layers.data("t1", shape=[-1, 5], dtype="float32")
          layer2 = fluid.layers.data("t2", shape=[-1, 4], dtype="float32")
          tensor = fluid.layers.bilinear_tensor_product(x=layer1, y=layer2, size=1000)
Q
Qiao Longfei 已提交
10724 10725
    """
    helper = LayerHelper('bilinear_tensor_product', **locals())
Q
Qiao Longfei 已提交
10726
    dtype = helper.input_dtype('x')
Q
Qiao Longfei 已提交
10727 10728 10729 10730

    param_shape = [size, x.shape[1], y.shape[1]]

    w = helper.create_parameter(
Q
Qiao Longfei 已提交
10731
        attr=helper.param_attr, shape=param_shape, dtype=dtype, is_bias=False)
Q
Qiao Longfei 已提交
10732 10733 10734 10735 10736 10737 10738 10739 10740 10741 10742 10743 10744 10745 10746 10747 10748

    if name is None:
        out = helper.create_variable_for_type_inference(dtype=dtype)
    else:
        out = helper.create_variable(name=name, dtype=dtype, persistable=False)

    inputs = {"X": x, "Y": y, "Weight": w}
    if helper.bias_attr:
        bias_size = [1, size]
        bias = helper.create_parameter(
            attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True)
        inputs["Bias"] = bias
    helper.append_op(
        type="bilinear_tensor_product", inputs=inputs, outputs={"Out": out})

    # add activation
    return helper.append_activation(out)
C
chengduo 已提交
10749 10750 10751 10752 10753 10754 10755 10756 10757 10758 10759 10760 10761


@templatedoc()
def get_tensor_from_selected_rows(x, name=None):
    """
    ${comment}

    Args:
        x(${x_type}): ${x_comment}
        name(basestring|None): Name of the output.

    Returns:
        out(${out_type}): ${out_comment}
B
bdzhuxiaoning 已提交
10762 10763 10764 10765 10766 10767 10768 10769

    Examples:
        .. code-block:: python
	    
            import paddle.fluid as fluid
            b = fluid.default_main_program().global_block()
            input = b.create_var(name="X", dtype="float32", persistable=True, type=fluid.core.VarDesc.VarType.SELECTED_ROWS)
            out = fluid.layers.get_tensor_from_selected_rows(input)
C
chengduo 已提交
10770 10771 10772 10773 10774 10775 10776 10777 10778 10779
    """

    helper = LayerHelper('get_tensor_from_selected_rows', **locals())
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
    helper.append_op(
        type='get_tensor_from_selected_rows',
        inputs={'X': x},
        outputs={'Out': out},
        attrs={})
    return out
10780 10781


S
shippingwang 已提交
10782
def shuffle_channel(x, group, name=None):
S
shippingwang 已提交
10783 10784
    """
    **Shuffle Channel Operator**
10785

S
shippingwang 已提交
10786 10787 10788 10789 10790 10791
    This operator shuffles the channels of input x.
    It divide the input channels in each group into :attr:`group` subgroups,
    and obtain a new order by selecting element from every subgroup one by one.

    Please refer to the paper
    https://arxiv.org/pdf/1707.01083.pdf
S
shippingwang 已提交
10792
    
S
shippingwang 已提交
10793
    .. code-block:: text
10794

S
shippingwang 已提交
10795 10796 10797 10798 10799 10800 10801 10802 10803 10804 10805 10806 10807 10808 10809 10810 10811 10812 10813 10814 10815 10816 10817 10818 10819 10820 10821 10822
        Given a 4-D tensor input with the shape (N, C, H, W):
            input.shape = (1, 4, 2, 2)
            input.data =[[[[0.1, 0.2],
                           [0.2, 0.3]],

                          [[0.3, 0.4],
                           [0.4, 0.5]],

                          [[0.5, 0.6],
                           [0.6, 0.7]],

                          [[0.7, 0.8],
                           [0.8, 0.9]]]]
            Given group: 2
            then we get a 4-D tensor out whth the same shape of input:
            out.shape = (1, 4, 2, 2)
            out.data = [[[[0.1, 0.2],
                          [0.2, 0.3]],
                          
                         [[0.5, 0.6],
                          [0.6, 0.7]],
                          
                         [[0.3, 0.4],
                          [0.4, 0.5]],
                          
                         [[0.7, 0.8],
                          [0.8, 0.9]]]]
                        
S
shippingwang 已提交
10823
    Args: 
S
shippingwang 已提交
10824 10825
        x(Variable): The input tensor variable. It should be a 4-D tensor with shape [N, C, H, W]
        group(int): Indicating the conuts of subgroups, It should divide the number of channels.
S
shippingwang 已提交
10826 10827

    Returns:
S
shippingwang 已提交
10828 10829
        out(Variable): the channels shuffling result is a tensor variable with the 
        same shape and same type as the input.
S
shippingwang 已提交
10830 10831

    Raises:
S
shippingwang 已提交
10832
        ValueError: If group is not an int type variable.
S
shippingwang 已提交
10833 10834 10835

    Examples:
        .. code-block:: python
10836 10837

            input = fluid.layers.data(name='input', shape=[4,2,2], dtype='float32')
S
shippingwang 已提交
10838
            out = fluid.layers.shuffle_channel(x=input, group=2)
S
shippingwang 已提交
10839 10840 10841
    """
    helper = LayerHelper("shuffle_channel", **locals())

S
shippingwang 已提交
10842
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
S
shippingwang 已提交
10843 10844 10845 10846 10847 10848 10849 10850 10851

    if not isinstance(group, int):
        raise TypeError("group must be int type")

    helper.append_op(
        type="shuffle_channel",
        inputs={"X": x},
        outputs={"Out": out},
        attrs={"group": group})
S
shippingwang 已提交
10852
    return out
S
Add  
shippingwang 已提交
10853 10854


10855
@templatedoc()
D
dengkaipeng 已提交
10856
def temporal_shift(x, seg_num, shift_ratio=0.25, name=None):
10857 10858 10859 10860 10861 10862 10863 10864
    """
    **Temporal Shift Operator**
    
    ${comment}
                        
    Args: 
        x(Variable): ${x_comment}
        seg_num(int): ${seg_num_comment}
D
dengkaipeng 已提交
10865
        shift_ratio(float): ${shift_ratio_comment}
D
dengkaipeng 已提交
10866
        name (str, default None): The name of this layer.
10867 10868 10869 10870 10871 10872 10873 10874 10875 10876 10877 10878

    Returns:
        out(Variable): The temporal shifting result is a tensor variable with the 
        same shape and same type as the input.

    Raises:
        TypeError: seg_num must be int type.

    Examples:
        .. code-block:: python

            input = fluid.layers.data(name='input', shape=[4,2,2], dtype='float32')
D
dengkaipeng 已提交
10879
            out = fluid.layers.temporal_shift(x=input, seg_num=2, shift_ratio=0.2)
10880 10881 10882 10883 10884 10885 10886 10887 10888 10889 10890 10891
    """
    helper = LayerHelper("temporal_shift", **locals())

    out = helper.create_variable_for_type_inference(dtype=x.dtype)

    if not isinstance(seg_num, int):
        raise TypeError("seg_num must be int type.")

    helper.append_op(
        type="temporal_shift",
        inputs={"X": x},
        outputs={"Out": out},
D
dengkaipeng 已提交
10892 10893
        attrs={"seg_num": seg_num,
               "shift_ratio": shift_ratio})
10894 10895 10896
    return out


S
sneaxiy 已提交
10897
class PyFuncRegistry(object):
S
sneaxiy 已提交
10898 10899 10900
    _register_funcs = []

    def __init__(self, func):
S
sneaxiy 已提交
10901
        if func is None or not callable(func):
S
sneaxiy 已提交
10902 10903 10904
            raise TypeError('func must be a Python function')

        self._func = func
M
minqiyang 已提交
10905
        # find named args using reflection
S
sneaxiy 已提交
10906 10907 10908 10909 10910 10911 10912
        args = inspect.getargspec(self._func)
        if len(args[0]) == 0 and args[1] is None and args[2] is None:
            # Function with no inputs
            self._named_args = None
        else:
            self._named_args = args[0]
        self._id = core._append_python_callable_object_and_return_id(self)
S
sneaxiy 已提交
10913 10914 10915
        '''
        Why record self here?

M
minqiyang 已提交
10916 10917
        1. For debug usage. Users can call
           :code:`py_func.registered_func(idx)` method
S
sneaxiy 已提交
10918
           to find the registered function corresponding
M
minqiyang 已提交
10919
           to :code:`idx`.
S
sneaxiy 已提交
10920

M
minqiyang 已提交
10921 10922
        2. For increasing reference count of self.
           It seems that to release Python object
S
sneaxiy 已提交
10923
           whose reference count is 1 would cause
M
minqiyang 已提交
10924
           segmentation fault error in C++ side.
S
sneaxiy 已提交
10925 10926
           May be lack of Python GC in C++ side?
        '''
S
sneaxiy 已提交
10927
        PyFuncRegistry._register_funcs.append(self)
S
sneaxiy 已提交
10928 10929 10930 10931 10932 10933 10934 10935 10936 10937 10938 10939 10940 10941

    @classmethod
    def registered_func(cls, idx):
        return cls._register_funcs[idx]._func

    @classmethod
    def registered_func_num(cls):
        return len(cls._register_funcs)

    @property
    def id(self):
        return self._id

    def __call__(self, *args):
S
sneaxiy 已提交
10942 10943 10944 10945 10946 10947 10948 10949 10950
        if self._named_args is None:
            func_ret = self._func()
        else:
            kwargs = dict()
            idx = 0
            for arg in self._named_args:
                kwargs[arg] = args[idx]
                idx += 1
            func_ret = self._func(*args[idx:], **kwargs)
S
sneaxiy 已提交
10951

S
sneaxiy 已提交
10952 10953
        if not isinstance(func_ret, (list, tuple)):
            func_ret = (func_ret, )
S
sneaxiy 已提交
10954 10955

        ret = []
S
sneaxiy 已提交
10956 10957 10958
        for each_ret in func_ret:
            if each_ret is None or isinstance(each_ret, core.LoDTensor):
                ret.append(each_ret)
S
sneaxiy 已提交
10959 10960
                continue

S
sneaxiy 已提交
10961 10962
            if not isinstance(each_ret, np.ndarray):
                each_ret = np.array(each_ret)
S
sneaxiy 已提交
10963

S
sneaxiy 已提交
10964 10965 10966
            tensor = core.LoDTensor()
            tensor.set(each_ret, core.CPUPlace())
            ret.append(tensor)
S
sneaxiy 已提交
10967

S
sneaxiy 已提交
10968
        return tuple(ret)
S
sneaxiy 已提交
10969 10970


S
sneaxiy 已提交
10971 10972 10973 10974
@templatedoc()
def py_func(func, x, out, backward_func=None, skip_vars_in_backward_input=None):
    """
    PyFunc Operator.
M
minqiyang 已提交
10975

S
sneaxiy 已提交
10976 10977 10978 10979 10980 10981 10982 10983
    User can use :code:`py_func` to register operators in Python side.
    The inputs of :code:`func` is :code:`LoDTensor` and outputs can be
    numpy array or :code:`LoDTensor`. Paddle would call the registered
    :code:`func` in forward part, and call :code:`backward_func` in
    backward part (if :code:`backward_func` is not None).

    User should set the right data type and shape of :code:`out` before
    calling this function. However, data types and shapes of gradients of
S
sneaxiy 已提交
10984
    :code:`out` and :code:`x` would be inferred automatically.
S
sneaxiy 已提交
10985

S
sneaxiy 已提交
10986 10987
    Input orders of :code:`backward_func` would be: forward inputs
    :code:`x`, forward outputs :code:`out` and backward input gradients of
S
sneaxiy 已提交
10988 10989 10990 10991
    :code:`out`. If some variables of :code:`out` have no gradient, the input
    tensor would be None in Python side. If some variables of :code:`in` have
    no gradient, users should return None.

S
sneaxiy 已提交
10992
    This function can also be used to debug the running network. User can
M
minqiyang 已提交
10993
    add a :code:`py_func` operator without output, and print input
S
sneaxiy 已提交
10994 10995
    :code:`x` inside :code:`func`.

S
sneaxiy 已提交
10996 10997 10998 10999 11000
    Args:
        func (callable): forward Python function.
        x (Variable|list(Variable)|tuple(Variable)): inputs of :code:`func`.
        out (Variable|list(Variable)|tuple(Variable)): outputs of :code:`func`.
            Paddle cannot infer shapes and data types of :code:`out`. Users
M
minqiyang 已提交
11001
            should create :code:`out` beforehand.
S
sneaxiy 已提交
11002
        backward_func (callable|None): backward Python function.
M
minqiyang 已提交
11003
                                       None means no backward. Default None.
S
sneaxiy 已提交
11004
        skip_vars_in_backward_input (Variable|list(Variable)|tuple(Variable)):
M
minqiyang 已提交
11005
            Variables that are not needed in :code:`backward_func` inputs.
S
sneaxiy 已提交
11006 11007
            These variables must be any of :code:`x` and :code:`out`.
            If set, these vars would not be inputs of :code:`backward_func`,
M
minqiyang 已提交
11008
            Only useful when :code:`backward_func` is not None. Default None.
S
sneaxiy 已提交
11009 11010 11011

    Returns:
        out (Variable|list(Variable)|tuple(Variable)): input :code:`out`
S
sneaxiy 已提交
11012 11013

    Examples:
M
minqiyang 已提交
11014

S
sneaxiy 已提交
11015 11016 11017 11018 11019
        >>> import paddle.fluid as fluid
        >>> import six
        >>>
        >>> def create_tmp_var(name, dtype, shape):
        >>>     return fluid.default_main_program().current_block().create_var(
M
minqiyang 已提交
11020
        >>>         name=name, dtype=dtype, shape=shape)
S
sneaxiy 已提交
11021 11022
        >>>
        >>> # tanh activation has been provided by Paddle C++ op
M
minqiyang 已提交
11023
        >>> # Here, we only use tanh to be an example to show the usage
S
sneaxiy 已提交
11024 11025 11026
        >>> # of py_func
        >>> def tanh(x):
        >>>     return np.tanh(x)
M
minqiyang 已提交
11027
        >>>
S
sneaxiy 已提交
11028 11029 11030 11031 11032
        >>> # forward input x is skipped
        >>> def tanh_grad(y, dy):
        >>>     return np.array(dy) * (1 - np.square(np.array(y)))
        >>>
        >>> def debug_func(x):
M
minqiyang 已提交
11033
        >>>     print(x)
S
sneaxiy 已提交
11034 11035 11036 11037 11038 11039
        >>>
        >>> def simple_net(img, label):
        >>>     hidden = img
        >>>     for idx in six.moves.range(4):
        >>>         hidden = fluid.layers.fc(hidden, size=200)
        >>>         new_hidden = create_tmp_var(name='hidden_{}'.format(idx),
M
minqiyang 已提交
11040
        >>>             dtype=hidden.dtype, shape=hidden.shape)
S
sneaxiy 已提交
11041 11042
        >>>
        >>>         # user-defined layers with forward and backward
M
minqiyang 已提交
11043 11044
        >>>         hidden = fluid.layers.py_func(func=tanh, x=hidden,
        >>>             out=new_hidden, backward_func=tanh_grad,
S
sneaxiy 已提交
11045 11046 11047 11048 11049 11050 11051 11052
        >>>             skip_vars_in_backward_input=hidden)
        >>>
        >>>         # user-defined debug layers to print variables
        >>>         fluid.layers.py_func(func=debug_func, x=hidden, out=None)
        >>>
        >>>     prediction = fluid.layers.fc(hidden, size=10, act='softmax')
        >>>     loss = fluid.layers.cross_entropy(input=prediction, label=label)
        >>>     return fluid.layers.mean(loss)
S
sneaxiy 已提交
11053
    """
S
sneaxiy 已提交
11054
    helper = LayerHelper('py_func', **locals())
S
sneaxiy 已提交
11055 11056 11057
    if x is None:
        x = []
    elif isinstance(x, Variable):
S
sneaxiy 已提交
11058
        x = [x]
S
sneaxiy 已提交
11059 11060
    elif not isinstance(x, (list, tuple)):
        raise TypeError('Input must be Variable/list(Variable)/tuple(Variable)')
S
sneaxiy 已提交
11061

S
sneaxiy 已提交
11062 11063 11064
    if out is None:
        out_list = []
    elif isinstance(out, Variable):
S
sneaxiy 已提交
11065
        out_list = [out]
S
sneaxiy 已提交
11066
    elif isinstance(out, (list, tuple)):
S
sneaxiy 已提交
11067
        out_list = out
S
sneaxiy 已提交
11068 11069 11070
    else:
        raise TypeError(
            'Output must be Variable/list(Variable)/tuple(Variable)')
S
sneaxiy 已提交
11071

S
sneaxiy 已提交
11072 11073
    fwd_func_id = PyFuncRegistry(func).id
    bwd_func_id = PyFuncRegistry(
S
sneaxiy 已提交
11074
        backward_func).id if backward_func is not None else -1
S
sneaxiy 已提交
11075 11076

    for each_out in out_list:
S
sneaxiy 已提交
11077 11078
        if len(each_out.shape) == 0:
            raise ValueError(
S
sneaxiy 已提交
11079 11080
                'Output shapes of py_func op should be provided by users manually'
            )
S
sneaxiy 已提交
11081

S
sneaxiy 已提交
11082 11083 11084 11085 11086 11087 11088 11089 11090 11091 11092 11093 11094 11095 11096
    backward_skip_vars = set()
    if backward_func is not None and skip_vars_in_backward_input is not None:
        if isinstance(skip_vars_in_backward_input, Variable):
            skip_vars_in_backward_input = [skip_vars_in_backward_input]

        fwd_in_out = [v.name for v in x]
        fwd_in_out.extend([v.name for v in out_list])
        fwd_in_out = set(fwd_in_out)
        backward_skip_vars = set()
        for v in skip_vars_in_backward_input:
            if not v.name in fwd_in_out:
                raise ValueError(
                    'Variable {} is not found in forward inputs and outputs'
                    .format(v.name))
            backward_skip_vars.add(v.name)
S
sneaxiy 已提交
11097 11098 11099 11100

    helper.append_op(
        type='py_func',
        inputs={'X': x},
S
sneaxiy 已提交
11101 11102
        outputs={'Out': out_list},
        attrs={
S
sneaxiy 已提交
11103 11104 11105
            'forward_callable_id': fwd_func_id,
            'backward_callable_id': bwd_func_id,
            'backward_skip_vars': list(backward_skip_vars)
S
sneaxiy 已提交
11106
        })
S
sneaxiy 已提交
11107
    return out
S
sneaxiy 已提交
11108 11109 11110


# For debug usage
S
sneaxiy 已提交
11111 11112 11113 11114
py_func.registered_func = PyFuncRegistry.registered_func
py_func.registered_func_num = PyFuncRegistry.registered_func_num


11115 11116 11117 11118 11119 11120 11121 11122 11123 11124 11125 11126 11127
@templatedoc()
def psroi_pool(input,
               rois,
               output_channels,
               spatial_scale,
               pooled_height,
               pooled_width,
               name=None):
    """
    ${comment}

    Args:
        input (Variable): ${x_comment}
S
SunGaofeng 已提交
11128 11129 11130 11131 11132
        rois (Variable): ROIs (Regions of Interest) to pool over.It should be
                         a 2-D LoDTensor of shape (num_rois, 4), the lod level
                         is 1. Given as [[x1, y1, x2, y2], ...], (x1, y1) is
                         the top left coordinates, and (x2, y2) is the bottom
                         right coordinates.
11133 11134 11135 11136 11137 11138 11139 11140 11141 11142 11143 11144
        output_channels (integer): ${output_channels_comment}
        spatial_scale (float): ${spatial_scale_comment} Default: 1.0
        pooled_height (integer): ${pooled_height_comment} Default: 1
        pooled_width (integer): ${pooled_width_comment} Default: 1
        name (str, default None): The name of this layer.

    Returns:
        Variable: ${out_comment}.

    Examples:
        .. code-block:: python

S
SunGaofeng 已提交
11145 11146 11147 11148
            import paddle.fluid as fluid
            x = fluid.layers.data(name='x', shape=[490, 28, 28], dtype='float32')
            rois = fluid.layers.data(name='rois', shape=[4], lod_level=1, dtype='float32')
            pool_out = fluid.layers.psroi_pool(x, rois, 10, 1.0, 7, 7)
11149 11150 11151 11152 11153 11154 11155 11156 11157 11158 11159 11160 11161 11162 11163 11164 11165 11166 11167 11168 11169 11170 11171 11172 11173
    """
    helper = LayerHelper('psroi_pool', **locals())
    # check attrs
    if not isinstance(output_channels, int):
        raise TypeError("output_channels must be int type")
    if not isinstance(spatial_scale, float):
        raise TypeError("spatial_scale must be float type")
    if not isinstance(pooled_height, int):
        raise TypeError("pooled_height must be int type")
    if not isinstance(pooled_width, int):
        raise TypeError("pooled_width must be int type")
    dtype = helper.input_dtype()
    out = helper.create_variable_for_type_inference(dtype)
    helper.append_op(
        type='psroi_pool',
        inputs={'X': input,
                'ROIs': rois},
        outputs={'Out': out},
        attrs={
            'output_channels': output_channels,
            'spatial_scale': spatial_scale,
            'pooled_height': pooled_height,
            'pooled_width': pooled_width
        })
    return out
11174

M
minqiyang 已提交
11175

M
minqiyang 已提交
11176
def huber_loss(input, label, delta):
11177
    """
M
minqiyang 已提交
11178 11179 11180
    Huber loss is a loss function used in robust.
    Huber loss can evaluate the fitness of input to label.
    Different from MSE loss, Huber loss is more robust for outliers.
11181 11182 11183 11184

    When the difference between input and label is large than delta
    .. math::

M
minqiyang 已提交
11185
        huber\_loss = delta * (label - input) - 0.5 * delta * delta
11186 11187 11188 11189

    When the difference between input and label is less than delta
    .. math::

M
minqiyang 已提交
11190
        huber\_loss = 0.5 * (label - input) * (label - input)
11191 11192 11193 11194 11195 11196 11197


    Args:
        input (Variable): This input is a probability computed by the previous operator.
                          The first dimension is batch size, and the last dimension is 1.
        label (Variable): The groud truth whose first dimension is batch size
                          and last dimension is 1.
M
minqiyang 已提交
11198
        delta (float): The parameter of huber loss, which controls
11199 11200 11201
                       the range of outliers

    Returns:
M
minqiyang 已提交
11202
        huber\_loss (Variable): The huber loss with shape [batch_size, 1].
11203 11204 11205 11206

    Examples:
        .. code-block:: python

11207 11208 11209 11210 11211 11212 11213 11214 11215
            import paddle.fluid as fluid

            x = fluid.layers.data(name='x', shape=[13], dtype='float32')
            predict = fluid.layers.fc(input=x, size=1)
            label = fluid.layers.data(
                name='label', shape=[1], dtype='float32')
            loss = fluid.layers.huber_loss(
                input=predict, label=label, delta=1.0)

11216
    """
M
minqiyang 已提交
11217
    helper = LayerHelper('huber_loss', **locals())
11218 11219 11220 11221 11222 11223 11224 11225 11226 11227 11228
    residual = helper.create_variable_for_type_inference(
        dtype=helper.input_dtype())
    out = helper.create_variable_for_type_inference(dtype=helper.input_dtype())
    helper.append_op(
        type='huber_loss',
        inputs={'X': input,
                'Y': label},
        outputs={'Out': out,
                 'Residual': residual},
        attrs={'delta': delta})
    return out
Z
zhaozhehao 已提交
11229 11230


D
dengkaipeng 已提交
11231 11232 11233 11234 11235 11236 11237 11238 11239 11240 11241 11242 11243 11244 11245 11246 11247 11248 11249 11250 11251 11252 11253 11254 11255 11256 11257 11258 11259 11260 11261 11262
@templatedoc()
def kldiv_loss(x, target, reduction='mean', name=None):
    """
    ${comment}

    Args:
        x (Variable): ${x_comment}
        target (Variable): ${target_comment}
        reduction (Variable): ${reduction_comment}
        name (str, default None): The name of this layer.

    Returns:
        kldiv\_loss (Variable): The KL divergence loss.

    Examples:
        .. code-block:: python

            x = fluid.layers.data(name='x', shape=[4,2,2], dtype='float32')
            target = fluid.layers.data(name='target', shape=[4,2,2], dtype='float32')
            loss = fluid.layers.kldiv_loss(x=x, target=target, reduction='batchmean')
    """
    helper = LayerHelper('kldiv_loss', **locals())
    loss = helper.create_variable_for_type_inference(dtype=x.dtype)
    helper.append_op(
        type='kldiv_loss',
        inputs={'X': x,
                'Target': target},
        outputs={'Loss': loss},
        attrs={'reduction': reduction})
    return loss


Z
zhaozhehao 已提交
11263 11264 11265 11266 11267 11268 11269 11270 11271 11272 11273 11274 11275 11276 11277 11278 11279 11280 11281 11282 11283 11284 11285 11286 11287 11288 11289 11290 11291 11292
@templatedoc()
def tree_conv(nodes_vector,
              edge_set,
              output_size,
              num_filters=1,
              max_depth=2,
              act='tanh',
              param_attr=None,
              bias_attr=None,
              name=None):
    """ 
    ${comment}
    		
    Args:
        nodes_vector(${nodes_vector_type}): ${nodes_vector_comment}
        edge_set(${edge_set_type}): ${edge_set_comment}
        output_size(int): output feature width
        num_filters(int): number of filters, Default 1
        max_depth(int): max depth of filters, Default 2
        act(str): activation function, Default tanh
        param_attr(ParamAttr): the parameter attribute for the filters, Default None
        bias_attr(ParamAttr): the parameter attribute for the bias of this layer, Default None
        name(str): a name of this layer(optional). If set None, the layer will be named automatically, Default None

    Returns:
        out(${out_type}): ${out_comment}

    Examples:
        .. code-block:: python

T
Tao Luo 已提交
11293 11294 11295
          # 10 for max_node_size of dataset, 5 for vector width
          nodes_vector = fluid.layers.data(name='vectors', shape=[10, 5], dtype='float32')
          # 10 for max_node_size of dataset, 2 for every edge has two nodes
Z
zhaozhehao 已提交
11296
          # edges must be directional
T
Tao Luo 已提交
11297 11298 11299 11300
          edge_set = fluid.layers.data(name='edge_set', shape=[10, 2], dtype='float32')
          # the shape of output will be [10, 6, 1],
          # 10 for max_node_size of dataset, 6 for output size, 1 for 1 filter
          out_vector = fluid.layers.tree_conv(nodes_vector, edge_set, 6, 1, 2)
Z
zhaozhehao 已提交
11301
          # After reshape, output tensor could be nodes_vector for next tree convolution
T
Tao Luo 已提交
11302 11303
          out_vector = fluid.layers.reshape(out_vector, shape=[-1, 10, 6])
          out_vector_2 = fluid.layers.tree_conv(out_vector, edge_set, 3, 4, 2)
Z
zhaozhehao 已提交
11304
          # also output tensor could be pooling(the pooling in paper called global pooling)
T
Tao Luo 已提交
11305
          pooled = fluid.layers.reduce_max(out_vector, dim=2) # global pooling
Z
zhaozhehao 已提交
11306 11307 11308 11309 11310 11311 11312 11313 11314 11315 11316 11317 11318 11319 11320 11321 11322 11323 11324 11325 11326 11327 11328
    """
    helper = LayerHelper("tree_conv", **locals())
    dtype = helper.input_dtype('nodes_vector')
    feature_size = nodes_vector.shape[2]
    W_shape = [feature_size, 3, output_size, num_filters]
    W = helper.create_parameter(
        attr=param_attr, shape=W_shape, dtype=dtype, is_bias=False)
    if name == None:
        out = helper.create_variable_for_type_inference(dtype=dtype)
    else:
        out = helper.create_variable(name=name, dtype=dtype, persistable=False)
    helper.append_op(
        type='tree_conv',
        inputs={'NodesVector': nodes_vector,
                'EdgeSet': edge_set,
                'Filter': W},
        outputs={'Out': out, },
        attrs={'max_depth': max_depth})
    if helper.bias_attr:
        pre_activation = helper.append_bias_op(out)
    else:
        pre_activation = out
    return helper.append_activation(pre_activation)
C
ceci3 已提交
11329 11330


C
ceci3 已提交
11331
from .ops import square
C
ceci3 已提交
11332
from .control_flow import equal
C
ceci3 已提交
11333 11334


C
ceci3 已提交
11335 11336 11337
def npair_loss(anchor, positive, labels, l2_reg=0.002):
    '''
  **Npair Loss Layer**
C
ceci3 已提交
11338

C
ceci3 已提交
11339
  Read `Improved Deep Metric Learning with Multi class N pair Loss Objective <http://www.nec-labs.com/uploads/images/Department-Images/MediaAnalytics/papers/nips16_npairmetriclearning.pdf>`_ .
C
ceci3 已提交
11340 11341

  Npair loss requires paired data. Npair loss has two parts: the first part is L2
C
ceci3 已提交
11342
  regularizer on the embedding vector; the second part is cross entropy loss which
C
ceci3 已提交
11343 11344 11345 11346 11347
  takes the similarity matrix of anchor and positive as logits.

  Args:
    anchor(Variable): embedding vector for the anchor image. shape=[batch_size, embedding_dims]
    positive(Variable): embedding vector for the positive image. shape=[batch_size, embedding_dims]
C
ceci3 已提交
11348 11349
    labels(Variable): 1-D tensor. shape=[batch_size]
    l2_reg(float32): L2 regularization term on embedding vector, default: 0.002
C
ceci3 已提交
11350 11351 11352 11353 11354 11355 11356

  Returns:
    npair loss(Variable): return npair loss, shape=[1]

  Examples:
    .. code-block:: python

C
ceci3 已提交
11357 11358 11359 11360 11361 11362 11363 11364
       anchor = fluid.layers.data(
                     name = 'anchor', shape = [18, 6], dtype = 'float32', append_batch_size=False)
       positive = fluid.layers.data(
                     name = 'positive', shape = [18, 6], dtype = 'float32', append_batch_size=False)
       labels = fluid.layers.data(
                     name = 'labels', shape = [18], dtype = 'float32', append_batch_size=False)

       npair_loss = fluid.layers.npair_loss(anchor, positive, labels, l2_reg = 0.002)
C
ceci3 已提交
11365 11366 11367 11368 11369 11370 11371
  '''
    Beta = 0.25
    batch_size = labels.shape[0]

    labels = reshape(labels, shape=[batch_size, 1], inplace=True)
    labels = expand(labels, expand_times=[1, batch_size])

C
ceci3 已提交
11372
    labels = equal(labels, transpose(labels, perm=[1, 0])).astype('float32')
C
ceci3 已提交
11373 11374
    labels = labels / reduce_sum(labels, dim=1, keep_dim=True)

C
ceci3 已提交
11375 11376
    l2loss = reduce_mean(reduce_sum(square(anchor), 1)) \
             + reduce_mean(reduce_sum(square(positive), 1))
C
ceci3 已提交
11377 11378 11379 11380
    l2loss = l2loss * Beta * l2_reg

    similarity_matrix = matmul(
        anchor, positive, transpose_x=False, transpose_y=True)
C
ceci3 已提交
11381 11382 11383
    softmax_ce = softmax_with_cross_entropy(
        logits=similarity_matrix, label=labels, soft_label=True)
    cross_entropy = reduce_sum(labels * softmax_ce, 0)
C
ceci3 已提交
11384 11385 11386
    celoss = reduce_mean(cross_entropy)

    return l2loss + celoss
11387 11388


R
ruri 已提交
11389 11390 11391 11392 11393 11394 11395 11396 11397 11398 11399 11400 11401 11402 11403 11404 11405 11406 11407 11408 11409 11410 11411 11412 11413 11414 11415 11416 11417
def pixel_shuffle(x, upscale_factor):
    """

    **Pixel Shuffle Layer**

    This layer rearranges elements in a tensor of shape [N, C, H, W]
    to a tensor of shape [N, C/r**2, H*r, W*r].
    This is useful for implementing efficient sub-pixel convolution
    with a stride of 1/r.
    Please refer to the paper: `Real-Time Single Image and Video Super-Resolution 
    Using an Efficient Sub-Pixel Convolutional Neural Network <https://arxiv.org/abs/1609.05158v2>`_ .
    by Shi et. al (2016) for more details.

        .. code-block:: text
        
            Given a 4-D tensor with the shape:
                x.shape = [1, 9, 4, 4]
            Given upscale_factor:
                upscale_factor= 3
            output shape is:
                [1, 1, 12, 12]
    
    Args:

        x(Variable): The input tensor variable.
        upscale_factor(int): factor to increase spatial resolution

    Returns:

11418
        Out(Variable): Reshaped tensor according to the new dimension.
R
ruri 已提交
11419 11420 11421 11422 11423 11424 11425 11426 11427

    Raises:

        ValueError: If the square of upscale_factor cannot divide the channels of input.

    Examples:

        .. code-block:: python

R
ruri 已提交
11428
            input = fluid.layers.data(name="input", shape=[9,4,4])
R
ruri 已提交
11429 11430 11431 11432 11433 11434 11435 11436 11437 11438 11439 11440 11441 11442 11443 11444 11445 11446 11447
            output = fluid.layers.pixel_shuffle(x=input, upscale_factor=3)

    """

    helper = LayerHelper("pixel_shuffle", **locals())

    out = helper.create_variable_for_type_inference(dtype=x.dtype)

    if not isinstance(upscale_factor, int):
        raise TypeError("upscale factor must be int type")

    helper.append_op(
        type="pixel_shuffle",
        inputs={"X": x},
        outputs={"Out": out},
        attrs={"upscale_factor": upscale_factor})
    return out


11448 11449 11450 11451 11452 11453 11454 11455 11456 11457 11458 11459 11460 11461 11462 11463 11464 11465 11466 11467 11468 11469 11470 11471 11472 11473 11474 11475 11476 11477 11478 11479 11480 11481 11482 11483 11484 11485 11486 11487 11488
def fsp_matrix(x, y):
    """

    **FSP matrix op**

    This op is used to calculate the flow of solution procedure (FSP) matrix of two feature maps.
    Given feature map x with shape [x_channel, h, w] and feature map y with shape
    [y_channel, h, w], we can get the fsp matrix of x and y in two steps:

    1. reshape x into matrix with shape [x_channel, h * w] and reshape and
       transpose y into matrix with shape [h * w, y_channel].
    2. multiply x and y to get fsp matrix with shape [x_channel, y_channel].

    The output is a batch of fsp matrices.

    Args:

        x (Variable): A feature map with shape [batch_size, x_channel, height, width].
        y (Variable): A feature map with shape [batch_size, y_channel, height, width].
                      The y_channel can be different with the x_channel of Input(X)
                      while the other dimensions must be the same with Input(X)'s.

    Returns:

        fsp matrix (Variable): The output of FSP op with shape [batch_size, x_channel, y_channel].
        The x_channel is the channel of x and the y_channel is the channel of y.

    Examples:

        .. code-block:: python

            feature_map_0 = fluid.layers.conv2d(x)
            feature_map_1 = fluid.layers.conv2d(feature_map_0)
            loss = fluid.layers.fsp_matrix(feature_map_0, feature_map_1)

    """
    helper = LayerHelper('fsp_matrix', **locals())
    out = helper.create_variable_for_type_inference(dtype=helper.input_dtype(
        input_param_name='x'))
    helper.append_op(type='fsp', inputs={'X': x, 'Y': y}, outputs={'Out': out})
    return out
H
heqiaozhi 已提交
11489 11490 11491 11492


def continuous_value_model(input, cvm, use_cvm=True):
    """
H
fix doc  
heqiaozhi 已提交
11493

H
heqiaozhi 已提交
11494
    **continuous_value_model layers**
H
fix doc  
heqiaozhi 已提交
11495

H
fix doc  
heqiaozhi 已提交
11496
    continuous value model(cvm). Now, it only considers show and click value in CTR project.
H
fix doc  
heqiaozhi 已提交
11497 11498 11499
    We assume that input is an embedding vector with cvm_feature, whose shape is [N * D] (D is 2 + embedding dim).
    If use_cvm is True, it will log(cvm_feature), and output shape is [N * D].
    If use_cvm is False, it will remove cvm_feature from input, and output shape is [N * (D - 2)].
H
heqiaozhi 已提交
11500
    
H
fix doc  
heqiaozhi 已提交
11501
    This layer accepts a tensor named input which is ID after embedded(lod level is 1), cvm is a show_click info.
H
fix doc  
heqiaozhi 已提交
11502

H
heqiaozhi 已提交
11503
    Args:
H
fix doc  
heqiaozhi 已提交
11504 11505

        input (Variable): a 2-D LodTensor with shape [N x D], where N is the batch size, D is 2 + the embedding dim. lod level = 1.
H
heqiaozhi 已提交
11506 11507
        cvm (Variable):   a 2-D Tensor with shape [N x 2], where N is the batch size, 2 is show and click.
        use_cvm  (bool):  use cvm or not. if use cvm, the output dim is the same as input
H
fix doc  
heqiaozhi 已提交
11508
                          if don't use cvm, the output dim is input dim - 2(remove show and click)
11509
                          (cvm op is a customized op, which input is a sequence has embed_with_cvm default, so we need an op named cvm to decided whever use it or not.)
H
fix doc  
heqiaozhi 已提交
11510

H
heqiaozhi 已提交
11511
    Returns:
H
fix doc  
heqiaozhi 已提交
11512 11513 11514

        Variable: A 2-D LodTensor with shape [N x D], if use cvm, D is equal to input dim, if don't use cvm, D is equal to input dim - 2. 

H
heqiaozhi 已提交
11515
    Examples:
H
fix doc  
heqiaozhi 已提交
11516

H
heqiaozhi 已提交
11517
        .. code-block:: python
H
fix doc  
heqiaozhi 已提交
11518

H
heqiaozhi 已提交
11519 11520 11521 11522 11523 11524 11525 11526 11527 11528
          input = fluid.layers.data(name="input", shape=[-1, 1], lod_level=1, append_batch_size=False, dtype="int64")#, stop_gradient=False)
          label = fluid.layers.data(name="label", shape=[-1, 1], append_batch_size=False, dtype="int64")
          embed = fluid.layers.embedding(
                            input=input,
                            size=[100, 11],
                            dtype='float32')
          ones = fluid.layers.fill_constant_batch_size_like(input=label, shape=[-1, 1], dtype="int64", value=1)
          show_clk = fluid.layers.cast(fluid.layers.concat([ones, label], axis=1), dtype='float32')
          show_clk.stop_gradient = True
          input_with_cvm = fluid.layers.continuous_value_model(embed, show_clk, True)
H
fix doc  
heqiaozhi 已提交
11529

H
heqiaozhi 已提交
11530 11531 11532 11533 11534 11535 11536 11537 11538
    """
    helper = LayerHelper('cvm', **locals())
    out = helper.create_variable(dtype=input.dtype)
    helper.append_op(
        type='cvm',
        inputs={'X': [input],
                'CVM': [cvm]},
        outputs={'Y': [out]},
        attrs={"use_cvm": use_cvm})
H
heqiaozhi 已提交
11539
    return out
Z
zhoukunsheng 已提交
11540 11541 11542 11543 11544 11545 11546 11547 11548 11549 11550 11551 11552 11553 11554 11555 11556 11557 11558 11559 11560 11561 11562 11563 11564 11565 11566 11567 11568 11569 11570 11571 11572 11573 11574


def where(condition):
    """
    Return an int64 tensor with rank 2, specifying the coordinate of true element in `condition`.

    Output's first dimension is the number of true element, second dimension is rank(number of dimension) of `condition`.
    If there is zero true element, then an empty tensor will be generated.  

    Args:
        condition(Variable): A bool tensor with rank at least 1.

    Returns:
        Variable: The tensor variable storing a 2-D tensor. 

    Examples:
        .. code-block:: python

             # condition is a tensor [True, False, True]
             out = fluid.layers.where(condition) # [[0], [2]]

             # condition is a tensor [[True, False], [False, True]]
             out = fluid.layers.where(condition) # [[0, 0], [1, 1]]

             # condition is a tensor [False, False, False]
             out = fluid.layers.where(condition) # [[]]
    """
    helper = LayerHelper("where", **locals())

    out = helper.create_variable_for_type_inference(
        dtype=core.VarDesc.VarType.INT64)

    helper.append_op(
        type='where', inputs={'Condition': condition}, outputs={'Out': [out]})
    return out
Z
zhoukunsheng 已提交
11575 11576 11577 11578 11579 11580 11581 11582 11583 11584 11585 11586 11587 11588 11589 11590 11591 11592 11593 11594 11595 11596 11597 11598 11599 11600 11601 11602 11603 11604 11605


def sign(x):
    """
    **sign**

    This function returns sign of every element in `x`: 1 for positive, -1 for negative and 0 for zero.

    Args:
        x(Variable|numpy.ndarray): The input tensor.

    Returns:
        Variable: The output sign tensor with identical shape and dtype to `x`.

    Examples:
        .. code-block:: python

          # [1, 0, -1]
          data = fluid.layers.sign(np.array([3, 0, -2])) 
    """

    helper = LayerHelper("sign", **locals())

    if not isinstance(x, Variable):
        x = assign(x)

    out = helper.create_variable_for_type_inference(dtype=x.dtype)

    helper.append_op(type='sign', inputs={'X': [x]}, outputs={'Out': [out]})

    return out