layers.py 221.0 KB
Newer Older
1
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
Z
zhangjinchao01 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import functools
15
import collections
Y
Yu Yang 已提交
16
import inspect
Z
zhangjinchao01 已提交
17

18
import paddle.trainer.config_parser as cp
Z
zhangjinchao01 已提交
19 20
from paddle.trainer.config_parser import *
from .activations import LinearActivation, SigmoidActivation, TanhActivation, \
Y
Yu Yang 已提交
21
    ReluActivation, IdentityActivation, SoftmaxActivation, BaseActivation
Z
zhangjinchao01 已提交
22
from .evaluators import *
23 24
from .poolings import MaxPooling, AvgPooling, BasePoolingType, \
    CudnnAvgPooling, CudnnMaxPooling
Z
zhangjinchao01 已提交
25 26
from .attrs import *
from .default_decorators import *
27

Z
zhangjinchao01 已提交
28 29 30 31 32 33
try:
    import cPickle as pickle
except ImportError:
    import pickle
import copy

Q
qijun 已提交
34
__all__ = [
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
    'full_matrix_projection',
    'AggregateLevel',
    'ExpandLevel',
    'identity_projection',
    'dotmul_projection',
    'dotmul_operator',
    'repeat_layer',
    'seq_reshape_layer',
    'table_projection',
    'mixed_layer',
    'data_layer',
    'embedding_layer',
    'fc_layer',
    'grumemory',
    'pooling_layer',
    'lstmemory',
    'last_seq',
    'first_seq',
    'cos_sim',
    'hsigmoid',
    'conv_projection',
56
    'square_error_cost',
57
    'regression_cost',
Q
qijun 已提交
58
    'classification_cost',
59
    'LayerOutput',
Q
qijun 已提交
60 61 62 63 64 65
    'img_conv_layer',
    'img_pool_layer',
    'batch_norm_layer',
    'img_cmrnorm_layer',
    'addto_layer',
    'concat_layer',
66
    'seq_concat_layer',
Q
qijun 已提交
67 68 69 70 71 72
    'lstm_step_layer',
    'recurrent_group',
    'memory',
    'StaticInput',
    'expand_layer',
    'scaling_layer',
X
xuwei06 已提交
73
    'scaling_projection',
Q
qijun 已提交
74 75 76 77
    'power_layer',
    'interpolation_layer',
    'bilinear_interp_layer',
    'trans_layer',
78
    'rotate_layer',
Q
qijun 已提交
79
    'sum_to_one_norm_layer',
G
guosheng 已提交
80
    'row_l2_norm_layer',
Q
qijun 已提交
81 82 83 84 85 86 87 88
    'get_output_layer',
    'LayerType',
    'context_projection',
    'beam_search',
    'maxid_layer',
    'GeneratedInput',
    'SubsequenceInput',
    'gru_step_layer',
Y
Yu Yang 已提交
89
    'gru_step_naive_layer',
Q
qijun 已提交
90 91 92 93 94 95 96 97 98 99 100 101
    'recurrent_layer',
    'BaseGeneratedInput',
    'conv_operator',
    'conv_shift_layer',
    'tensor_layer',
    'selective_fc_layer',
    'sampling_id_layer',
    'slope_intercept_layer',
    'trans_full_matrix_projection',
    'linear_comb_layer',
    'convex_comb_layer',
    'ctc_layer',
102
    'warp_ctc_layer',
Q
qijun 已提交
103 104 105 106 107
    'crf_layer',
    'crf_decoding_layer',
    'nce_layer',
    'cross_entropy_with_selfnorm',
    'cross_entropy',
C
caoying03 已提交
108
    'BeamInput',
C
caoying03 已提交
109
    'cross_entropy_over_beam',
Q
qijun 已提交
110 111 112 113
    'multi_binary_label_cross_entropy',
    'sum_cost',
    'rank_cost',
    'lambda_cost',
L
Luo Tao 已提交
114
    'huber_regression_cost',
115
    'huber_classification_cost',
Q
qijun 已提交
116 117 118
    'block_expand_layer',
    'maxout_layer',
    'out_prod_layer',
X
xuwei06 已提交
119
    'printer_layer',
Q
qijun 已提交
120
    'print_layer',
Y
yuan 已提交
121
    'priorbox_layer',
122
    'cross_channel_norm_layer',
123 124
    'multibox_loss_layer',
    'detection_output_layer',
Q
qijun 已提交
125
    'spp_layer',
D
dangqingqing 已提交
126
    'pad_layer',
L
Luo Tao 已提交
127
    'eos_layer',
128
    'smooth_l1_cost',
129
    'layer_support',
W
wwhu 已提交
130
    'multiplex_layer',
D
dangqingqing 已提交
131
    'row_conv_layer',
132
    'dropout_layer',
133
    'prelu_layer',
134
    'switch_order_layer',
135
    'gated_unit_layer',
136
    'crop_layer',
137
    'sub_nested_seq_layer',
138
    'clip_layer',
139
    'slice_projection',
140
    'seq_slice_layer',
141
    'kmax_seq_score_layer',
C
chengduoZH 已提交
142
    'img_pool3d_layer',
G
guosheng 已提交
143
    'scale_shift_layer',
C
chengduoZH 已提交
144
    'img_conv3d_layer',
Q
qijun 已提交
145
]
Z
zhangjinchao01 已提交
146 147 148 149 150 151 152


class LayerType(object):
    """
    Layer type enumerations.
    """

153 154 155 156 157 158 159 160
    DATA = 'data'
    MIXED_LAYER = 'mixed'
    LSTMEMORY = 'lstmemory'
    GRUMEMORY = 'gated_recurrent'
    SEQUENCE_LAST_INSTANCE = 'seqlastins'
    SEQUENCE_FIRST_INSTANCE = 'seqfirstins'
    SEQUENCE_RESHAPE = 'seqreshape'
    POOLING_MAX = 'max'
Z
zhangjinchao01 已提交
161
    POOLING_AVG = 'average'
162
    FC_LAYER = 'fc'
Z
zhangjinchao01 已提交
163
    COST = 'cost'
164 165
    COSINE_SIM_VEC = 'cos_vm'
    COSINE_SIM = 'cos'
Z
zhangjinchao01 已提交
166
    HSIGMOID = 'hsigmoid'
167 168 169 170 171
    CONV_LAYER = 'conv'
    CONVTRANS_LAYER = 'convt'
    EXCONV_LAYER = 'exconv'
    EXCONVTRANS_LAYER = 'exconvt'
    CUDNNCONV_LAYER = 'cudnn_conv'
C
chengduoZH 已提交
172
    CUDNNCONVTRANS_LAYER = 'cudnn_convt'
173
    POOL_LAYER = 'pool'
C
chengduoZH 已提交
174
    POOL3D_LAYER = 'pool3d'
Z
zhangjinchao01 已提交
175 176 177
    BATCH_NORM_LAYER = 'batch_norm'
    NORM_LAYER = 'norm'
    SUM_TO_ONE_NORM_LAYER = 'sum_to_one_norm'
G
guosheng 已提交
178
    ROW_L2_NORM_LAYER = 'row_l2_norm'
Z
zhangjinchao01 已提交
179 180 181 182
    ADDTO_LAYER = 'addto'

    CONCAT_LAYER = 'concat'
    CONCAT_PROJ_LAYER = 'concat2'
183
    SEQUENCE_CONCAT_LAYER = 'seqconcat'
Z
zhangjinchao01 已提交
184 185 186 187 188 189 190

    LSTM_STEP_LAYER = 'lstm_step'
    GRU_STEP_LAYER = 'gru_step'
    GET_OUTPUT_LAYER = 'get_output'

    EXPAND_LAYER = 'expand'
    INTERPOLATION_LAYER = 'interpolation'
L
liaogang 已提交
191
    BILINEAR_INTERP_LAYER = 'bilinear_interp'
Z
zhangjinchao01 已提交
192 193 194
    POWER_LAYER = 'power'
    SCALING_LAYER = 'scaling'
    TRANS_LAYER = 'trans'
195
    ROTATE_LAYER = 'rotate'
H
Haonan 已提交
196
    OUT_PROD_LAYER = 'out_prod'
X
xuwei06 已提交
197
    FEATURE_MAP_EXPAND_LAYER = 'featmap_expand'
Z
zhangjinchao01 已提交
198 199 200 201 202 203 204 205 206 207 208

    MEMORY = 'memory'
    MAXID_LAYER = 'maxid'
    EOSID_LAYER = 'eos_id'
    RECURRENT_LAYER = 'recurrent'

    CONV_SHIFT_LAYER = "conv_shift"
    TENSOR_LAYER = "tensor"
    SEL_FC_LAYER = "selective_fc"
    SAMPLING_ID_LAYER = "sampling_id"
    SLOPE_INTERCEPT_LAYER = "slope_intercept"
209
    LINEAR_COMBINATION_LAYER = "convex_comb"
Z
zhangjinchao01 已提交
210
    BLOCK_EXPAND = "blockexpand"
211
    MAXOUT = "maxout"
Q
qijun 已提交
212
    SPP_LAYER = "spp"
D
dangqingqing 已提交
213
    PAD_LAYER = "pad"
W
wwhu 已提交
214
    MULTIPLEX_LAYER = "multiplex"
D
dangqingqing 已提交
215
    ROW_CONV_LAYER = "row_conv"
D
dangqingqing 已提交
216 217 218

    PRINT_LAYER = 'print'
    PRIORBOX_LAYER = 'priorbox'
219 220
    MULTIBOX_LOSS_LAYER = 'multibox_loss'
    DETECTION_OUTPUT_LAYER = 'detection_output'
D
dangqingqing 已提交
221 222 223 224 225

    CTC_LAYER = 'ctc'
    WARP_CTC_LAYER = 'warp_ctc'
    CRF_LAYER = 'crf'
    CRF_DECODING_LAYER = 'crf_decoding'
226
    NCE_LAYER = 'nce'
Z
zhangjinchao01 已提交
227

228 229 230
    CONV3D_LAYER = 'conv3d'
    DECONV3D_LAYER = 'deconv3d'

231 232
    RANK_COST = 'rank-cost'
    LAMBDA_COST = 'lambda_cost'
L
Luo Tao 已提交
233
    HUBER_REGRESSION = 'huber_regression'
234
    HUBER_CLASSIFICATION = 'huber_classification'
235 236
    CROSS_ENTROPY = 'multi-class-cross-entropy'
    CROSS_ENTROPY_WITH_SELFNORM = 'multi_class_cross_entropy_with_selfnorm'
C
caoying03 已提交
237
    CROSS_ENTROPY_OVER_BEAM = 'cross_entropy_over_beam'
238 239 240 241 242 243
    SOFT_BIN_CLASS_CROSS_ENTROPY = 'soft_binary_class_cross_entropy'
    MULTI_BIN_LABEL_CROSS_ENTROPY = 'multi_binary_label_cross_entropy'
    SUM_COST = 'sum_cost'
    SMOOTH_L1 = 'smooth_l1'

    PRELU = 'prelu'
244
    SWITCH_ORDER_LAYER = 'switch_order'
245
    CROP_LAYER = 'crop'
C
caoying03 已提交
246
    SUB_NESTED_SEQ = 'sub_nested_seq'
G
guosheng 已提交
247
    CLIP_LAYER = 'clip'
248
    SEQ_SLICE = 'seq_slice'
Z
zhangjinchao01 已提交
249

250
    KMAX_SEQ_SCORE = 'kmax_seq_score'
G
guosheng 已提交
251
    SCALE_SHIFT_LAYER = 'scale_shift'
Z
zhangjinchao01 已提交
252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272

    @staticmethod
    def is_layer_type(type_name):
        """
        If type_name is a layer type.

        :param type_name: layer type name. Because layer type enumerations are
                          strings.
        :type type_name: basestring
        :return: True if is a layer_type
        :rtype: bool
        """
        for key in dir(LayerType):
            if key.isupper():
                att = getattr(LayerType, key)
                if isinstance(att, basestring) and type_name == att:
                    return True
        return False


class AggregateLevel(object):
273
    """
L
Luo Tao 已提交
274
    PaddlePaddle supports three sequence types:
275 276 277

    - :code:`SequenceType.NO_SEQUENCE` means the sample is not a sequence.
    - :code:`SequenceType.SEQUENCE` means the sample is a sequence.
L
Luo Tao 已提交
278 279
    - :code:`SequenceType.SUB_SEQUENCE` means the sample is a nested sequence,
      each timestep of which is also a sequence.
280

L
Luo Tao 已提交
281
    Accordingly, AggregateLevel supports two modes:
282

L
Luo Tao 已提交
283
    - :code:`AggregateLevel.TO_NO_SEQUENCE` means the aggregation acts on each
L
Luo Tao 已提交
284
      timestep of a sequence, both :code:`SUB_SEQUENCE` and :code:`SEQUENCE` will
285 286
      be aggregated to :code:`NO_SEQUENCE`.

L
Luo Tao 已提交
287
    - :code:`AggregateLevel.TO_SEQUENCE` means the aggregation acts on each
288 289 290
      sequence of a nested sequence, :code:`SUB_SEQUENCE` will be aggregated to
      :code:`SEQUENCE`.
    """
L
Luo Tao 已提交
291 292
    TO_NO_SEQUENCE = 'non-seq'
    TO_SEQUENCE = 'seq'
293 294 295
    # compatible with previous configuration
    EACH_TIMESTEP = TO_NO_SEQUENCE
    EACH_SEQUENCE = TO_SEQUENCE
Z
zhangjinchao01 已提交
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317


class LayerOutput(object):
    """
    LayerOutput is output for layer function. It is used internally by several
    reasons.

    - Check layer connection make sense.

        - FC(Softmax) => Cost(MSE Error) is not good for example.

    - Tracking layer connection.

    - Pass to layer methods as input.

    :param name: Layer output name.
    :type name: basestring
    :param layer_type: Current Layer Type. One of LayerType enumeration.
    :type layer_type: basestring
    :param activation: Layer Activation.
    :type activation: BaseActivation.
    :param parents: Layer's parents.
318
    :type parents: list|tuple|collections.Sequence
Z
zhangjinchao01 已提交
319 320
    """

Q
qijun 已提交
321 322 323 324 325 326 327 328 329
    def __init__(self,
                 name,
                 layer_type,
                 parents=None,
                 activation=None,
                 num_filters=None,
                 img_norm_type=None,
                 size=None,
                 outputs=None,
330
                 reverse=None):
Z
zhangjinchao01 已提交
331 332
        assert isinstance(name, basestring)
        assert isinstance(layer_type, basestring)
X
xuwei06 已提交
333
        assert size is not None
Z
zhangjinchao01 已提交
334 335
        assert LayerType.is_layer_type(layer_type)
        self.name = name
X
xuwei06 已提交
336
        self.full_name = MakeLayerNameInSubmodel(name)
Z
zhangjinchao01 已提交
337
        self.layer_type = layer_type
338 339
        if parents is not None and type(parents) != list:
            parents = [parents]
Z
zhangjinchao01 已提交
340 341 342 343 344 345 346 347
        self.parents = [] if parents is None else parents
        self.activation = activation
        self.num_filters = num_filters
        self.img_norm_type = img_norm_type
        self.size = size
        if outputs is None:
            outputs = ['default']
        self.outputs = outputs
348
        self.reverse = reverse
Z
zhangjinchao01 已提交
349

350 351 352 353 354 355 356 357
    @property
    def width(self):
        return cp.g_layer_map[self.full_name].width

    @property
    def height(self):
        return cp.g_layer_map[self.full_name].height

358 359 360 361
    @property
    def depth(self):
        return cp.g_layer_map[self.full_name].depth

362 363 364 365 366 367 368 369
    def set_input(self, input):
        """
        Set the input for a memory layer. Can only be used for memory layer
        """
        assert isinstance(input, LayerOutput)
        assert self.layer_type == LayerType.MEMORY
        SetMemoryInput(self.name, input.name)

Z
zhangjinchao01 已提交
370 371 372

ERROR_CLIPPING = 'error_clipping_threshold'
DROPOUT = 'drop_rate'
373
DEVICE = 'device'
Z
zhangjinchao01 已提交
374 375 376


def layer_support(*attrs):
377
    attrs_list = list(attrs)
378
    attrs_list.append(DEVICE)
Q
qijun 已提交
379

Z
zhangjinchao01 已提交
380 381 382
    def decorator(method):
        @functools.wraps(method)
        def wrapper(*args, **kwargs):
383
            for attr in attrs_list:
Z
zhangjinchao01 已提交
384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399
                for each in args:
                    if isinstance(each, ExtraLayerAttribute):
                        setattr(each, '_'.join(['can', attr]), True)
                for key in kwargs:
                    val = kwargs[key]
                    if isinstance(val, ExtraLayerAttribute):
                        setattr(val, '_'.join(['can', attr]), True)
            for each in args:
                if isinstance(each, ExtraLayerAttribute):
                    each.check(method.__name__)
            for key in kwargs:
                val = kwargs[key]
                if isinstance(val, ExtraLayerAttribute):
                    val.check(method.__name__)
            return method(*args, **kwargs)

Y
Yu Yang 已提交
400 401 402 403 404
        if hasattr(method, 'argspec'):
            wrapper.argspec = method.argspec
        else:
            wrapper.argspec = inspect.getargspec(method)

Z
zhangjinchao01 已提交
405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443
        return wrapper

    return decorator


@wrap_param_attr_default()
def full_matrix_projection(input, size=0, param_attr=None):
    """
    Full Matrix Projection. It performs full matrix multiplication.

    ..  math::
        out.row[i] += in.row[i] * weight

    There are two styles of usage.

    1. When used in mixed_layer like this, you can only set the input:

    .. code-block:: python

       with mixed_layer(size=100) as m:
           m += full_matrix_projection(input=layer)

    2. When used as an independant object like this, you must set the size:

    .. code-block:: python

       proj = full_matrix_projection(input=layer,
                                     size=100,
                                     param_attr=ParamAttr(name='_proj'))

    :param input: input layer
    :type input: LayerOutput
    :param size: The parameter size. Means the width of parameter.
    :type size: int
    :param param_attr: Parameter config, None if use default.
    :type param_attr: ParameterAttribute
    :return: A FullMatrixProjection Object.
    :rtype: FullMatrixProjection
    """
Q
qijun 已提交
444 445
    proj = FullMatrixProjection(
        input_layer_name=input.name, size=size, **param_attr.attr)
Z
zhangjinchao01 已提交
446 447 448 449
    proj.origin = input
    return proj


450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
@wrap_param_attr_default()
def trans_full_matrix_projection(input, size=0, param_attr=None):
    """
    Different from full_matrix_projection, this projection performs matrix
    multiplication, using transpose of weight.

    ..  math::
        out.row[i] += in.row[i] * w^\mathrm{T}

    :math:`w^\mathrm{T}` means transpose of weight.
    The simply usage is:

    .. code-block:: python

       proj = trans_full_matrix_projection(input=layer,
                                           size=100,
                                           param_attr=ParamAttr(
                                                name='_proj',
                                                initial_mean=0.0,
                                                initial_std=0.01))

    :param input: input layer
    :type input: LayerOutput
    :param size: The parameter size. Means the width of parameter.
    :type size: int
    :param param_attr: Parameter config, None if use default.
    :type param_attr: ParameterAttribute
    :return: A TransposedFullMatrixProjection Object.
    :rtype: TransposedFullMatrixProjection
    """
Q
qijun 已提交
480 481
    proj = TransposedFullMatrixProjection(
        input_layer_name=input.name, size=size, **param_attr.attr)
482 483 484 485
    proj.origin = input
    return proj


Z
zhangjinchao01 已提交
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
@wrap_param_attr_default()
def table_projection(input, size=0, param_attr=None):
    """
    Table Projection. It selects rows from parameter where row\_id
    is in input\_ids.

    .. math::
       out.row[i] += table.row[ids[i]]

    where :math:`out` is output, :math:`table` is parameter, :math:`ids` is input\_ids,
    and :math:`i` is row\_id.

    There are two styles of usage.

    1. When used in mixed_layer like this, you can only set the input:

    .. code-block:: python

       with mixed_layer(size=100) as m:
           m += table_projection(input=layer)

    2. When used as an independant object like this, you must set the size:

    .. code-block:: python

       proj = table_projection(input=layer,
                               size=100,
                               param_attr=ParamAttr(name='_proj'))


    :param input: Input layer, which must contains id fields.
    :type input: LayerOutput
    :param size: The parameter size. Means the width of parameter.
    :type size: int
    :param param_attr: Parameter config, None if use default.
    :type param_attr: ParameterAttribute
    :return: A TableProjection Object.
    :rtype: TableProjection
    """
Q
qijun 已提交
525 526
    proj = TableProjection(
        input_layer_name=input.name, size=size, **param_attr.attr)
Z
zhangjinchao01 已提交
527 528 529 530
    proj.origin = input
    return proj


531
def identity_projection(input, offset=None, size=None):
Z
zhangjinchao01 已提交
532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561
    """
    1. IdentityProjection if offset=None. It performs:

    .. math::
       out.row[i] += in.row[i]

    The example usage is:

    .. code-block:: python

       proj = identity_projection(input=layer)


    2. IdentityOffsetProjection if offset!=None. It likes IdentityProjection,
    but layer size may be smaller than input size.
    It select dimesions [offset, offset+layer_size) from input:

    .. math::
       out.row[i] += in.row[i + \\textrm{offset}]

    The example usage is:

    .. code-block:: python

       proj = identity_projection(input=layer,
                                  offset=10)

    Note that both of two projections should not have any parameter.

    :param input: Input Layer.
562
    :type input: LayerOutput
Z
zhangjinchao01 已提交
563 564
    :param offset: Offset, None if use default.
    :type offset: int
X
xuwei06 已提交
565
    :return: A IdentityProjection or IdentityOffsetProjection object
Z
zhangjinchao01 已提交
566 567 568 569 570 571
    :rtype: IdentityProjection or IdentityOffsetProjection
    """
    if offset is None:
        proj = IdentityProjection(input_layer_name=input.name)
        proj.origin = input
    else:
572 573
        if size is None:
            size = input.size - offset
Q
qijun 已提交
574
        proj = IdentityOffsetProjection(
575
            input_layer_name=input.name, offset=offset, size=size)
Z
zhangjinchao01 已提交
576 577 578 579
        proj.origin = input
    return proj


580 581
def slice_projection(input, slices):
    """
582 583
    slice_projection can slice the input value into multiple parts,
    and then select some of them to merge into a new output.
584 585

    .. math::
586
       output = [input.slices()]
587 588 589 590 591 592 593 594 595 596 597 598 599 600

    The example usage is:

    .. code-block:: python

       proj = slice_projection(input=layer, slices=[(0, 10), (20, 30)])

    Note that slice_projection should not have any parameter.

    :param input: Input Layer.
    :type input: LayerOutput
    :param slices: An array of slice parameters.
                   Each slice contains the start and end offsets based
                   on the input.
H
hedaoyuan 已提交
601
    :type slices: pair of int
602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618
    :return: A SliceProjection object
    :rtype: SliceProjection
    """
    assert len(slices) >= 1
    start = 0
    for i in xrange(len(slices)):
        assert len(slices[i]) == 2
        # The start position of the next slice needs to be greater than
        # or equal to the end position of the previous slice.
        assert slices[i][0] >= start
        assert slices[i][1] >= slices[i][0]
        start = slices[i][1]
    proj = SliceProjection(input_layer_name=input.name, slices=slices)
    proj.origin = input
    return proj


X
xuwei06 已提交
619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
@wrap_param_attr_default()
def scaling_projection(input, param_attr=None):
    """
    scaling_projection multiplies the input with a scalar parameter and add to
    the output.

    .. math::
       out += w * in

    The example usage is:

    .. code-block:: python

       proj = scaling_projection(input=layer)

    :param input: Input Layer.
    :type input: LayerOutput
    :param param_attr: Parameter config, None if use default.
    :type param_attr: ParameterAttribute
    :return: A ScalingProjection object
    :rtype: ScalingProjection
    """
L
Luo Tao 已提交
641
    proj = ScalingProjection(input_layer_name=input.name, **param_attr.attr)
X
xuwei06 已提交
642 643 644 645
    proj.origin = input
    return proj


Z
zhangjinchao01 已提交
646
@wrap_param_attr_default()
647
def dotmul_projection(input, param_attr=None):
Z
zhangjinchao01 已提交
648
    """
649
    DotMulProjection with a layer as input.
Z
zhangjinchao01 已提交
650 651 652 653 654 655 656 657 658 659 660 661 662
    It performs element-wise multiplication with weight.

    ..  math::
        out.row[i] += in.row[i] .* weight

    where :math:`.*` means element-wise multiplication.

    The example usage is:

    .. code-block:: python

       proj = dotmul_projection(input=layer)

663 664 665 666 667 668 669
    :param input: Input layer.
    :type input: LayerOutput
    :param param_attr: Parameter config, None if use default.
    :type param_attr: ParameterAttribute
    :return: A DotMulProjection Object.
    :rtype: DotMulProjection
    """
Q
qijun 已提交
670 671
    proj = DotMulProjection(
        input_layer_name=input.name, size=input.size, **param_attr.attr)
672
    proj.origin = input
673
    return proj
Z
zhangjinchao01 已提交
674

675 676

def dotmul_operator(a=None, b=None, scale=1, **kwargs):
677 678
    """
    DotMulOperator takes two inputs and performs element-wise multiplication:
679

Z
zhangjinchao01 已提交
680
    .. math::
L
Luo Tao 已提交
681
       out.row[i] += scale * (a.row[i] .* b.row[i])
682

Z
zhangjinchao01 已提交
683 684
    where :math:`.*` means element-wise multiplication, and
    scale is a config scalar, its default value is one.
685

Z
zhangjinchao01 已提交
686
    The example usage is:
687

Z
zhangjinchao01 已提交
688
    .. code-block:: python
689

L
Luo Tao 已提交
690
       op = dotmul_operator(a=layer1, b=layer2, scale=0.5)
691

692 693 694 695
    :param a: Input layer1
    :type a: LayerOutput
    :param b: Input layer2
    :type b: LayerOutput
Z
zhangjinchao01 已提交
696 697
    :param scale: config scalar, default value is one.
    :type scale: float
698 699
    :return: A DotMulOperator Object.
    :rtype: DotMulOperator
Z
zhangjinchao01 已提交
700
    """
701 702 703
    if 'x' in kwargs or 'y' in kwargs:
        logger.warning('x and y arguments for dotmul_operator is deprecated. '
                       'Please use a and b as parameter.')
Q
qijun 已提交
704
    a = kwargs.get('x', a)  # For Backward capacity.
705 706 707 708 709 710
    b = kwargs.get('y', b)
    assert isinstance(a, LayerOutput)
    assert isinstance(b, LayerOutput)
    if a.size is not None and b.size is not None:
        assert a.size == b.size

Q
qijun 已提交
711
    op = DotMulOperator(input_layer_names=[a.name, b.name], scale=scale)
712
    op.origin = [a, b]
713
    return op
Z
zhangjinchao01 已提交
714

715

Z
zhangjinchao01 已提交
716
@wrap_bias_attr_default(['padding_attr'])
Q
qijun 已提交
717 718 719
def context_projection(input,
                       context_len,
                       context_start=None,
Z
zhangjinchao01 已提交
720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755
                       padding_attr=False):
    """
    Context Projection.

    It just simply reorganizes input sequence, combines "context_len" sequence
    to one context from context_start. "context_start" will be set to
    -(context_len - 1) / 2 by default. If context position out of sequence
    length, padding will be filled as zero if padding_attr = False, otherwise
    it is trainable.

    For example, origin sequence is [A B C D E F G], context len is 3, then
    after context projection and not set padding_attr, sequence will
    be [ 0AB ABC BCD CDE DEF EFG FG0 ].

    :param input: Input Sequence.
    :type input: LayerOutput
    :param context_len: context length.
    :type context_len: int
    :param context_start: context start position. Default is
                          -(context_len - 1)/2
    :type context_start: int
    :param padding_attr: Padding Parameter Attribute. If false, it means padding
                         always be zero. Otherwise Padding is learnable, and
                         parameter attribute is set by this parameter.
    :type padding_attr: bool|ParameterAttribute
    :return: Projection
    :rtype: Projection
    """
    context_start = -(
        context_len - 1) / 2 if context_start is None else context_start

    extra_dict = dict()
    trainable = isinstance(padding_attr, ParameterAttribute)
    if trainable:
        extra_dict = padding_attr.attr

Q
qijun 已提交
756 757 758 759 760 761
    proj = ContextProjection(
        input_layer_name=input.name,
        context_length=context_len,
        context_start=context_start,
        trainable_padding=trainable,
        **extra_dict)
Z
zhangjinchao01 已提交
762 763 764 765 766 767 768 769 770 771 772 773 774
    proj.origin = input
    return proj


class MixedLayerType(LayerOutput):
    """
    The internal object for trainer_helpers.
    """

    class AddToSealedMixedLayerException(Exception):
        def __init__(self):
            Exception.__init__(self)

Q
qijun 已提交
775
    def __init__(self, name, size, act, bias_attr, layer_attr, parents=None):
Z
zhangjinchao01 已提交
776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
        """
        Ctor.
        :param name: layer name.
        :type name: basestring
        :param size: layer size.
        :type size: int
        :param act: activation type.
        :type act: BaseActivation
        :param bias_attr: The Bias Attribute. If no bias, then pass False or
                          something not type of ParameterAttribute. None will
                          get a default Bias.
        :type bias_attr: ParameterAttribute or None means has bias. Any other
                         type means no bias.
        :param layer_attr: Extra Layer Attribute.
        :type layer_attr: ExtraLayerAttribute or None
        """
Q
qijun 已提交
792 793 794 795 796 797 798
        LayerOutput.__init__(
            self,
            name,
            LayerType.MIXED_LAYER,
            parents,
            size=size,
            activation=act)
Z
zhangjinchao01 已提交
799 800 801 802 803
        self.bias_attr = bias_attr
        self.layer_attr = layer_attr
        self.inputs = []
        self.finalized = False

804
    def __iadd__(self, other):
Z
zhangjinchao01 已提交
805 806 807 808 809 810 811 812
        """
        + += operator
        :param other: Other projection.
        :type other: Projection
        :return: self.
        :rtype: MixedLayerType
        """
        if not self.finalized:
813
            assert isinstance(other, Projection) or isinstance(other, Operator)
Z
zhangjinchao01 已提交
814
            self.inputs.append(other)
815 816 817 818
            if isinstance(other, Projection):
                self.parents.append(other.origin)
            else:
                self.parents.extend(other.origin)
Z
zhangjinchao01 已提交
819 820 821 822 823 824 825 826
            return self
        else:
            raise MixedLayerType.AddToSealedMixedLayerException()

    def __enter__(self):
        assert len(self.inputs) == 0
        return self

827
    def __exit__(self, exc_type, exc_value, tb):
W
wangyang59 已提交
828 829
        if exc_value is not None:
            raise exc_value
Z
zhangjinchao01 已提交
830
        assert len(self.inputs) != 0
831
        ml = MixedLayer(
Z
zhangjinchao01 已提交
832 833 834 835 836
            name=self.name,
            size=self.size,
            active_type=self.activation.name,
            bias=ParamAttr.to_bias(self.bias_attr),
            inputs=self.inputs,
Q
qijun 已提交
837
            **ExtraLayerAttribute.to_kwargs(self.layer_attr))
838 839 840
        # update the size which might be computed inside MixedLayer
        # according to the operator's output size
        self.size = ml.config.size
841
        self.finalized = True
Z
zhangjinchao01 已提交
842 843 844 845 846 847


@wrap_name_default("mixed")
@wrap_act_default(act=LinearActivation())
@wrap_bias_attr_default(has_bias=False)
@layer_support(ERROR_CLIPPING, DROPOUT)
Q
qijun 已提交
848 849 850 851 852
def mixed_layer(size=0,
                input=None,
                name=None,
                act=None,
                bias_attr=False,
Z
zhangjinchao01 已提交
853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896
                layer_attr=None):
    """
    Mixed Layer. A mixed layer will add all inputs together, then activate.
    Each inputs is a projection or operator.

    There are two styles of usages.

    1. When not set inputs parameter, use mixed_layer like this:

    .. code-block:: python

       with mixed_layer(size=256) as m:
           m += full_matrix_projection(input=layer1)
           m += identity_projection(input=layer2)

    2. You can also set all inputs when invoke mixed_layer as follows:

    .. code-block:: python

       m = mixed_layer(size=256,
                       input=[full_matrix_projection(input=layer1),
                              full_matrix_projection(input=layer2)])

    :param name: mixed layer name. Can be referenced by other layer.
    :type name: basestring
    :param size: layer size.
    :type size: int
    :param input: inputs layer. It is an optional parameter. If set,
                  then this function will just return layer's name.
    :param act: Activation Type.
    :type act: BaseActivation
    :param bias_attr: The Bias Attribute. If no bias, then pass False or
                      something not type of ParameterAttribute. None will get a
                      default Bias.
    :type bias_attr: ParameterAttribute or None or bool
    :param layer_attr: The extra layer config. Default is None.
    :type layer_attr: ExtraLayerAttribute
    :return: MixedLayerType object can add inputs or layer name.
    :rtype: MixedLayerType
    """

    if input is None:
        return MixedLayerType(name, size, act, bias_attr, layer_attr)
    else:
Q
qijun 已提交
897 898 899 900 901 902
        with mixed_layer(
                name=name,
                size=size,
                act=act,
                bias_attr=bias_attr,
                layer_attr=layer_attr) as m:
903
            if isinstance(input, collections.Sequence):
Z
zhangjinchao01 已提交
904 905 906 907 908 909 910 911
                for each in input:
                    m += each
            else:
                m += input
        return m


@layer_support()
C
chengduoZH 已提交
912 913
def data_layer(name, size, depth=None, height=None, width=None,
               layer_attr=None):
Z
zhangjinchao01 已提交
914 915 916 917 918 919 920
    """
    Define DataLayer For NeuralNetwork.

    The example usage is:

    ..  code-block:: python

Y
Yu Yang 已提交
921
        data = data_layer(name="input", size=1000)
Z
zhangjinchao01 已提交
922 923 924 925 926

    :param name: Name of this data layer.
    :type name: basestring
    :param size: Size of this data layer.
    :type size: int
L
Luo Tao 已提交
927
    :param height: Height of this data layer, used for image
Y
Yu Yang 已提交
928
    :type height: int|None
L
Luo Tao 已提交
929
    :param width: Width of this data layer, used for image
Y
Yu Yang 已提交
930
    :type width: int|None
Z
zhangjinchao01 已提交
931 932
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute.
D
dangqingqing 已提交
933
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
934 935
    :rtype: LayerOutput
    """
Q
qijun 已提交
936 937 938 939
    Layer(
        type=LayerType.DATA,
        name=name,
        size=size,
C
chengduoZH 已提交
940
        depth=depth,
L
Luo Tao 已提交
941 942
        height=height,
        width=width,
Q
qijun 已提交
943
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
944

C
chengduoZH 已提交
945 946
    if depth is None:
        depth = 1
947 948
    num_filters = None
    if height is not None and width is not None:
C
chengduoZH 已提交
949 950
        num_filters = size / (width * height * depth)
        assert num_filters * width * height * depth == size, \
C
chengduoZH 已提交
951
                "size=%s width=%s height=%s depth=%s" % (size, width, height, depth)
952 953

    return LayerOutput(name, LayerType.DATA, size=size, num_filters=num_filters)
Z
zhangjinchao01 已提交
954 955 956 957


@wrap_name_default("embedding")
@wrap_param_attr_default()
958
@layer_support(ERROR_CLIPPING, DROPOUT)
Z
zhangjinchao01 已提交
959 960 961 962 963 964 965 966 967 968 969 970 971 972 973
def embedding_layer(input, size, name=None, param_attr=None, layer_attr=None):
    """
    Define a embedding Layer.

    :param name: Name of this embedding layer.
    :type name: basestring
    :param input: The input layer for this embedding. NOTE: must be Index Data.
    :type input: LayerOutput
    :param size: The embedding dimension.
    :type size: int
    :param param_attr: The embedding parameter attribute. See ParameterAttribute
                      for details.
    :type param_attr: ParameterAttribute|None
    :param layer_attr: Extra layer Config. Default is None.
    :type layer_attr: ExtraLayerAttribute|None
D
dangqingqing 已提交
974
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
975 976
    :rtype: LayerOutput
    """
Q
qijun 已提交
977 978 979 980 981 982
    with mixed_layer(
            name=name,
            size=size,
            act=LinearActivation(),
            bias_attr=False,
            layer_attr=layer_attr) as mix:
Z
zhangjinchao01 已提交
983 984 985 986 987 988 989 990 991
        mix += table_projection(input=input, size=size, param_attr=param_attr)
    return mix


@wrap_name_default()
@wrap_param_attr_default()
@wrap_bias_attr_default()
@wrap_act_default()
@layer_support(ERROR_CLIPPING, DROPOUT)
Q
qijun 已提交
992 993 994 995 996 997 998
def fc_layer(input,
             size,
             act=None,
             name=None,
             param_attr=None,
             bias_attr=None,
             layer_attr=None):
Z
zhangjinchao01 已提交
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010
    """
    Helper for declare fully connected layer.

    The example usage is:

    .. code-block:: python

       fc = fc_layer(input=layer,
                     size=1024,
                     act=LinearActivation(),
                     bias_attr=False)

L
luotao02 已提交
1011
    which is equal to:
Z
zhangjinchao01 已提交
1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033

    .. code-block:: python

       with mixed_layer(size=1024) as fc:
           fc += full_matrix_projection(input=layer)

    :param name: The Layer Name.
    :type name: basestring
    :param input: The input layer. Could be a list/tuple of input layer.
    :type input: LayerOutput|list|tuple
    :param size: The layer dimension.
    :type size: int
    :param act: Activation Type. Default is tanh.
    :type act: BaseActivation
    :param param_attr: The Parameter Attribute|list.
    :type param_attr: ParameterAttribute
    :param bias_attr: The Bias Attribute. If no bias, then pass False or
                      something not type of ParameterAttribute. None will get a
                      default Bias.
    :type bias_attr: ParameterAttribute|None|Any
    :param layer_attr: Extra Layer config.
    :type layer_attr: ExtraLayerAttribute|None
D
dangqingqing 已提交
1034
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
1035 1036 1037 1038
    :rtype: LayerOutput
    """
    if isinstance(input, LayerOutput):
        input = [input]
1039
        assert not isinstance(param_attr, collections.Sequence)
Z
zhangjinchao01 已提交
1040 1041
        param_attr = [param_attr]
    else:
1042
        if isinstance(param_attr, collections.Sequence):
Z
zhangjinchao01 已提交
1043 1044 1045 1046
            assert len(input) == len(param_attr)
        else:
            param_attr = [copy.deepcopy(param_attr) for _ in range(len(input))]

1047
    assert isinstance(input, collections.Sequence)
Z
zhangjinchao01 已提交
1048 1049

    Layer(
Q
qijun 已提交
1050 1051 1052
        inputs=[
            Input(ipt.name, **attr.attr) for ipt, attr in zip(input, param_attr)
        ],
Z
zhangjinchao01 已提交
1053 1054 1055 1056 1057
        name=name,
        type=LayerType.FC_LAYER,
        size=size,
        bias=ParamAttr.to_bias(bias_attr),
        active_type=act.name,
Q
qijun 已提交
1058 1059 1060
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.FC_LAYER, input, activation=act, size=size)
Z
zhangjinchao01 已提交
1061

1062

1063
@wrap_name_default("print")
1064
def printer_layer(input, format=None, name=None):
1065 1066
    """
    Print the output value of input layers. This layer is useful for debugging.
1067 1068 1069 1070 1071

    :param name: The Layer Name.
    :type name: basestring
    :param input: The input layer. Could be a list/tuple of input layer.
    :type input: LayerOutput|list|tuple
1072
    :return: LayerOutput
1073
    """
1074 1075 1076 1077 1078
    if isinstance(input, LayerOutput):
        input = [input]
    assert isinstance(input, collections.Sequence)  # list or tuple
    for each in input:
        assert isinstance(each, LayerOutput)
1079 1080 1081

    Layer(
        name=name,
1082
        format=format,
1083
        type=LayerType.PRINT_LAYER,
Q
qijun 已提交
1084
        inputs=[l.name for l in input], )
1085
    # this layer don't return anything, can not be input of other layer.
1086

X
xuwei06 已提交
1087 1088 1089 1090 1091 1092 1093
# Keep print_layer for compatibility with V1 API.
# 'print_layer' does not work for V2 API because it will be changed to
# 'print' for V2 API. But 'print' is a reserved key word in python.


print_layer = printer_layer

Z
zhangjinchao01 已提交
1094

Y
yuan 已提交
1095
@wrap_name_default("priorbox")
G
gaoyuan 已提交
1096
def priorbox_layer(input,
G
gaoyuan 已提交
1097
                   image,
G
gaoyuan 已提交
1098 1099 1100 1101 1102
                   aspect_ratio,
                   variance,
                   min_size,
                   max_size=[],
                   name=None):
Y
yuan 已提交
1103 1104 1105 1106 1107 1108 1109
    """
    Compute the priorbox and set the variance. This layer is necessary for ssd.

    :param name: The Layer Name.
    :type name: basestring
    :param input: The input layer.
    :type input: LayerOutput
G
gaoyuan 已提交
1110 1111
    :param image: The network input image.
    :type image: LayerOutput
Y
yuan 已提交
1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
    :param aspect_ratio: The aspect ratio.
    :type aspect_ratio: list
    :param variance: The bounding box variance.
    :type min_size: The min size of the priorbox width/height.
    :param min_size: list
    :type max_size: The max size of the priorbox width/height. Could be NULL.
    :param max_size: list
    :return: LayerOutput
    """
    # plus one for ratio 1.
    num_filters = (len(aspect_ratio) * 2 + 1 + len(max_size)) * 4
G
gaoyuan 已提交
1123
    size = (input.size / input.num_filters) * num_filters * 2
Y
yuan 已提交
1124 1125 1126
    Layer(
        name=name,
        type=LayerType.PRIORBOX_LAYER,
G
gaoyuan 已提交
1127
        inputs=[input.name, image.name],
Y
yuan 已提交
1128 1129 1130 1131 1132 1133
        size=size,
        min_size=min_size,
        max_size=max_size,
        aspect_ratio=aspect_ratio,
        variance=variance)
    return LayerOutput(
G
gaoyuan 已提交
1134 1135
        name,
        LayerType.PRIORBOX_LAYER,
G
gaoyuan 已提交
1136
        parents=[input, image],
G
gaoyuan 已提交
1137 1138 1139
        num_filters=num_filters,
        size=size)

Z
zhangjinchao01 已提交
1140

1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156
@wrap_name_default("multibox_loss")
def multibox_loss_layer(input_loc,
                        input_conf,
                        priorbox,
                        label,
                        num_classes,
                        overlap_threshold=0.5,
                        neg_pos_ratio=3.0,
                        neg_overlap=0.5,
                        background_id=0,
                        name=None):
    """
    Compute the location loss and the confidence loss for ssd.

    :param name: The Layer Name.
    :type name: basestring
Y
yangyaming 已提交
1157 1158
    :param input_loc: The input predict locations.
    :type input_loc: LayerOutput | List of LayerOutput
1159
    :param input_conf: The input priorbox confidence.
Y
yangyaming 已提交
1160
    :type input_conf: LayerOutput | List of LayerOutput
1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181
    :param priorbox: The input priorbox location and the variance.
    :type priorbox: LayerOutput
    :param label: The input label.
    :type label: LayerOutput
    :param num_classes: The number of the classification.
    :type num_classes: int
    :param overlap_threshold: The threshold of the overlap.
    :type overlap_threshold: float
    :param neg_pos_ratio: The ratio of the negative bbox to the positive bbox.
    :type neg_pos_ratio: float
    :param neg_overlap: The negative bbox overlap threshold.
    :type neg_overlap: float
    :param background_id: The background class index.
    :type background_id: int
    :return: LayerOutput
    """
    if isinstance(input_loc, LayerOutput):
        input_loc = [input_loc]
    assert isinstance(input_loc, collections.Sequence)  # list or tuple
    for each in input_loc:
        assert isinstance(each, LayerOutput)
1182
    input_loc_num = len(input_loc)
1183 1184 1185 1186 1187 1188

    if isinstance(input_conf, LayerOutput):
        input_conf = [input_conf]
    assert isinstance(input_conf, collections.Sequence)  # list or tuple
    for each in input_conf:
        assert isinstance(each, LayerOutput)
1189
    input_conf_num = len(input_conf)
1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226
    # Check the input layer number.
    assert input_loc_num == input_conf_num

    inputs = [priorbox.name, label.name]
    inputs.extend([l.name for l in input_loc])
    inputs.extend([l.name for l in input_conf])
    parents = [priorbox, label]
    parents.extend(input_loc)
    parents.extend(input_conf)

    Layer(
        name=name,
        type=LayerType.MULTIBOX_LOSS_LAYER,
        inputs=inputs,
        input_num=input_loc_num,
        num_classes=num_classes,
        overlap_threshold=overlap_threshold,
        neg_pos_ratio=neg_pos_ratio,
        neg_overlap=neg_overlap,
        background_id=background_id)
    return LayerOutput(
        name, LayerType.MULTIBOX_LOSS_LAYER, parents=parents, size=1)


@wrap_name_default("detection_output")
def detection_output_layer(input_loc,
                           input_conf,
                           priorbox,
                           num_classes,
                           nms_threshold=0.45,
                           nms_top_k=400,
                           keep_top_k=200,
                           confidence_threshold=0.01,
                           background_id=0,
                           name=None):
    """
    Apply the NMS to the output of network and compute the predict bounding
G
gaoyuan 已提交
1227 1228
    box location. The output's shape of this layer could be zero if there is
    no valid bounding box.
1229 1230 1231

    :param name: The Layer Name.
    :type name: basestring
Y
yangyaming 已提交
1232 1233
    :param input_loc: The input predict locations.
    :type input_loc: LayerOutput | List of LayerOutput.
1234
    :param input_conf: The input priorbox confidence.
Y
yangyaming 已提交
1235
    :type input_conf: LayerOutput | List of LayerOutput.
1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256
    :param priorbox: The input priorbox location and the variance.
    :type priorbox: LayerOutput
    :param num_classes: The number of the classification.
    :type num_classes: int
    :param nms_threshold: The Non-maximum suppression threshold.
    :type nms_threshold: float
    :param nms_top_k: The bbox number kept of the NMS's output
    :type nms_top_k: int
    :param keep_top_k: The bbox number kept of the layer's output
    :type keep_top_k: int
    :param confidence_threshold: The classification confidence threshold
    :type confidence_threshold: float
    :param background_id: The background class index.
    :type background_id: int
    :return: LayerOutput
    """
    if isinstance(input_loc, LayerOutput):
        input_loc = [input_loc]
    assert isinstance(input_loc, collections.Sequence)  # list or tuple
    for each in input_loc:
        assert isinstance(each, LayerOutput)
Y
yangyaming 已提交
1257
    input_loc_num = len(input_loc)
1258 1259 1260 1261 1262 1263

    if isinstance(input_conf, LayerOutput):
        input_conf = [input_conf]
    assert isinstance(input_conf, collections.Sequence)  # list or tuple
    for each in input_conf:
        assert isinstance(each, LayerOutput)
Y
yangyaming 已提交
1264 1265
    input_conf_num = len(input_conf)

1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293
    # Check the input layer number.
    assert input_loc_num == input_conf_num

    inputs = [priorbox.name]
    inputs.extend([l.name for l in input_loc])
    inputs.extend([l.name for l in input_conf])
    parents = [priorbox]
    parents.extend(input_loc)
    parents.extend(input_conf)

    size = keep_top_k * 7

    Layer(
        name=name,
        type=LayerType.DETECTION_OUTPUT_LAYER,
        inputs=inputs,
        size=size,
        input_num=input_loc_num,
        num_classes=num_classes,
        nms_threshold=nms_threshold,
        nms_top_k=nms_top_k,
        keep_top_k=keep_top_k,
        confidence_threshold=confidence_threshold,
        background_id=background_id)
    return LayerOutput(
        name, LayerType.DETECTION_OUTPUT_LAYER, parents=parents, size=size)


1294 1295
@wrap_name_default("cross_channel_norm")
def cross_channel_norm_layer(input, name=None, param_attr=None):
G
gaoyuan 已提交
1296 1297 1298 1299 1300
    """
    Normalize a layer's output. This layer is necessary for ssd.
    This layer applys normalize across the channels of each sample to
    a conv layer's output and scale the output by a group of trainable
    factors which dimensions equal to the channel's number.
G
gaoyuan 已提交
1301

G
gaoyuan 已提交
1302 1303 1304 1305 1306 1307 1308 1309
    :param name: The Layer Name.
    :type name: basestring
    :param input: The input layer.
    :type input: LayerOutput
    :param param_attr: The Parameter Attribute|list.
    :type param_attr: ParameterAttribute
    :return: LayerOutput
    """
1310
    assert input.num_filters is not None
G
gaoyuan 已提交
1311 1312
    Layer(
        name=name,
1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325
        type=LayerType.NORM_LAYER,
        inputs=[
            Input(
                input.name,
                norm=Norm(
                    norm_type="cross-channel-norm",
                    channels=input.num_filters,
                    size=input.size,
                    scale=0,
                    pow=0,
                    blocked=0),
                **param_attr.attr)
        ])
G
gaoyuan 已提交
1326 1327
    return LayerOutput(
        name,
1328
        LayerType.NORM_LAYER,
G
gaoyuan 已提交
1329 1330 1331 1332 1333
        parents=input,
        num_filters=input.num_filters,
        size=input.size)


Z
zhangjinchao01 已提交
1334 1335 1336 1337
@wrap_name_default("seq_pooling")
@wrap_bias_attr_default(has_bias=False)
@wrap_param_default(['pooling_type'], default_factory=lambda _: MaxPooling())
@layer_support()
Q
qijun 已提交
1338 1339 1340 1341
def pooling_layer(input,
                  pooling_type=None,
                  name=None,
                  bias_attr=None,
L
Luo Tao 已提交
1342
                  agg_level=AggregateLevel.TO_NO_SEQUENCE,
1343
                  stride=-1,
Z
zhangjinchao01 已提交
1344 1345 1346 1347
                  layer_attr=None):
    """
    Pooling layer for sequence inputs, not used for Image.

1348 1349
    If stride > 0, this layer slides a window whose size is determined by stride,
    and return the pooling value of the window as the output. Thus, a long sequence
X
xuwei06 已提交
1350 1351 1352
    will be shorten.

    The parameter stride specifies the intervals at which to apply the pooling
L
Luo Tao 已提交
1353
    operation. Note that for sequence with sub-sequence, the default value
1354 1355
    of stride is -1.

Z
zhangjinchao01 已提交
1356 1357 1358 1359 1360 1361
    The example usage is:

    .. code-block:: python

       seq_pool = pooling_layer(input=layer,
                                pooling_type=AvgPooling(),
L
Luo Tao 已提交
1362
                                agg_level=AggregateLevel.TO_NO_SEQUENCE)
Z
zhangjinchao01 已提交
1363

L
Luo Tao 已提交
1364 1365
    :param agg_level: AggregateLevel.TO_NO_SEQUENCE or
                      AggregateLevel.TO_SEQUENCE
Z
zhangjinchao01 已提交
1366 1367 1368 1369 1370 1371 1372 1373
    :type agg_level: AggregateLevel
    :param name: layer name.
    :type name: basestring
    :param input: input layer name.
    :type input: LayerOutput
    :param pooling_type: Type of pooling, MaxPooling(default), AvgPooling,
                         SumPooling, SquareRootNPooling.
    :type pooling_type: BasePoolingType|None
L
Luo Tao 已提交
1374
    :param stride: The step size between successive pooling regions.
1375
    :type stride: Int
Z
zhangjinchao01 已提交
1376 1377 1378 1379
    :param bias_attr: Bias parameter attribute. False if no bias.
    :type bias_attr: ParameterAttribute|None|False
    :param layer_attr: The Extra Attributes for layer, such as dropout.
    :type layer_attr: ExtraLayerAttribute|None
D
dangqingqing 已提交
1380
    :return: LayerOutput object.
Y
Yu Yang 已提交
1381
    :rtype: LayerOutput
Z
zhangjinchao01 已提交
1382 1383
    """
    extra_dict = dict()
1384
    # noinspection PyUnresolvedReferences
Z
zhangjinchao01 已提交
1385 1386
    if isinstance(pooling_type, AvgPooling):
        extra_dict['average_strategy'] = pooling_type.strategy
1387 1388 1389 1390
    elif isinstance(pooling_type, MaxPooling) and \
                    pooling_type.output_max_index is not None:
        assert isinstance(pooling_type.output_max_index, bool)
        extra_dict['output_max_index'] = pooling_type.output_max_index
Z
zhangjinchao01 已提交
1391 1392
    extra_dict.update(ExtraLayerAttribute.to_kwargs(layer_attr))

1393 1394 1395
    if agg_level == AggregateLevel.TO_SEQUENCE:
        assert stride == -1

Z
zhangjinchao01 已提交
1396 1397 1398 1399 1400 1401
    Layer(
        name=name,
        type=pooling_type.name,
        inputs=[Input(input.name)],
        bias=ParamAttr.to_bias(bias_attr),
        trans_type=agg_level,
1402
        stride=stride,
Q
qijun 已提交
1403
        **extra_dict)
Z
zhangjinchao01 已提交
1404

Q
qijun 已提交
1405 1406
    return LayerOutput(
        name, pooling_type.name, parents=[input], size=input.size)
Z
zhangjinchao01 已提交
1407

Q
qijun 已提交
1408

Z
zhangjinchao01 已提交
1409 1410
@wrap_bias_attr_default()
@wrap_param_attr_default()
Q
qijun 已提交
1411
@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation())
Z
zhangjinchao01 已提交
1412 1413
@wrap_act_default(param_names=["act", 'state_act'], act=TanhActivation())
@wrap_name_default("lstmemory")
1414
@layer_support()
Q
qijun 已提交
1415 1416
def lstmemory(input,
              name=None,
1417
              size=None,
Q
qijun 已提交
1418 1419 1420 1421 1422 1423
              reverse=False,
              act=None,
              gate_act=None,
              state_act=None,
              bias_attr=None,
              param_attr=None,
Z
zhangjinchao01 已提交
1424 1425 1426 1427 1428 1429 1430 1431
              layer_attr=None):
    """
    Long Short-term Memory Cell.

    The memory cell was implemented as follow equations.

    ..  math::

L
luotao02 已提交
1432
        i_t & = \\sigma(W_{xi}x_{t} + W_{hi}h_{t-1} + W_{ci}c_{t-1} + b_i)
Z
zhangjinchao01 已提交
1433

L
luotao02 已提交
1434
        f_t & = \\sigma(W_{xf}x_{t} + W_{hf}h_{t-1} + W_{cf}c_{t-1} + b_f)
Z
zhangjinchao01 已提交
1435

L
luotao02 已提交
1436
        c_t & = f_tc_{t-1} + i_t tanh (W_{xc}x_t+W_{hc}h_{t-1} + b_c)
Z
zhangjinchao01 已提交
1437

L
luotao02 已提交
1438
        o_t & = \\sigma(W_{xo}x_{t} + W_{ho}h_{t-1} + W_{co}c_t + b_o)
Z
zhangjinchao01 已提交
1439

L
luotao02 已提交
1440
        h_t & = o_t tanh(c_t)
Z
zhangjinchao01 已提交
1441 1442


C
caoying03 已提交
1443
    NOTE: In PaddlePaddle's implementation, the multiplications
Z
zhangjinchao01 已提交
1444
    :math:`W_{xi}x_{t}` , :math:`W_{xf}x_{t}`,
C
caoying03 已提交
1445 1446 1447 1448
    :math:`W_{xc}x_t`, :math:`W_{xo}x_{t}` are not done in the lstmemory layer,
    so an additional mixed_layer with full_matrix_projection or a fc_layer must
    be included in the configuration file to complete the input-to-hidden
    mappings before lstmemory is called.
Z
zhangjinchao01 已提交
1449

C
caoying03 已提交
1450
    NOTE: This is a low level user interface. You can use network.simple_lstm
Z
zhangjinchao01 已提交
1451 1452
    to config a simple plain lstm layer.

C
caoying03 已提交
1453 1454 1455 1456
    Please refer to **Generating Sequences With Recurrent Neural Networks** for
    more details about LSTM.

    Link_ goes as below.
Z
zhangjinchao01 已提交
1457 1458 1459 1460 1461

    .. _Link: http://arxiv.org/abs/1308.0850

    :param name: The lstmemory layer name.
    :type name: basestring
1462 1463
    :param size: DEPRECATED. size of the lstm cell
    :type size: int
Z
zhangjinchao01 已提交
1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481
    :param input: input layer name.
    :type input: LayerOutput
    :param reverse: is sequence process reversed or not.
    :type reverse: bool
    :param act: activation type, TanhActivation by default. :math:`h_t`
    :type act: BaseActivation
    :param gate_act: gate activation type, SigmoidActivation by default.
    :type gate_act: BaseActivation
    :param state_act: state activation type, TanhActivation by default.
    :type state_act: BaseActivation

    :param bias_attr: Bias attribute. None means default bias. False means no
                      bias.
    :type bias_attr: ParameterAttribute|None|False
    :param param_attr: Parameter Attribute.
    :type param_attr: ParameterAttribute|None|False
    :param layer_attr: Extra Layer attribute
    :type layer_attr: ExtraLayerAttribute|None
D
dangqingqing 已提交
1482
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
1483 1484 1485 1486 1487 1488
    :rtype: LayerOutput
    """

    assert gate_act.support_hppl
    assert state_act.support_hppl
    assert act.support_hppl
1489
    assert input.size is not None and input.size % 4 == 0
1490

1491 1492 1493 1494 1495
    if size is not None:
        if input.size / 4 == size:
            plog = logger.warning
        else:
            plog = logger.fatal
1496 1497 1498
        plog("size of lstmemory layer: %s is automatically set to "
             "size of input layer / 4. The parameter size passing to "
             "this layer is ignored." % (name))
Z
zhangjinchao01 已提交
1499

Q
qijun 已提交
1500 1501 1502 1503 1504 1505 1506 1507 1508 1509
    Layer(
        name=name,
        type=LayerType.LSTMEMORY,
        active_type=act.name,
        active_state_type=state_act.name,
        active_gate_type=gate_act.name,
        reversed=reverse,
        bias=ParamAttr.to_bias(bias_attr),
        inputs=[Input(input.name, **param_attr.attr)],
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
1510

Q
qijun 已提交
1511 1512 1513 1514 1515
    return LayerOutput(
        name,
        LayerType.LSTMEMORY, [input],
        size=input.size / 4,
        reverse=reverse)
1516

Z
zhangjinchao01 已提交
1517 1518 1519

@wrap_bias_attr_default()
@wrap_param_attr_default()
Q
qijun 已提交
1520
@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation())
Z
zhangjinchao01 已提交
1521 1522
@wrap_act_default(param_names=["act"], act=TanhActivation())
@wrap_name_default("gru")
1523
@layer_support()
Q
qijun 已提交
1524
def grumemory(input,
1525
              size=None,
Q
qijun 已提交
1526 1527 1528 1529 1530 1531
              name=None,
              reverse=False,
              act=None,
              gate_act=None,
              bias_attr=None,
              param_attr=None,
Z
zhangjinchao01 已提交
1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552
              layer_attr=None):
    """
    Gate Recurrent Unit Layer.

    The memory cell was implemented as follow equations.

    1. update gate :math:`z`: defines how much of the previous memory to
    keep around or the unit updates its activations. The update gate
    is computed by:

    ..  math::

        z_t = \\sigma(W_{z}x_{t} + U_{z}h_{t-1} + b_z)

    2. reset gate :math:`r`: determines how to combine the new input with the
    previous memory. The reset gate is computed similarly to the update gate:

    ..  math::

        r_t = \\sigma(W_{r}x_{t} + U_{r}h_{t-1} + b_r)

C
caoying03 已提交
1553 1554
    3. The candidate activation :math:`\\tilde{h_t}` is computed similarly to
    that of the traditional recurrent unit:
Z
zhangjinchao01 已提交
1555 1556 1557 1558 1559

    ..  math::

        {\\tilde{h_t}} = tanh(W x_{t} + U (r_{t} \odot h_{t-1}) + b)

C
caoying03 已提交
1560 1561 1562
    4. The hidden activation :math:`h_t` of the GRU at time t is a linear
    interpolation between the previous activation :math:`h_{t-1}` and the
    candidate activation :math:`\\tilde{h_t}`:
Z
zhangjinchao01 已提交
1563 1564 1565 1566 1567

    ..  math::

        h_t = (1 - z_t) h_{t-1} + z_t {\\tilde{h_t}}

C
caoying03 已提交
1568
    NOTE: In PaddlePaddle's implementation, the multiplication operations
Z
zhangjinchao01 已提交
1569
    :math:`W_{r}x_{t}`, :math:`W_{z}x_{t}` and :math:`W x_t` are not computed in
C
caoying03 已提交
1570 1571 1572
    gate_recurrent layer. Consequently, an additional mixed_layer with
    full_matrix_projection or a fc_layer must be included before grumemory
    is called.
Z
zhangjinchao01 已提交
1573

C
caoying03 已提交
1574 1575 1576
    More details can be found by referring to `Empirical Evaluation of Gated
    Recurrent Neural Networks on Sequence Modeling.
    <https://arxiv.org/abs/1412.3555>`_
Z
zhangjinchao01 已提交
1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587

    The simple usage is:

    .. code-block:: python

       gru = grumemory(input)

    :param name: The gru layer name.
    :type name: None|basestring
    :param input: input layer.
    :type input: LayerOutput.
1588 1589
    :param size: DEPRECATED. size of the gru cell
    :type size: int
1590
    :param reverse: Whether sequence process is reversed or not.
Z
zhangjinchao01 已提交
1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605
    :type reverse: bool
    :param act: activation type, TanhActivation by default. This activation
                affects the :math:`{\\tilde{h_t}}`.
    :type act: BaseActivation
    :param gate_act: gate activation type, SigmoidActivation by default.
                     This activation affects the :math:`z_t` and :math:`r_t`. It is the
                     :math:`\\sigma` in the above formula.
    :type gate_act: BaseActivation
    :param bias_attr: Bias attribute. None means default bias. False means no
                      bias.
    :type bias_attr: ParameterAttribute|None|False
    :param param_attr: Parameter Attribute.
    :type param_attr: ParameterAttribute|None|False
    :param layer_attr: Extra Layer attribute
    :type layer_attr: ExtraLayerAttribute|None
D
dangqingqing 已提交
1606
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
1607 1608 1609 1610
    :rtype: LayerOutput
    """
    assert act.support_hppl
    assert gate_act.support_hppl
1611 1612 1613 1614 1615 1616
    assert input.size is not None and input.size % 3 == 0
    if size is not None:
        if input.size / 3 == size:
            plog = logger.warning
        else:
            plog = logger.fatal
1617 1618 1619
        plog("size of grumemory layer: %s is automatically set to "
             "size of input layer / 3. The parameter size passing to this "
             "layer is ignored." % (name))
Z
zhangjinchao01 已提交
1620

Q
qijun 已提交
1621 1622 1623 1624 1625 1626 1627 1628 1629
    Layer(
        name=name,
        type=LayerType.GRUMEMORY,
        active_type=act.name,
        active_gate_type=gate_act.name,
        reversed=reverse,
        bias=ParamAttr.to_bias(bias_attr),
        inputs=[Input(input.name, **param_attr.attr)],
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
1630

Q
qijun 已提交
1631 1632 1633 1634 1635
    return LayerOutput(
        name,
        LayerType.GRUMEMORY, [input],
        size=input.size / 3,
        reverse=reverse)
1636

Z
zhangjinchao01 已提交
1637 1638 1639

@wrap_name_default()
@layer_support()
Q
qijun 已提交
1640 1641
def last_seq(input,
             name=None,
L
Luo Tao 已提交
1642
             agg_level=AggregateLevel.TO_NO_SEQUENCE,
1643
             stride=-1,
Z
zhangjinchao01 已提交
1644 1645 1646 1647
             layer_attr=None):
    """
    Get Last Timestamp Activation of a sequence.

1648 1649 1650
    If stride > 0, this layer slides a window whose size is determined by stride,
    and return the last value of the window as the output. Thus, a long sequence
    will be shorten. Note that for sequence with sub-sequence, the default value
L
Luo Tao 已提交
1651
    of stride is -1.
1652

L
Luo Tao 已提交
1653 1654 1655 1656 1657 1658
    The simple usage is:

    .. code-block:: python

       seq = last_seq(input=layer)

Z
zhangjinchao01 已提交
1659 1660 1661 1662 1663
    :param agg_level: Aggregated level
    :param name: Layer name.
    :type name: basestring
    :param input: Input layer name.
    :type input: LayerOutput
L
Luo Tao 已提交
1664
    :param stride: The step size between successive pooling regions.
1665
    :type stride: Int
Z
zhangjinchao01 已提交
1666 1667
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
D
dangqingqing 已提交
1668
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
1669 1670
    :rtype: LayerOutput
    """
1671 1672 1673 1674 1675 1676
    if input.reverse is not None and input.reverse:
        logger.warning("You are getting the last instance of a sequence that"
                       " is a output of a REVERSED layer. There is no time"
                       " series information at all. Maybe you want to use"
                       " first_seq instead.")

L
Luo Tao 已提交
1677
    if agg_level == AggregateLevel.TO_SEQUENCE:
1678 1679
        assert stride == -1

Z
zhangjinchao01 已提交
1680 1681 1682 1683 1684
    Layer(
        name=name,
        type=LayerType.SEQUENCE_LAST_INSTANCE,
        inputs=[input.name],
        trans_type=agg_level,
1685
        stride=stride,
Q
qijun 已提交
1686 1687 1688 1689 1690 1691
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name,
        LayerType.SEQUENCE_LAST_INSTANCE,
        parents=[input],
        size=input.size)
Z
zhangjinchao01 已提交
1692 1693 1694 1695


@wrap_name_default()
@layer_support()
Q
qijun 已提交
1696 1697
def first_seq(input,
              name=None,
L
Luo Tao 已提交
1698
              agg_level=AggregateLevel.TO_NO_SEQUENCE,
1699
              stride=-1,
Z
zhangjinchao01 已提交
1700 1701 1702 1703
              layer_attr=None):
    """
    Get First Timestamp Activation of a sequence.

1704 1705 1706
    If stride > 0, this layer slides a window whose size is determined by stride,
    and return the first value of the window as the output. Thus, a long sequence
    will be shorten. Note that for sequence with sub-sequence, the default value
L
Luo Tao 已提交
1707
    of stride is -1.
1708

L
Luo Tao 已提交
1709 1710 1711 1712 1713 1714
    The simple usage is:

    .. code-block:: python

       seq = first_seq(input=layer)

Z
zhangjinchao01 已提交
1715 1716 1717 1718 1719
    :param agg_level: aggregation level
    :param name: Layer name.
    :type name: basestring
    :param input: Input layer name.
    :type input: LayerOutput
L
Luo Tao 已提交
1720
    :param stride: The step size between successive pooling regions.
1721
    :type stride: Int
Z
zhangjinchao01 已提交
1722 1723
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
D
dangqingqing 已提交
1724
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
1725 1726
    :rtype: LayerOutput
    """
1727 1728 1729 1730 1731 1732 1733

    if input.reverse is not None and not input.reverse:
        logger.warning('You are getting the first instance for a time series,'
                       ' and it is a normal recurrent layer output. There is no'
                       ' time series information at all. Maybe you want to use'
                       ' last_seq instead.')

L
Luo Tao 已提交
1734
    if agg_level == AggregateLevel.TO_SEQUENCE:
1735 1736
        assert stride == -1

Z
zhangjinchao01 已提交
1737 1738 1739 1740 1741
    Layer(
        name=name,
        type=LayerType.SEQUENCE_FIRST_INSTANCE,
        inputs=[input.name],
        trans_type=agg_level,
1742
        stride=stride,
Q
qijun 已提交
1743 1744 1745 1746 1747 1748
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name,
        LayerType.SEQUENCE_FIRST_INSTANCE,
        parents=[input],
        size=input.size)
Z
zhangjinchao01 已提交
1749 1750 1751


class ExpandLevel(object):
1752 1753 1754 1755 1756
    """
    Please refer to AggregateLevel first.

    ExpandLevel supports two modes:

L
Luo Tao 已提交
1757 1758
    - :code:`ExpandLevel.FROM_NO_SEQUENCE` means the expansion acts on
      :code:`NO_SEQUENCE`, which will be expanded to
1759 1760
      :code:`SEQUENCE` or :code:`SUB_SEQUENCE`.

L
Luo Tao 已提交
1761 1762
    - :code:`ExpandLevel.FROM_SEQUENCE` means the expansion acts on
      :code:`SEQUENCE`, which will be expanded to
1763 1764
      :code:`SUB_SEQUENCE`.
    """
L
Luo Tao 已提交
1765 1766
    FROM_NO_SEQUENCE = AggregateLevel.TO_NO_SEQUENCE
    FROM_SEQUENCE = AggregateLevel.TO_SEQUENCE
1767 1768
    # compatible with previous configuration
    FROM_TIMESTEP = FROM_NO_SEQUENCE
Z
zhangjinchao01 已提交
1769

1770

Z
zhangjinchao01 已提交
1771 1772
@wrap_name_default()
@layer_support()
Q
qijun 已提交
1773 1774
def expand_layer(input,
                 expand_as,
Z
zhangjinchao01 已提交
1775 1776
                 name=None,
                 bias_attr=False,
L
Luo Tao 已提交
1777
                 expand_level=ExpandLevel.FROM_NO_SEQUENCE,
Z
zhangjinchao01 已提交
1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788
                 layer_attr=None):
    """
    A layer for "Expand Dense data or (sequence data where the length of each
    sequence is one) to sequence data."

    The example usage is:

    .. code-block:: python

       expand = expand_layer(input=layer1,
                             expand_as=layer2,
L
Luo Tao 已提交
1789
                             expand_level=ExpandLevel.FROM_NO_SEQUENCE)
Z
zhangjinchao01 已提交
1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803

    :param input: Input layer
    :type input: LayerOutput
    :param expand_as: Expand as this layer's sequence info.
    :type expand_as: LayerOutput
    :param name: Layer name.
    :type name: basestring
    :param bias_attr: Bias attribute. None means default bias. False means no
                      bias.
    :type bias_attr: ParameterAttribute|None|False
    :param expand_level: whether input layer is timestep(default) or sequence.
    :type expand_level: ExpandLevel
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
D
dangqingqing 已提交
1804
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
1805 1806 1807 1808 1809 1810 1811 1812 1813
    :rtype: LayerOutput
    """

    Layer(
        inputs=[input.name, expand_as.name],
        name=name,
        bias=ParamAttr.to_bias(bias_attr=bias_attr),
        type=LayerType.EXPAND_LAYER,
        trans_type=expand_level,
Q
qijun 已提交
1814 1815 1816 1817 1818 1819
        **ExtraAttr.to_kwargs(layer_attr))
    return LayerOutput(
        name=name,
        size=input.size,
        layer_type=LayerType.EXPAND_LAYER,
        parents=[input, expand_as])
Z
zhangjinchao01 已提交
1820 1821


X
xuwei06 已提交
1822
@wrap_name_default()
X
xuwei06 已提交
1823
@wrap_act_default(act=IdentityActivation())
X
xuwei06 已提交
1824
@layer_support()
X
xuwei06 已提交
1825 1826 1827
def repeat_layer(input,
                 num_repeats,
                 as_row_vector=True,
X
xuwei06 已提交
1828
                 act=None,
X
xuwei06 已提交
1829 1830
                 name=None,
                 layer_attr=None):
X
xuwei06 已提交
1831
    """
X
xuwei06 已提交
1832
    A layer for repeating the input for num_repeats times.
X
xuwei06 已提交
1833

X
xuwei06 已提交
1834
    If as_row_vector:
X
xuwei06 已提交
1835
    .. math::
X
xuwei06 已提交
1836 1837 1838 1839 1840
       y  = [x_1,\cdots, x_n, \cdots, x_1, \cdots, x_n]
    If not as_row_vector:
    .. math::
       y  = [x_1,\cdots, x_1, \cdots, x_n, \cdots, x_n]

X
xuwei06 已提交
1841 1842 1843 1844 1845

    The example usage is:

    .. code-block:: python

L
Luo Tao 已提交
1846
       expand = repeat_layer(input=layer, num_repeats=4)
X
xuwei06 已提交
1847 1848 1849 1850 1851 1852

    :param input: Input layer
    :type input: LayerOutput
    :param num_repeats: Repeat the input so many times
    :type num_repeats: int
    :param name: Layer name.
X
xuwei06 已提交
1853 1854 1855 1856 1857 1858
    :param as_row_vector: True for treating input as row vector and repeating
                          in the column direction.  This is equivalent to apply
                          concat_layer() with num_repeats same input.
                          False for treating input as column vector and repeating
                          in the row direction.
    :type as_row_vector: bool
X
xuwei06 已提交
1859 1860
    :param act: Activation type.
    :type act: BaseActivation
X
xuwei06 已提交
1861 1862 1863 1864 1865 1866 1867 1868 1869 1870
    :type name: basestring
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
    :return: LayerOutput object.
    :rtype: LayerOutput
    """

    l = Layer(
        inputs=[input.name],
        name=name,
X
xuwei06 已提交
1871
        active_type=act.name,
X
xuwei06 已提交
1872
        num_filters=num_repeats,
X
xuwei06 已提交
1873
        as_row_vector=as_row_vector,
X
xuwei06 已提交
1874
        type=LayerType.FEATURE_MAP_EXPAND_LAYER,
Q
qijun 已提交
1875 1876 1877 1878 1879
        **ExtraAttr.to_kwargs(layer_attr))
    return LayerOutput(
        name=name,
        size=l.config.size,
        layer_type=LayerType.FEATURE_MAP_EXPAND_LAYER,
X
xuwei06 已提交
1880
        activation=act,
Q
qijun 已提交
1881 1882
        parents=[input])

X
xuwei06 已提交
1883

1884 1885 1886
@wrap_name_default("seqreshape")
@wrap_act_default(act=IdentityActivation())
@wrap_bias_attr_default(has_bias=False)
1887
@layer_support(ERROR_CLIPPING, DROPOUT)
1888 1889 1890 1891 1892 1893 1894 1895
def seq_reshape_layer(input,
                      reshape_size,
                      act=None,
                      name=None,
                      layer_attr=None,
                      bias_attr=None):
    """
    A layer for reshaping the sequence. Assume the input sequence has T instances,
1896
    the dimension of each instance is M, and the input reshape_size is N, then the
1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938
    output sequence has T*M/N instances, the dimension of each instance is N.

    Note that T*M/N must be an integer.

    The example usage is:

    .. code-block:: python

       reshape = seq_reshape_layer(input=layer, reshape_size=4)

    :param input: Input layer.
    :type input: LayerOutput
    :param reshape_size: the size of reshaped sequence.
    :type reshape_size: int
    :param name: Layer name.
    :type name: basestring
    :param act: Activation type.
    :type act: BaseActivation
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
    :param bias_attr: The Bias Attribute. If no bias, then pass False or
                      something not type of ParameterAttribute. None will get a
                      default Bias.
    :type bias_attr: ParameterAttribute or None or bool
    :return: LayerOutput object.
    :rtype: LayerOutput
    """

    Layer(
        inputs=[input.name],
        name=name,
        size=reshape_size,
        type=LayerType.SEQUENCE_RESHAPE,
        bias=ParamAttr.to_bias(bias_attr),
        **ExtraAttr.to_kwargs(layer_attr))
    return LayerOutput(
        name=name,
        size=reshape_size,
        layer_type=LayerType.SEQUENCE_RESHAPE,
        parents=[input])


Z
zhangjinchao01 已提交
1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966
@wrap_name_default()
@layer_support()
def interpolation_layer(input, weight, name=None, layer_attr=None):
    """
    This layer is for linear interpolation with two inputs,
    which is used in NEURAL TURING MACHINE.

    .. math::
       y.row[i] = w[i] * x_1.row[i] + (1 - w[i]) * x_2.row[i]

    where :math:`x_1` and :math:`x_2` are two (batchSize x dataDim) inputs,
    :math:`w` is (batchSize x 1) weight vector, and :math:`y` is
    (batchSize x dataDim) output.

    The example usage is:

    .. code-block:: python

       interpolation = interpolation_layer(input=[layer1, layer2], weight=layer3)

    :param input: Input layer.
    :type input: list|tuple
    :param weight: Weight layer.
    :type weight: LayerOutput
    :param name: Layer name.
    :type name: basestring
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
D
dangqingqing 已提交
1967
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
1968 1969
    :rtype: LayerOutput
    """
1970
    assert isinstance(input, collections.Sequence)
Z
zhangjinchao01 已提交
1971
    assert len(input) == 2
1972 1973 1974 1975 1976 1977 1978
    assert isinstance(input[0], LayerOutput) and isinstance(input[1],
                                                            LayerOutput)
    if input[0].size is not None and input[1].size is not None:
        assert input[0].size == input[1].size
    assert isinstance(weight, LayerOutput)
    if weight.size is not None:
        assert weight.size == 1
Z
zhangjinchao01 已提交
1979 1980 1981 1982
    Layer(
        name=name,
        type=LayerType.INTERPOLATION_LAYER,
        inputs=[weight.name, input[0].name, input[1].name],
Q
qijun 已提交
1983 1984 1985 1986 1987 1988
        **ExtraAttr.to_kwargs(layer_attr))
    return LayerOutput(
        name,
        LayerType.INTERPOLATION_LAYER,
        parents=[weight, input[0], input[1]],
        size=input[0].size)
Z
zhangjinchao01 已提交
1989 1990


L
liaogang 已提交
1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006
@wrap_name_default()
@layer_support()
def bilinear_interp_layer(input,
                          out_size_x=None,
                          out_size_y=None,
                          name=None,
                          layer_attr=None):
    """
    This layer is to implement bilinear interpolation on conv layer output.

    Please refer to Wikipedia: https://en.wikipedia.org/wiki/Bilinear_interpolation

    The simple usage is:

    .. code-block:: python

L
liaogang 已提交
2007
       bilinear = bilinear_interp_layer(input=layer1, out_size_x=64, out_size_y=64)
X
xuwei06 已提交
2008

L
liaogang 已提交
2009
    :param   input:        A input layer.
L
liaogang 已提交
2010
    :type    input:        LayerOutput.
L
liaogang 已提交
2011
    :param   out_size_x:   bilinear interpolation output width.
X
xuwei06 已提交
2012
    :type    out_size_x:   int|None
L
liaogang 已提交
2013
    :param   out_size_y:   bilinear interpolation output height.
L
liaogang 已提交
2014
    :type    out_size_y:   int|None
L
liaogang 已提交
2015
    :param   name:         The layer's name, which cna not be specified.
L
liaogang 已提交
2016
    :type    name:         None|basestring
L
liaogang 已提交
2017
    :param   layer_attr:   Extra Layer attribute.
L
liaogang 已提交
2018 2019 2020 2021 2022 2023 2024
    :type    layer_attr:   ExtraLayerAttribute
    :return: LayerOutput object.
    :rtype:  LayerOutput
    """
    assert input.layer_type == LayerType.CONV_LAYER
    assert isinstance(input.activation, LinearActivation)
    assert out_size_x > 0 and out_size_y > 0
L
liaogang 已提交
2025
    assert input.num_filters is not None
L
liaogang 已提交
2026
    num_channels = input.num_filters
Q
qijun 已提交
2027 2028 2029 2030 2031 2032 2033
    l = Layer(
        name=name,
        inputs=Input(
            input.name,
            bilinear_interp=BilinearInterp(
                out_size_x=out_size_x,
                out_size_y=out_size_y,
L
Luo Tao 已提交
2034
                channels=num_channels)),
Q
qijun 已提交
2035 2036 2037 2038 2039 2040 2041 2042 2043
        type=LayerType.BILINEAR_INTERP_LAYER,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name,
        LayerType.BILINEAR_INTERP_LAYER,
        parents=[input],
        num_filters=num_channels,
        size=l.config.size)

L
liaogang 已提交
2044

Z
zhangjinchao01 已提交
2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071
@wrap_name_default()
@layer_support()
def power_layer(input, weight, name=None, layer_attr=None):
    """
    This layer applies a power function to a vector element-wise,
    which is used in NEURAL TURING MACHINE.

    .. math::
       y = x^w

    where :math:`x` is a input vector, :math:`w` is scalar weight,
    and :math:`y` is a output vector.

    The example usage is:

    .. code-block:: python

       power = power_layer(input=layer1, weight=layer2)

    :param input: Input layer.
    :type input: LayerOutput
    :param weight: Weight layer.
    :type weight: LayerOutput
    :param name: Layer name.
    :type name: basestring
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
D
dangqingqing 已提交
2072
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
2073 2074
    :rtype: LayerOutput
    """
2075 2076 2077
    assert isinstance(input, LayerOutput) and isinstance(weight, LayerOutput)
    if weight.size is not None:
        assert weight.size == 1
Z
zhangjinchao01 已提交
2078 2079 2080
    Layer(
        name=name,
        type=LayerType.POWER_LAYER,
2081
        inputs=[weight.name, input.name],
Q
qijun 已提交
2082 2083 2084
        **ExtraAttr.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.POWER_LAYER, parents=[input, weight], size=input.size)
Z
zhangjinchao01 已提交
2085 2086 2087 2088 2089 2090


@wrap_name_default()
@layer_support()
def scaling_layer(input, weight, name=None, layer_attr=None):
    """
2091
    A layer for multiplying input vector by weight scalar.
Z
zhangjinchao01 已提交
2092 2093

    .. math::
2094
       y  = w x
Z
zhangjinchao01 已提交
2095

2096 2097 2098 2099 2100
    where :math:`x` is size=dataDim input, :math:`w` is size=1 weight,
    and :math:`y` is size=dataDim output.

    Note that the above computation is for one sample. Multiple samples are
    processed in one batch.
Z
zhangjinchao01 已提交
2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115

    The example usage is:

    .. code-block:: python

       scale = scaling_layer(input=layer1, weight=layer2)

    :param input: Input layer.
    :type input: LayerOutput
    :param weight: Weight layer.
    :type weight: LayerOutput
    :param name: Layer name.
    :type name: basestring
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
D
dangqingqing 已提交
2116
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
2117 2118
    :rtype: LayerOutput
    """
2119 2120 2121
    assert isinstance(weight, LayerOutput) and isinstance(input, LayerOutput)
    if weight.size is not None:
        assert weight.size == 1
Z
zhangjinchao01 已提交
2122 2123 2124 2125
    Layer(
        name=name,
        type=LayerType.SCALING_LAYER,
        inputs=[weight.name, input.name],
Q
qijun 已提交
2126 2127 2128
        **ExtraAttr.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.SCALING_LAYER, parents=[weight, input], size=input.size)
Z
zhangjinchao01 已提交
2129 2130 2131 2132 2133 2134


@wrap_name_default()
@layer_support()
def trans_layer(input, name=None, layer_attr=None):
    """
2135
    A layer for transposing a minibatch matrix.
Z
zhangjinchao01 已提交
2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153

    .. math::
       y = x^\mathrm{T}

    where :math:`x` is (M x N) input, and :math:`y` is (N x M) output.

    The example usage is:

    .. code-block:: python

       trans = trans_layer(input=layer)

    :param input: Input layer.
    :type input: LayerOutput
    :param name: Layer name.
    :type name: basestring
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
D
dangqingqing 已提交
2154
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
2155 2156 2157 2158 2159 2160
    :rtype: LayerOutput
    """
    Layer(
        name=name,
        type=LayerType.TRANS_LAYER,
        inputs=[input.name],
Q
qijun 已提交
2161 2162 2163
        **ExtraAttr.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.TRANS_LAYER, parents=[input], size=input.size)
Z
zhangjinchao01 已提交
2164 2165


2166 2167
@wrap_name_default()
@layer_support()
H
Haonan 已提交
2168
def rotate_layer(input, height, width, name=None, layer_attr=None):
2169
    """
H
Haonan 已提交
2170 2171
    A layer for rotating 90 degrees (clock-wise) for each feature channel,
    usually used when the input sample is some image or feature map.
2172 2173

    .. math::
H
Haonan 已提交
2174
       y(j,i,:) = x(M-i-1,j,:)
2175

H
Haonan 已提交
2176
    where :math:`x` is (M x N x C) input, and :math:`y` is (N x M x C) output.
2177 2178 2179 2180 2181 2182

    The example usage is:

    .. code-block:: python

       rot = rotate_layer(input=layer,
H
Haonan 已提交
2183 2184
                          height=100,
                          width=100)
2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197

    :param input: Input layer.
    :type input: LayerOutput
    :param height: The height of the sample matrix
    :type height: int
    :param name: Layer name.
    :type name: basestring
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
    assert isinstance(input, LayerOutput)
H
Haonan 已提交
2198 2199 2200
    l = Layer(
        name=name,
        height=height,
H
Haonan 已提交
2201
        width=width,
H
Haonan 已提交
2202 2203 2204 2205 2206 2207 2208 2209
        type=LayerType.ROTATE_LAYER,
        inputs=[input.name],
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name=name,
        layer_type=LayerType.ROTATE_LAYER,
        parents=[input],
        size=l.config.size)
2210 2211


Z
zhangjinchao01 已提交
2212 2213
@wrap_name_default()
@layer_support()
2214
def cos_sim(a, b, scale=1, size=1, name=None, layer_attr=None):
Z
zhangjinchao01 已提交
2215 2216 2217 2218
    """
    Cosine Similarity Layer. The cosine similarity equation is here.

    ..  math::
D
dangqingqing 已提交
2219
        similarity = cos(\\theta) = {\\mathbf{a} \\cdot \\mathbf{b}
2220 2221 2222 2223 2224
        \\over \\|\\mathbf{a}\\| \\|\\mathbf{b}\\|}

    The size of a is M, size of b is M*N,
    Similarity will be calculated N times by step M. The output size is
    N. The scale will be multiplied to similarity.
Z
zhangjinchao01 已提交
2225

2226 2227
    Note that the above computation is for one sample. Multiple samples are
    processed in one batch.
Z
zhangjinchao01 已提交
2228

L
Luo Tao 已提交
2229 2230 2231 2232 2233 2234
    The example usage is:

    .. code-block:: python

       cos = cos_sim(a=layer1, b=layer2, size=3)

Z
zhangjinchao01 已提交
2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246
    :param name: layer name
    :type name: basestring
    :param a: input layer a
    :type a: LayerOutput
    :param b: input layer b
    :type b: LayerOutput
    :param scale: scale for cosine value. default is 5.
    :type scale: float
    :param size: layer size. NOTE size_a * size should equal size_b.
    :type size: int
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
2247
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
2248 2249
    :rtype: LayerOutput
    """
2250
    assert isinstance(a, LayerOutput) and isinstance(b, LayerOutput)
2251 2252 2253 2254 2255 2256
    if size == 1:
        Layer(
            name=name,
            type=LayerType.COSINE_SIM,
            cos_scale=scale,
            inputs=[a.name, b.name],
Q
qijun 已提交
2257
            **ExtraLayerAttribute.to_kwargs(layer_attr))
2258
    else:
2259 2260
        if a.size is not None and b.size is not None:
            assert size == b.size / a.size
2261 2262 2263 2264 2265 2266
        Layer(
            name=name,
            type=LayerType.COSINE_SIM_VEC,
            size=size,
            cos_scale=scale,
            inputs=[a.name, b.name],
Q
qijun 已提交
2267
            **ExtraLayerAttribute.to_kwargs(layer_attr))
X
xuwei06 已提交
2268
    return LayerOutput(name, LayerType.COSINE_SIM, parents=[a, b], size=size)
Z
zhangjinchao01 已提交
2269

2270

Z
zhangjinchao01 已提交
2271 2272
@wrap_name_default()
@wrap_bias_attr_default(has_bias=True)
2273
@wrap_param_attr_default()
Z
zhangjinchao01 已提交
2274
@layer_support()
Q
qijun 已提交
2275 2276
def hsigmoid(input,
             label,
2277
             num_classes=None,
Q
qijun 已提交
2278 2279 2280 2281
             name=None,
             bias_attr=None,
             param_attr=None,
             layer_attr=None):
Z
zhangjinchao01 已提交
2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292
    """
    Organize the classes into a binary tree. At each node, a sigmoid function
    is used to calculate the probability of belonging to the right branch.
    This idea is from "F. Morin, Y. Bengio (AISTATS 05):
    Hierarchical Probabilistic Neural Network Language Model."

    The example usage is:

    ..  code-block:: python

        cost = hsigmoid(input=[layer1, layer2],
2293
                        label=data_layer)
Z
zhangjinchao01 已提交
2294 2295 2296 2297 2298 2299 2300

    :param input: Input layers. It could be a LayerOutput or list/tuple of
                 LayerOutput.
    :type input: LayerOutput|list|tuple
    :param label: Label layer.
    :type label: LayerOutput
    :param num_classes: number of classes.
2301
    :type num_classes: int|None
L
luotao02 已提交
2302 2303
    :param name: layer name
    :type name: basestring
Z
zhangjinchao01 已提交
2304 2305 2306
    :param bias_attr: Bias attribute. None means default bias.
                      False means no bias.
    :type bias_attr: ParameterAttribute|False
2307 2308
    :param param_attr: Parameter Attribute. None means default parameter.
    :type param_attr: ParameterAttribute|None
Z
zhangjinchao01 已提交
2309 2310
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
2311
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
2312 2313 2314 2315
    :rtype: LayerOutput
    """
    if isinstance(input, LayerOutput):
        input = [input]
2316 2317 2318 2319 2320 2321 2322 2323 2324
        if not isinstance(param_attr, collections.Sequence):
            param_attr = [param_attr]
    else:
        if not isinstance(param_attr, collections.Sequence):
            param_attr = [param_attr] * len(input)
        else:
            assert len(param_attr) == len(input)

    assert isinstance(input, collections.Sequence)
Z
zhangjinchao01 已提交
2325 2326 2327
    assert isinstance(label, LayerOutput)
    assert label.layer_type == LayerType.DATA

2328 2329 2330 2331 2332
    if num_classes is None:
        num_classes = label.size
    if num_classes is None or num_classes <= 2:
        raise ValueError("hsigmoid label size must larger than 2.")

Z
zhangjinchao01 已提交
2333 2334
    ipts_for_layer = []
    parents = []
2335
    for each_input, each_param_attr in zip(input, param_attr):
Z
zhangjinchao01 已提交
2336
        assert isinstance(each_input, LayerOutput)
2337
        ipts_for_layer.append(Input(each_input.name, **each_param_attr.attr))
Z
zhangjinchao01 已提交
2338 2339 2340 2341
        parents.append(each_input)
    ipts_for_layer.append(label.name)
    parents.append(label)

X
xuwei06 已提交
2342
    l = Layer(
Z
zhangjinchao01 已提交
2343 2344 2345 2346 2347
        name=name,
        type=LayerType.HSIGMOID,
        num_classes=num_classes,
        bias=ParamAttr.to_bias(bias_attr),
        inputs=ipts_for_layer,
Q
qijun 已提交
2348 2349 2350
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.HSIGMOID, parents=parents, size=l.config.size)
Z
zhangjinchao01 已提交
2351

2352

Z
zhangjinchao01 已提交
2353 2354 2355 2356 2357
@wrap_name_default("conv")
@wrap_param_attr_default()
@wrap_bias_attr_default()
@wrap_act_default(act=ReluActivation())
@layer_support(DROPOUT)
Q
qijun 已提交
2358 2359 2360 2361 2362 2363 2364 2365 2366
def img_conv_layer(input,
                   filter_size,
                   num_filters,
                   name=None,
                   num_channels=None,
                   act=None,
                   groups=1,
                   stride=1,
                   padding=0,
W
wanghaoshuang 已提交
2367
                   dilation=1,
Q
qijun 已提交
2368 2369 2370 2371 2372 2373 2374
                   bias_attr=None,
                   param_attr=None,
                   shared_biases=True,
                   layer_attr=None,
                   filter_size_y=None,
                   stride_y=None,
                   padding_y=None,
2375
                   dilation_y=None,
2376 2377
                   trans=False,
                   layer_type=None):
Z
zhangjinchao01 已提交
2378
    """
2379
    Convolution layer for image. Paddle can support both square and non-square
2380
    input currently.
Z
zhangjinchao01 已提交
2381 2382 2383 2384

    The details of convolution layer, please refer UFLDL's `convolution
    <http://ufldl.stanford.edu/tutorial/supervised/
    FeatureExtractionUsingConvolution/>`_ .
X
xuwei06 已提交
2385

2386
    Convolution Transpose (deconv) layer for image. Paddle can support both square
2387
    and non-square input currently.
2388

X
xuwei06 已提交
2389
    The details of convolution transpose layer,
2390 2391 2392
    please refer to the following explanation and references therein
    <http://datascience.stackexchange.com/questions/6107/
    what-are-deconvolutional-layers/>`_ .
Z
zhangjinchao01 已提交
2393 2394 2395 2396
    The num_channel means input image's channel number. It may be 1 or 3 when
    input is raw pixels of image(mono or RGB), or it may be the previous layer's
    num_filters * num_group.

C
caoying03 已提交
2397 2398 2399
    There are several group of filter in PaddlePaddle implementation.
    Each group will process some channel of the inputs. For example, if an input
    num_channel = 256, group = 4, num_filter=32, the PaddlePaddle will create
Z
zhangjinchao01 已提交
2400
    32*4 = 128 filters to process inputs. The channels will be split into 4
C
caoying03 已提交
2401 2402
    pieces. First 256/4 = 64 channels will process by first 32 filters. The
    rest channels will be processed by rest group of filters.
Z
zhangjinchao01 已提交
2403

L
Luo Tao 已提交
2404 2405 2406 2407 2408 2409 2410 2411 2412 2413
    The example usage is:

    ..  code-block:: python

        conv = img_conv_layer(input=data, filter_size=1, filter_size_y=1,
                              num_channels=8,
                              num_filters=16, stride=1,
                              bias_attr=False,
                              act=ReluActivation())

Z
zhangjinchao01 已提交
2414 2415 2416 2417
    :param name: Layer name.
    :type name: basestring
    :param input: Layer Input.
    :type input: LayerOutput
2418 2419 2420
    :param filter_size: The x dimension of a filter kernel. Or input a tuple for
                        two image dimension.
    :type filter_size: int|tuple|list
C
caoying03 已提交
2421 2422 2423
    :param filter_size_y: The y dimension of a filter kernel. Since PaddlePaddle
                        currently supports rectangular filters, the filter's
                        shape will be (filter_size, filter_size_y).
2424
    :type filter_size_y: int|None
Z
zhangjinchao01 已提交
2425 2426 2427 2428 2429
    :param num_filters: Each filter group's number of filter
    :param act: Activation type. Default is tanh
    :type act: BaseActivation
    :param groups: Group size of filters.
    :type groups: int
2430 2431 2432
    :param stride: The x dimension of the stride. Or input a tuple for two image
                   dimension.
    :type stride: int|tuple|list
Z
zhangjinchao01 已提交
2433 2434
    :param stride_y: The y dimension of the stride.
    :type stride_y: int
2435 2436 2437
    :param padding: The x dimension of the padding. Or input a tuple for two
                    image dimension
    :type padding: int|tuple|list
Z
zhangjinchao01 已提交
2438 2439
    :param padding_y: The y dimension of the padding.
    :type padding_y: int
2440 2441 2442
    :param dilation: The x dimension of the dilation. Or input a tuple for two
                    image dimension
    :type dilation: int|tuple|list
W
wanghaoshuang 已提交
2443 2444
    :param dilation_y: The y dimension of the dilation.
    :type dilation_y: int
Z
zhangjinchao01 已提交
2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456
    :param bias_attr: Convolution bias attribute. None means default bias.
                      False means no bias.
    :type bias_attr: ParameterAttribute|False
    :param num_channels: number of input channels. If None will be set
                        automatically from previous output.
    :type num_channels: int
    :param param_attr: Convolution param attribute. None means default attribute
    :type param_attr: ParameterAttribute
    :param shared_biases: Is biases will be shared between filters or not.
    :type shared_biases: bool
    :param layer_attr: Layer Extra Attribute.
    :type layer_attr: ExtraLayerAttribute
2457 2458
    :param trans: true if it is a convTransLayer, false if it is a convLayer
    :type trans: bool
2459
    :param layer_type: specify the layer_type, default is None. If trans=True,
2460 2461
                       layer_type has to be "exconvt" or "cudnn_convt",
                       otherwise layer_type has to be either "exconv" or
2462
                       "cudnn_conv"
2463
    :type layer_type: String
D
dangqingqing 已提交
2464
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
2465 2466 2467 2468 2469
    :rtype: LayerOutput
    """
    if num_channels is None:
        assert input.num_filters is not None
        num_channels = input.num_filters
2470

Z
zhangjinchao01 已提交
2471
    if filter_size_y is None:
2472 2473 2474 2475 2476 2477
        if isinstance(filter_size, collections.Sequence):
            assert len(filter_size) == 2
            filter_size, filter_size_y = filter_size
        else:
            filter_size_y = filter_size

Z
zhangjinchao01 已提交
2478
    if stride_y is None:
2479 2480 2481 2482 2483 2484
        if isinstance(stride, collections.Sequence):
            assert len(stride) == 2
            stride, stride_y = stride
        else:
            stride_y = stride

Z
zhangjinchao01 已提交
2485
    if padding_y is None:
2486 2487 2488 2489 2490 2491
        if isinstance(padding, collections.Sequence):
            assert len(padding) == 2
            padding, padding_y = padding
        else:
            padding_y = padding

2492 2493 2494 2495 2496 2497 2498
    if dilation_y is None:
        if isinstance(dilation, collections.Sequence):
            assert len(dilation) == 2
            dilation, dilation_y = dilation
        else:
            dilation_y = dilation

2499 2500
    if param_attr.attr.get('initial_smart'):
        # special initial for conv layers.
Q
qijun 已提交
2501
        init_w = (2.0 / (filter_size**2 * num_channels))**0.5
2502 2503 2504 2505
        param_attr.attr["initial_mean"] = 0.0
        param_attr.attr["initial_std"] = init_w
        param_attr.attr["initial_strategy"] = 0
        param_attr.attr["initial_smart"] = False
2506

2507
    if layer_type:
W
wanghaoshuang 已提交
2508 2509
        if dilation > 1 or dilation_y > 1:
            assert layer_type in ["cudnn_conv", "cudnn_convt"]
2510
        if trans:
2511
            assert layer_type in ["exconvt", "cudnn_convt"]
2512 2513 2514 2515 2516
        else:
            assert layer_type in ["exconv", "cudnn_conv"]
        lt = layer_type
    else:
        lt = LayerType.CONVTRANS_LAYER if trans else LayerType.CONV_LAYER
Q
qijun 已提交
2517

X
xuwei06 已提交
2518
    l = Layer(
Z
zhangjinchao01 已提交
2519
        name=name,
Q
qijun 已提交
2520 2521 2522 2523 2524
        inputs=Input(
            input.name,
            conv=Conv(
                filter_size=filter_size,
                padding=padding,
2525
                dilation=dilation,
Q
qijun 已提交
2526 2527 2528 2529 2530
                stride=stride,
                channels=num_channels,
                groups=groups,
                filter_size_y=filter_size_y,
                padding_y=padding_y,
2531
                dilation_y=dilation_y,
Q
qijun 已提交
2532 2533
                stride_y=stride_y),
            **param_attr.attr),
Z
zhangjinchao01 已提交
2534 2535 2536 2537
        active_type=act.name,
        num_filters=num_filters,
        bias=ParamAttr.to_bias(bias_attr),
        shared_biases=shared_biases,
2538
        type=lt,
Q
qijun 已提交
2539 2540 2541 2542 2543 2544 2545 2546
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name,
        lt,
        parents=[input],
        activation=act,
        num_filters=num_filters,
        size=l.config.size)
Z
zhangjinchao01 已提交
2547 2548 2549 2550


@wrap_name_default("pool")
@layer_support()
Q
qijun 已提交
2551 2552 2553 2554 2555 2556 2557 2558 2559 2560
def img_pool_layer(input,
                   pool_size,
                   name=None,
                   num_channels=None,
                   pool_type=None,
                   stride=1,
                   padding=0,
                   layer_attr=None,
                   pool_size_y=None,
                   stride_y=None,
2561 2562
                   padding_y=None,
                   ceil_mode=True):
Z
zhangjinchao01 已提交
2563 2564 2565 2566 2567 2568 2569
    """
    Image pooling Layer.

    The details of pooling layer, please refer ufldl's pooling_ .

    .. _pooling: http://ufldl.stanford.edu/tutorial/supervised/Pooling/

L
Luo Tao 已提交
2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597
    - ceil_mode=True:

    ..  math::

        w = 1 + int(ceil(input\_width + 2 * padding - pool\_size) / float(stride))
        h = 1 + int(ceil(input\_height + 2 * padding\_y - pool\_size\_y) / float(stride\_y))

    - ceil_mode=False:

    ..  math::

        w = 1 + int(floor(input\_width + 2 * padding - pool\_size) / float(stride))
        h = 1 + int(floor(input\_height + 2 * padding\_y - pool\_size\_y) / float(stride\_y))

    The example usage is:

    ..  code-block:: python

        maxpool = img_pool_layer(input=conv,
                                 pool_size=3,
                                 pool_size_y=5,
                                 num_channels=8,
                                 stride=1,
                                 stride_y=2,
                                 padding=1,
                                 padding_y=2,
                                 pool_type=MaxPooling())

2598
    :param padding: pooling padding width.
Z
zhangjinchao01 已提交
2599
    :type padding: int
2600 2601
    :param padding_y: pooling padding height. It's equal to padding by default.
    :type padding_y: int|None
Z
zhangjinchao01 已提交
2602 2603 2604 2605
    :param name: name of pooling layer
    :type name: basestring.
    :param input: layer's input
    :type input: LayerOutput
2606
    :param pool_size: pooling window width
Z
zhangjinchao01 已提交
2607
    :type pool_size: int
2608 2609
    :param pool_size_y: pooling window height. It's eaqual to pool_size by default.
    :type pool_size_y: int|None
Z
zhangjinchao01 已提交
2610 2611
    :param num_channels: number of input channel.
    :type num_channels: int
2612
    :param pool_type: pooling type. MaxPooling or AvgPooling. Default is
Z
zhangjinchao01 已提交
2613 2614
                      MaxPooling.
    :type pool_type: BasePoolingType
2615
    :param stride: stride width of pooling.
Z
zhangjinchao01 已提交
2616
    :type stride: int
2617 2618
    :param stride_y: stride height of pooling. It is equal to stride by default.
    :type stride_y: int|None
Z
zhangjinchao01 已提交
2619 2620
    :param layer_attr: Extra Layer attribute.
    :type layer_attr: ExtraLayerAttribute
2621 2622 2623 2624
    :param ceil_mode: Wether to use ceil mode to calculate output height and with.
                      Defalut is True. If set false, Otherwise use floor.

    :type ceil_mode: bool
D
dangqingqing 已提交
2625 2626
    :return: LayerOutput object.
    :rtype: LayerOutput
Z
zhangjinchao01 已提交
2627 2628 2629 2630 2631 2632 2633 2634 2635 2636
    """
    if num_channels is None:
        assert input.num_filters is not None
        num_channels = input.num_filters

    if pool_type is None:
        pool_type = MaxPooling()
    elif isinstance(pool_type, AvgPooling):
        pool_type.name = 'avg'

W
wanghaoshuang 已提交
2637 2638 2639 2640
    assert type(pool_type) in [AvgPooling, MaxPooling, CudnnAvgPooling,
                               CudnnMaxPooling], \
        "only (Cudnn)AvgPooling, (Cudnn)MaxPooling are supported"

2641
    type_name = pool_type.name + '-projection' \
Y
Yu Yang 已提交
2642
        if (
Y
Yu Yang 已提交
2643
        isinstance(pool_type, AvgPooling) or isinstance(pool_type, MaxPooling)) \
Y
Yu Yang 已提交
2644
        else pool_type.name
2645 2646 2647 2648
    pool_size_y = pool_size if pool_size_y is None else pool_size_y
    stride_y = stride if stride_y is None else stride_y
    padding_y = padding if padding_y is None else padding_y

X
xuwei06 已提交
2649
    l = Layer(
Z
zhangjinchao01 已提交
2650 2651
        name=name,
        type=LayerType.POOL_LAYER,
Q
qijun 已提交
2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663
        inputs=[
            Input(
                input.name,
                pool=Pool(
                    pool_type=type_name,
                    channels=num_channels,
                    size_x=pool_size,
                    start=None,
                    stride=stride,
                    padding=padding,
                    size_y=pool_size_y,
                    stride_y=stride_y,
L
Luo Tao 已提交
2664
                    padding_y=padding_y))
Q
qijun 已提交
2665
        ],
2666
        ceil_mode=ceil_mode,
Q
qijun 已提交
2667 2668 2669 2670 2671 2672 2673
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name,
        LayerType.POOL_LAYER,
        parents=[input],
        num_filters=num_channels,
        size=l.config.size)
Z
zhangjinchao01 已提交
2674 2675


C
chengduoZH 已提交
2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815
@wrap_name_default("pool3d")
@layer_support()
def img_pool3d_layer(input,
                     pool_size,
                     name=None,
                     num_channels=None,
                     pool_type=None,
                     stride=1,
                     padding=0,
                     layer_attr=None,
                     pool_size_y=None,
                     stride_y=None,
                     padding_y=None,
                     pool_size_z=None,
                     stride_z=None,
                     padding_z=None,
                     ceil_mode=True):
    """
    Image pooling Layer.

    The details of pooling layer, please refer ufldl's pooling_ .

    .. _pooling: http://ufldl.stanford.edu/tutorial/supervised/Pooling/

    - ceil_mode=True:

    ..  math::

        w = 1 + int(ceil(input\_width + 2 * padding - pool\_size) / float(stride))
        h = 1 + int(ceil(input\_height + 2 * padding\_y - pool\_size\_y) / float(stride\_y))
        d = 1 + int(ceil(input\_depth + 2 * padding\_z - pool\_size\_z) / float(stride\_z))

    - ceil_mode=False:

    ..  math::

        w = 1 + int(floor(input\_width + 2 * padding - pool\_size) / float(stride))
        h = 1 + int(floor(input\_height + 2 * padding\_y - pool\_size\_y) / float(stride\_y))
        d = 1 + int(floor(input\_depth + 2 * padding\_z - pool\_size\_z) / float(stride\_z))

    The example usage is:

    ..  code-block:: python

        maxpool = img_pool3d_layer(input=conv,
                                 pool_size=3,
                                 num_channels=8,
                                 stride=1,
                                 padding=1,
                                 pool_type=MaxPooling())

    :param padding: pooling padding width.
    :type padding: int|tuple|list
    :param name: name of pooling layer
    :type name: basestring.
    :param input: layer's input
    :type input: LayerOutput
    :param pool_size: pooling window width
    :type pool_size: int|tuple|list
    :param num_channels: number of input channel.
    :type num_channels: int
    :param pool_type: pooling type. MaxPooling or AvgPooling. Default is
                      MaxPooling.
    :type pool_type: BasePoolingType
    :param stride: stride width of pooling.
    :type stride: int|tuple|list
    :param layer_attr: Extra Layer attribute.
    :type layer_attr: ExtraLayerAttribute
    :param ceil_mode: Wether to use ceil mode to calculate output height and with.
                      Defalut is True. If set false, Otherwise use floor.

    :type ceil_mode: bool
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
    if num_channels is None:
        assert input.num_filters is not None
        num_channels = input.num_filters

    if pool_type is None:
        pool_type = MaxPooling()
    elif isinstance(pool_type, AvgPooling):
        pool_type.name = 'avg'

    type_name = pool_type.name + '-projection' \
        if (
        isinstance(pool_type, AvgPooling) or isinstance(pool_type, MaxPooling)) \
        else pool_type.name

    if isinstance(pool_size, collections.Sequence):
        assert len(pool_size) == 3
        pool_size, pool_size_y, pool_size_z = pool_size
    else:
        pool_size_y = pool_size
        pool_size_z = pool_size

    if isinstance(stride, collections.Sequence):
        assert len(stride) == 3
        stride, stride_y, stride_z = stride
    else:
        stride_y = stride
        stride_z = stride

    if isinstance(padding, collections.Sequence):
        assert len(padding) == 3
        padding, padding_y, padding_y = padding
    else:
        padding_y = padding
        padding_z = padding

    l = Layer(
        name=name,
        type=LayerType.POOL3D_LAYER,
        inputs=[
            Input(
                input.name,
                pool=Pool3d(
                    pool_type=type_name,
                    channels=num_channels,
                    size_x=pool_size,
                    start=None,
                    stride=stride,
                    padding=padding,
                    size_y=pool_size_y,
                    stride_y=stride_y,
                    padding_y=padding_y,
                    size_z=pool_size_z,
                    stride_z=stride_z,
                    padding_z=padding_z))
        ],
        ceil_mode=ceil_mode,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name,
        LayerType.POOL_LAYER,
        parents=[input],
        num_filters=num_channels,
        size=l.config.size)


Q
qijun 已提交
2816 2817
@wrap_name_default("spp")
@layer_support()
Q
qijun 已提交
2818 2819 2820 2821 2822 2823
def spp_layer(input,
              name=None,
              num_channels=None,
              pool_type=None,
              pyramid_height=None,
              layer_attr=None):
Q
qijun 已提交
2824 2825 2826 2827 2828
    """
    Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition.
    The details please refer to
    `Kaiming He's paper <https://arxiv.org/abs/1406.4729>`_.

L
Luo Tao 已提交
2829 2830 2831 2832
    The example usage is:

    ..  code-block:: python

2833 2834 2835
        spp = spp_layer(input=data,
                        pyramid_height=2,
                        num_channels=16,
L
Luo Tao 已提交
2836 2837
                        pool_type=MaxPooling())

Q
qijun 已提交
2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865
    :param name: layer name.
    :type name: basestring
    :param input: layer's input.
    :type input: LayerOutput
    :param num_channels: number of input channel.
    :type num_channels: int
    :param pool_type: Pooling type. MaxPooling or AveragePooling. Default is MaxPooling.
    :type scale: BasePoolingType
    :param pyramid_height: pyramid height.
    :type pyramid_height: int
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
    if num_channels is None:
        assert input.num_filters is not None
        num_channels = input.num_filters

    if pool_type is None:
        pool_type = MaxPooling()
    elif isinstance(pool_type, AvgPooling):
        pool_type.name = 'avg'

    type_name = pool_type.name
    if (isinstance(pool_type, AvgPooling) or isinstance(pool_type, MaxPooling)):
        type_name += '-projection'

Q
qijun 已提交
2866
    l = Layer(
Q
qijun 已提交
2867 2868
        name=name,
        type=LayerType.SPP_LAYER,
Q
qijun 已提交
2869 2870 2871 2872 2873
        inputs=Input(
            input.name,
            spp=SpatialPyramidPool(
                pool_type=type_name,
                channels=num_channels,
L
Luo Tao 已提交
2874
                pyramid_height=pyramid_height)),
Q
qijun 已提交
2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name,
        layer_type=LayerType.SPP_LAYER,
        parents=[input],
        num_filters=num_channels,
        size=l.config.size)


def __img_norm_layer__(name, input, size, norm_type, scale, power, num_channels,
                       blocked, layer_attr):
Z
zhangjinchao01 已提交
2886 2887 2888 2889
    if num_channels is None:
        assert input.num_filters is not None
        num_channels = input.num_filters

X
xuwei06 已提交
2890
    l = Layer(
Q
qijun 已提交
2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909
        name=name,
        type=LayerType.NORM_LAYER,
        inputs=Input(
            input.name,
            norm=Norm(
                norm_type=norm_type,
                channels=num_channels,
                size=size,
                scale=scale,
                pow=power,
                blocked=blocked)),
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name,
        layer_type=LayerType.NORM_LAYER,
        parents=[input],
        num_filters=num_channels,
        img_norm_type=norm_type,
        size=l.config.size)
Z
zhangjinchao01 已提交
2910 2911 2912 2913


@wrap_name_default("crmnorm")
@layer_support()
Q
qijun 已提交
2914 2915 2916 2917 2918 2919
def img_cmrnorm_layer(input,
                      size,
                      scale=0.0128,
                      power=0.75,
                      name=None,
                      num_channels=None,
2920
                      layer_attr=None):
Z
zhangjinchao01 已提交
2921
    """
2922
    Response normalization across feature maps.
D
dangqingqing 已提交
2923 2924
    The details please refer to
    `Alex's paper <http://www.cs.toronto.edu/~fritz/absps/imagenet.pdf>`_.
Z
zhangjinchao01 已提交
2925

L
Luo Tao 已提交
2926 2927 2928
    The example usage is:

    ..  code-block:: python
2929

L
Luo Tao 已提交
2930 2931
        norm = img_cmrnorm_layer(input=net, size=5)

Z
zhangjinchao01 已提交
2932
    :param name: layer name.
D
dangqingqing 已提交
2933
    :type name: None|basestring
Z
zhangjinchao01 已提交
2934 2935
    :param input: layer's input.
    :type input: LayerOutput
2936
    :param size: Normalize in number of :math:`size` feature maps.
Z
zhangjinchao01 已提交
2937
    :type size: int
D
dangqingqing 已提交
2938
    :param scale: The hyper-parameter.
Z
zhangjinchao01 已提交
2939
    :type scale: float
D
dangqingqing 已提交
2940
    :param power: The hyper-parameter.
Z
zhangjinchao01 已提交
2941 2942 2943 2944 2945
    :type power: float
    :param num_channels: input layer's filers number or channels. If
                         num_channels is None, it will be set automatically.
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
2946
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
2947 2948 2949
    :rtype: LayerOutput
    """
    return __img_norm_layer__(name, input, size, "cmrnorm-projection", scale,
2950
                              power, num_channels, 0, layer_attr)
Z
zhangjinchao01 已提交
2951 2952 2953


@wrap_bias_attr_default()
2954 2955
@wrap_param_attr_default(
    default_factory=lambda _: ParamAttr(initial_mean=1.0, initial_std=0.))
Z
zhangjinchao01 已提交
2956 2957
@wrap_act_default(act=ReluActivation())
@wrap_name_default("batch_norm")
2958
@layer_support(DROPOUT, ERROR_CLIPPING)
Q
qijun 已提交
2959 2960 2961
def batch_norm_layer(input,
                     act=None,
                     name=None,
C
chengduoZH 已提交
2962
                     img3D=False,
Q
qijun 已提交
2963 2964 2965 2966
                     num_channels=None,
                     bias_attr=None,
                     param_attr=None,
                     layer_attr=None,
Z
zhangjinchao01 已提交
2967 2968
                     batch_norm_type=None,
                     moving_average_fraction=0.9,
C
chengduoZH 已提交
2969 2970
                     use_global_stats=None,
                     mean_var_names=None):
Z
zhangjinchao01 已提交
2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988
    """
    Batch Normalization Layer. The notation of this layer as follow.

    :math:`x` is the input features over a mini-batch.

    ..  math::

        \\mu_{\\beta} &\\gets \\frac{1}{m} \\sum_{i=1}^{m} x_i \\qquad &//\\
        \ mini-batch\ mean \\\\
        \\sigma_{\\beta}^{2} &\\gets \\frac{1}{m} \\sum_{i=1}^{m}(x_i - \\
        \\mu_{\\beta})^2 \\qquad &//\ mini-batch\ variance \\\\
        \\hat{x_i} &\\gets \\frac{x_i - \\mu_\\beta} {\\sqrt{\\
        \\sigma_{\\beta}^{2} + \\epsilon}} \\qquad &//\ normalize \\\\
        y_i &\\gets \\gamma \\hat{x_i} + \\beta \\qquad &//\ scale\ and\ shift

    The details of batch normalization please refer to this
    `paper <http://arxiv.org/abs/1502.03167>`_.

L
Luo Tao 已提交
2989 2990 2991
    The example usage is:

    ..  code-block:: python
2992

L
Luo Tao 已提交
2993 2994
        norm = batch_norm_layer(input=net, act=ReluActivation())

Z
zhangjinchao01 已提交
2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008
    :param name: layer name.
    :type name: basestring
    :param input: batch normalization input. Better be linear activation.
                Because there is an activation inside batch_normalization.
    :type input: LayerOutput
    :param batch_norm_type: We have batch_norm and cudnn_batch_norm. batch_norm
                            supports both CPU and GPU. cudnn_batch_norm requires
                            cuDNN version greater or equal to v4 (>=v4). But
                            cudnn_batch_norm is faster and needs less memory
                            than batch_norm. By default (None), we will
                            automaticly select cudnn_batch_norm for GPU and
                            batch_norm for CPU. Otherwise, select batch norm
                            type based on the specified type. If you use cudnn_batch_norm,
                            we suggested you use latest version, such as v5.1.
3009
    :type batch_norm_type: None|string, None or "batch_norm" or "cudnn_batch_norm"
Z
zhangjinchao01 已提交
3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036
    :param act: Activation Type. Better be relu. Because batch
                     normalization will normalize input near zero.
    :type act: BaseActivation
    :param num_channels: num of image channels or previous layer's number of
                         filters. None will automatically get from layer's
                         input.
    :type num_channels: int
    :param bias_attr: :math:`\\beta`, better be zero when initialize. So the
                      initial_std=0, initial_mean=1 is best practice.
    :type bias_attr: ParameterAttribute
    :param param_attr: :math:`\\gamma`, better be one when initialize. So the
                       initial_std=0, initial_mean=1 is best practice.
    :type param_attr: ParameterAttribute
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
    :param use_global_stats: whether use moving mean/variance statistics
                             during testing peroid. If None or True,
                             it will use moving mean/variance statistics during
                             testing. If False, it will use the mean
                             and variance of current batch of test data for
                             testing.
    :type use_global_stats: bool|None.
    :param moving_average_fraction: Factor used in the moving average
                                   computation, referred to as facotr,
                                   :math:`runningMean = newMean*(1-factor)
                                   + runningMean*factor`
    :type moving_average_fraction: float.
C
chengduoZH 已提交
3037 3038
    :param mean_var_names: [mean name, variance name]
    :type mean_var_names: string list
D
dangqingqing 已提交
3039
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
3040 3041 3042 3043 3044 3045 3046 3047 3048 3049
    :rtype: LayerOutput
    """

    if num_channels is None:
        if input.num_filters is not None:
            num_channels = input.num_filters
        else:
            num_channels = input.size
    assert (batch_norm_type is None) or (batch_norm_type == "batch_norm") or \
           (batch_norm_type == "cudnn_batch_norm")
X
xuwei06 已提交
3050
    l = Layer(
Z
zhangjinchao01 已提交
3051
        name=name,
C
chengduoZH 已提交
3052
        img3D=img3D,
Q
qijun 已提交
3053 3054
        inputs=Input(
            input.name, image=Image(channels=num_channels), **param_attr.attr),
Z
zhangjinchao01 已提交
3055 3056 3057 3058 3059 3060
        active_type=act.name,
        type=LayerType.BATCH_NORM_LAYER,
        batch_norm_type=batch_norm_type,
        bias=ParamAttr.to_bias(bias_attr),
        moving_average_fraction=moving_average_fraction,
        use_global_stats=use_global_stats,
C
chengduoZH 已提交
3061
        mean_var_names=mean_var_names,
Q
qijun 已提交
3062
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
3063

Q
qijun 已提交
3064 3065 3066 3067 3068 3069 3070
    return LayerOutput(
        name=name,
        layer_type=LayerType.BATCH_NORM_LAYER,
        parents=[input],
        activation=act,
        num_filters=num_channels,
        size=l.config.size)
Z
zhangjinchao01 已提交
3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097


@wrap_name_default()
@layer_support()
def sum_to_one_norm_layer(input, name=None, layer_attr=None):
    """
    A layer for sum-to-one normalization,
    which is used in NEURAL TURING MACHINE.

    .. math::
       out[i] = \\frac {in[i]} {\sum_{k=1}^N in[k]}

    where :math:`in` is a (batchSize x dataDim) input vector,
    and :math:`out` is a (batchSize x dataDim) output vector.

    The example usage is:

    .. code-block:: python

       sum_to_one_norm = sum_to_one_norm_layer(input=layer)

    :param input: Input layer.
    :type input: LayerOutput
    :param name: Layer name.
    :type name: basestring
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
D
dangqingqing 已提交
3098
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
3099 3100 3101 3102 3103 3104
    :rtype: LayerOutput
    """
    Layer(
        name=name,
        type=LayerType.SUM_TO_ONE_NORM_LAYER,
        inputs=[input.name],
Q
qijun 已提交
3105 3106 3107
        **ExtraAttr.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.SUM_TO_ONE_NORM_LAYER, parents=[input], size=input.size)
Z
zhangjinchao01 已提交
3108 3109


G
guosheng 已提交
3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145
@wrap_name_default()
@layer_support()
def row_l2_norm_layer(input, name=None, layer_attr=None):
    """
    A layer for L2-normalization in each row.

    .. math::
       out[i] = \frac{in[i]}{\sqrt{\sum_{k=1}^N in[k]^{2}}}

    where the size of :math:`in` is (batchSize x dataDim) ,
    and the size of :math:`out` is a (batchSize x dataDim) .

    The example usage is:

    .. code-block:: python

       row_l2_norm_layer = row_l2_norm_layer(input=layer)

    :param input: Input layer.
    :type input: LayerOutput
    :param name: Layer name.
    :type name: basestring
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
    Layer(
        name=name,
        type=LayerType.ROW_L2_NORM_LAYER,
        inputs=[input.name],
        **ExtraAttr.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.ROW_L2_NORM_LAYER, parents=[input], size=input.size)


Z
zhangjinchao01 已提交
3146 3147 3148
@wrap_name_default("addto")
@wrap_act_default(act=LinearActivation())
@wrap_bias_attr_default(has_bias=False)
3149
@layer_support(DROPOUT, ERROR_CLIPPING)
Q
qijun 已提交
3150
def addto_layer(input, act=None, name=None, bias_attr=None, layer_attr=None):
Z
zhangjinchao01 已提交
3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172
    """
    AddtoLayer.

    ..  math::

        y = f(\\sum_{i} x_i + b)

    where :math:`y` is output, :math:`x` is input, :math:`b` is bias,
    and :math:`f` is activation function.

    The example usage is:

    ..  code-block:: python

        addto = addto_layer(input=[layer1, layer2],
                            act=ReluActivation(),
                            bias_attr=False)

    This layer just simply add all input layers together, then activate the sum
    inputs. Each input of this layer should be the same size, which is also the
    output size of this layer.

C
caoying03 已提交
3173 3174 3175
    There is no weight matrix for each input, because it just a simple add
    operation. If you want a complicated operation before add, please use
    mixed_layer.
Z
zhangjinchao01 已提交
3176 3177

    It is a very good way to set dropout outside the layers. Since not all
C
caoying03 已提交
3178 3179
    PaddlePaddle layer support dropout, you can add an add_to layer, set
    dropout here.
Z
zhangjinchao01 已提交
3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193
    Please refer to dropout_layer for details.

    :param name: Layer name.
    :type name: basestring
    :param input: Input layers. It could be a LayerOutput or list/tuple of
                 LayerOutput.
    :type input: LayerOutput|list|tuple
    :param act: Activation Type, default is tanh.
    :type act: BaseActivation
    :param bias_attr: Bias attribute. If False, means no bias. None is default
                      bias.
    :type bias_attr: ParameterAttribute|bool
    :param layer_attr: Extra Layer attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
3194
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
3195 3196 3197 3198 3199 3200
    :rtype: LayerOutput
    """
    num_filters = None
    if isinstance(input, LayerOutput):
        input = [input]

3201
    assert isinstance(input, collections.Sequence)
Z
zhangjinchao01 已提交
3202 3203 3204 3205 3206 3207 3208
    ipts_for_layer = []
    for each_input in input:
        assert isinstance(each_input, LayerOutput)
        ipts_for_layer.append(Input(each_input.name))
        if each_input.num_filters is not None:
            num_filters = each_input.num_filters

X
xuwei06 已提交
3209
    l = Layer(
Q
qijun 已提交
3210 3211 3212
        name=name,
        type=LayerType.ADDTO_LAYER,
        inputs=ipts_for_layer,
Z
zhangjinchao01 已提交
3213 3214
        bias=ParamAttr.to_bias(bias_attr),
        active_type=act.name,
Q
qijun 已提交
3215
        **ExtraLayerAttribute.to_kwargs(layer_attr))
3216

Q
qijun 已提交
3217 3218 3219 3220 3221 3222 3223
    return LayerOutput(
        name,
        LayerType.ADDTO_LAYER,
        parents=input,
        activation=act,
        num_filters=num_filters,
        size=l.config.size)
Z
zhangjinchao01 已提交
3224 3225 3226 3227


@wrap_act_default(act=IdentityActivation())
@wrap_name_default("concat")
3228
@layer_support(DROPOUT, ERROR_CLIPPING)
3229
def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None):
Z
zhangjinchao01 已提交
3230 3231 3232 3233
    """
    Concat all input vector into one huge vector.
    Inputs can be list of LayerOutput or list of projection.

3234 3235 3236 3237 3238 3239
    The example usage is:

    ..  code-block:: python

        concat = concat_layer(input=[layer1, layer2])

Z
zhangjinchao01 已提交
3240 3241 3242
    :param name: Layer name.
    :type name: basestring
    :param input: input layers or projections
3243
    :type input: list|tuple|collections.Sequence
Z
zhangjinchao01 已提交
3244 3245 3246 3247
    :param act: Activation type.
    :type act: BaseActivation
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
3248
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
3249 3250 3251 3252 3253 3254 3255 3256
    :rtype: LayerOutput
    """

    if isinstance(input, LayerOutput):
        input = [input]
    elif isinstance(input, Projection):
        input = [input]
    else:
3257
        assert isinstance(input, collections.Sequence)
Z
zhangjinchao01 已提交
3258 3259

    def __is_type__(o, tp):
3260
        if not isinstance(o, collections.Sequence):
Z
zhangjinchao01 已提交
3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281
            if o == tp:
                return True
            elif len(o.__bases__) == 0:
                return False
            else:
                for bs in o.__bases__:
                    if __is_type__(bs, tp):
                        return True
                return False
        else:
            tmp = map(lambda _x: __is_type__(_x, tp), o)
            a = tmp[0]
            for b in tmp[1:]:
                assert a == b
            return a

    def __reduce_concat_type__(a, b):
        assert __is_type__([a, b], Projection) or __is_type__([a, b],
                                                              LayerOutput)
        return a

Q
qijun 已提交
3282 3283
    is_concat_layer = __is_type__(
        reduce(__reduce_concat_type__, map(type, input)), LayerOutput)
Z
zhangjinchao01 已提交
3284

Q
qijun 已提交
3285 3286
    layer_type = (LayerType.CONCAT_LAYER
                  if is_concat_layer else LayerType.CONCAT_PROJ_LAYER)
Z
zhangjinchao01 已提交
3287

3288 3289
    if layer_type == LayerType.CONCAT_LAYER:
        assert not bias_attr
3290

3291
    layer = Layer(
Q
qijun 已提交
3292 3293
        name=name,
        type=layer_type,
Z
zhangjinchao01 已提交
3294 3295
        inputs=[x.name for x in input] if is_concat_layer else input,
        active_type=act.name,
3296
        bias=ParamAttr.to_bias(bias_attr),
Q
qijun 已提交
3297
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
3298

3299
    sz = layer.config.size
Z
zhangjinchao01 已提交
3300

Q
qijun 已提交
3301 3302 3303 3304 3305 3306 3307 3308
    return LayerOutput(
        name,
        layer_type=layer_type,
        parents=input if is_concat_layer else [x.origin for x in input],
        activation=act,
        size=sz)


3309 3310
@wrap_name_default("seqconcat")
@wrap_act_default(act=IdentityActivation())
3311
@wrap_bias_attr_default(has_bias=False)
3312
@layer_support(DROPOUT, ERROR_CLIPPING)
3313 3314 3315 3316
def seq_concat_layer(a, b, act=None, name=None, layer_attr=None,
                     bias_attr=None):
    """
    Concat sequence a with sequence b.
3317

3318
    Inputs:
X
xuwei06 已提交
3319
      - a = [a1, a2, ..., am]
3320
      - b = [b1, b2, ..., bn]
3321

X
xuwei06 已提交
3322 3323 3324 3325
    Output: [a1, ..., am, b1, ..., bn]

    Note that the above computation is for one sample. Multiple samples are
    processed in one batch.
3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342

    The example usage is:

    ..  code-block:: python

        concat = seq_concat_layer(a=layer1, b=layer2)

    :param name: Layer name.
    :type name: basestring
    :param a: input sequence layer
    :type a: LayerOutput
    :param b: input sequence layer
    :type b: LayerOutput
    :param act: Activation type.
    :type act: BaseActivation
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
3343 3344 3345 3346
    :param bias_attr: The Bias Attribute. If no bias, then pass False or
                      something not type of ParameterAttribute. None will get a
                      default Bias.
    :type bias_attr: ParameterAttribute or None or bool
3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
    assert isinstance(a, LayerOutput) and isinstance(b, LayerOutput)
    assert a.size == b.size
    Layer(
        name=name,
        type=LayerType.SEQUENCE_CONCAT_LAYER,
        inputs=[a.name, b.name],
        active_type=act.name,
        bias=ParamAttr.to_bias(bias_attr),
        **ExtraLayerAttribute.to_kwargs(layer_attr))

    return LayerOutput(
        name,
        layer_type=LayerType.SEQUENCE_CONCAT_LAYER,
        parents=[a, b],
        activation=act,
        size=a.size)


3368
@wrap_name_default("memory", "memory_name")
Q
qijun 已提交
3369 3370
def memory(name,
           size,
3371
           memory_name=None,
Q
qijun 已提交
3372 3373 3374 3375
           is_seq=False,
           boot_layer=None,
           boot_bias=None,
           boot_bias_active_type=None,
Z
zhangjinchao01 已提交
3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395
           boot_with_const_id=None):
    """
    The memory layers is a layer cross each time step. Reference this output
    as previous time step layer :code:`name` 's output.

    The default memory is zero in first time step, previous time step's
    output in the rest time steps.

    If boot_bias, the first time step value is this bias and
    with activation.

    If boot_with_const_id, then the first time stop is a IndexSlot, the
    Arguments.ids()[0] is this :code:`cost_id`.

    If boot_layer is not null, the memory is just the boot_layer's output.
    Set :code:`is_seq` is true boot layer is sequence.

    The same name layer in recurrent group will set memory on each time
    step.

3396 3397 3398 3399 3400 3401 3402 3403 3404
    .. code-block:: python

       mem = memory(size=256, name='state')
       state = fc_layer(input=mem, size=256, name='state')

    If you do not want to specify the name, you can equivalently use set_input()
    to specify the layer needs to be remembered as the following:

    .. code-block:: python
L
Liu Yiqun 已提交
3405

3406 3407 3408 3409 3410 3411 3412
       mem = memory(size=256)
       state = fc_layer(input=mem, size=256)
       mem.set_input(mem)

    :param name: the name of the layer which this memory remembers.
                 If name is None, user should call set_input() to specify the
                 name of the layer which this memory remembers.
Z
zhangjinchao01 已提交
3413 3414 3415
    :type name: basestring
    :param size: size of memory.
    :type size: int
3416 3417 3418
    :param memory_name: the name of the memory.
                        It is ignored when name is provided.
    :type memory_name: basestring
3419
    :param is_seq: DEPRECATED. is sequence for boot_layer
Z
zhangjinchao01 已提交
3420 3421 3422 3423 3424 3425 3426 3427 3428
    :type is_seq: bool
    :param boot_layer: boot layer of memory.
    :type boot_layer: LayerOutput|None
    :param boot_bias: boot layer's bias
    :type boot_bias: ParameterAttribute|None
    :param boot_bias_active_type: boot layer's active type.
    :type boot_bias_active_type: BaseActivation
    :param boot_with_const_id: boot layer's id.
    :type boot_with_const_id: int
D
dangqingqing 已提交
3429
    :return: LayerOutput object which is a memory.
Z
zhangjinchao01 已提交
3430 3431 3432 3433 3434 3435 3436 3437 3438 3439
    :rtype: LayerOutput
    """
    if boot_bias_active_type is None:
        boot_bias_active_type = LinearActivation()

    assert boot_bias is None or isinstance(boot_bias, ParameterAttribute)
    if isinstance(boot_bias, ParameterAttribute):
        boot_bias = ParamAttr.to_bias(boot_bias)

    assert boot_layer is None or isinstance(boot_layer, LayerOutput)
3440 3441
    if name is not None:
        memory_name = None
Z
zhangjinchao01 已提交
3442

3443 3444 3445 3446 3447 3448 3449 3450
    memory_name = Memory(
        name,
        size,
        boot_layer=boot_layer.name if boot_layer is not None else None,
        boot_bias=boot_bias,
        boot_bias_active_type=boot_bias_active_type.name,
        boot_with_const_id=boot_with_const_id,
        memory_name=memory_name)
Q
qijun 已提交
3451 3452

    lout = LayerOutput(
3453
        name=memory_name,
Q
qijun 已提交
3454 3455 3456
        size=size,
        layer_type=LayerType.MEMORY,
        parents=[boot_layer] if boot_layer is not None else None)
Z
zhangjinchao01 已提交
3457 3458 3459 3460
    return lout


@wrap_bias_attr_default()
3461 3462
@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation())
@wrap_act_default(param_names=['state_act'], act=TanhActivation())
Z
zhangjinchao01 已提交
3463 3464 3465
@wrap_act_default(act=TanhActivation())
@wrap_name_default('lstm_step')
@layer_support()
Q
qijun 已提交
3466 3467
def lstm_step_layer(input,
                    state,
3468
                    size=None,
Q
qijun 已提交
3469 3470 3471 3472 3473 3474
                    act=None,
                    name=None,
                    gate_act=None,
                    state_act=None,
                    bias_attr=None,
                    layer_attr=None):
Z
zhangjinchao01 已提交
3475
    """
3476 3477
    LSTM Step Layer. This function is used only in recurrent_group.
    The lstm equations are shown as follows.
Z
zhangjinchao01 已提交
3478 3479 3480

    ..  math::

3481
        i_t & = \\sigma(W_{x_i}x_{t} + W_{h_i}h_{t-1} + W_{c_i}c_{t-1} + b_i)
Z
zhangjinchao01 已提交
3482

3483
        f_t & = \\sigma(W_{x_f}x_{t} + W_{h_f}h_{t-1} + W_{c_f}c_{t-1} + b_f)
Z
zhangjinchao01 已提交
3484

3485
        c_t & = f_tc_{t-1} + i_t tanh (W_{x_c}x_t+W_{h_c}h_{t-1} + b_c)
Z
zhangjinchao01 已提交
3486

3487
        o_t & = \\sigma(W_{x_o}x_{t} + W_{h_o}h_{t-1} + W_{c_o}c_t + b_o)
Z
zhangjinchao01 已提交
3488

L
luotao02 已提交
3489
        h_t & = o_t tanh(c_t)
Z
zhangjinchao01 已提交
3490 3491


L
luotao02 已提交
3492
    The input of lstm step is :math:`Wx_t + Wh_{t-1}`, and user should use
Z
zhangjinchao01 已提交
3493
    :code:`mixed_layer` and :code:`full_matrix_projection` to calculate these
3494
    input vectors.
Z
zhangjinchao01 已提交
3495 3496 3497 3498 3499 3500 3501 3502 3503 3504

    The state of lstm step is :math:`c_{t-1}`. And lstm step layer will do

    ..  math::

        i_t = \\sigma(input + W_{ci}c_{t-1} + b_i)

        ...


3505 3506
    This layer has two outputs. Default output is :math:`h_t`. The other
    output is :math:`o_t`, whose name is 'state' and can use
Z
zhangjinchao01 已提交
3507 3508 3509 3510
    :code:`get_output_layer` to extract this output.

    :param name: Layer's name.
    :type name: basestring
3511 3512
    :param size: Layer's size. NOTE: lstm layer's size, should be equal to
                 :code:`input.size/4`, and should be equal to
Z
zhangjinchao01 已提交
3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530
                 :code:`state.size`.
    :type size: int
    :param input: input layer. :math:`Wx_t + Wh_{t-1}`
    :type input: LayerOutput
    :param state: State Layer. :math:`c_{t-1}`
    :type state: LayerOutput
    :param act: Activation type. Default is tanh
    :type act: BaseActivation
    :param gate_act: Gate Activation Type. Default is sigmoid, and should
                          be sigmoid only.
    :type gate_act: BaseActivation
    :param state_act: State Activation Type. Default is sigmoid, and should
                           be sigmoid only.
    :type state_act: BaseActivation
    :param bias_attr: Bias Attribute.
    :type bias_attr: ParameterAttribute
    :param layer_attr: layer's extra attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
3531
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
3532 3533
    :rtype: LayerOutput
    """
3534 3535 3536

    assert size is None or state.size == size
    size = state.size
Z
zhangjinchao01 已提交
3537 3538 3539 3540 3541 3542 3543
    Layer(
        name=name,
        type=LayerType.LSTM_STEP_LAYER,
        active_type=act.name,
        active_gate_type=gate_act.name,
        active_state_type=state_act.name,
        bias=ParamAttr.to_bias(bias_attr),
3544
        size=state.size,
Q
qijun 已提交
3545 3546
        inputs=[input.name, state.name],
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
3547

Q
qijun 已提交
3548 3549 3550 3551 3552 3553 3554
    return LayerOutput(
        name=name,
        layer_type=LayerType.LSTM_STEP_LAYER,
        parents=[input, state],
        activation=act,
        size=size,
        outputs=['default', 'state'])
Z
zhangjinchao01 已提交
3555 3556 3557


@wrap_bias_attr_default()
W
wangyang59 已提交
3558
@wrap_param_attr_default()
Q
qijun 已提交
3559
@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation())
Z
zhangjinchao01 已提交
3560 3561 3562
@wrap_act_default(act=TanhActivation())
@wrap_name_default('gru_step')
@layer_support()
Q
qijun 已提交
3563 3564 3565 3566 3567 3568 3569
def gru_step_layer(input,
                   output_mem,
                   size=None,
                   act=None,
                   name=None,
                   gate_act=None,
                   bias_attr=None,
W
wangyang59 已提交
3570
                   param_attr=None,
Q
qijun 已提交
3571
                   layer_attr=None):
Z
zhangjinchao01 已提交
3572 3573 3574 3575 3576 3577 3578 3579 3580 3581
    """

    :param input:
    :type input: LayerOutput
    :param output_mem:
    :param size:
    :param act:
    :param name:
    :param gate_act:
    :param bias_attr:
3582 3583
    :param param_attr: the parameter_attribute for transforming the output_mem
                       from previous step.
Z
zhangjinchao01 已提交
3584
    :param layer_attr:
D
dangqingqing 已提交
3585
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
3586 3587 3588 3589 3590 3591 3592 3593
    :rtype: LayerOutput
    """
    assert input.size % 3 == 0
    if size is None:
        size = input.size / 3
    Layer(
        name=name,
        type=LayerType.GRU_STEP_LAYER,
3594 3595 3596 3597
        # The parameter here is for transforming the output_mem. The input has
        # already been transformed outside this module so it does not need
        # parameter associated with it.
        # The parameter here is instead grouped with input is due to
3598
        # backward model compatibility.
3599
        inputs=[Input(input.name, **param_attr.attr), output_mem.name],
Z
zhangjinchao01 已提交
3600 3601 3602 3603
        bias=ParamAttr.to_bias(bias_attr),
        size=size,
        active_type=act.name,
        active_gate_type=gate_act.name,
Q
qijun 已提交
3604
        **ExtraAttr.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
3605
    return LayerOutput(
Q
qijun 已提交
3606 3607
        name=name,
        layer_type=LayerType.GRU_STEP_LAYER,
Z
zhangjinchao01 已提交
3608
        parents=[input, output_mem],
Q
qijun 已提交
3609 3610
        size=size,
        activation=act)
Z
zhangjinchao01 已提交
3611 3612


Y
Yu Yang 已提交
3613 3614 3615 3616
@wrap_bias_attr_default()
@wrap_param_attr_default()
@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation())
@wrap_act_default(act=TanhActivation())
Q
qijun 已提交
3617
@wrap_name_default('gru_step_naive')
Y
Yu Yang 已提交
3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684
@layer_support(ERROR_CLIPPING, DROPOUT)
def gru_step_naive_layer(input,
                         output_mem,
                         size=None,
                         name=None,
                         act=None,
                         gate_act=None,
                         bias_attr=None,
                         param_attr=None,
                         layer_attr=None):
    """
    GRU Step Layer, but using MixedLayer to generate. It support ERROR_CLIPPING
    and DROPOUT.

    :param input:
    :param output_mem:
    :param size:
    :param name:
    :param act:
    :param gate_act:
    :param bias_attr:
    :param param_attr:
    :param layer_attr:
    :return:
    """
    if input.size % 3 != 0:
        raise ValueError("GruStep input size must be divided by 3")
    if size is None:
        size = input.size / 3

    def __gate__(gate_name, offset):
        with mixed_layer(
                name=name + "_" + gate_name,
                size=size,
                layer_attr=layer_attr,
                bias_attr=bias_attr,
                act=gate_act) as gate:
            gate += identity_projection(input=input, offset=offset)
            gate += full_matrix_projection(
                input=output_mem, param_attr=param_attr)
        return gate

    update_gate = __gate__("update", 0)
    reset_gate = __gate__("reset", size)

    with mixed_layer(
            name=name + "_reset_output", bias_attr=False) as reset_output:
        reset_output += dotmul_operator(a=output_mem, b=reset_gate)

    with mixed_layer(
            name=name + "_output_candidate",
            size=size,
            layer_attr=layer_attr,
            bias_attr=bias_attr,
            act=act) as output_candidate:
        output_candidate += identity_projection(input=input, offset=2 * size)
        output_candidate += full_matrix_projection(
            input=reset_output, param_attr=param_attr)

    with mixed_layer(name=name) as output:
        output += identity_projection(output_mem)
        output += dotmul_operator(a=output_mem, b=update_gate, scale=-1.0)
        output += dotmul_operator(a=output_candidate, b=update_gate)

    return output


Z
zhangjinchao01 已提交
3685 3686 3687 3688
@wrap_name_default()
@layer_support()
def get_output_layer(input, arg_name, name=None, layer_attr=None):
    """
C
caoying03 已提交
3689 3690 3691 3692
    Get layer's output by name. In PaddlePaddle, a layer might return multiple
    values, but returns one layer's output. If the user wants to use another
    output besides the default one, please use get_output_layer first to get
    the output from input.
Z
zhangjinchao01 已提交
3693 3694 3695 3696 3697 3698 3699 3700 3701

    :param name: Layer's name.
    :type name: basestring
    :param input: get output layer's input. And this layer should contains
                   multiple outputs.
    :type input: LayerOutput
    :param arg_name: Output name from input.
    :type arg_name: basestring
    :param layer_attr: Layer's extra attribute.
D
dangqingqing 已提交
3702
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
3703 3704 3705 3706 3707 3708 3709
    :rtype: LayerOutput
    """
    # GetOutputLayer
    assert arg_name in input.outputs, 'Get Output From an not existed input.' \
                                      ' The get output name is %s, which not' \
                                      ' in %s' % (
                                          arg_name, ",".join(input.outputs))
Q
qijun 已提交
3710 3711 3712 3713 3714 3715 3716
    Layer(
        name=name,
        type=LayerType.GET_OUTPUT_LAYER,
        inputs=[Input(
            input.name, input_layer_argument=arg_name)],
        size=input.size,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
3717

Q
qijun 已提交
3718 3719 3720 3721 3722
    return LayerOutput(
        name=name,
        layer_type=LayerType.GET_OUTPUT_LAYER,
        parents=[input],
        size=input.size)
Z
zhangjinchao01 已提交
3723 3724 3725 3726 3727 3728 3729


@wrap_name_default()
@wrap_act_default()
@wrap_bias_attr_default()
@wrap_param_attr_default()
@layer_support()
Q
qijun 已提交
3730 3731 3732 3733 3734 3735 3736
def recurrent_layer(input,
                    act=None,
                    bias_attr=None,
                    param_attr=None,
                    name=None,
                    reverse=False,
                    layer_attr=None):
Z
zhangjinchao01 已提交
3737
    """
3738 3739
    Simple recurrent unit layer. It is just a fully connect layer through both
    time and neural network.
Z
zhangjinchao01 已提交
3740

3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767
    For each sequence [start, end] it performs the following computation\:

    ..  math::

        out_{i} = act(in_{i})     \\      \\      \\text{for} \\ i = start \\\\
        out_{i} = act(in_{i} + out_{i-1} * W) \\ \\ \\text{for} \\ start < i <= end

    If reversed is true, the order is reversed\:

    ..  math::

        out_{i} = act(in_{i})           \\    \\   \\text{for} \\ i = end  \\\\
        out_{i} = act(in_{i} + out_{i+1} * W) \\ \\ \\text{for} \\ start <= i < end


    :param input: Input Layer
    :type input: LayerOutput
    :param act: activation.
    :type act: BaseActivation
    :param bias_attr: bias attribute.
    :type bias_attr: ParameterAttribute
    :param param_attr: parameter attribute.
    :type param_attr: ParameterAttribute
    :param name: name of the layer
    :type name: basestring
    :param layer_attr: Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
3768
    :return: LayerOutput object.
3769
    :rtype: LayerOutput
Z
zhangjinchao01 已提交
3770
    """
Q
qijun 已提交
3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785
    Layer(
        name=name,
        type=LayerType.RECURRENT_LAYER,
        inputs=Input(input.name, **param_attr.attr),
        active_type=act.name,
        bias=ParamAttr.to_bias(bias_attr),
        reversed=reverse,
        **ExtraAttr.to_kwargs(layer_attr))
    return LayerOutput(
        name=name,
        layer_type=LayerType.RECURRENT_LAYER,
        parents=[input],
        size=input.size,
        activation=act,
        reverse=reverse)
Z
zhangjinchao01 已提交
3786 3787 3788 3789 3790 3791


class StaticInput(object):
    """
    StaticInput is only used in recurrent_group which defines a read-only memory
    that can be a sequence or non-sequence.
3792 3793
    :param size: DEPRECATED
    :param is_seq: DEPRECATED
Z
zhangjinchao01 已提交
3794
    """
3795

Z
zhangjinchao01 已提交
3796 3797 3798
    def __init__(self, input, is_seq=False, size=None):
        assert isinstance(input, LayerOutput)
        self.input = input
3799
        assert input.size is not None
Z
zhangjinchao01 已提交
3800
        if size is not None:
3801
            assert input.size == size
Z
zhangjinchao01 已提交
3802 3803


3804
def SubsequenceInput(input):
Z
zhangjinchao01 已提交
3805
    """
3806
    DEPRECATED.
Z
zhangjinchao01 已提交
3807 3808 3809 3810 3811 3812 3813 3814
    Input sequence has sub-sequence, used in recurrent_group.

    The example usage is:

    .. code-block:: python

       input = SubsequenceInput(layer)
    """
3815
    return input
Z
zhangjinchao01 已提交
3816 3817 3818


@wrap_name_default("recurrent_group")
3819
def recurrent_group(step, input, reverse=False, name=None, targetInlink=None):
Z
zhangjinchao01 已提交
3820
    """
C
caoying03 已提交
3821 3822 3823 3824 3825
    Recurrent layer group is an extremely flexible recurrent unit in
    PaddlePaddle. As long as the user defines the calculation done within a
    time step, PaddlePaddle will iterate such a recurrent calculation over
    sequence input. This is extremely usefull for attention based model, or
    Neural Turning Machine like models.
Z
zhangjinchao01 已提交
3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869

    The basic usage (time steps) is:

    .. code-block:: python

       def step(input):
           output = fc_layer(input=layer,
                             size=1024,
                             act=LinearActivation(),
                             bias_attr=False)
           return output

       group = recurrent_group(input=layer,
                               step=step)

    You can see following configs for further usages:

    - time steps: lstmemory_group, paddle/gserver/tests/sequence_layer_group.conf, \
                  demo/seqToseq/seqToseq_net.py
    - sequence steps: paddle/gserver/tests/sequence_nest_layer_group.conf

    :param step: recurrent one time step function.The input of this function is
                 input of the group. The return of this function will be
                 recurrent group's return value.

                 The recurrent group scatter a sequence into time steps. And
                 for each time step, will invoke step function, and return
                 a time step result. Then gather each time step of output into
                 layer group's output.

    :type step: callable

    :param name: recurrent_group's name.
    :type name: basestring

    :param input: Input links array.

                  LayerOutput will be scattered into time steps.
                  SubsequenceInput will be scattered into sequence steps.
                  StaticInput will be imported to each time step, and doesn't change
                  through time. It's a mechanism to access layer outside step function.

    :type input: LayerOutput|StaticInput|SubsequenceInput|list|tuple

3870 3871
    :param reverse: If reverse is set true, the recurrent unit will process the
                    input sequence in a reverse order.
Z
zhangjinchao01 已提交
3872
    :type reverse: bool
3873

3874 3875
    :param targetInlink: DEPRECATED.
                         The input layer which share info with layer group's output
3876 3877 3878 3879 3880 3881 3882 3883 3884

                         Param input specifies multiple input layers. For
                         SubsequenceInput inputs, config should assign one input
                         layer that share info(the number of sentences and the number
                         of words in each sentence) with all layer group's outputs.
                         targetInlink should be one of the layer group's input.

    :type targetInlink: LayerOutput|SubsequenceInput

D
dangqingqing 已提交
3885
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
3886 3887 3888 3889
    :rtype: LayerOutput
    """
    model_type('recurrent_nn')

3890
    if isinstance(input, LayerOutput) or isinstance(input, StaticInput):
Z
zhangjinchao01 已提交
3891
        input = [input]
3892
    assert isinstance(input, collections.Sequence)
Z
zhangjinchao01 已提交
3893 3894

    def is_in_links(x):
3895
        return isinstance(x, LayerOutput)
Z
zhangjinchao01 已提交
3896 3897 3898 3899

    in_links = filter(is_in_links, input)

    RecurrentLayerGroupWithoutOutLinksBegin(
Q
qijun 已提交
3900
        name=name,
3901 3902
        in_links=map(lambda x: x.name, in_links),
        seq_reversed=reverse)
Z
zhangjinchao01 已提交
3903 3904
    in_args = []
    for each_input in input:
3905
        if isinstance(each_input, StaticInput):  # StaticInput
Z
zhangjinchao01 已提交
3906
            mem_name = "__%s_memory__" % each_input.input.name
Q
qijun 已提交
3907
            mem = memory(
3908
                name=None,
Q
qijun 已提交
3909 3910
                size=each_input.input.size,
                boot_layer=each_input.input)
3911
            mem.set_input(mem)
Z
zhangjinchao01 已提交
3912
            in_args.append(mem)
3913 3914
        else:
            in_args.append(each_input)
L
Luo Tao 已提交
3915

Z
zhangjinchao01 已提交
3916 3917 3918 3919 3920
    layer_outs = step(*in_args)

    if isinstance(layer_outs, LayerOutput):
        layer_outs = [layer_outs]

3921 3922 3923 3924 3925 3926
    for layer_out in layer_outs:
        assert isinstance(
            layer_out, LayerOutput
        ), "Type of step function's return value must be LayerOutput."
        layer_out.reverse = reverse
        RecurrentLayerGroupSetOutLink(layer_out.name)
Z
zhangjinchao01 已提交
3927 3928 3929

    RecurrentLayerGroupEnd(name=name)

X
xuwei06 已提交
3930
    for layer_out in layer_outs:
3931 3932
        # The previous full_name is the name inside the recurrent group.
        # We need a full_name outside the recurrent group.
X
xuwei06 已提交
3933 3934
        layer_out.full_name = MakeLayerNameInSubmodel(layer_out.name)

Z
zhangjinchao01 已提交
3935 3936 3937 3938 3939
    if len(layer_outs) == 1:
        return layer_outs[0]
    else:
        return layer_outs

3940

Z
zhangjinchao01 已提交
3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954
class BaseGeneratedInput(object):
    def __init__(self):
        self.bos_id = None
        self.eos_id = None

    def before_real_step(self):
        raise NotImplementedError()

    def after_real_step(self, *args):
        raise NotImplementedError()


class GeneratedInput(BaseGeneratedInput):
    def after_real_step(self, input):
3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968
        if isinstance(input, LayerOutput):
            input = [input]
        elif isinstance(input, collections.Sequence):
            input = list(input)
            if len(input) > 1:
                logger.info(
                    ("More than one layers inside the recurrent_group "
                     "are returned as outputs of the entire recurrent_group "
                     "PLEASE garantee the first output is probability of "
                     "the predicted next word."))

        return [maxid_layer(
            input=input[0], name='__beam_search_predict__')] + (
                input[1:] if len(input) > 1 else [])
Z
zhangjinchao01 已提交
3969 3970

    def before_real_step(self):
Q
qijun 已提交
3971 3972 3973 3974 3975 3976 3977 3978 3979
        predict_id = memory(
            name='__beam_search_predict__',
            size=self.size,
            boot_with_const_id=self.bos_id)

        trg_emb = embedding_layer(
            input=predict_id,
            size=self.embedding_size,
            param_attr=ParamAttr(name=self.embedding_name))
Z
zhangjinchao01 已提交
3980 3981 3982
        return trg_emb

    def __init__(self, size, embedding_name, embedding_size):
3983
        super(GeneratedInput, self).__init__()
Z
zhangjinchao01 已提交
3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006
        self.size = size
        self.embedding_name = embedding_name
        self.embedding_size = embedding_size


@wrap_name_default()
def maxid_layer(input, name=None, layer_attr=None):
    """
    A layer for finding the id which has the maximal value for each sample.
    The result is stored in output.ids.

    The example usage is:

    .. code-block:: python

       maxid = maxid_layer(input=layer)

    :param input: Input layer name.
    :type input: LayerOutput
    :param name: Layer name.
    :type name: basestring
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
D
dangqingqing 已提交
4007
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
4008 4009 4010 4011
    :rtype: LayerOutput
    """

    assert isinstance(input, LayerOutput)
Q
qijun 已提交
4012 4013 4014 4015 4016 4017 4018 4019 4020 4021
    l = Layer(
        name=name,
        type='maxid',
        inputs=[input.name],
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name=name,
        layer_type=LayerType.MAXID_LAYER,
        parents=[input],
        size=l.config.size)
Z
zhangjinchao01 已提交
4022

4023

H
Haonan 已提交
4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049
@wrap_name_default()
def out_prod_layer(input1, input2, name=None, layer_attr=None):
    """
    A layer for computing the outer product of two vectors
    The result is a matrix of size(input1) x size(input2)

    The example usage is:

    .. code-block:: python

       out_prod = out_prod_layer(input1=vec1, input2=vec2)

    :param name: Layer name.
    :type name: basestring
    :param input1: The first input layer name.
    :type input: LayerOutput
    :param input2: The second input layer name.
    :type input2: LayerOutput
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
    :return: LayerOutput object.
    :rtype: LayerOutput
    """

    assert isinstance(input1, LayerOutput)
    assert isinstance(input2, LayerOutput)
Q
qijun 已提交
4050 4051 4052 4053 4054 4055 4056 4057 4058 4059
    l = Layer(
        name=name,
        type=LayerType.OUT_PROD_LAYER,
        inputs=[input1.name, input2.name],
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name=name,
        layer_type=LayerType.OUT_PROD_LAYER,
        parents=[input1, input2],
        size=l.config.size)
4060

Z
zhangjinchao01 已提交
4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076

@wrap_name_default()
def eos_layer(input, eos_id, name=None, layer_attr=None):
    """
    A layer for checking EOS for each sample:
    - output_id = (input_id == conf.eos_id)

    The result is stored in output\_.ids.
    It is used by recurrent layer group.

    The example usage is:

    .. code-block:: python

       eos = eos_layer(input=layer, eos_id=id)

L
luotao02 已提交
4077 4078
    :param name: Layer name.
    :type name: basestring
Z
zhangjinchao01 已提交
4079 4080 4081 4082 4083 4084
    :param input: Input layer name.
    :type input: LayerOutput
    :param eos_id: end id of sequence
    :type eos_id: int
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
D
dangqingqing 已提交
4085
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
4086 4087
    :rtype: LayerOutput
    """
Q
qijun 已提交
4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098
    l = Layer(
        name=name,
        type=LayerType.EOSID_LAYER,
        eos_id=eos_id,
        inputs=[input.name],
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name=name,
        layer_type=LayerType.EOSID_LAYER,
        parents=[input],
        size=l.config.size)
Z
zhangjinchao01 已提交
4099 4100 4101


@wrap_name_default()
Q
qijun 已提交
4102 4103 4104 4105 4106 4107 4108
def beam_search(step,
                input,
                bos_id,
                eos_id,
                beam_size,
                max_length=500,
                name=None,
Z
zhangjinchao01 已提交
4109
                num_results_per_sample=None):
4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120
    """
    Beam search is a heuristic search algorithm used in sequence generation.
    It explores a graph by expanding the most promising nodes in a limited set
    to maintain tractability.

    The example usage is:

    .. code-block:: python

        def rnn_step(input):
            last_time_step_output = memory(name='rnn', size=512)
4121
            with mixed_layer(size=512, name='rnn') as simple_rnn:
4122 4123 4124 4125
                simple_rnn += full_matrix_projection(input)
                simple_rnn += last_time_step_output
            return simple_rnn

4126 4127 4128 4129 4130
        generated_word_embedding = GeneratedInput(
                               size=target_dictionary_dim,
                               embedding_name="target_language_embedding",
                               embedding_size=word_vector_dim)

4131 4132
        beam_gen = beam_search(name="decoder",
                               step=rnn_step,
4133 4134
                               input=[StaticInput(encoder_last),
                                      generated_word_embedding],
4135 4136
                               bos_id=0,
                               eos_id=1,
4137
                               beam_size=5)
4138 4139 4140 4141 4142 4143 4144 4145 4146

    Please see the following demo for more details:

    - machine translation : demo/seqToseq/translation/gen.conf \
                            demo/seqToseq/seqToseq_net.py

    :param name: Name of the recurrent unit that generates sequences.
    :type name: base string
    :param step: A callable function that defines the calculation in a time
4147
                 step, and it is applied to sequences with arbitrary length by
4148 4149 4150 4151 4152
                 sharing a same set of weights.

                 You can refer to the first parameter of recurrent_group, or
                 demo/seqToseq/seqToseq_net.py for more details.
    :type step: callable
4153 4154
    :param input: Input data for the recurrent unit, which should include the
                  previously generated words as a GeneratedInput object.
4155
                  In beam_search, none of the input's type should be LayerOutput.
4156
    :type input: list
4157 4158 4159
    :param bos_id: Index of the start symbol in the dictionary. The start symbol
                   is a special token for NLP task, which indicates the
                   beginning of a sequence. In the generation task, the start
4160
                   symbol is essential, since it is used to initialize the RNN
4161 4162 4163 4164 4165 4166 4167 4168
                   internal state.
    :type bos_id: int
    :param eos_id: Index of the end symbol in the dictionary. The end symbol is
                   a special token for NLP task, which indicates the end of a
                   sequence. The generation process will stop once the end
                   symbol is generated, or a pre-defined max iteration number
                   is exceeded.
    :type eos_id: int
4169 4170
    :param max_length: Max generated sequence length.
    :type max_length: int
4171 4172 4173 4174 4175 4176 4177 4178 4179 4180
    :param beam_size: Beam search for sequence generation is an iterative search
                      algorithm. To maintain tractability, every iteration only
                      only stores a predetermined number, called the beam_size,
                      of the most promising next words. The greater the beam
                      size, the fewer candidate words are pruned.
    :type beam_size: int
    :param num_results_per_sample: Number of the generated results per input
                                  sequence. This number must always be less than
                                  beam size.
    :type num_results_per_sample: int
4181 4182
    :return: The generated word index.
    :rtype: LayerOutput
4183 4184
    """

Z
zhangjinchao01 已提交
4185 4186 4187 4188 4189
    if num_results_per_sample is None:
        num_results_per_sample = beam_size
    if num_results_per_sample > beam_size:
        logger.warning("num_results_per_sample should be less than beam_size")

Q
qijun 已提交
4190
    if isinstance(input, StaticInput) or isinstance(input, BaseGeneratedInput):
Z
zhangjinchao01 已提交
4191 4192 4193 4194 4195 4196
        input = [input]

    generated_input_index = -1

    real_input = []
    for i, each_input in enumerate(input):
4197 4198 4199
        assert not isinstance(each_input, LayerOutput), (
            "in beam_search, "
            "none of the input should has a type of LayerOutput.")
Z
zhangjinchao01 已提交
4200
        if isinstance(each_input, BaseGeneratedInput):
4201 4202
            assert generated_input_index == -1, ("recurrent_group accepts "
                                                 "only one GeneratedInput.")
Z
zhangjinchao01 已提交
4203
            generated_input_index = i
4204

Z
zhangjinchao01 已提交
4205 4206 4207
        else:
            real_input.append(each_input)

4208
    assert generated_input_index != -1, "No GeneratedInput is given."
Z
zhangjinchao01 已提交
4209 4210 4211 4212 4213 4214 4215 4216

    gipt = input[generated_input_index]

    gipt.bos_id = bos_id
    gipt.eos_id = eos_id

    def __real_step__(*args):
        eos_name = "__%s_eos_layer__" % name
Q
qijun 已提交
4217 4218 4219 4220 4221 4222
        RecurrentLayerGroupSetGenerator(
            Generator(
                eos_layer_name=eos_name,
                max_num_frames=max_length,
                beam_size=beam_size,
                num_results_per_sample=num_results_per_sample))
Z
zhangjinchao01 已提交
4223 4224 4225 4226 4227 4228

        args = list(args)
        args.insert(generated_input_index, gipt.before_real_step())

        predict = gipt.after_real_step(step(*args))

4229
        eos_layer(input=predict[0], eos_id=eos_id, name=eos_name)
Z
zhangjinchao01 已提交
4230 4231
        return predict

4232 4233
    return recurrent_group(
        step=__real_step__, input=real_input, reverse=False, name=name)
Z
zhangjinchao01 已提交
4234

Q
qijun 已提交
4235

4236 4237
def __cost_input__(input, label, weight=None):
    """
4238
    inputs and parents for cost layers.
4239
    """
C
caoying03 已提交
4240 4241 4242 4243 4244 4245
    if isinstance(input, LayerOutput):
        input = [input]
    if isinstance(label, LayerOutput):
        label = [label]
    ipts = [Input(ipt.name) for ipt in (input + label)]
    parents = [ipt for ipt in (input + label)]
4246
    if weight is not None:
4247
        assert weight.size == 1
4248 4249 4250
        ipts.append(Input(weight.name))
        parents.append(weight)
    return ipts, parents
4251

Z
zhangjinchao01 已提交
4252 4253

@wrap_name_default()
L
luotao1 已提交
4254
@layer_support()
4255 4256 4257 4258 4259 4260
def square_error_cost(input,
                      label,
                      weight=None,
                      name=None,
                      coeff=1.0,
                      layer_attr=None):
Z
zhangjinchao01 已提交
4261
    """
4262
    sum of square error cost:
L
Luo Tao 已提交
4263 4264 4265

    ..  math::

4266
        cost = \\sum_{i=1}^N(t_i-y_i)^2
Z
zhangjinchao01 已提交
4267 4268

    :param name: layer name.
4269
    :type name: basestring
Z
zhangjinchao01 已提交
4270
    :param input: Network prediction.
4271
    :type input: LayerOutput
Z
zhangjinchao01 已提交
4272
    :param label: Data label.
4273 4274 4275 4276
    :type label: LayerOutput
    :param weight: The weight affects the cost, namely the scale of cost.
                   It is an optional argument.
    :type weight: LayerOutput
4277 4278
    :param coeff: The coefficient affects the gradient in the backward.
    :type coeff: float
L
luotao1 已提交
4279 4280
    :param layer_attr: layer's extra attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
4281
    :return: LayerOutput object.
4282
    :rtype: LayerOutput
Z
zhangjinchao01 已提交
4283
    """
4284 4285
    ipts, parents = __cost_input__(input, label, weight)

Q
qijun 已提交
4286 4287 4288 4289
    Layer(
        inputs=ipts,
        type="square_error",
        name=name,
4290
        coeff=coeff,
Q
qijun 已提交
4291
        **ExtraLayerAttribute.to_kwargs(layer_attr))
X
xuwei06 已提交
4292
    return LayerOutput(name, LayerType.COST, parents=parents, size=1)
Z
zhangjinchao01 已提交
4293 4294


4295
regression_cost = square_error_cost
L
Luo Tao 已提交
4296 4297


Z
zhangjinchao01 已提交
4298
@wrap_name_default("cost")
4299
@layer_support()
Q
qijun 已提交
4300 4301 4302 4303
def classification_cost(input,
                        label,
                        weight=None,
                        name=None,
4304
                        evaluator=classification_error_evaluator,
4305 4306
                        layer_attr=None,
                        coeff=1.):
Z
zhangjinchao01 已提交
4307 4308 4309 4310 4311 4312 4313 4314 4315
    """
    classification cost Layer.

    :param name: layer name.
    :type name: basestring
    :param input: input layer name. network output.
    :type input: LayerOutput
    :param label: label layer name. data_layer often.
    :type label: LayerOutput
4316 4317 4318
    :param weight: The weight affects the cost, namely the scale of cost.
                   It is an optional argument.
    :type weight: LayerOutput
Z
zhangjinchao01 已提交
4319
    :param evaluator: Evaluator method.
4320 4321
    :param layer_attr: layer's extra attribute.
    :type layer_attr: ExtraLayerAttribute
4322 4323
    :param coeff: The coefficient affects the gradient in the backward.
    :type coeff: float
D
dangqingqing 已提交
4324
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
4325 4326 4327 4328 4329
    :rtype: LayerOutput
    """
    assert input.layer_type != LayerType.DATA
    assert isinstance(input.activation, SoftmaxActivation)
    assert label.layer_type == LayerType.DATA
4330 4331 4332

    ipts, parents = __cost_input__(input, label, weight)

Q
qijun 已提交
4333 4334 4335 4336
    Layer(
        name=name,
        type="multi-class-cross-entropy",
        inputs=ipts,
4337
        coeff=coeff,
Q
qijun 已提交
4338
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
4339 4340 4341 4342 4343 4344 4345 4346 4347 4348

    def __add_evaluator__(e):
        assert callable(e)
        assert hasattr(e, 'is_evaluator')
        assert isinstance(e.is_evaluator, bool)
        assert e.is_evaluator
        assert hasattr(e, "for_classification")
        assert isinstance(e.for_classification, bool)
        assert e.for_classification

4349
        e(name=e.__name__, input=input, label=label, weight=weight)
Z
zhangjinchao01 已提交
4350

4351
    if not isinstance(evaluator, collections.Sequence):
Z
zhangjinchao01 已提交
4352 4353 4354 4355 4356
        evaluator = [evaluator]

    for each_evaluator in evaluator:
        __add_evaluator__(each_evaluator)

X
xuwei06 已提交
4357
    return LayerOutput(name, LayerType.COST, parents=parents, size=1)
Z
zhangjinchao01 已提交
4358

4359

Q
qijun 已提交
4360 4361 4362 4363 4364 4365 4366 4367 4368
def conv_operator(img,
                  filter,
                  filter_size,
                  num_filters,
                  num_channels=None,
                  stride=1,
                  padding=0,
                  filter_size_y=None,
                  stride_y=None,
4369 4370
                  padding_y=None,
                  trans=False):
Z
zhangjinchao01 已提交
4371 4372 4373 4374 4375 4376 4377 4378 4379 4380
    """
    Different from img_conv_layer, conv_op is an Operator, which can be used
    in mixed_layer. And conv_op takes two inputs to perform convolution.
    The first input is the image and the second is filter kernel. It only
    support GPU mode.

    The example usage is:

    .. code-block:: python

4381 4382
       op = conv_operator(img=input1,
                          filter=input2,
4383
                          filter_size=3,
Z
zhangjinchao01 已提交
4384 4385 4386
                          num_filters=64,
                          num_channels=64)

4387 4388 4389 4390
    :param img: input image
    :type img: LayerOutput
    :param filter: input filter
    :type filter: LayerOutput
Z
zhangjinchao01 已提交
4391 4392
    :param filter_size: The x dimension of a filter kernel.
    :type filter_size: int
C
caoying03 已提交
4393 4394 4395
    :param filter_size_y: The y dimension of a filter kernel. Since
                        PaddlePaddle now supports rectangular filters,
                        the filter's shape can be (filter_size, filter_size_y).
Z
zhangjinchao01 已提交
4396
    :type filter_size_y: int
4397 4398
    :param num_filters: channel of output data.
    :type num_filters: int
4399 4400
    :param num_channels: channel of input data.
    :type num_channels: int
Z
zhangjinchao01 已提交
4401
    :param stride: The x dimension of the stride.
L
luotao02 已提交
4402
    :type stride: int
Z
zhangjinchao01 已提交
4403
    :param stride_y: The y dimension of the stride.
L
luotao02 已提交
4404
    :type stride_y: int
Z
zhangjinchao01 已提交
4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417
    :param padding: The x dimension of padding.
    :type padding: int
    :param padding_y: The y dimension of padding.
    :type padding_y: int
    :return: A ConvOperator Object.
    :rtype: ConvOperator
    """
    if filter_size_y is None:
        filter_size_y = filter_size
    if stride_y is None:
        stride_y = stride
    if padding_y is None:
        padding_y = padding
4418

4419 4420
    if num_channels is None:
        num_channels = img.num_filters
4421 4422

    assert isinstance(filter, LayerOutput)
4423
    assert filter.size is not None
4424

4425 4426 4427
    opCls = ConvTransOperator if trans else ConvOperator

    op = opCls(
Q
qijun 已提交
4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438
        input_layer_names=[img.name, filter.name],
        num_filters=num_filters,
        conv_conf=Conv(
            filter_size=filter_size,
            padding=padding,
            stride=stride,
            channels=num_channels,
            filter_size_y=filter_size_y,
            padding_y=padding_y,
            stride_y=stride_y,
            groups=1))
4439

4440
    op.origin = [img, filter]
Z
zhangjinchao01 已提交
4441 4442
    return op

Q
qijun 已提交
4443

4444
@wrap_param_attr_default()
Q
qijun 已提交
4445 4446 4447 4448 4449 4450 4451 4452 4453 4454
def conv_projection(input,
                    filter_size,
                    num_filters,
                    num_channels=None,
                    stride=1,
                    padding=0,
                    filter_size_y=None,
                    stride_y=None,
                    padding_y=None,
                    groups=1,
4455 4456
                    param_attr=None,
                    trans=False):
4457 4458 4459 4460 4461 4462 4463 4464 4465
    """
    Different from img_conv_layer and conv_op, conv_projection is an Projection,
    which can be used in mixed_layer and conat_layer. It use cudnn to implement
    conv and only support GPU mode.

    The example usage is:

    .. code-block:: python

D
dangqingqing 已提交
4466
       proj = conv_projection(input=input1,
4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480
                              filter_size=3,
                              num_filters=64,
                              num_channels=64)

    :param input: input layer
    :type input: LayerOutput
    :param filter_size: The x dimension of a filter kernel.
    :type filter_size: int
    :param filter_size_y: The y dimension of a filter kernel. Since
                          PaddlePaddle now supports rectangular filters,
                          the filter's shape can be (filter_size, filter_size_y).
    :type filter_size_y: int
    :param num_filters: channel of output data.
    :type num_filters: int
4481 4482
    :param num_channels: channel of input data.
    :type num_channels: int
4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494
    :param stride: The x dimension of the stride.
    :type stride: int
    :param stride_y: The y dimension of the stride.
    :type stride_y: int
    :param padding: The x dimension of padding.
    :type padding: int
    :param padding_y: The y dimension of padding.
    :type padding_y: int
    :param groups: The group number.
    :type groups: int
    :param param_attr: Convolution param attribute. None means default attribute
    :type param_attr: ParameterAttribute
4495 4496
    :param trans: whether it is convTrans or conv
    :type trans: boolean
4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526
    :return: A DotMulProjection Object.
    :rtype: DotMulProjection
    """
    if num_channels is None:
        assert input.num_filters is not None
        num_channels = input.num_filters

    if filter_size_y is None:
        if isinstance(filter_size, collections.Sequence):
            assert len(filter_size) == 2
            filter_size, filter_size_y = filter_size
        else:
            filter_size_y = filter_size

    if stride_y is None:
        if isinstance(stride, collections.Sequence):
            assert len(stride) == 2
            stride, stride_y = stride
        else:
            stride_y = stride

    if padding_y is None:
        if isinstance(padding, collections.Sequence):
            assert len(padding) == 2
            padding, padding_y = padding
        else:
            padding_y = padding

    if param_attr.attr.get('initial_smart'):
        # special initial for conv layers.
Q
qijun 已提交
4527
        init_w = (2.0 / (filter_size**2 * num_channels))**0.5
4528 4529 4530 4531 4532
        param_attr.attr["initial_mean"] = 0.0
        param_attr.attr["initial_std"] = init_w
        param_attr.attr["initial_strategy"] = 0
        param_attr.attr["initial_smart"] = False

4533 4534 4535
    projCls = ConvTransProjection if trans else ConvProjection

    proj = projCls(
Q
qijun 已提交
4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547
        input_layer_name=input.name,
        num_filters=num_filters,
        conv_conf=Conv(
            filter_size=filter_size,
            padding=padding,
            stride=stride,
            channels=num_channels,
            filter_size_y=filter_size_y,
            padding_y=padding_y,
            stride_y=stride_y,
            groups=groups),
        **param_attr.attr)
4548 4549 4550 4551

    proj.origin = input
    return proj

Z
zhangjinchao01 已提交
4552

D
dangqingqing 已提交
4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569
@wrap_name_default("pad")
@layer_support()
def pad_layer(input,
              pad_c=None,
              pad_h=None,
              pad_w=None,
              name=None,
              layer_attr=None):
    """
    This operation pads zeros to the input data according to pad_c,pad_h
    and pad_w. pad_c, pad_h, pad_w specifies the which dimension and size
    of padding. And the input data shape is NCHW.

    For example, pad_c=[2,3] means padding 2 zeros before the
    input data and 3 zeros after the input data in channel dimension.
    pad_h means padding zeros in height dimension. pad_w means padding zeros
    in width dimension.
4570

D
dangqingqing 已提交
4571
    For example,
4572

4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593
    .. code-block:: python

       input(2,2,2,3)  = [
                           [ [[1,2,3], [3,4,5]],
                             [[2,3,5], [1,6,7]] ],
                           [ [[4,3,1], [1,8,7]],
                             [[3,8,9], [2,3,5]] ]
                         ]

       pad_c=[1,1], pad_h=[0,0], pad_w=[0,0]

       output(2,4,2,3) = [
                           [ [[0,0,0], [0,0,0]],
                             [[1,2,3], [3,4,5]],
                             [[2,3,5], [1,6,7]],
                             [[0,0,0], [0,0,0]] ],
                           [ [[0,0,0], [0,0,0]],
                             [[4,3,1], [1,8,7]],
                             [[3,8,9], [2,3,5]],
                             [[0,0,0], [0,0,0]] ]
                         ]
D
dangqingqing 已提交
4594 4595

    The simply usage is:
D
dangqingqing 已提交
4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656

    .. code-block:: python

       pad = pad_layer(input=ipt,
                       pad_c=[4,4],
                       pad_h=[0,0],
                       pad_w=[2,2])

    :param input: layer's input.
    :type input: LayerOutput
    :param pad_c: padding size in channel dimension.
    :type pad_c: list|None
    :param pad_h: padding size in height dimension.
    :type pad_h: list|None
    :param pad_w: padding size in width dimension.
    :type pad_w: list|None
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
    :param name: layer name.
    :type name: basestring
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
    if pad_c is not None:
        assert isinstance(pad_c, collections.Sequence) and len(pad_c) == 2
    else:
        pad_c = [0, 0]

    if pad_h is not None:
        assert isinstance(pad_h, collections.Sequence) and len(pad_h) == 2
    else:
        pad_h = [0, 0]

    if pad_w is not None:
        assert isinstance(pad_w, collections.Sequence) and len(pad_w) == 2
    else:
        pad_w = [0, 0]

    assert input.num_filters is not None
    in_ch = input.num_filters
    out_ch = in_ch + pad_c[0] + pad_c[1]

    l = Layer(
        name=name,
        type=LayerType.PAD_LAYER,
        inputs=Input(
            input.name,
            pad=Pad(
                channels=in_ch,
                pad_c=pad_c,
                pad_h=pad_h,
                pad_w=pad_w, )),
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name,
        layer_type=LayerType.PAD_LAYER,
        parents=[input],
        num_filters=out_ch,
        size=l.config.size)


Z
zhangjinchao01 已提交
4657
@wrap_name_default()
L
luotao1 已提交
4658 4659
@layer_support()
def conv_shift_layer(a, b, name=None, layer_attr=None):
Z
zhangjinchao01 已提交
4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670
    """
    This layer performs cyclic convolution for two input. For example:
      - a[in]: contains M elements.
      - b[in]: contains N elements (N should be odd).
      - c[out]: contains M elements.

    .. math::

        c[i] = \sum_{j=-(N-1)/2}^{(N-1)/2}a_{i+j} * b_{j}

    In this formular:
4671 4672 4673 4674
     - a's index is computed modulo M. When it is negative, then get item from
       the right side (which is the end of array) to the left.
     - b's index is computed modulo N. When it is negative, then get item from
       the right size (which is the end of array) to the left.
Z
zhangjinchao01 已提交
4675 4676 4677 4678 4679

    The example usage is:

    .. code-block:: python

L
Luo Tao 已提交
4680
       conv_shift = conv_shift_layer(a=layer1, b=layer2)
Z
zhangjinchao01 已提交
4681 4682 4683

    :param name: layer name
    :type name: basestring
4684 4685
    :param a: Input layer a.
    :type a: LayerOutput
L
Luo Tao 已提交
4686
    :param b: input layer b.
4687
    :type b: LayerOutput
L
luotao1 已提交
4688 4689
    :param layer_attr: layer's extra attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
4690
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
4691 4692
    :rtype: LayerOutput
    """
4693 4694
    assert isinstance(a, LayerOutput) and isinstance(b, LayerOutput)
    assert b.size is None or b.size % 2 == 1  # size of b must be odd.
Z
zhangjinchao01 已提交
4695 4696 4697
    Layer(
        name=name,
        type=LayerType.CONV_SHIFT_LAYER,
4698
        inputs=[a.name, b.name],
Q
qijun 已提交
4699
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
4700

Q
qijun 已提交
4701 4702
    return LayerOutput(
        name, LayerType.CONV_SHIFT_LAYER, parents=[a, b], size=a.size)
Z
zhangjinchao01 已提交
4703 4704 4705 4706 4707


@wrap_name_default()
@wrap_param_attr_default()
@wrap_bias_attr_default()
4708
@wrap_act_default(act=LinearActivation())
Z
zhangjinchao01 已提交
4709
@layer_support(ERROR_CLIPPING, DROPOUT)
Q
qijun 已提交
4710 4711 4712 4713 4714 4715 4716 4717
def tensor_layer(a,
                 b,
                 size,
                 act=None,
                 name=None,
                 param_attr=None,
                 bias_attr=None,
                 layer_attr=None):
Z
zhangjinchao01 已提交
4718 4719 4720 4721 4722
    """
    This layer performs tensor operation for two input.
    For example, each sample:

    .. math::
4723
       y_{i} = a * W_{i} * {b^\mathrm{T}}, i=0,1,...,K-1
Z
zhangjinchao01 已提交
4724 4725

    In this formular:
4726 4727
      - :math:`a`: the first input contains M elements.
      - :math:`b`: the second input contains N elements.
Z
zhangjinchao01 已提交
4728 4729
      - :math:`y_{i}`: the i-th element of y.
      - :math:`W_{i}`: the i-th learned weight, shape if [M, N]
4730
      - :math:`b^\mathrm{T}`: the transpose of :math:`b_{2}`.
Z
zhangjinchao01 已提交
4731 4732 4733 4734 4735

    The simple usage is:

    .. code-block:: python

4736
       tensor = tensor_layer(a=layer1, b=layer2, size=1000)
Z
zhangjinchao01 已提交
4737 4738 4739

    :param name: layer name
    :type name: basestring
4740 4741 4742 4743
    :param a: Input layer a.
    :type a: LayerOutput
    :param b: input layer b.
    :type b: LayerOutput
Z
zhangjinchao01 已提交
4744
    :param size: the layer dimension.
L
luotao02 已提交
4745
    :type size: int.
Z
zhangjinchao01 已提交
4746 4747 4748
    :param act: Activation Type. Default is tanh.
    :type act: BaseActivation
    :param param_attr: The Parameter Attribute.
4749
    :type param_attr: ParameterAttribute
Z
zhangjinchao01 已提交
4750 4751 4752 4753 4754 4755
    :param bias_attr: The Bias Attribute. If no bias, then pass False or
                      something not type of ParameterAttribute. None will get a
                      default Bias.
    :type bias_attr: ParameterAttribute|None|Any
    :param layer_attr: Extra Layer config.
    :type layer_attr: ExtraLayerAttribute|None
D
dangqingqing 已提交
4756
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
4757 4758
    :rtype: LayerOutput
    """
4759
    assert isinstance(a, LayerOutput) and isinstance(b, LayerOutput)
Z
zhangjinchao01 已提交
4760 4761 4762 4763 4764 4765
    Layer(
        name=name,
        size=size,
        type=LayerType.TENSOR_LAYER,
        active_type=act.name,
        bias=ParamAttr.to_bias(bias_attr),
Q
qijun 已提交
4766 4767 4768 4769
        inputs=[Input(a.name, **param_attr.attr), Input(b.name)],
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.TENSOR_LAYER, parents=[a, b], activation=act, size=size)
Z
zhangjinchao01 已提交
4770 4771 4772 4773 4774 4775


@wrap_name_default()
@wrap_param_attr_default()
@wrap_bias_attr_default()
@wrap_act_default()
4776
@layer_support(DROPOUT, ERROR_CLIPPING)
Q
qijun 已提交
4777 4778
def selective_fc_layer(input,
                       size,
L
Luo Tao 已提交
4779
                       select=None,
Q
qijun 已提交
4780 4781
                       act=None,
                       name=None,
Z
zhangjinchao01 已提交
4782 4783 4784
                       pass_generation=False,
                       has_selected_colums=True,
                       mul_ratio=0.02,
Q
qijun 已提交
4785 4786 4787
                       param_attr=None,
                       bias_attr=None,
                       layer_attr=None):
Z
zhangjinchao01 已提交
4788 4789 4790 4791 4792 4793 4794 4795 4796 4797
    """
    Selectived fully connected layer. Different from fc_layer, the output
    of this layer maybe sparse. It requires an additional input to indicate
    several selected columns for output. If the selected columns is not
    specified, selective_fc_layer acts exactly like fc_layer.

    The simple usage is:

    .. code-block:: python

4798
       sel_fc = selective_fc_layer(input=input, size=128, act=TanhActivation())
Z
zhangjinchao01 已提交
4799 4800 4801 4802 4803

    :param name: The Layer Name.
    :type name: basestring
    :param input: The input layer.
    :type input: LayerOutput|list|tuple
4804 4805
    :param select: The select layer. The output of select layer should be a
                   sparse binary matrix, and treat as the mask of selective fc.
L
Luo Tao 已提交
4806
                   If is None, acts exactly like fc_layer.
4807
    :type select: LayerOutput
Z
zhangjinchao01 已提交
4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819
    :param size: The layer dimension.
    :type size: int
    :param act: Activation Type. Default is tanh.
    :type act: BaseActivation
    :param param_attr: The Parameter Attribute.
    :type param_attr: ParameterAttribute
    :param bias_attr: The Bias Attribute. If no bias, then pass False or
                      something not type of ParameterAttribute. None will get a
                      default Bias.
    :type bias_attr: ParameterAttribute|None|Any
    :param layer_attr: Extra Layer config.
    :type layer_attr: ExtraLayerAttribute|None
D
dangqingqing 已提交
4820
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
4821 4822 4823 4824
    :rtype: LayerOutput
    """
    if isinstance(input, LayerOutput):
        input = [input]
4825
        assert not isinstance(param_attr, collections.Sequence)
Z
zhangjinchao01 已提交
4826 4827
        param_attr = [param_attr]
    else:
4828
        if isinstance(param_attr, collections.Sequence):
Z
zhangjinchao01 已提交
4829 4830 4831 4832
            assert len(input) == len(param_attr)
        else:
            param_attr = [copy.deepcopy(param_attr) for _ in range(len(input))]

4833 4834 4835 4836
    assert isinstance(input, collections.Sequence)
    assert isinstance(select, LayerOutput)
    if select.size is not None:
        assert select.size == size
Z
zhangjinchao01 已提交
4837
    Layer(
Q
qijun 已提交
4838 4839 4840
        inputs=[
            Input(ipt.name, **attr.attr) for ipt, attr in zip(input, param_attr)
        ] + [select.name],
Z
zhangjinchao01 已提交
4841 4842 4843
        name=name,
        type=LayerType.SEL_FC_LAYER,
        size=size,
4844
        bias=ParameterAttribute.to_bias(bias_attr),
Z
zhangjinchao01 已提交
4845 4846 4847 4848
        active_type=act.name,
        selective_fc_pass_generation=pass_generation,
        has_selected_colums=has_selected_colums,
        selective_fc_full_mul_ratio=mul_ratio,
Q
qijun 已提交
4849 4850 4851 4852 4853 4854 4855
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name,
        LayerType.SEL_FC_LAYER,
        list(input) + [select],
        activation=act,
        size=size)
Z
zhangjinchao01 已提交
4856 4857 4858


@wrap_name_default()
L
luotao1 已提交
4859 4860
@layer_support()
def sampling_id_layer(input, name=None, layer_attr=None):
Z
zhangjinchao01 已提交
4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874
    """
    A layer for sampling id from multinomial distribution from the input layer.
    Sampling one id for one sample.

    The simple usage is:

    .. code-block:: python

       samping_id = sampling_id_layer(input=input)

    :param input: The input layer.
    :type input: LayerOutput
    :param name: The Layer Name.
    :type name: basestring
L
luotao1 已提交
4875 4876
    :param layer_attr: Extra Layer config.
    :type layer_attr: ExtraLayerAttribute|None
D
dangqingqing 已提交
4877
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
4878 4879
    :rtype: LayerOutput
    """
X
xuwei06 已提交
4880
    l = Layer(
Z
zhangjinchao01 已提交
4881 4882 4883
        name=name,
        type=LayerType.SAMPLING_ID_LAYER,
        inputs=[Input(input.name)],
Q
qijun 已提交
4884 4885 4886
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.SAMPLING_ID_LAYER, input, size=l.config.size)
Z
zhangjinchao01 已提交
4887 4888 4889


@wrap_name_default()
L
luotao1 已提交
4890
@layer_support()
Q
qijun 已提交
4891 4892 4893 4894
def slope_intercept_layer(input,
                          name=None,
                          slope=1.0,
                          intercept=0.0,
L
luotao1 已提交
4895
                          layer_attr=None):
Z
zhangjinchao01 已提交
4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916
    """
    This layer for applying a slope and an intercept to the input
    element-wise. There is no activation and weight.

    ..  math::
        y = slope * x + intercept

    The simple usage is:

    .. code-block:: python

       scale = slope_intercept_layer(input=input, slope=-1.0, intercept=1.0)

    :param input: The input layer.
    :type input: LayerOutput
    :param name: The Layer Name.
    :type name: basestring
    :param slope: the scale factor.
    :type slope: float.
    :param intercept: the offset.
    :type intercept: float.
L
luotao1 已提交
4917 4918
    :param layer_attr: Extra Layer config.
    :type layer_attr: ExtraLayerAttribute|None
D
dangqingqing 已提交
4919
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
4920 4921 4922 4923 4924 4925 4926 4927
    :rtype: LayerOutput
    """
    Layer(
        name=name,
        type=LayerType.SLOPE_INTERCEPT_LAYER,
        slope=slope,
        intercept=intercept,
        inputs=[Input(input.name)],
Q
qijun 已提交
4928 4929 4930
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.SLOPE_INTERCEPT_LAYER, input, size=input.size)
Z
zhangjinchao01 已提交
4931 4932 4933


@wrap_name_default()
L
luotao1 已提交
4934
@layer_support()
Q
qijun 已提交
4935
def linear_comb_layer(weights, vectors, size=None, name=None, layer_attr=None):
Z
zhangjinchao01 已提交
4936
    """
4937 4938 4939 4940
    A layer for weighted sum of vectors takes two inputs.
      - Input: size of weights is M
               size of vectors is M*N
      - Output: a vector of size=N
Z
zhangjinchao01 已提交
4941 4942 4943

    .. math::

4944
       z(i) = \sum_{j=0}^{M-1} x(j) y(i+Nj)
4945

4946 4947 4948 4949 4950
    where :math:`0 \le i \le N-1`

    Or in the matrix notation:

    .. math::
Z
zhangjinchao01 已提交
4951

4952
       z = x^\mathrm{T} Y
Z
zhangjinchao01 已提交
4953 4954

    In this formular:
4955 4956 4957 4958 4959 4960
      - :math:`x`: weights
      - :math:`y`: vectors.
      - :math:`z`: the output.

    Note that the above computation is for one sample. Multiple samples are
    processed in one batch.
Z
zhangjinchao01 已提交
4961 4962 4963 4964 4965

    The simple usage is:

    .. code-block:: python

4966
       linear_comb = linear_comb_layer(weights=weight, vectors=vectors,
Z
zhangjinchao01 已提交
4967 4968
                                       size=elem_dim)

4969 4970 4971 4972
    :param weights: The weight layer.
    :type weights: LayerOutput
    :param vectors: The vector layer.
    :type vectors: LayerOutput
Z
zhangjinchao01 已提交
4973 4974 4975 4976
    :param size: the dimension of this layer.
    :type size: int
    :param name: The Layer Name.
    :type name: basestring
L
luotao1 已提交
4977 4978
    :param layer_attr: Extra Layer config.
    :type layer_attr: ExtraLayerAttribute|None
D
dangqingqing 已提交
4979
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
4980 4981
    :rtype: LayerOutput
    """
4982 4983 4984 4985
    assert isinstance(weights, LayerOutput) and isinstance(vectors, LayerOutput)
    if vectors.size is not None and weights.size is not None:
        assert vectors.size % weights.size == 0
        if size is None:
Q
qijun 已提交
4986
            size = vectors.size / weights.size
4987 4988
        else:
            assert size == vectors.size / weights.size
Z
zhangjinchao01 已提交
4989 4990
    Layer(
        name=name,
4991
        type=LayerType.LINEAR_COMBINATION_LAYER,
Z
zhangjinchao01 已提交
4992
        size=size,
4993
        inputs=[Input(weights.name), Input(vectors.name)],
Q
qijun 已提交
4994 4995 4996
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.LINEAR_COMBINATION_LAYER, [weights, vectors], size=size)
4997

4998

4999
convex_comb_layer = linear_comb_layer
Z
zhangjinchao01 已提交
5000

5001

Z
zhangjinchao01 已提交
5002
@wrap_name_default()
L
luotao1 已提交
5003
@layer_support()
Z
zhangjinchao01 已提交
5004 5005 5006 5007 5008 5009 5010
def block_expand_layer(input,
                       block_x=0,
                       block_y=0,
                       stride_x=0,
                       stride_y=0,
                       padding_x=0,
                       padding_y=0,
5011
                       num_channels=None,
L
luotao1 已提交
5012 5013
                       name=None,
                       layer_attr=None):
Z
zhangjinchao01 已提交
5014 5015
    """
    Expand feature map to minibatch matrix.
5016
       - matrix width is: block_y * block_x * num_channels
L
luotao02 已提交
5017
       - matirx height is: outputH * outputW
Z
zhangjinchao01 已提交
5018 5019 5020 5021 5022 5023 5024 5025 5026 5027

    .. math::

       outputH = 1 + (2 * padding_y + imgSizeH - block_y + stride_y - 1) / stride_y

       outputW = 1 + (2 * padding_x + imgSizeW - block_x + stride_x - 1) / stride_x

    The expand method is the same with ExpandConvLayer, but saved the transposed
    value. After expanding, output.sequenceStartPositions will store timeline.
    The number of time steps are outputH * outputW and the dimension of each
5028
    time step is block_y * block_x * num_channels. This layer can be used after
Z
zhangjinchao01 已提交
5029 5030
    convolution neural network, and before recurrent neural network.

5031 5032 5033 5034
    The simple usage is:

    .. code-block:: python

L
Luo Tao 已提交
5035
       block_expand = block_expand_layer(input=layer,
5036
                                         num_channels=128,
5037 5038 5039 5040 5041
                                         stride_x=1,
                                         stride_y=1,
                                         block_x=1,
                                         block_x=3)

Z
zhangjinchao01 已提交
5042 5043
    :param input: The input layer.
    :type input: LayerOutput
5044 5045
    :param num_channels: The channel number of input layer.
    :type num_channels: int|None
Z
zhangjinchao01 已提交
5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059
    :param block_x: The width of sub block.
    :type block_x: int
    :param block_y: The width of sub block.
    :type block_y: int
    :param stride_x: The stride size in horizontal direction.
    :type stride_x: int
    :param stride_y: The stride size in vertical direction.
    :type stride_y: int
    :param padding_x: The padding size in horizontal direction.
    :type padding_x: int
    :param padding_y: The padding size in vertical direction.
    :type padding_y: int
    :param name: The name of this layer, which can not specify.
    :type name: None|basestring.
L
luotao1 已提交
5060 5061
    :param layer_attr: Extra Layer config.
    :type layer_attr: ExtraLayerAttribute|None
D
dangqingqing 已提交
5062
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
5063 5064
    :rtype: LayerOutput
    """
5065 5066 5067
    if num_channels is None:
        assert input.num_filters is not None
        num_channels = input.num_filters
Q
qijun 已提交
5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084
    l = Layer(
        name=name,
        inputs=Input(
            input.name,
            block_expand=BlockExpand(
                channels=num_channels,
                block_x=block_x,
                block_y=block_y,
                stride_x=stride_x,
                stride_y=stride_y,
                padding_x=padding_x,
                padding_y=padding_y)),
        type=LayerType.BLOCK_EXPAND,
        **ExtraLayerAttribute.to_kwargs(layer_attr))

    return LayerOutput(
        name, LayerType.BLOCK_EXPAND, parents=[input], size=l.config.size)
Z
zhangjinchao01 已提交
5085 5086


5087 5088
@wrap_name_default()
@layer_support()
5089
def maxout_layer(input, groups, num_channels=None, name=None, layer_attr=None):
5090 5091 5092 5093 5094
    """
    A layer to do max out on conv layer output.
      - Input: output of a conv layer.
      - Output: feature map size same as input. Channel is (input channel) / groups.

5095
    So groups should be larger than 1, and the num of channels should be able
5096 5097
    to devided by groups.

X
xuwei06 已提交
5098 5099 5100 5101 5102 5103 5104 5105
    .. math::
       y_{si+j} = \max_k x_{gsi + sk + j}
       g = groups
       s = input.size / num_channels
       0 \le i < num_channels / groups
       0 \le j < s
       0 \le k < groups

5106
    Please refer to Paper:
5107 5108 5109 5110
      - Maxout Networks: http://www.jmlr.org/proceedings/papers/v28/goodfellow13.pdf
      - Multi-digit Number Recognition from Street View \
        Imagery using Deep Convolutional Neural Networks: \
        https://arxiv.org/pdf/1312.6082v4.pdf
5111

5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139
    The simple usage is:

    .. code-block:: python

       maxout = maxout_layer(input,
                             num_channels=128,
                             groups=4)

    :param input: The input layer.
    :type input: LayerOutput
    :param num_channels: The channel number of input layer. If None will be set
                     automatically from previous output.
    :type num_channels: int|None
    :param groups: The group number of input layer.
    :type groups: int
    :param name: The name of this layer, which can not specify.
    :type name: None|basestring.
    :param layer_attr: Extra Layer attribute.
    :type layer_attr: ExtraLayerAttribute
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
    assert isinstance(input.activation, LinearActivation)
    assert groups > 1
    if num_channels is None:
        assert input.num_filters is not None
        num_channels = input.num_filters
    assert num_channels % groups == 0
Q
qijun 已提交
5140 5141 5142 5143 5144 5145 5146 5147 5148
    l = Layer(
        name=name,
        inputs=Input(
            input.name, maxout=MaxOut(
                channels=num_channels, groups=groups)),
        type=LayerType.MAXOUT,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.MAXOUT, parents=[input], size=l.config.size)
5149 5150


Z
zhangjinchao01 已提交
5151
@wrap_name_default()
L
luotao1 已提交
5152
@layer_support()
Q
qijun 已提交
5153 5154 5155 5156 5157
def ctc_layer(input,
              label,
              size=None,
              name=None,
              norm_by_times=False,
L
luotao1 已提交
5158
              layer_attr=None):
Z
zhangjinchao01 已提交
5159 5160 5161 5162 5163
    """
    Connectionist Temporal Classification (CTC) is designed for temporal
    classication task. That is, for sequence labeling problems where the
    alignment between the inputs and the target labels is unknown.

5164 5165
    More details can be found by referring to `Connectionist Temporal
    Classification: Labelling Unsegmented Sequence Data with Recurrent
5166 5167
    Neural Networks <http://machinelearning.wustl.edu/mlpapers/paper_files/
    icml2006_GravesFGS06.pdf>`_
5168 5169 5170 5171 5172 5173 5174 5175

    Note:
        Considering the 'blank' label needed by CTC, you need to use
        (num_classes + 1) as the input size. num_classes is the category number.
        And the 'blank' is the last category index. So the size of 'input' layer, such as
        fc_layer with softmax activation, should be num_classes + 1. The size of ctc_layer
        should also be num_classes + 1.

C
caoying03 已提交
5176
    The example usage is:
Z
zhangjinchao01 已提交
5177 5178 5179 5180 5181 5182 5183 5184

    .. code-block:: python

      ctc = ctc_layer(input=input,
                      label=label,
                      size=9055,
                      norm_by_times=True)

5185
    :param input: The input layer.
Z
zhangjinchao01 已提交
5186 5187 5188
    :type input: LayerOutput
    :param label: The data layer of label with variable length.
    :type label: LayerOutput
5189
    :param size: category numbers + 1.
Z
zhangjinchao01 已提交
5190
    :type size: int
5191 5192
    :param name: The name of this layer
    :type name: basestring|None
Z
zhangjinchao01 已提交
5193 5194
    :param norm_by_times: Whether to normalization by times. False by default.
    :type norm_by_times: bool
L
luotao1 已提交
5195 5196
    :param layer_attr: Extra Layer config.
    :type layer_attr: ExtraLayerAttribute|None
D
dangqingqing 已提交
5197
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
5198 5199 5200 5201
    :rtype: LayerOutput
    """
    assert isinstance(input, LayerOutput)
    assert isinstance(label, LayerOutput)
5202 5203 5204 5205 5206
    if label.size is not None:
        if size is not None:
            assert size == label.size + 1
        else:
            size = label.size + 1
Z
zhangjinchao01 已提交
5207
    Layer(
5208 5209 5210 5211
        name=name,
        type=LayerType.CTC_LAYER,
        size=size,
        norm_by_times=norm_by_times,
L
luotao1 已提交
5212
        inputs=[input.name, label.name],
Q
qijun 已提交
5213
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
5214 5215
    return LayerOutput(name, LayerType.CTC_LAYER, [input, label], size=size)

5216

5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227
@wrap_name_default()
@layer_support()
def warp_ctc_layer(input,
                   label,
                   size=None,
                   name=None,
                   blank=0,
                   norm_by_times=False,
                   layer_attr=None):
    """
    A layer intergrating the open-source `warp-ctc
L
Liu Yiqun 已提交
5228
    <https://github.com/baidu-research/warp-ctc>`_ library, which is used in
5229
    `Deep Speech 2: End-toEnd Speech Recognition in English and Mandarin
L
Liu Yiqun 已提交
5230 5231 5232 5233 5234 5235 5236
    <https://arxiv.org/pdf/1512.02595v1.pdf>`_, to compute Connectionist Temporal
    Classification (CTC) loss. Besides, another `warp-ctc
    <https://github.com/gangliao/warp-ctc>`_ repository, which is forked from
    the official one, is maintained to enable more compiling options. During the
    building process, PaddlePaddle will clone the source codes, build and
    install it to :code:`third_party/install/warpctc` directory.

5237 5238 5239
    More details of CTC can be found by referring to `Connectionist Temporal
    Classification: Labelling Unsegmented Sequence Data with Recurrent
    Neural Networks <http://machinelearning.wustl.edu/mlpapers/paper_files/
L
Liu Yiqun 已提交
5240
    icml2006_GravesFGS06.pdf>`_.
5241 5242 5243

    Note:
        - Let num_classes represent the category number. Considering the 'blank'
L
Liu Yiqun 已提交
5244 5245 5246
          label needed by CTC, you need to use (num_classes + 1) as the input size.
          Thus, the size of both warp_ctc layer and 'input' layer should be set to
          num_classes + 1.
5247 5248
        - You can set 'blank' to any value ranged in [0, num_classes], which
          should be consistent as that used in your labels.
5249
        - As a native 'softmax' activation is interated to the warp-ctc library,
L
Luo Tao 已提交
5250
          'linear' activation is expected instead in the 'input' layer.
5251

C
caoying03 已提交
5252
    The example usage is:
5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297

    .. code-block:: python

      ctc = warp_ctc_layer(input=input,
                           label=label,
                           size=1001,
                           blank=1000,
                           norm_by_times=False)

    :param input: The input layer.
    :type input: LayerOutput
    :param label: The data layer of label with variable length.
    :type label: LayerOutput
    :param size: category numbers + 1.
    :type size: int
    :param name: The name of this layer, which can not specify.
    :type name: basestring|None
    :param blank: the 'blank' label used in ctc
    :type blank: int
    :param norm_by_times: Whether to normalization by times. False by default.
    :type norm_by_times: bool
    :param layer_attr: Extra Layer config.
    :type layer_attr: ExtraLayerAttribute|None
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
    assert isinstance(input, LayerOutput)
    assert isinstance(label, LayerOutput)
    if label.size is not None:
        if size is not None:
            assert size == label.size + 1
        else:
            size = label.size + 1
    Layer(
        name=name,
        type=LayerType.WARP_CTC_LAYER,
        size=size,
        blank=blank,
        norm_by_times=norm_by_times,
        inputs=[input.name, label.name],
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.WARP_CTC_LAYER, parents=[input, label], size=size)


Z
zhangjinchao01 已提交
5298
@wrap_name_default()
5299
@wrap_param_attr_default()
L
luotao1 已提交
5300
@layer_support()
Q
qijun 已提交
5301 5302 5303 5304 5305 5306
def crf_layer(input,
              label,
              size=None,
              weight=None,
              param_attr=None,
              name=None,
5307
              coeff=1.0,
L
luotao1 已提交
5308
              layer_attr=None):
Z
zhangjinchao01 已提交
5309 5310 5311 5312
    """
    A layer for calculating the cost of sequential conditional random
    field model.

C
caoying03 已提交
5313
    The example usage is:
Z
zhangjinchao01 已提交
5314 5315 5316 5317 5318 5319 5320 5321 5322 5323

    .. code-block:: python

      crf = crf_layer(input=input,
                      label=label,
                      size=label_dim)

    :param input: The first input layer is the feature.
    :type input: LayerOutput
    :param label: The second input layer is label.
5324
    :type label: LayerOutput
Z
zhangjinchao01 已提交
5325 5326 5327 5328 5329 5330 5331 5332 5333
    :param size: The category number.
    :type size: int
    :param weight: The third layer is "weight" of each sample, which is an
                  optional argument.
    :type weight: LayerOutput
    :param param_attr: Parameter attribute. None means default attribute
    :type param_attr: ParameterAttribute
    :param name: The name of this layers. It is not necessary.
    :type name: None|basestring
5334 5335
    :param coeff: The coefficient affects the gradient in the backward.
    :type coeff: float
L
luotao1 已提交
5336 5337
    :param layer_attr: Extra Layer config.
    :type layer_attr: ExtraLayerAttribute|None
D
dangqingqing 已提交
5338
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
5339 5340 5341 5342 5343
    :rtype: LayerOutput
    """
    assert isinstance(input, LayerOutput)
    assert isinstance(label, LayerOutput)
    assert weight is None or isinstance(weight, LayerOutput)
5344 5345 5346 5347 5348 5349
    if input.size is not None and label.size is not None:
        assert input.size == label.size
        if size is None:
            size = input.size
        else:
            assert size == input.size
Z
zhangjinchao01 已提交
5350

Q
qijun 已提交
5351
    ipts = [Input(input.name, **param_attr.attr), Input(label.name)]
Z
zhangjinchao01 已提交
5352 5353 5354 5355
    if weight is not None:
        ipts.append(Input(weight.name))

    Layer(
5356 5357 5358 5359
        name=name,
        type=LayerType.CRF_LAYER,
        size=size,
        inputs=ipts,
5360
        coeff=coeff,
Q
qijun 已提交
5361
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
5362 5363 5364
    parents = [input, label]
    if weight is not None:
        parents.append(weight)
X
xuwei06 已提交
5365 5366 5367 5368
    # The size for LayerOutput means the dimension of the output.
    # It's different from the meaning of crf layer, which is the number of
    # classes.
    return LayerOutput(name, LayerType.CRF_LAYER, parents, size=1)
Z
zhangjinchao01 已提交
5369

5370

Z
zhangjinchao01 已提交
5371
@wrap_name_default()
5372
@wrap_param_attr_default()
L
luotao1 已提交
5373
@layer_support()
Q
qijun 已提交
5374 5375 5376 5377 5378
def crf_decoding_layer(input,
                       size,
                       label=None,
                       param_attr=None,
                       name=None,
L
luotao1 已提交
5379
                       layer_attr=None):
Z
zhangjinchao01 已提交
5380 5381 5382 5383 5384 5385 5386
    """
    A layer for calculating the decoding sequence of sequential conditional
    random field model. The decoding sequence is stored in output.ids.
    If a second input is provided, it is treated as the ground-truth label, and
    this layer will also calculate error. output.value[i] is 1 for incorrect
    decoding or 0 for correct decoding.

C
caoying03 已提交
5387
    The example usage is:
L
Luo Tao 已提交
5388 5389 5390 5391 5392 5393

    .. code-block:: python

      crf_decoding = crf_decoding_layer(input=input,
                                        size=label_dim)

Z
zhangjinchao01 已提交
5394 5395 5396 5397 5398 5399 5400 5401 5402 5403
    :param input: The first input layer.
    :type input: LayerOutput
    :param size: size of this layer.
    :type size: int
    :param label: None or ground-truth label.
    :type label: LayerOutput or None
    :param param_attr: Parameter attribute. None means default attribute
    :type param_attr: ParameterAttribute
    :param name: The name of this layers. It is not necessary.
    :type name: None|basestring
L
luotao1 已提交
5404 5405
    :param layer_attr: Extra Layer config.
    :type layer_attr: ExtraLayerAttribute|None
D
dangqingqing 已提交
5406
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
5407 5408 5409 5410 5411 5412
    :rtype: LayerOutput
    """

    assert isinstance(input, LayerOutput)
    assert label is None or isinstance(label, LayerOutput)

5413
    ipts = [Input(input.name, **param_attr.attr)]
Z
zhangjinchao01 已提交
5414 5415 5416 5417
    if label is not None:
        ipts.append(Input(label.name))

    Layer(
5418 5419 5420 5421
        name=name,
        type=LayerType.CRF_DECODING_LAYER,
        size=size,
        inputs=ipts,
Q
qijun 已提交
5422
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
5423 5424 5425
    parents = [input]
    if label is not None:
        parents.append(label)
X
xuwei06 已提交
5426 5427 5428 5429
    # The size for LayerOutput means the dimension of the output.
    # It's different from the meaning of crf layer, which is the number of
    # classes.
    return LayerOutput(name, LayerType.CRF_DECODING_LAYER, parents, size=1)
Z
zhangjinchao01 已提交
5430

Q
qijun 已提交
5431

Y
Yu Yang 已提交
5432
@wrap_act_default(act=SigmoidActivation())
5433
@wrap_bias_attr_default(has_bias=True)
5434
@wrap_param_attr_default()
5435 5436
@wrap_name_default()
@layer_support()
Q
qijun 已提交
5437 5438
def nce_layer(input,
              label,
C
caoying03 已提交
5439
              num_classes=None,
Y
Yu Yang 已提交
5440
              act=None,
5441
              param_attr=None,
Q
qijun 已提交
5442 5443 5444 5445 5446 5447
              weight=None,
              num_neg_samples=10,
              neg_distribution=None,
              name=None,
              bias_attr=None,
              layer_attr=None):
5448 5449 5450 5451 5452 5453 5454 5455 5456
    """
    Noise-contrastive estimation.
    Implements the method in the following paper:
    A fast and simple algorithm for training neural probabilistic language models.

    The example usage is:

    .. code-block:: python

C
caoying03 已提交
5457 5458
       cost = nce_layer(input=[layer1, layer2], label=layer2,
                        param_attr=[attr1, attr2], weight=layer3,
5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469
                        num_classes=3, neg_distribution=[0.1,0.3,0.6])

    :param name: layer name
    :type name: basestring
    :param input: input layers. It could be a LayerOutput of list/tuple of LayerOutput.
    :type input: LayerOutput|list|tuple|collections.Sequence
    :param label: label layer
    :type label: LayerOutput
    :param weight: weight layer, can be None(default)
    :type weight: LayerOutput
    :param num_classes: number of classes.
5470
    :type num_classes: int
Y
Yu Yang 已提交
5471 5472
    :param act: Activation, default is Sigmoid.
    :type act: BaseActivation
5473 5474
    :param param_attr: The Parameter Attribute|list.
    :type param_attr: ParameterAttribute
5475
    :param num_neg_samples: number of negative samples. Default is 10.
5476
    :type num_neg_samples: int
5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489
    :param neg_distribution: The distribution for generating the random negative labels.
                             A uniform distribution will be used if not provided.
                             If not None, its length must be equal to num_classes.
    :type neg_distribution: list|tuple|collections.Sequence|None
    :param bias_attr: Bias parameter attribute. True if no bias.
    :type bias_attr: ParameterAttribute|None|False
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
    :return: layer name.
    :rtype: LayerOutput
    """
    if isinstance(input, LayerOutput):
        input = [input]
5490 5491 5492 5493 5494 5495 5496 5497
        assert not isinstance(param_attr, collections.Sequence)
        param_attr = [param_attr]
    else:
        if isinstance(param_attr, collections.Sequence):
            assert len(input) == len(param_attr)
        else:
            param_attr = [copy.deepcopy(param_attr) for _ in range(len(input))]

5498
    assert isinstance(input, collections.Sequence)
5499

5500 5501
    assert isinstance(label, LayerOutput)
    assert label.layer_type == LayerType.DATA
C
caoying03 已提交
5502 5503
    if num_classes is None:
        num_classes = label.size
5504 5505 5506
    if neg_distribution is not None:
        assert isinstance(neg_distribution, collections.Sequence)
        assert len(neg_distribution) == num_classes
5507
        assert abs(sum(neg_distribution) - 1.0) < 1e-5
Y
Yu Yang 已提交
5508 5509
    if not isinstance(act, BaseActivation):
        raise TypeError()
5510

5511 5512
    ipts_for_layer = []
    parents = []
5513
    for each_input, attr in zip(input, param_attr):
5514
        assert isinstance(each_input, LayerOutput)
5515
        ipts_for_layer.append(Input(each_input.name, **attr.attr))
5516 5517 5518 5519 5520 5521 5522 5523 5524 5525
        parents.append(each_input)
    ipts_for_layer.append(label.name)
    parents.append(label)

    if weight is not None:
        assert isinstance(weight, LayerOutput)
        assert weight.layer_type == LayerType.DATA
        ipts_for_layer.append(weight.name)
        parents.append(weight)

X
xuwei06 已提交
5526
    l = Layer(
5527 5528 5529 5530
        name=name,
        type=LayerType.NCE_LAYER,
        num_classes=num_classes,
        neg_sampling_dist=neg_distribution,
Y
Yu Yang 已提交
5531
        active_type=act.name,
5532 5533 5534
        num_neg_samples=num_neg_samples,
        inputs=ipts_for_layer,
        bias=ParamAttr.to_bias(bias_attr),
Q
qijun 已提交
5535 5536
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
Y
Yu Yang 已提交
5537 5538 5539 5540 5541
        name,
        LayerType.NCE_LAYER,
        parents=parents,
        size=l.config.size,
        activation=act)
Q
qijun 已提交
5542

5543

Z
zhangjinchao01 已提交
5544 5545 5546
"""
following are cost Layers.
"""
5547 5548


Z
zhangjinchao01 已提交
5549
@wrap_name_default()
L
luotao1 已提交
5550
@layer_support()
Q
qijun 已提交
5551 5552 5553 5554 5555 5556 5557
def rank_cost(left,
              right,
              label,
              weight=None,
              name=None,
              coeff=1.0,
              layer_attr=None):
Z
zhangjinchao01 已提交
5558
    """
5559
    A cost Layer for learning to rank using gradient descent. Details can refer
5560 5561
    to `papers <http://research.microsoft.com/en-us/um/people/cburges/papers/
    ICML_ranking.pdf>`_.
Z
zhangjinchao01 已提交
5562 5563 5564 5565 5566
    This layer contains at least three inputs. The weight is an optional
    argument, which affects the cost.

    .. math::

L
luotao02 已提交
5567
       C_{i,j} & = -\\tilde{P_{ij}} * o_{i,j} + log(1 + e^{o_{i,j}})
Z
zhangjinchao01 已提交
5568

L
luotao02 已提交
5569
       o_{i,j} & =  o_i - o_j
Z
zhangjinchao01 已提交
5570

L
luotao02 已提交
5571
       \\tilde{P_{i,j}} & = \\{0, 0.5, 1\\} \ or \ \\{0, 1\\}
Z
zhangjinchao01 已提交
5572 5573 5574 5575 5576 5577 5578 5579

    In this formula:
      - :math:`C_{i,j}` is the cross entropy cost.
      - :math:`\\tilde{P_{i,j}}` is the label. 1 means positive order
        and 0 means reverse order.
      - :math:`o_i` and :math:`o_j`: the left output and right output.
        Their dimension is one.

C
caoying03 已提交
5580
    The example usage is:
Z
zhangjinchao01 已提交
5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600

    .. code-block:: python

      cost = rank_cost(left=out_left,
                       right=out_right,
                       label=label)

    :param left: The first input, the size of this layer is 1.
    :type left: LayerOutput
    :param right: The right input, the size of this layer is 1.
    :type right: LayerOutput
    :param label: Label is 1 or 0, means positive order and reverse order.
    :type label: LayerOutput
    :param weight: The weight affects the cost, namely the scale of cost.
                   It is an optional argument.
    :type weight: LayerOutput
    :param name: The name of this layers. It is not necessary.
    :type name: None|basestring
    :param coeff: The coefficient affects the gradient in the backward.
    :type coeff: float
L
luotao1 已提交
5601 5602
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
5603
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615
    :rtype: LayerOutput
    """
    assert left.size == 1
    assert right.size == 1
    assert label.size == 1

    ipts = [left.name, right.name, label.name]
    parents = [left, right, label]
    if weight is not None:
        ipts.append(weight.name)
        parents.append(weight)

Q
qijun 已提交
5616 5617 5618 5619 5620 5621
    Layer(
        name=name,
        type=LayerType.RANK_COST,
        inputs=ipts,
        coeff=coeff,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
5622

X
xuwei06 已提交
5623
    return LayerOutput(name, LayerType.RANK_COST, parents=parents, size=1)
Z
zhangjinchao01 已提交
5624

5625

Z
zhangjinchao01 已提交
5626
@wrap_name_default()
L
luotao1 已提交
5627
@layer_support()
Q
qijun 已提交
5628 5629 5630 5631 5632 5633
def lambda_cost(input,
                score,
                name,
                NDCG_num=5,
                max_sort_size=-1,
                layer_attr=None):
Z
zhangjinchao01 已提交
5634 5635 5636
    """
    lambdaCost for lambdaRank LTR approach.

C
caoying03 已提交
5637
    The example usage is:
Z
zhangjinchao01 已提交
5638 5639 5640 5641 5642 5643 5644 5645

    .. code-block:: python

      cost = lambda_cost(input=input,
                         score=score,
                         NDCG_num=8,
                         max_sort_size=-1)

5646
    :param input: Samples of the same query should be loaded as sequence.
Z
zhangjinchao01 已提交
5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657
    :type input: LayerOutput
    :param score: The 2nd input. Score of each sample.
    :type input: LayerOutput
    :param NDCG_num: The size of NDCG (Normalized Discounted Cumulative Gain),
                     e.g., 5 for NDCG@5. It must be less than for equal to the
                     minimum size of lists.
    :type NDCG_num: int
    :param max_sort_size: The size of partial sorting in calculating gradient.
                          If max_sort_size = -1, then for each list, the
                          algorithm will sort the entire list to get gradient.
                          In other cases, max_sort_size must be greater than or
C
caoying03 已提交
5658 5659 5660
                          equal to NDCG_num. And if max_sort_size is greater
                          than the size of a list, the algorithm will sort the
                          entire list of get gradient.
Z
zhangjinchao01 已提交
5661 5662 5663
    :type max_sort_size: int
    :param name: The name of this layers. It is not necessary.
    :type name: None|basestring
L
luotao1 已提交
5664 5665
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
5666
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
5667 5668
    :rtype: LayerOutput
    """
5669 5670 5671
    assert isinstance(input, LayerOutput) and isinstance(score, LayerOutput)
    if score.size is not None:
        assert score.size == 1
Q
qijun 已提交
5672 5673 5674 5675 5676 5677 5678
    Layer(
        name=name,
        type=LayerType.LAMBDA_COST,
        inputs=[input.name, score.name],
        NDCG_num=NDCG_num,
        max_sort_size=max_sort_size,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
5679

Q
qijun 已提交
5680 5681
    return LayerOutput(
        name, LayerType.LAMBDA_COST, parents=[input, score], size=1)
Z
zhangjinchao01 已提交
5682

5683

Z
zhangjinchao01 已提交
5684
@wrap_name_default()
L
luotao1 已提交
5685
@layer_support()
5686 5687 5688 5689 5690 5691
def cross_entropy(input,
                  label,
                  name=None,
                  coeff=1.0,
                  weight=None,
                  layer_attr=None):
Z
zhangjinchao01 已提交
5692 5693 5694
    """
    A loss layer for multi class entropy.

C
caoying03 已提交
5695 5696
    The example usage is:

Z
zhangjinchao01 已提交
5697 5698
    .. code-block:: python

X
xuwei06 已提交
5699
       cost = cross_entropy(input=input_layer,
L
Luo Tao 已提交
5700
                            label=label_layer)
Z
zhangjinchao01 已提交
5701 5702 5703 5704 5705 5706 5707

    :param input: The first input layer.
    :type input: LayerOutput.
    :param label: The input label.
    :type input: LayerOutput.
    :param name: The name of this layers. It is not necessary.
    :type name: None|basestring.
5708 5709
    :param coeff: The cost is multiplied with coeff.
                  The coefficient affects the gradient in the backward.
Z
zhangjinchao01 已提交
5710
    :type coeff: float.
5711 5712 5713 5714
    :param weight: The cost of each sample is multiplied with each weight.
                   The weight should be a layer with size=1. Note that gradient
                   will not be calculated for weight.
    :type weight: LayerOutout
L
luotao1 已提交
5715 5716
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
5717
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
5718 5719 5720
    :rtype: LayerOutput.
    """

5721
    ipts, parents = __cost_input__(input, label, weight)
Q
qijun 已提交
5722 5723 5724
    Layer(
        name=name,
        type=LayerType.CROSS_ENTROPY,
5725
        inputs=ipts,
Q
qijun 已提交
5726 5727
        coeff=coeff,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
5728
    return LayerOutput(name, LayerType.CROSS_ENTROPY, parents=parents, size=1)
Z
zhangjinchao01 已提交
5729

5730

Z
zhangjinchao01 已提交
5731
@wrap_name_default()
L
luotao1 已提交
5732
@layer_support()
Q
qijun 已提交
5733 5734 5735 5736
def cross_entropy_with_selfnorm(input,
                                label,
                                name=None,
                                coeff=1.0,
L
luotao1 已提交
5737 5738
                                softmax_selfnorm_alpha=0.1,
                                layer_attr=None):
Z
zhangjinchao01 已提交
5739 5740
    """
    A loss layer for multi class entropy with selfnorm.
5741
    Input should be a vector of positive numbers, without normalization.
Z
zhangjinchao01 已提交
5742

C
caoying03 已提交
5743 5744
    The example usage is:

Z
zhangjinchao01 已提交
5745 5746
    .. code-block:: python

X
xuwei06 已提交
5747
       cost = cross_entropy_with_selfnorm(input=input_layer,
L
Luo Tao 已提交
5748
                                          label=label_layer)
Z
zhangjinchao01 已提交
5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759

    :param input: The first input layer.
    :type input: LayerOutput.
    :param label: The input label.
    :type input: LayerOutput.
    :param name: The name of this layers. It is not necessary.
    :type name: None|basestring.
    :param coeff: The coefficient affects the gradient in the backward.
    :type coeff: float.
    :param softmax_selfnorm_alpha: The scale factor affects the cost.
    :type softmax_selfnorm_alpha: float.
L
luotao1 已提交
5760 5761
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
5762
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
5763 5764
    :rtype: LayerOutput.
    """
Q
qijun 已提交
5765 5766 5767 5768 5769 5770 5771
    Layer(
        name=name,
        type=LayerType.CROSS_ENTROPY_WITH_SELFNORM,
        inputs=[input.name, label.name],
        coeff=coeff,
        softmax_selfnorm_alpha=softmax_selfnorm_alpha,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
Z
zhangjinchao01 已提交
5772

Q
qijun 已提交
5773 5774 5775 5776 5777
    return LayerOutput(
        name,
        LayerType.CROSS_ENTROPY_WITH_SELFNORM,
        parents=[input, label],
        size=1)
Z
zhangjinchao01 已提交
5778

5779

X
xuwei06 已提交
5780 5781 5782 5783 5784 5785
@wrap_name_default()
@layer_support()
def sum_cost(input, name=None, layer_attr=None):
    """
    A loss layer which calculate the sum of the input as loss

C
caoying03 已提交
5786 5787
    The example usage is:

X
xuwei06 已提交
5788 5789
    .. code-block:: python

L
Luo Tao 已提交
5790
       cost = sum_cost(input=input_layer)
X
xuwei06 已提交
5791 5792 5793 5794 5795 5796 5797 5798 5799 5800

    :param input: The first input layer.
    :type input: LayerOutput.
    :param name: The name of this layers. It is not necessary.
    :type name: None|basestring.
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
    :return: LayerOutput object.
    :rtype: LayerOutput.
    """
L
Luo Tao 已提交
5801
    assert isinstance(input, LayerOutput)
Q
qijun 已提交
5802 5803 5804 5805 5806
    Layer(
        name=name,
        type=LayerType.SUM_COST,
        inputs=[input.name],
        **ExtraLayerAttribute.to_kwargs(layer_attr))
X
xuwei06 已提交
5807

Q
qijun 已提交
5808
    return LayerOutput(name, LayerType.SUM_COST, parents=[input], size=1)
X
xuwei06 已提交
5809 5810


Z
zhangjinchao01 已提交
5811
@wrap_name_default()
L
luotao1 已提交
5812
@layer_support()
L
Luo Tao 已提交
5813 5814 5815 5816 5817 5818
def huber_regression_cost(input,
                          label,
                          name=None,
                          delta=1.0,
                          coeff=1.0,
                          layer_attr=None):
Z
zhangjinchao01 已提交
5819
    """
5820 5821 5822
    In statistics, the Huber loss is a loss function used in robust regression,
    that is less sensitive to outliers in data than the squared error loss.
    Given a prediction f(x), a label y and :math:`\delta`, the loss function
L
Luo Tao 已提交
5823 5824 5825 5826 5827
    is defined as:

    .. math:
       loss = 0.5*\left ( y-f(x) \right )^2, \left | y-f(x) \right |\leq \delta
       loss = \delta \left | y-f(x) \right |-0.5\delta ^2, otherwise
Z
zhangjinchao01 已提交
5828

C
caoying03 已提交
5829 5830
    The example usage is:

Z
zhangjinchao01 已提交
5831 5832
    .. code-block:: python

L
Luo Tao 已提交
5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861
       cost = huber_regression_cost(input=input_layer, label=label_layer)

    :param input: The first input layer.
    :type input: LayerOutput.
    :param label: The input label.
    :type input: LayerOutput.
    :param name: The name of this layers. It is not necessary.
    :type name: None|basestring.
    :param delta: The difference between the observed and predicted values.
    :type delta: float.
    :param coeff: The coefficient affects the gradient in the backward.
    :type coeff: float.
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
    :return: LayerOutput object.
    :rtype: LayerOutput.
    """
    assert isinstance(input, LayerOutput)
    Layer(
        name=name,
        type=LayerType.HUBER_REGRESSION,
        inputs=[input.name, label.name],
        delta=delta,
        coeff=coeff,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.HUBER_REGRESSION, parents=[input, label], size=1)


Z
zhangjinchao01 已提交
5862
@wrap_name_default()
L
luotao1 已提交
5863
@layer_support()
5864 5865 5866 5867 5868
def huber_classification_cost(input,
                              label,
                              name=None,
                              coeff=1.0,
                              layer_attr=None):
Z
zhangjinchao01 已提交
5869
    """
5870 5871 5872
    For classification purposes, a variant of the Huber loss called modified Huber
    is sometimes used. Given a prediction f(x) (a real-valued classifier score) and
    a true binary class label :math:`y\in \left \{-1, 1 \right \}`, the modified Huber
5873 5874 5875
    loss is defined as:

    .. math:
5876
       loss = \max \left ( 0, 1-yf(x) \right )^2, yf(x)\geq 1
5877
       loss = -4yf(x), \text{otherwise}
Z
zhangjinchao01 已提交
5878

C
caoying03 已提交
5879 5880
    The example usage is:

Z
zhangjinchao01 已提交
5881 5882
    .. code-block:: python

5883
       cost = huber_classification_cost(input=input_layer, label=label_layer)
Z
zhangjinchao01 已提交
5884 5885 5886 5887 5888 5889 5890 5891 5892

    :param input: The first input layer.
    :type input: LayerOutput.
    :param label: The input label.
    :type input: LayerOutput.
    :param name: The name of this layers. It is not necessary.
    :type name: None|basestring.
    :param coeff: The coefficient affects the gradient in the backward.
    :type coeff: float.
L
luotao1 已提交
5893 5894
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
5895
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
5896 5897
    :rtype: LayerOutput.
    """
5898 5899 5900
    assert isinstance(input, LayerOutput)
    if input.size is not None:
        assert input.size == 1
Q
qijun 已提交
5901 5902
    Layer(
        name=name,
5903
        type=LayerType.HUBER_CLASSIFICATION,
Q
qijun 已提交
5904 5905 5906
        inputs=[input.name, label.name],
        coeff=coeff,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
5907 5908
    return LayerOutput(
        name, LayerType.HUBER_CLASSIFICATION, parents=[input, label], size=1)
Z
zhangjinchao01 已提交
5909

5910

Z
zhangjinchao01 已提交
5911
@wrap_name_default()
L
luotao1 已提交
5912
@layer_support()
Q
qijun 已提交
5913 5914 5915 5916
def multi_binary_label_cross_entropy(input,
                                     label,
                                     name=None,
                                     coeff=1.0,
L
luotao1 已提交
5917
                                     layer_attr=None):
Z
zhangjinchao01 已提交
5918 5919 5920
    """
    A loss layer for multi binary label cross entropy.

C
caoying03 已提交
5921 5922
    The example usage is:

Z
zhangjinchao01 已提交
5923 5924
    .. code-block:: python

X
xuwei06 已提交
5925
       cost = multi_binary_label_cross_entropy(input=input_layer,
L
Luo Tao 已提交
5926
                                               label=label_layer)
Z
zhangjinchao01 已提交
5927 5928 5929 5930 5931 5932 5933 5934 5935

    :param input: The first input layer.
    :type input: LayerOutput
    :param label: The input label.
    :type input: LayerOutput
    :param name: The name of this layers. It is not necessary.
    :type name: None|basestring
    :param coeff: The coefficient affects the gradient in the backward.
    :type coeff: float
L
luotao1 已提交
5936 5937
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
D
dangqingqing 已提交
5938
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
5939 5940 5941
    :rtype: LayerOutput
    """

5942 5943
    if input.activation is None or \
            not isinstance(input.activation, SigmoidActivation):
C
caoying03 已提交
5944 5945 5946 5947
        logger.log(logging.WARN,
                   ("%s is not a recommended activation for "
                    "multi_binary_label_cross_entropy, sigmoid is better") %
                   repr(input.activation))
Q
qijun 已提交
5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959

    Layer(
        name=name,
        type=LayerType.MULTI_BIN_LABEL_CROSS_ENTROPY,
        inputs=[input.name, label.name],
        coeff=coeff,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name,
        LayerType.MULTI_BIN_LABEL_CROSS_ENTROPY,
        parents=[input, label],
        size=1)
D
dangqingqing 已提交
5960 5961


C
caoying03 已提交
5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983
class BeamInput(object):
    """
    Define the input for cross_entropy_over_beam layer.

    A beam is made up of a triple: the first one is scores over all
    candidates; the second one is indices of top k selected candidates; the
    third one is the index of ground truth, which is also always called
    gold.
    """

    def __init__(self, candidate_scores, selected_candidates, gold):
        assert isinstance(candidate_scores, LayerOutput)
        self.candidate_scores = candidate_scores
        assert candidate_scores.size == 1

        assert isinstance(selected_candidates, LayerOutput)
        self.selected_candidates = selected_candidates

        assert isinstance(gold, LayerOutput)
        self.gold = gold


C
caoying03 已提交
5984 5985
@wrap_name_default()
@layer_support()
C
caoying03 已提交
5986
def cross_entropy_over_beam(input, name=None):
C
caoying03 已提交
5987
    """
C
caoying03 已提交
5988 5989 5990
    This layer is used in learning to search models, which is to solve complex
    joint prediction problems based on learning to search through a
    problem-defined search space.
C
caoying03 已提交
5991

C
caoying03 已提交
5992 5993 5994 5995 5996
    Specifically, the learning to search process for this layer begins with
    searching a target sequence from a nested sequence. In the first search
    step, top beam size sequences with highest scores, indices of these top k
    sequences in the original nested sequence, and the ground truth (also
    called gold) altogether (a triple) make up of the first beam.
C
caoying03 已提交
5997

C
caoying03 已提交
5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015
    Then, several special positions, for example, start and end positions
    that define meaningful segments are searched. In these searches, top k
    positions with highest scores are selected, and then sequence, starting
    from the selected starts till ends of the sequences (or a fixed position)
    are taken to search next.

    We call the possible top k results returned in one search the beam. This
    search process can be repeated for pre-defined turns and leads to several
    beam expansions.

    Finally, the layer cross_entropy_over_beam takes all the beam expansions
    which contain several candidate targets found along the multi-step search.
    cross_entropy_over_beam calculates cross entropy over the expanded beams
    which all the candidates in the beam as the normalized factor.

    Note that, if gold falls off the beam at search step t, then the cost is
    calculated over the beam at step t.

6016
    This cost layer always works together with kmax_seq_score_layer,
C
caoying03 已提交
6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065
    sub_nested_seq_layer, and sequence_slice_layer to trim the input to form a
    sub-search space.


    The example usage is:

    .. code-block:: python

       cost = cross_entropy_over_beam(input=[
           BeamInput(
               candidate_scores=beam1_candidates,
               selected_candidates=beam1_topk,
               gold=gold1),
           BeamInput(
               candidate_scores=beam2_candidates,
               selected_candidates=beam2_topk,
               gold=gold2),
       ])


    :param input: input beams for this layer.
    :type input: BeamInput
    :param name: input beams for this layer.
    :type name: basestring
    :return: LayerOutput object.
    :rtype: LayerOutput
    """

    if isinstance(input, BeamInput):
        input = [input]
    else:
        assert isinstance(input, list), (
            'input for cross_entropy_over_beam shold be a python list '
            'of BeamInput object.')
        for ipt in input:
            assert isinstance(ipt, BeamInput), (
                'input for cross_entropy_over_beam '
                'should be a BeamInput object.')

    ipts = []
    parents = []
    for beam in input:
        parents += [beam.candidate_scores, beam.selected_candidates, beam.gold]
        ipts += [
            beam.candidate_scores.name, beam.selected_candidates.name,
            beam.gold.name
        ]

    Layer(name=name, type=LayerType.CROSS_ENTROPY_OVER_BEAM, inputs=ipts)
C
caoying03 已提交
6066 6067 6068
    return LayerOutput(name, LayerType.CROSS_ENTROPY, parents=parents, size=1)


D
dangqingqing 已提交
6069 6070
@wrap_name_default()
@layer_support()
6071
def smooth_l1_cost(input, label, name=None, coeff=1.0, layer_attr=None):
D
dangqingqing 已提交
6072 6073
    """
    This is a L1 loss but more smooth. It requires that the
D
dangqingqing 已提交
6074
    size of input and label are equal. The formula is as follows,
D
dangqingqing 已提交
6075 6076 6077 6078 6079 6080 6081 6082 6083

    .. math::

        L = \sum_{i} smooth_{L1}(input_i - label_i)

    in which

    .. math::

6084
        smooth_{L1}(x) = \\begin{cases} 0.5x^2& \\text{if}  \\ |x| < 1 \\\\ |x|-0.5& \\text{otherwise} \end{cases}
D
dangqingqing 已提交
6085

D
dangqingqing 已提交
6086 6087 6088
    More details can be found by referring to `Fast R-CNN
    <https://arxiv.org/pdf/1504.08083v2.pdf>`_

C
caoying03 已提交
6089 6090
    The example usage is:

D
dangqingqing 已提交
6091 6092
    .. code-block:: python

6093 6094
       cost = smooth_l1_cost(input=input_layer,
                             label=label_layer)
D
dangqingqing 已提交
6095 6096 6097 6098 6099 6100 6101

    :param input: The input layer.
    :type input: LayerOutput
    :param label: The input label.
    :type input: LayerOutput
    :param name: The name of this layers. It is not necessary.
    :type name: None|basestring
6102 6103
    :param coeff: The coefficient affects the gradient in the backward.
    :type coeff: float
D
dangqingqing 已提交
6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116
    :param layer_attr: Extra Layer Attribute.
    :type layer_attr: ExtraLayerAttribute
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
    assert isinstance(input, LayerOutput)
    assert isinstance(label, LayerOutput)
    assert input.size == label.size

    Layer(
        name=name,
        type=LayerType.SMOOTH_L1,
        inputs=[input.name, label.name],
6117
        coeff=coeff,
D
dangqingqing 已提交
6118 6119 6120
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.SMOOTH_L1, parents=[input, label], size=1)
W
wwhu 已提交
6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139


@wrap_name_default()
def multiplex_layer(input, name=None, layer_attr=None):
    """
    This layer multiplex multiple layers according to the index,
    which is provided by the first input layer.
    inputs[0]: the index of the layer to output of size batchSize.
    inputs[1:N]; the candidate output data.
    For each index i from 0 to batchSize -1, the output is the i-th row of the
    (index[i] + 1)-th layer.

    For each i-th row of output:
    .. math::
        y[i][j] = x_{x_{0}[i] + 1}[i][j], j = 0,1, ... , (x_{1}.width - 1)

    where, y is output. :math:`x_{k}` is the k-th input layer and
    :math:`k = x_{0}[i] + 1`.

C
caoying03 已提交
6140 6141
    The example usage is:

W
wwhu 已提交
6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173
    .. code-block:: python

       maxid = multiplex_layer(input=layers)

    :param input: Input layers.
    :type input: list of LayerOutput
    :param name: Layer name.
    :type name: basestring
    :param layer_attr: extra layer attributes.
    :type layer_attr: ExtraLayerAttribute.
    :return: LayerOutput object.
    :rtype: LayerOutput
    """

    assert isinstance(input, collections.Sequence)
    assert len(input) > 2, 'multiplex_layer should have more than 2 inputs'
    for i in range(1, len(input)):
        assert isinstance(input[i], LayerOutput)
        assert input[i].size == input[1].size, \
            "All the input layers except the first one should have the same size"

    l = Layer(
        name=name,
        type='multiplex',
        inputs=[x.name for x in input],
        size=input[1].size,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name=name,
        layer_type=LayerType.MULTIPLEX_LAYER,
        parents=input,
        size=l.config.size)
D
dangqingqing 已提交
6174 6175


6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191
@wrap_name_default("dropout")
def dropout_layer(input, dropout_rate, name=None):
    """
    @TODO(yuyang18): Add comments.

    :param name:
    :param input:
    :param dropout_rate:
    :return:
    """
    return addto_layer(
        name=name,
        input=input,
        act=LinearActivation(),
        bias_attr=False,
        layer_attr=ExtraAttr(drop_rate=dropout_rate))
6192 6193


D
dangqingqing 已提交
6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215
@wrap_name_default()
@wrap_act_default(act=LinearActivation())
@wrap_param_attr_default()
@layer_support(DROPOUT)
def row_conv_layer(input,
                   context_len,
                   act=None,
                   name=None,
                   param_attr=None,
                   layer_attr=None):
    """

    The row convolution is called lookahead convolution. It is firstly
    introduced in paper of `Deep Speech 2: End-toEnd Speech Recognition
    in English and Mandarin <https://arxiv.org/pdf/1512.02595v1.pdf>`_ .

    The bidirectional RNN that learns representation for a sequence by
    performing a forward and a backward pass through the entire sequence.
    However, unlike unidirectional RNNs, bidirectional RNNs are challenging
    to deploy in an online and low-latency setting. The lookahead convolution
    incorporates information from future subsequences in a computationally
    efficient manner to improve unidirectional recurrent neural networks.
6216

D
dangqingqing 已提交
6217 6218 6219 6220 6221
    The connection of row convolution is different form the 1D sequence
    convolution. Assumed that, the future context-length is k, that is to say,
    it can get the output at timestep t by using the the input feature from t-th
    timestep to (t+k+1)-th timestep. Assumed that the hidden dim of input
    activations are d, the activations r_t for the new layer at time-step t are:
6222

D
dangqingqing 已提交
6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265
    .. math::

        r_{t,r} = \sum_{j=1}^{k + 1} {w_{i,j}h_{t+j-1, i}}
                  \quad \text{for} \quad  (1 \leq i \leq d)

    Note:
        The `context_len` is `k + 1`. That is to say, the lookahead step
        number plus one equals context_len.


    .. code-block:: python

       row_conv = row_conv_layer(input=input_layer, context_len=3)


    :param input: The input layer.
    :type input: LayerOutput
    :param context_len: The context length equals the lookahead step number
                        plus one.
    :type context_len: int
    :param act: Activation Type. Default is linear activation.
    :type act: BaseActivation
    :param param_attr: The Parameter Attribute. If None, the parameter will be
                       initialized smartly. It's better set it by yourself.
    :type param_attr: ParameterAttribute
    :param layer_attr: Extra Layer config.
    :type layer_attr: ExtraLayerAttribute|None
    :return: LayerOutput object.
    :rtype: LayerOutput

    """
    assert isinstance(input, LayerOutput)
    assert context_len > 0, "the context_len must be greatet than 0."

    Layer(
        inputs=[Input(input.name, **param_attr.attr)],
        name=name,
        context_length=context_len,
        type=LayerType.ROW_CONV_LAYER,
        active_type=act.name,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name, LayerType.ROW_CONV_LAYER, input, activation=act, size=input.size)
D
dangqingqing 已提交
6266 6267


6268 6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286
@layer_support()
@wrap_name_default()
@wrap_param_attr_default()
def prelu_layer(input,
                name=None,
                partial_sum=1,
                param_attr=None,
                layer_attr=None):
    """
    The Parameter Relu activation that actives outputs with a learnable weight.

    Reference:
        Delving Deep into Rectifiers: Surpassing Human-Level Performance on
        ImageNet Classification http://arxiv.org/pdf/1502.01852v1.pdf

    .. math::
       z_i &\\quad if \\quad z_i > 0 \\\\
       a_i * z_i  &\\quad \\mathrm{otherwise}

C
caoying03 已提交
6287 6288 6289 6290 6291 6292
    The example usage is:

    .. code-block:: python

       prelu = prelu_layer(input=layers, partial_sum=1)

6293 6294 6295 6296 6297
    :param name: Name of this layer.
    :type name: basestring
    :param input: The input layer.
    :type input: LayerOutput
    :param partial_sum: this parameter makes a group of inputs share a same weight.
C
caoying03 已提交
6298 6299 6300 6301 6302 6303

        - partial_sum = 1, indicates the element-wise activation: each element has a weight.
        - partial_sum = number of elements in one channel, indicates the channel-wise activation, elements in a channel share a same weight.
        - partial_sum = number of outputs, indicates all elements share a same weight.

    :type partial_sum: int
6304 6305 6306 6307 6308 6309 6310 6311
    :param param_attr: The parameter attribute. See ParameterAttribute for details.
    :type param_attr: ParameterAttribute|None
    :param layer_attr: Extra layer configurations. Default is None.
    :type layer_attr: ExtraLayerAttribute|None
    :return: LayerOutput object.
    :rtype: LayerOutput
    """

6312
    assert isinstance(input, LayerOutput), 'prelu_layer accepts only one input.'
C
caoying03 已提交
6313
    assert isinstance(param_attr, ParameterAttribute)
6314 6315 6316

    l = Layer(
        name=name,
C
caoying03 已提交
6317
        type=LayerType.PRELU,
C
caoying03 已提交
6318
        inputs=Input(input.name, **param_attr.attr),
6319 6320 6321 6322 6323 6324 6325
        partial_sum=partial_sum,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name=name,
        layer_type=LayerType.PRELU,
        parents=input,
        size=l.config.size)
6326 6327


6328
@wrap_name_default()
C
caoying03 已提交
6329
@layer_support(ERROR_CLIPPING, DROPOUT)
6330 6331 6332 6333 6334 6335 6336
@wrap_act_default(act=LinearActivation())
def gated_unit_layer(input,
                     size,
                     act=None,
                     name=None,
                     gate_attr=None,
                     gate_param_attr=None,
C
caoying03 已提交
6337 6338
                     gate_bias_attr=True,
                     inproj_attr=None,
6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368 6369 6370 6371 6372 6373 6374
                     inproj_param_attr=None,
                     inproj_bias_attr=True,
                     layer_attr=None):
    """
    The gated unit layer implements a simple gating mechanism over the input.
    The input :math:`X` is first projected into a new space :math:`X'`, and
    it is also used to produce a gate weight :math:`\sigma`. Element-wise
    prodict between :match:`X'` and :math:`\sigma` is finally returned.

    Reference:
        Language Modeling with Gated Convolutional Networks
        https://arxiv.org/abs/1612.08083

    .. math::
       y=\\text{act}(X \cdot W + b)\otimes \sigma(X \cdot V + c)

    The example usage is:

    .. code-block:: python
        gated_unit = gated_unit_layer(size=128, input=input_layer))

    :param input: input for this layer.
    :type input: LayerOutput
    :param size: output size of the gated unit.
    :type size: int
    :param act: activation type of the projected input.
    :type act: BaseActivation
    :param name: name of this layer.
    :type name: basestring
    :param gate_attr: Attributes to tune the gate output, for example, error
        clipping threshold, dropout and so on. See ExtraLayerAttribute for
        more details.
    :type gate_attr: ExtraLayerAttribute|None
    :param gate_param_attr: Attributes to tune the learnable projected matrix
        parameter of the gate.
    :type gate_param_attr: ParameterAttribute|None
C
caoying03 已提交
6375 6376 6377 6378 6379 6380
    :param gate_bias_attr: Attributes to tune the learnable bias of the gate.
    :type gate_bias_attr: ParameterAttribute|None
    :param inproj_attr: Attributes to the tune the projected input, for
        example, error clipping threshold, dropout and so on. See
        ExtraLayerAttribute for more details.
    :type inproj_attr: ExtraLayerAttribute|None
6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402
    :param inproj_param_attr: Attributes to tune the learnable parameter of
        the projection of input.
    :type inproj_param_attr: ParameterAttribute|None
    :param inproj_bias_attr: Attributes to tune the learnable bias of
        projection of the input.
    :type inproj_bias_attr: ParameterAttribute|None
    :param layer_attr: Attributes to tune the final output of the gated unit,
        for example, error clipping threshold, dropout and so on. See
        ExtraLayerAttribute for more details.
    :type layer_attr: ExtraLayerAttribute|None
    :return: LayerOutput object.
    :rtype: LayerOutput
    """

    assert isinstance(
        input, LayerOutput), 'The gated linear unit accepts only one input.'

    input_proj = fc_layer(
        input=input,
        name="%s_input_proj" % name,
        size=size,
        act=act,
C
caoying03 已提交
6403
        layer_attr=inproj_attr,
6404 6405 6406 6407 6408 6409 6410 6411 6412
        param_attr=inproj_param_attr,
        bias_attr=inproj_bias_attr)

    gate = fc_layer(
        size=size,
        name="%s_gate" % name,
        act=SigmoidActivation(),
        input=input,
        layer_attr=gate_attr,
C
caoying03 已提交
6413
        param_attr=gate_param_attr,
6414 6415 6416 6417 6418
        bias_attr=gate_bias_attr)
    return mixed_layer(
        name="%s_gated_act" % name,
        input=dotmul_operator(input_proj, gate),
        layer_attr=layer_attr)
6419 6420


6421
@layer_support()
6422
@wrap_name_default('switch_order')
W
wanghaoshuang 已提交
6423 6424
def switch_order_layer(input,
                       name=None,
6425
                       reshape_axis=None,
W
wanghaoshuang 已提交
6426 6427
                       act=None,
                       layer_attr=None):
6428
    """
6429 6430 6431
    This layer switch dimension order of image input. 
    From order "batchSize, channels, height, width"
    to order "batchSize, height, width, channels".
6432 6433 6434 6435

    The example usage is:

    .. code-block:: python
6436 6437
       reshape_axis = 3
       switch = switch_order(input=layer, name='switch', reshape_axis=reshape_axis)
6438
       reshape = {'height':[ 0, 1, 2], 'width':[3]}
6439 6440 6441

    :param input: The input layer.
    :type input: LayerOutput
6442 6443 6444 6445
    :param name: Name of this layer.
    :type name: basestring
    :param reshape: reshape matrix by axises.
    :type reshape: Dict
6446 6447 6448
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
6449
    assert isinstance(input, LayerOutput)
6450 6451 6452 6453 6454
    assert reshape_axis != None and (reshape_axis > 0 and reshape_axis < 4)
    height = [ele for ele in xrange(reshape_axis)]
    width = [ele for ele in range(reshape_axis, 4)]
    reshape = {'height': height, 'width': width}

6455 6456
    l = Layer(
        name=name,
W
wanghaoshuang 已提交
6457
        inputs=input.name,
6458 6459
        reshape=reshape,
        type=LayerType.SWITCH_ORDER_LAYER,
W
wanghaoshuang 已提交
6460
        active_type=act.name,
6461 6462 6463
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name=name,
6464
        layer_type=LayerType.SWITCH_ORDER_LAYER,
6465
        activation=act,
6466 6467
        parents=input,
        size=l.config.size)
W
wanghaoshuang 已提交
6468 6469


6470 6471
@wrap_name_default()
@layer_support()
6472
def crop_layer(input, offset, axis=2, shape=None, name=None, layer_attr=None):
6473
    """
6474
    The crop layer crops images by offset and shape. User can set crop shape by
6475
    args 'shape' explicitly or by reference input layer.
6476

6477 6478 6479
    The example usage is:

    .. code-block:: python
W
whs 已提交
6480
    crop = crop_layer(input=[image_input, reference_input], axis=2, offset=[2, 3])
6481 6482 6483 6484

    :param input: The input layer.If two inputs were setted,
                    the second input will be regarded as reference input
    :type input: LayerOutput or Sequence
6485 6486
    :param offset: The crop offset
    :type offset: Sequence
6487 6488 6489 6490 6491 6492 6493
    :param axis: start axis to be cropped. To image input layer:
        - 0: batch size
        - 1: channels
        - 2: height
        - 3: width
    :type partial_sum: int
    :param shape: The shape to be cropped. Default is None.
6494
    :type shape: Sequence | None
6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505 6506 6507 6508 6509 6510 6511 6512 6513 6514 6515 6516
    :param name: Name of this layer.
    :type name: basestring
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
    if isinstance(input, LayerOutput):
        input = [input]
    else:
        assert isinstance(input, collections.Sequence)
    l = Layer(
        inputs=[x.name for x in input],
        axis=axis,
        offset=offset,
        shape=shape,
        name=name,
        type=LayerType.CROP_LAYER,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name=name,
        layer_type=LayerType.CROP_LAYER,
        parents=input,
        size=l.config.size)
G
guosheng 已提交
6517 6518


C
caoying03 已提交
6519 6520
@wrap_name_default()
@layer_support()
6521
def sub_nested_seq_layer(input, selected_indices, name=None):
C
caoying03 已提交
6522
    """
6523
    The sub_nested_seq_layer accepts two inputs: the first one is a nested
6524
    sequence; the second one is a set of selceted indices in the nested sequence.
C
caoying03 已提交
6525

C
caoying03 已提交
6526 6527 6528
    Then sub_nest_seq_layer trims the first nested sequence input according
    to the selected indices to form a new output. This layer is useful in
    beam training.
C
caoying03 已提交
6529 6530 6531 6532

    The example usage is:

    .. code-block:: python
C
caoying03 已提交
6533 6534

        sub_nest_seq = sub_nested_seq_layer(input=[data, selected_indices])
6535

C
caoying03 已提交
6536

6537 6538 6539
    :param input: A nested sequence.
    :type input: LayerOutput
    :param selected_indices: a set of sequence indices in the nested sequence.
C
caoying03 已提交
6540 6541 6542 6543 6544 6545
    :type input: LayerOutput
    :param name: name of this layer.
    :type name: basestring
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
C
caoying03 已提交
6546

6547 6548 6549 6550 6551 6552 6553
    assert isinstance(input, LayerOutput), (
        'The first input of '
        'sub_nested_seq_layer must be a Paddle layer.')
    assert isinstance(selected_indices, LayerOutput), (
        'The second input of '
        'sub_nested_seq_layer must be a Paddle layer.')

C
caoying03 已提交
6554
    l = Layer(
6555 6556
        inputs=input.name,
        selected_indices=selected_indices.name,
C
caoying03 已提交
6557 6558 6559 6560 6561 6562 6563
        name=name,
        type=LayerType.SUB_NESTED_SEQ)
    return LayerOutput(
        name=name,
        layer_type=LayerType.SUB_NESTED_SEQ,
        parents=input,
        size=l.config.size)
6564 6565


G
guosheng 已提交
6566
@wrap_name_default("clip")
6567
def clip_layer(input, min, max, name=None):
G
guosheng 已提交
6568 6569 6570 6571 6572 6573 6574 6575 6576
    """
    A layer for clipping the input value by the threshold.

    .. math::

        out[i] = \min\left(\max\left(in[i],p_{1}\right),p_{2}\right)

    .. code-block:: python

6577
        clip = clip_layer(input=input_layer, min=-10, max=10)
G
guosheng 已提交
6578 6579 6580 6581 6582

    :param name: The Layer Name.
    :type name: basestring
    :param input: The input layer.
    :type input: LayerOutput.
6583 6584 6585 6586
    :param min: The lower threshold for clipping.
    :type min: double
    :param max: The upper threshold for clipping.
    :type max: double
6587 6588
    :return: LayerOutput object.
    :rtype: LayerOutput
G
guosheng 已提交
6589 6590 6591 6592 6593
    """
    Layer(
        name=name,
        type=LayerType.CLIP_LAYER,
        inputs=[input.name],
6594 6595
        min=min,
        max=max)
G
guosheng 已提交
6596 6597
    return LayerOutput(
        name, LayerType.CLIP_LAYER, parents=[input], size=input.size)
6598 6599


6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663
@wrap_name_default()
def seq_slice_layer(input, starts, ends, name=None):
    """
    seq_slice_layer will return one or several sub-sequences from the
    input sequence layer given start and end indices.

        - If only start indices are given, and end indices are set to None,
          this layer slices the input sequence from the given start indices
          to its end.
        - If only end indices are given, and start indices are set to None,
          this layer slices the input sequence from its beginning to the
          given end indices.
        - If start and end indices are both given, they should have the same
          number of elements.

    If start or end indices contains more than one elements, the input sequence
    will be sliced for multiple times.


    .. code-block:: python

        seq_silce = seq_slice_layer(input=input_seq,
                                    starts=start_pos, ends=end_pos)

    :param name: name of this layer.
    :type name: basestring
    :param input: input for this layer, it should be a sequence.
    :type input: LayerOutput
    :param starts: start indices to slice the input sequence.
    :type starts: LayerOutput|None
    :param ends: end indices to slice the input sequence.
    :type ends: LayerOutput|None
    :return: LayerOutput object.
    :rtype: LayerOutput

    """

    assert isinstance(input, LayerOutput), (
        'The first input of seq_slice layer must be a PaddlePaddle layer.')

    if starts is not None:
        assert isinstance(starts, LayerOutput), (
            'The start indices for seq_slice layer '
            'must be a PaddlePaddle layer.')
    if ends is not None:
        assert isinstance(ends, LayerOutput), (
            'The end indices for seq_slice layer must be a PaddlePaddle layer.')
    assert starts is not None or ends is not None, (
        'start and end indices '
        'cannot be set to None at the same time, at least one of '
        'them should be given.')
    if starts is not None and ends is not None:
        assert starts.size == ends.size, (
            'If start and end indices are both given to seq_slice_layer, '
            'they should have the same width.')

    Layer(
        name=name,
        type=LayerType.SEQ_SLICE,
        inputs=input.name,
        starts=starts.name if starts is not None else None,
        ends=ends.name if ends is not None else None)
    return LayerOutput(
        name, LayerType.SEQ_SLICE, parents=[input], size=input.size)
6664 6665


6666 6667
@wrap_name_default()
@layer_support()
6668
def kmax_seq_score_layer(input, name=None, beam_size=1):
6669
    """
C
caoying03 已提交
6670
    This layer accepts one input which are scores over a sequence or a nested
6671 6672 6673 6674
    sequence, and returns indices of beam_size sequences with highest scores.

    .. code-block:: python

6675
        kmax_indices = kmax_seq_score_layer(input=input_layer, beam_size)
6676 6677 6678 6679


    :param name: The Layer Name.
    :type name: basestring
C
caoying03 已提交
6680
    :param input: The input layer. It stores scores over a sequence or a nested
6681 6682 6683 6684 6685 6686 6687
        sequence and its size must be 1.
    :type input: LayerOutput.
    :param beam_size: squence indices with top beam_size scores are returned.
    :type beam_size: double
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
6688
    assert isinstance(input, LayerOutput), ("kmax_seq_score_layer "
6689
                                            "accepts only one input.")
6690
    assert input.size == 1, (
6691
        "input of kmax_seq_score_layer is a score "
6692 6693 6694 6695 6696 6697 6698 6699 6700 6701
        "over a sequence or a nested sequence, so its width must be 1.")

    Layer(
        name=name,
        type=LayerType.KMAX_SEQ_SCORE,
        inputs=[input.name],
        beam_size=beam_size)

    return LayerOutput(
        name, LayerType.KMAX_SEQ_SCORE, parents=[input], size=input.size)
G
guosheng 已提交
6702 6703


6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721 6722 6723 6724 6725 6726 6727 6728 6729
@wrap_name_default("conv3d")
@wrap_param_attr_default()
@wrap_bias_attr_default()
@wrap_act_default(act=ReluActivation())
@layer_support(DROPOUT)
def img_conv3d_layer(input,
                     filter_size,
                     num_filters,
                     name=None,
                     num_channels=None,
                     act=None,
                     groups=1,
                     stride=1,
                     padding=0,
                     bias_attr=None,
                     param_attr=None,
                     shared_biases=True,
                     layer_attr=None,
                     trans=False,
                     layer_type=None):
    """

    The example usage is:

    ..  code-block:: python

C
chengduoZH 已提交
6730
        conv = img_conv3d_layer(input=data, filter_size=1,
6731 6732 6733 6734 6735 6736 6737 6738 6739
                              num_channels=8,
                              num_filters=16, stride=1,
                              bias_attr=False,
                              act=ReluActivation())

    :param name: Layer name.
    :type name: basestring
    :param input: Layer Input.
    :type input: LayerOutput
C
chengduoZH 已提交
6740
    :param filter_size: The x dimension of a filter kernel. Or input a list.
6741 6742 6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778
    :type filter_size: int|tuple|list
    :param num_filters: Each filter group's number of filter
    :param act: Activation type. Default is tanh
    :type act: BaseActivation
    :param groups: Group size of filters.
    :type groups: int
    :param stride: The x dimension of the stride. Or input a tuple for two image
                   dimension.
    :type stride: int|tuple|list
    :param padding: The x dimension of the padding. Or input a tuple for two
                    image dimension
    :type padding: int|tuple|list
    :param bias_attr: Convolution bias attribute. None means default bias.
                      False means no bias.
    :type bias_attr: ParameterAttribute|False
    :param num_channels: number of input channels. If None will be set
                        automatically from previous output.
    :type num_channels: int
    :param param_attr: Convolution param attribute. None means default attribute
    :type param_attr: ParameterAttribute
    :param shared_biases: Is biases will be shared between filters or not.
    :type shared_biases: bool
    :param layer_attr: Layer Extra Attribute.
    :type layer_attr: ExtraLayerAttribute
    :param trans: true if it is a convTransLayer, false if it is a convLayer
    :type trans: bool
    :param layer_type: specify the layer_type, default is None. If trans=True,
                       layer_type has to be "exconvt" or "cudnn_convt",
                       otherwise layer_type has to be either "exconv" or
                       "cudnn_conv"
    :type layer_type: String
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
    if num_channels is None:
        assert input.num_filters is not None
        num_channels = input.num_filters

C
chengduoZH 已提交
6779 6780 6781 6782 6783 6784
    if isinstance(filter_size, collections.Sequence):
        assert len(filter_size) == 3
        filter_size, filter_size_y, filter_size_z = filter_size
    else:
        filter_size_y = filter_size
        filter_size_z = filter_size
6785

C
chengduoZH 已提交
6786 6787 6788 6789 6790 6791
    if isinstance(stride, collections.Sequence):
        assert len(stride) == 3
        stride, stride_y, stride_z = stride
    else:
        stride_y = stride
        stride_z = stride
6792

C
chengduoZH 已提交
6793 6794 6795 6796 6797 6798
    if isinstance(padding, collections.Sequence):
        assert len(padding) == 3
        padding, padding_y, padding_z = padding
    else:
        padding_y = padding
        padding_z = padding
6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844

    if param_attr.attr.get('initial_smart'):
        # special initial for conv layers.
        init_w = (2.0 / (filter_size**2 * num_channels))**0.5
        param_attr.attr["initial_mean"] = 0.0
        param_attr.attr["initial_std"] = init_w
        param_attr.attr["initial_strategy"] = 0
        param_attr.attr["initial_smart"] = False

    if layer_type:
        if trans:
            assert layer_type in ["deconv3d"]
        lt = layer_type
    else:
        lt = LayerType.DECONV3D_LAYER if trans else LayerType.CONV3D_LAYER

    l = Layer(
        name=name,
        inputs=Input(
            input.name,
            conv=Conv3D(
                filter_size=filter_size,
                padding=padding,
                stride=stride,
                channels=num_channels,
                groups=groups,
                filter_size_y=filter_size_y,
                padding_y=padding_y,
                stride_y=stride_y,
                filter_size_z=filter_size_z,
                padding_z=padding_z,
                stride_z=stride_z),
            **param_attr.attr),
        active_type=act.name,
        num_filters=num_filters,
        bias=ParamAttr.to_bias(bias_attr),
        shared_biases=shared_biases,
        type=lt,
        **ExtraLayerAttribute.to_kwargs(layer_attr))
    return LayerOutput(
        name,
        lt,
        parents=[input],
        activation=act,
        num_filters=num_filters,
        size=l.config.size)
C
chengduoZH 已提交
6845 6846


G
guosheng 已提交
6847 6848 6849 6850 6851
@wrap_name_default("scale_shift")
@wrap_param_attr_default()
@wrap_bias_attr_default()
def scale_shift_layer(input, name=None, param_attr=None, bias_attr=None):
    """
X
xuwei06 已提交
6852 6853
    A layer applies a linear transformation to each element in each row of
    the input matrix. For each element, the layer first re-scale it and then
6854 6855
    adds a bias to it.

X
xuwei06 已提交
6856
    This layer is very like the SlopeInterceptLayer, except the scale and
6857 6858
    bias are trainable.

G
guosheng 已提交
6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884
    .. math::

        y = w * x + b

    .. code-block:: python

        scale_shift = scale_shift_layer(input=input_layer, bias_attr=False)

    :param name: The Layer Name.
    :type name: basestring
    :param input: The input layer.
    :type input: LayerOutput.
    :param param_attr: The parameter attribute of scaling.
    :type param_attr: ParameterAttribute
    :param bias_attr: The parameter attribute of shifting.
    :type bias_attr: ParameterAttribute
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
    Layer(
        name=name,
        type=LayerType.SCALE_SHIFT_LAYER,
        inputs=Input(input.name, **param_attr.attr),
        bias=ParamAttr.to_bias(bias_attr))
    return LayerOutput(
        name, LayerType.SCALE_SHIFT_LAYER, parents=[input], size=input.size)