networks.py 60.6 KB
Newer Older
1
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
Z
zhangjinchao01 已提交
2 3 4 5 6 7 8 9 10 11 12 13
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
R
ranqiu 已提交
14
import math
P
peterzhang2029 已提交
15

Z
zhangjinchao01 已提交
16 17 18 19
from activations import LinearActivation, ReluActivation, SoftmaxActivation, \
    IdentityActivation, TanhActivation, SequenceSoftmaxActivation
from attrs import ExtraAttr
from default_decorators import wrap_name_default, wrap_act_default, \
Y
Yu Yang 已提交
20
    wrap_param_default, wrap_bias_attr_default, wrap_param_attr_default
Z
zhangjinchao01 已提交
21 22 23 24
from layers import *  # There are too many layers used in network, so import *
from poolings import MaxPooling, SumPooling
from paddle.trainer.config_parser import *

Q
qijun 已提交
25 26
__all__ = [
    'sequence_conv_pool', 'simple_lstm', "simple_img_conv_pool",
27 28
    "img_conv_bn_pool", 'lstmemory_group', 'lstmemory_unit', 'small_vgg',
    'img_conv_group', 'vgg_16_network', 'gru_unit', 'gru_group', 'simple_gru',
R
ranqiu 已提交
29 30 31
    'simple_attention', 'dot_product_attention', 'multi_head_attention',
    'simple_gru2', 'bidirectional_gru', 'text_conv_pool', 'bidirectional_lstm',
    'inputs', 'outputs'
Q
qijun 已提交
32
]
Z
zhangjinchao01 已提交
33 34 35 36 37

######################################################
#                     Text CNN                       #
######################################################

Q
qijun 已提交
38

Z
zhangjinchao01 已提交
39 40
@wrap_name_default("sequence_conv_pooling")
def sequence_conv_pool(input,
Q
qijun 已提交
41 42
                       context_len,
                       hidden_size,
Z
zhangjinchao01 已提交
43 44
                       name=None,
                       context_start=None,
Q
qijun 已提交
45 46
                       pool_type=None,
                       context_proj_layer_name=None,
Z
zhangjinchao01 已提交
47 48 49
                       context_proj_param_attr=False,
                       fc_layer_name=None,
                       fc_param_attr=None,
Q
qijun 已提交
50 51
                       fc_bias_attr=None,
                       fc_act=None,
Z
zhangjinchao01 已提交
52 53 54 55 56
                       pool_bias_attr=None,
                       fc_attr=None,
                       context_attr=None,
                       pool_attr=None):
    """
57
    Text convolution pooling group.
Z
zhangjinchao01 已提交
58 59 60

    Text input => Context Projection => FC Layer => Pooling => Output.

61
    :param name: group name.
Z
zhangjinchao01 已提交
62
    :type name: basestring
63
    :param input: input layer.
Z
zhangjinchao01 已提交
64 65 66 67 68 69
    :type input: LayerOutput
    :param context_len: context projection length. See
                        context_projection's document.
    :type context_len: int
    :param hidden_size: FC Layer size.
    :type hidden_size: int
70
    :param context_start: context start position. See
Z
zhangjinchao01 已提交
71
                          context_projection's context_start.
72
    :type context_start: int|None
Z
zhangjinchao01 已提交
73
    :param pool_type: pooling layer type. See pooling_layer's document.
74
    :type pool_type: BasePoolingType
Z
zhangjinchao01 已提交
75 76 77
    :param context_proj_layer_name: context projection layer name.
                                    None if user don't care.
    :type context_proj_layer_name: basestring
78 79 80
    :param context_proj_param_attr: padding parameter attribute of context projection layer.
                                    If false, it means padding always be zero.
    :type context_proj_param_attr: ParameterAttribute|None
Z
zhangjinchao01 已提交
81 82 83
    :param fc_layer_name: fc layer name. None if user don't care.
    :type fc_layer_name: basestring
    :param fc_param_attr: fc layer parameter attribute. None if user don't care.
84
    :type fc_param_attr: ParameterAttribute|None
Z
zhangjinchao01 已提交
85 86
    :param fc_bias_attr: fc bias parameter attribute. False if no bias,
                         None if user don't care.
87 88
    :type fc_bias_attr: ParameterAttribute|False|None
    :param fc_act: fc layer activation type. None means tanh.
Z
zhangjinchao01 已提交
89
    :type fc_act: BaseActivation
90 91 92
    :param pool_bias_attr: pooling layer bias attr. False if no bias.
                           None if user don't care.
    :type pool_bias_attr: ParameterAttribute|False|None
Z
zhangjinchao01 已提交
93 94 95 96 97 98
    :param fc_attr: fc layer extra attribute.
    :type fc_attr: ExtraLayerAttribute
    :param context_attr: context projection layer extra attribute.
    :type context_attr: ExtraLayerAttribute
    :param pool_attr: pooling layer extra attribute.
    :type pool_attr: ExtraLayerAttribute
99
    :return: layer's output.
Z
zhangjinchao01 已提交
100 101 102 103 104 105
    :rtype: LayerOutput
    """
    # Set Default Value to param
    context_proj_layer_name = "%s_conv_proj" % name \
        if context_proj_layer_name is None else context_proj_layer_name

Q
qijun 已提交
106 107 108 109 110 111 112 113 114 115
    with mixed_layer(
            name=context_proj_layer_name,
            size=input.size * context_len,
            act=LinearActivation(),
            layer_attr=context_attr) as m:
        m += context_projection(
            input,
            context_len=context_len,
            context_start=context_start,
            padding_attr=context_proj_param_attr)
Z
zhangjinchao01 已提交
116 117 118

    fc_layer_name = "%s_conv_fc" % name \
        if fc_layer_name is None else fc_layer_name
Q
qijun 已提交
119 120 121 122 123 124 125 126
    fl = fc_layer(
        name=fc_layer_name,
        input=m,
        size=hidden_size,
        act=fc_act,
        layer_attr=fc_attr,
        param_attr=fc_param_attr,
        bias_attr=fc_bias_attr)
Z
zhangjinchao01 已提交
127

Q
qijun 已提交
128 129 130 131 132 133
    return pooling_layer(
        name=name,
        input=fl,
        pooling_type=pool_type,
        bias_attr=pool_bias_attr,
        layer_attr=pool_attr)
Z
zhangjinchao01 已提交
134 135 136 137 138 139 140 141


text_conv_pool = sequence_conv_pool

############################################################################
#                       Images                                             #
############################################################################

Q
qijun 已提交
142

Z
zhangjinchao01 已提交
143
@wrap_name_default("conv_pool")
Q
qijun 已提交
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
def simple_img_conv_pool(input,
                         filter_size,
                         num_filters,
                         pool_size,
                         name=None,
                         pool_type=None,
                         act=None,
                         groups=1,
                         conv_stride=1,
                         conv_padding=0,
                         bias_attr=None,
                         num_channel=None,
                         param_attr=None,
                         shared_bias=True,
                         conv_layer_attr=None,
                         pool_stride=1,
                         pool_padding=0,
                         pool_layer_attr=None):
Z
zhangjinchao01 已提交
162 163 164
    """
    Simple image convolution and pooling group.

165
    Img input => Conv => Pooling => Output.
Z
zhangjinchao01 已提交
166

167
    :param name: group name.
Z
zhangjinchao01 已提交
168
    :type name: basestring
169
    :param input: input layer.
Z
zhangjinchao01 已提交
170
    :type input: LayerOutput
171
    :param filter_size: see img_conv_layer for details.
Z
zhangjinchao01 已提交
172
    :type filter_size: int
173
    :param num_filters: see img_conv_layer for details.
Z
zhangjinchao01 已提交
174
    :type num_filters: int
175
    :param pool_size: see img_pool_layer for details.
Z
zhangjinchao01 已提交
176
    :type pool_size: int
177
    :param pool_type: see img_pool_layer for details.
Z
zhangjinchao01 已提交
178
    :type pool_type: BasePoolingType
179
    :param act: see img_conv_layer for details.
Z
zhangjinchao01 已提交
180
    :type act: BaseActivation
181
    :param groups: see img_conv_layer for details.
Z
zhangjinchao01 已提交
182
    :type groups: int
183
    :param conv_stride: see img_conv_layer for details.
Z
zhangjinchao01 已提交
184
    :type conv_stride: int
185
    :param conv_padding: see img_conv_layer for details.
Z
zhangjinchao01 已提交
186
    :type conv_padding: int
187
    :param bias_attr: see img_conv_layer for details.
Z
zhangjinchao01 已提交
188
    :type bias_attr: ParameterAttribute
189
    :param num_channel: see img_conv_layer for details.
Z
zhangjinchao01 已提交
190
    :type num_channel: int
191
    :param param_attr: see img_conv_layer for details.
Z
zhangjinchao01 已提交
192
    :type param_attr: ParameterAttribute
193
    :param shared_bias: see img_conv_layer for details.
Z
zhangjinchao01 已提交
194
    :type shared_bias: bool
195
    :param conv_layer_attr: see img_conv_layer for details.
Z
zhangjinchao01 已提交
196
    :type conv_layer_attr: ExtraLayerAttribute
197
    :param pool_stride: see img_pool_layer for details.
Z
zhangjinchao01 已提交
198
    :type pool_stride: int
199
    :param pool_padding: see img_pool_layer for details.
Z
zhangjinchao01 已提交
200
    :type pool_padding: int
201
    :param pool_layer_attr: see img_pool_layer for details.
Z
zhangjinchao01 已提交
202
    :type pool_layer_attr: ExtraLayerAttribute
203
    :return: layer's output
Z
zhangjinchao01 已提交
204 205
    :rtype: LayerOutput
    """
Q
qijun 已提交
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
    _conv_ = img_conv_layer(
        name="%s_conv" % name,
        input=input,
        filter_size=filter_size,
        num_filters=num_filters,
        num_channels=num_channel,
        act=act,
        groups=groups,
        stride=conv_stride,
        padding=conv_padding,
        bias_attr=bias_attr,
        param_attr=param_attr,
        shared_biases=shared_bias,
        layer_attr=conv_layer_attr)
    return img_pool_layer(
        name="%s_pool" % name,
        input=_conv_,
        pool_size=pool_size,
        pool_type=pool_type,
        stride=pool_stride,
        padding=pool_padding,
        layer_attr=pool_layer_attr)
Z
zhangjinchao01 已提交
228 229 230


@wrap_name_default("conv_bn_pool")
Q
qijun 已提交
231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251
def img_conv_bn_pool(input,
                     filter_size,
                     num_filters,
                     pool_size,
                     name=None,
                     pool_type=None,
                     act=None,
                     groups=1,
                     conv_stride=1,
                     conv_padding=0,
                     conv_bias_attr=None,
                     num_channel=None,
                     conv_param_attr=None,
                     shared_bias=True,
                     conv_layer_attr=None,
                     bn_param_attr=None,
                     bn_bias_attr=None,
                     bn_layer_attr=None,
                     pool_stride=1,
                     pool_padding=0,
                     pool_layer_attr=None):
Z
zhangjinchao01 已提交
252 253
    """
    Convolution, batch normalization, pooling group.
254 255
    
    Img input => Conv => BN => Pooling => Output.
Z
zhangjinchao01 已提交
256

257
    :param name: group name.
Z
zhangjinchao01 已提交
258
    :type name: basestring
259 260 261
    :param input: input layer.
    :type input: LayerOutput 
    :param filter_size: see img_conv_layer for details.
Z
zhangjinchao01 已提交
262
    :type filter_size: int
263
    :param num_filters: see img_conv_layer for details.
Z
zhangjinchao01 已提交
264
    :type num_filters: int
265
    :param pool_size: see img_pool_layer for details.
Z
zhangjinchao01 已提交
266
    :type pool_size: int
267
    :param pool_type: see img_pool_layer for details.
Z
zhangjinchao01 已提交
268
    :type pool_type: BasePoolingType
269
    :param act: see batch_norm_layer for details.
Z
zhangjinchao01 已提交
270
    :type act: BaseActivation
271
    :param groups: see img_conv_layer for details.
Z
zhangjinchao01 已提交
272
    :type groups: int
273
    :param conv_stride: see img_conv_layer for details.
Z
zhangjinchao01 已提交
274
    :type conv_stride: int
275
    :param conv_padding: see img_conv_layer for details.
Z
zhangjinchao01 已提交
276
    :type conv_padding: int
277
    :param conv_bias_attr: see img_conv_layer for details.
Z
zhangjinchao01 已提交
278
    :type conv_bias_attr: ParameterAttribute
279
    :param num_channel: see img_conv_layer for details.
Z
zhangjinchao01 已提交
280
    :type num_channel: int
281
    :param conv_param_attr: see img_conv_layer for details.
Z
zhangjinchao01 已提交
282
    :type conv_param_attr: ParameterAttribute
283
    :param shared_bias: see img_conv_layer for details.
Z
zhangjinchao01 已提交
284
    :type shared_bias: bool
285
    :param conv_layer_attr: see img_conv_layer for details.
Z
zhangjinchao01 已提交
286
    :type conv_layer_attr: ExtraLayerOutput
287 288 289 290 291 292 293
    :param bn_param_attr: see batch_norm_layer for details.
    :type bn_param_attr: ParameterAttribute
    :param bn_bias_attr: see batch_norm_layer for details.
    :type bn_bias_attr: ParameterAttribute
    :param bn_layer_attr: see batch_norm_layer for details.
    :type bn_layer_attr: ExtraLayerAttribute
    :param pool_stride: see img_pool_layer for details.
Z
zhangjinchao01 已提交
294
    :type pool_stride: int
295
    :param pool_padding: see img_pool_layer for details.
Z
zhangjinchao01 已提交
296
    :type pool_padding: int
297
    :param pool_layer_attr: see img_pool_layer for details.
Z
zhangjinchao01 已提交
298
    :type pool_layer_attr: ExtraLayerAttribute
299
    :return: layer's output
Z
zhangjinchao01 已提交
300 301
    :rtype: LayerOutput
    """
Q
qijun 已提交
302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
    __conv__ = img_conv_layer(
        name="%s_conv" % name,
        input=input,
        filter_size=filter_size,
        num_filters=num_filters,
        num_channels=num_channel,
        act=LinearActivation(),
        groups=groups,
        stride=conv_stride,
        padding=conv_padding,
        bias_attr=conv_bias_attr,
        param_attr=conv_param_attr,
        shared_biases=shared_bias,
        layer_attr=conv_layer_attr)
    __bn__ = batch_norm_layer(
        name="%s_bn" % name,
        input=__conv__,
        act=act,
        bias_attr=bn_bias_attr,
        param_attr=bn_param_attr,
        layer_attr=bn_layer_attr)
    return img_pool_layer(
        name="%s_pool" % name,
        input=__bn__,
        pool_type=pool_type,
        pool_size=pool_size,
        stride=pool_stride,
        padding=pool_padding,
        layer_attr=pool_layer_attr)


@wrap_act_default(param_names=['conv_act'], act=ReluActivation())
@wrap_param_default(
    param_names=['pool_type'], default_factory=lambda _: MaxPooling())
def img_conv_group(input,
                   conv_num_filter,
Z
zhangjinchao01 已提交
338 339 340 341 342 343 344 345
                   pool_size,
                   num_channels=None,
                   conv_padding=1,
                   conv_filter_size=3,
                   conv_act=None,
                   conv_with_batchnorm=False,
                   conv_batchnorm_drop_rate=0,
                   pool_stride=1,
Z
zlx 已提交
346 347
                   pool_type=None,
                   param_attr=None):
Z
zhangjinchao01 已提交
348 349 350
    """
    Image Convolution Group, Used for vgg net.

Z
zlx 已提交
351 352 353
    :param conv_batchnorm_drop_rate: if conv_with_batchnorm[i] is true,
        conv_batchnorm_drop_rate[i] represents the drop rate of each batch norm.
    :type conv_batchnorm_drop_rate: list
354
    :param input: input layer.
Z
zlx 已提交
355
    :type input: LayerOutput
356 357
    :param conv_num_filter: list of output channels num.
    :type conv_num_filter: list|tuple
Z
zlx 已提交
358 359 360 361 362 363 364 365 366 367
    :param pool_size: pooling filter size.
    :type pool_size: int
    :param num_channels: input channels num.
    :type num_channels: int
    :param conv_padding: convolution padding size.
    :type conv_padding: int
    :param conv_filter_size: convolution filter size.
    :type conv_filter_size: int
    :param conv_act: activation funciton after convolution.
    :type conv_act: BaseActivation
368 369
    :param conv_with_batchnorm: if conv_with_batchnorm[i] is true,
        there is a batch normalization operation after each convolution.
Z
zlx 已提交
370 371 372 373 374
    :type conv_with_batchnorm: list
    :param pool_stride: pooling stride size.
    :type pool_stride: int
    :param pool_type: pooling type.
    :type pool_type: BasePoolingType
375 376
    :param param_attr: param attribute of convolution layer,
                       None means default attribute.
Z
zlx 已提交
377
    :type param_attr: ParameterAttribute
378 379
    :return: layer's output
    :rtype: LayerOutput
Z
zhangjinchao01 已提交
380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
    """
    tmp = input

    # Type checks
    assert isinstance(tmp, LayerOutput)
    assert isinstance(conv_num_filter, list) or isinstance(conv_num_filter,
                                                           tuple)
    for each_num_filter in conv_num_filter:
        assert isinstance(each_num_filter, int)

    assert isinstance(pool_size, int)

    def __extend_list__(obj):
        if not hasattr(obj, '__len__'):
            return [obj] * len(conv_num_filter)
        else:
            return obj

    conv_padding = __extend_list__(conv_padding)
    conv_filter_size = __extend_list__(conv_filter_size)
    conv_act = __extend_list__(conv_act)
    conv_with_batchnorm = __extend_list__(conv_with_batchnorm)
    conv_batchnorm_drop_rate = __extend_list__(conv_batchnorm_drop_rate)

    for i in xrange(len(conv_num_filter)):
        extra_kwargs = dict()
        if num_channels is not None:
            extra_kwargs['num_channels'] = num_channels
            num_channels = None
        if conv_with_batchnorm[i]:
            extra_kwargs['act'] = LinearActivation()
        else:
            extra_kwargs['act'] = conv_act[i]

Q
qijun 已提交
414 415 416 417 418
        tmp = img_conv_layer(
            input=tmp,
            padding=conv_padding[i],
            filter_size=conv_filter_size[i],
            num_filters=conv_num_filter[i],
Z
zlx 已提交
419
            param_attr=param_attr,
Q
qijun 已提交
420
            **extra_kwargs)
Z
zhangjinchao01 已提交
421 422 423 424 425 426 427 428

        # logger.debug("tmp.num_filters = %d" % tmp.num_filters)

        if conv_with_batchnorm[i]:
            dropout = conv_batchnorm_drop_rate[i]
            if dropout == 0 or abs(dropout) < 1e-5:  # dropout not set
                tmp = batch_norm_layer(input=tmp, act=conv_act[i])
            else:
Q
qijun 已提交
429 430 431 432
                tmp = batch_norm_layer(
                    input=tmp,
                    act=conv_act[i],
                    layer_attr=ExtraAttr(drop_rate=dropout))
Z
zhangjinchao01 已提交
433

Q
qijun 已提交
434 435
    return img_pool_layer(
        input=tmp, stride=pool_stride, pool_size=pool_size, pool_type=pool_type)
Z
zhangjinchao01 已提交
436 437 438 439


def small_vgg(input_image, num_channels, num_classes):
    def __vgg__(ipt, num_filter, times, dropouts, num_channels_=None):
Q
qijun 已提交
440 441 442 443 444 445 446 447 448 449 450
        return img_conv_group(
            input=ipt,
            num_channels=num_channels_,
            pool_size=2,
            pool_stride=2,
            conv_num_filter=[num_filter] * times,
            conv_filter_size=3,
            conv_act=ReluActivation(),
            conv_with_batchnorm=True,
            conv_batchnorm_drop_rate=dropouts,
            pool_type=MaxPooling())
Z
zhangjinchao01 已提交
451 452 453 454 455

    tmp = __vgg__(input_image, 64, 2, [0.3, 0], num_channels)
    tmp = __vgg__(tmp, 128, 2, [0.4, 0])
    tmp = __vgg__(tmp, 256, 3, [0.4, 0.4, 0])
    tmp = __vgg__(tmp, 512, 3, [0.4, 0.4, 0])
Q
qijun 已提交
456 457
    tmp = img_pool_layer(
        input=tmp, stride=2, pool_size=2, pool_type=MaxPooling())
Z
zhangjinchao01 已提交
458
    tmp = dropout_layer(input=tmp, dropout_rate=0.5)
Q
qijun 已提交
459 460 461 462 463
    tmp = fc_layer(
        input=tmp,
        size=512,
        layer_attr=ExtraAttr(drop_rate=0.5),
        act=LinearActivation())
Z
zhangjinchao01 已提交
464 465 466 467 468 469 470 471
    tmp = batch_norm_layer(input=tmp, act=ReluActivation())
    return fc_layer(input=tmp, size=num_classes, act=SoftmaxActivation())


def vgg_16_network(input_image, num_channels, num_classes=1000):
    """
    Same model from https://gist.github.com/ksimonyan/211839e770f7b538e2d8

472 473 474
    :param num_classes: number of class.
    :type num_classes: int
    :param input_image: input layer.
Z
zhangjinchao01 已提交
475
    :type input_image: LayerOutput
476
    :param num_channels: input channels num.
Z
zhangjinchao01 已提交
477
    :type num_channels: int
478 479
    :return: layer's output
    :rtype: LayerOutput
Z
zhangjinchao01 已提交
480 481
    """

Q
qijun 已提交
482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542
    tmp = img_conv_group(
        input=input_image,
        num_channels=num_channels,
        conv_padding=1,
        conv_num_filter=[64, 64],
        conv_filter_size=3,
        conv_act=ReluActivation(),
        pool_size=2,
        pool_stride=2,
        pool_type=MaxPooling())

    tmp = img_conv_group(
        input=tmp,
        conv_num_filter=[128, 128],
        conv_padding=1,
        conv_filter_size=3,
        conv_act=ReluActivation(),
        pool_stride=2,
        pool_type=MaxPooling(),
        pool_size=2)

    tmp = img_conv_group(
        input=tmp,
        conv_num_filter=[256, 256, 256],
        conv_padding=1,
        conv_filter_size=3,
        conv_act=ReluActivation(),
        pool_stride=2,
        pool_type=MaxPooling(),
        pool_size=2)

    tmp = img_conv_group(
        input=tmp,
        conv_num_filter=[512, 512, 512],
        conv_padding=1,
        conv_filter_size=3,
        conv_act=ReluActivation(),
        pool_stride=2,
        pool_type=MaxPooling(),
        pool_size=2)
    tmp = img_conv_group(
        input=tmp,
        conv_num_filter=[512, 512, 512],
        conv_padding=1,
        conv_filter_size=3,
        conv_act=ReluActivation(),
        pool_stride=2,
        pool_type=MaxPooling(),
        pool_size=2)

    tmp = fc_layer(
        input=tmp,
        size=4096,
        act=ReluActivation(),
        layer_attr=ExtraAttr(drop_rate=0.5))

    tmp = fc_layer(
        input=tmp,
        size=4096,
        act=ReluActivation(),
        layer_attr=ExtraAttr(drop_rate=0.5))
Z
zhangjinchao01 已提交
543 544 545 546 547 548 549 550

    return fc_layer(input=tmp, size=num_classes, act=SoftmaxActivation())


############################################################################
#                       Recurrent                                          #
############################################################################

Q
qijun 已提交
551

Z
zhangjinchao01 已提交
552
@wrap_name_default("lstm")
Q
qijun 已提交
553 554 555 556 557 558 559 560 561 562 563
def simple_lstm(input,
                size,
                name=None,
                reverse=False,
                mat_param_attr=None,
                bias_param_attr=None,
                inner_param_attr=None,
                act=None,
                gate_act=None,
                state_act=None,
                mixed_layer_attr=None,
Z
zhangjinchao01 已提交
564 565 566 567
                lstm_cell_attr=None):
    """
    Simple LSTM Cell.

568 569
    It just combines a mixed layer with fully_matrix_projection and a lstmemory
    layer. The simple lstm cell was implemented with follow equations.
Z
zhangjinchao01 已提交
570 571 572

    ..  math::

L
luotao02 已提交
573
        i_t & = \\sigma(W_{xi}x_{t} + W_{hi}h_{t-1} + W_{ci}c_{t-1} + b_i)
Z
zhangjinchao01 已提交
574

L
luotao02 已提交
575
        f_t & = \\sigma(W_{xf}x_{t} + W_{hf}h_{t-1} + W_{cf}c_{t-1} + b_f)
Z
zhangjinchao01 已提交
576

L
luotao02 已提交
577
        c_t & = f_tc_{t-1} + i_t tanh (W_{xc}x_t+W_{hc}h_{t-1} + b_c)
Z
zhangjinchao01 已提交
578

L
luotao02 已提交
579
        o_t & = \\sigma(W_{xo}x_{t} + W_{ho}h_{t-1} + W_{co}c_t + b_o)
Z
zhangjinchao01 已提交
580

L
luotao02 已提交
581
        h_t & = o_t tanh(c_t)
Z
zhangjinchao01 已提交
582

583 584
    Please refer to **Generating Sequences With Recurrent Neural Networks** for more
    details about lstm. Link_ is here.
Z
zhangjinchao01 已提交
585 586 587 588 589

    .. _Link: http://arxiv.org/abs/1308.0850

    :param name: lstm layer name.
    :type name: basestring
590
    :param input: layer's input.
Z
zhangjinchao01 已提交
591 592 593
    :type input: LayerOutput
    :param size: lstm layer size.
    :type size: int
594
    :param reverse: process the input in a reverse order or not.
Z
zhangjinchao01 已提交
595
    :type reverse: bool
596
    :param mat_param_attr: parameter attribute of matrix projection in mixed layer.
Z
zhangjinchao01 已提交
597 598 599 600
    :type mat_param_attr: ParameterAttribute
    :param bias_param_attr: bias parameter attribute. False means no bias, None
                            means default bias.
    :type bias_param_attr: ParameterAttribute|False
601
    :param inner_param_attr: parameter attribute of lstm cell.
Z
zhangjinchao01 已提交
602
    :type inner_param_attr: ParameterAttribute
603
    :param act: last activiation type of lstm.
Z
zhangjinchao01 已提交
604
    :type act: BaseActivation
605
    :param gate_act: gate activiation type of lstm.
Z
zhangjinchao01 已提交
606
    :type gate_act: BaseActivation
607
    :param state_act: state activiation type of lstm.
Z
zhangjinchao01 已提交
608
    :type state_act: BaseActivation
609
    :param mixed_layer_attr: extra attribute of mixed layer.
Z
zhangjinchao01 已提交
610
    :type mixed_layer_attr: ExtraLayerAttribute
611
    :param lstm_cell_attr: extra attribute of lstm.
Z
zhangjinchao01 已提交
612
    :type lstm_cell_attr: ExtraLayerAttribute
613
    :return: layer's output.
Z
zhangjinchao01 已提交
614 615 616
    :rtype: LayerOutput
    """
    fc_name = 'lstm_transform_%s' % name
Q
qijun 已提交
617 618 619 620 621 622
    with mixed_layer(
            name=fc_name,
            size=size * 4,
            act=IdentityActivation(),
            layer_attr=mixed_layer_attr,
            bias_attr=False) as m:
Z
zhangjinchao01 已提交
623 624
        m += full_matrix_projection(input, param_attr=mat_param_attr)

Q
qijun 已提交
625 626 627 628 629 630 631 632 633 634
    return lstmemory(
        name=name,
        input=m,
        reverse=reverse,
        bias_attr=bias_param_attr,
        param_attr=inner_param_attr,
        act=act,
        gate_act=gate_act,
        state_act=state_act,
        layer_attr=lstm_cell_attr)
Z
zhangjinchao01 已提交
635 636 637


@wrap_name_default('lstm_unit')
Q
qijun 已提交
638
def lstmemory_unit(input,
639
                   out_memory=None,
Q
qijun 已提交
640 641 642 643 644 645
                   name=None,
                   size=None,
                   param_attr=None,
                   act=None,
                   gate_act=None,
                   state_act=None,
646 647
                   input_proj_bias_attr=None,
                   input_proj_layer_attr=None,
Q
qijun 已提交
648
                   lstm_bias_attr=None,
649
                   lstm_layer_attr=None):
Z
zhangjinchao01 已提交
650
    """
651 652 653
    lstmemory_unit defines the caculation process of a LSTM unit during a 
    single time step. This function is not a recurrent layer, so it can not be
    directly used to process sequence input. This function is always used in
C
caoying03 已提交
654 655 656 657 658 659 660 661 662
    recurrent_group (see layers.py for more details) to implement attention
    mechanism.

    Please refer to  **Generating Sequences With Recurrent Neural Networks**
    for more details about LSTM. The link goes as follows:
    .. _Link: https://arxiv.org/abs/1308.0850

    ..  math::

663
        i_t & = \\sigma(W_{x_i}x_{t} + W_{h_i}h_{t-1} + W_{c_i}c_{t-1} + b_i)
C
caoying03 已提交
664

665
        f_t & = \\sigma(W_{x_f}x_{t} + W_{h_f}h_{t-1} + W_{c_f}c_{t-1} + b_f)
C
caoying03 已提交
666

667
        c_t & = f_tc_{t-1} + i_t tanh (W_{x_c}x_t+W_{h_c}h_{t-1} + b_c)
C
caoying03 已提交
668

669
        o_t & = \\sigma(W_{x_o}x_{t} + W_{h_o}h_{t-1} + W_{c_o}c_t + b_o)
C
caoying03 已提交
670 671 672 673 674 675 676 677 678 679 680 681 682

        h_t & = o_t tanh(c_t)

    The example usage is:

    ..  code-block:: python

        lstm_step = lstmemory_unit(input=[layer1],
                                   size=256,
                                   act=TanhActivation(),
                                   gate_act=SigmoidActivation(),
                                   state_act=TanhActivation())

Z
zhangjinchao01 已提交
683

684
    :param input: input layer.
L
luotao02 已提交
685
    :type input: LayerOutput
686 687
    :param out_memory: output of previous time step
    :type out_memory: LayerOutput | None
L
luotao02 已提交
688 689 690 691
    :param name: lstmemory unit name.
    :type name: basestring
    :param size: lstmemory unit size.
    :type size: int
692
    :param param_attr: parameter attribute, None means default attribute.
L
luotao02 已提交
693
    :type param_attr: ParameterAttribute
694
    :param act: last activiation type of lstm.
L
luotao02 已提交
695
    :type act: BaseActivation
696
    :param gate_act: gate activiation type of lstm.
L
luotao02 已提交
697
    :type gate_act: BaseActivation
698
    :param state_act: state activiation type of lstm.
L
luotao02 已提交
699
    :type state_act: BaseActivation
700
    :param input_proj_bias_attr: bias attribute for input to hidden projection.
701 702 703 704 705
                False means no bias, None means default bias.
    :type input_proj_bias_attr: ParameterAttribute|False|None
    :param input_proj_layer_attr: extra layer attribute for input to hidden
                projection of the LSTM unit, such as dropout, error clipping.
    :type input_proj_layer_attr: ExtraLayerAttribute
L
luotao02 已提交
706
    :param lstm_bias_attr: bias parameter attribute of lstm layer.
707
                False means no bias, None means default bias.
708 709
    :type lstm_bias_attr: ParameterAttribute|False|None
    :param lstm_layer_attr: extra attribute of lstm layer.
L
luotao02 已提交
710 711 712
    :type lstm_layer_attr: ExtraLayerAttribute
    :return: lstmemory unit name.
    :rtype: LayerOutput
Z
zhangjinchao01 已提交
713 714 715 716
    """
    if size is None:
        assert input.size % 4 == 0
        size = input.size / 4
717 718 719 720 721
    if out_memory is None:
        out_mem = memory(name=name, size=size)
    else:
        out_mem = out_memory

Z
zhangjinchao01 已提交
722 723
    state_mem = memory(name="%s_state" % name, size=size)

Q
qijun 已提交
724 725 726
    with mixed_layer(
            name="%s_input_recurrent" % name,
            size=size * 4,
727 728
            bias_attr=input_proj_bias_attr,
            layer_attr=input_proj_layer_attr,
Q
qijun 已提交
729
            act=IdentityActivation()) as m:
Z
zhangjinchao01 已提交
730 731 732 733 734 735 736 737 738 739 740 741
        m += identity_projection(input=input)
        m += full_matrix_projection(input=out_mem, param_attr=param_attr)

    lstm_out = lstm_step_layer(
        name=name,
        input=m,
        state=state_mem,
        size=size,
        bias_attr=lstm_bias_attr,
        act=act,
        gate_act=gate_act,
        state_act=state_act,
Q
qijun 已提交
742
        layer_attr=lstm_layer_attr)
743
    get_output_layer(name='%s_state' % name, input=lstm_out, arg_name='state')
Z
zhangjinchao01 已提交
744 745 746 747 748

    return lstm_out


@wrap_name_default('lstm_group')
Q
qijun 已提交
749 750 751
def lstmemory_group(input,
                    size=None,
                    name=None,
752
                    out_memory=None,
Q
qijun 已提交
753 754 755 756 757
                    reverse=False,
                    param_attr=None,
                    act=None,
                    gate_act=None,
                    state_act=None,
758 759
                    input_proj_bias_attr=None,
                    input_proj_layer_attr=None,
Q
qijun 已提交
760
                    lstm_bias_attr=None,
761
                    lstm_layer_attr=None):
Z
zhangjinchao01 已提交
762
    """
763
    lstm_group is a recurrent_group version of Long Short Term Memory. It
C
caoying03 已提交
764 765
    does exactly the same calculation as the lstmemory layer (see lstmemory in
    layers.py for the maths) does. A promising benefit is that LSTM memory
766
    cell states(or hidden states) in every time step are accessible to the
C
caoying03 已提交
767
    user. This is especially useful in attention model. If you do not need to
768
    access the internal states of the lstm and merely use its outputs,
769
    it is recommended to use the lstmemory, which is relatively faster than
C
caoying03 已提交
770 771 772 773
    lstmemory_group.

    NOTE: In PaddlePaddle's implementation, the following input-to-hidden
    multiplications:
774 775
    :math:`W_{x_i}x_{t}` , :math:`W_{x_f}x_{t}`,
    :math:`W_{x_c}x_t`, :math:`W_{x_o}x_{t}` are not done in lstmemory_unit to
C
caoying03 已提交
776 777 778 779 780 781 782 783 784 785 786 787
    speed up the calculations. Consequently, an additional mixed_layer with
    full_matrix_projection must be included before lstmemory_unit is called.

    The example usage is:

    ..  code-block:: python

        lstm_step = lstmemory_group(input=[layer1],
                                    size=256,
                                    act=TanhActivation(),
                                    gate_act=SigmoidActivation(),
                                    state_act=TanhActivation())
Z
zhangjinchao01 已提交
788

789
    :param input: input layer.
L
luotao02 已提交
790 791 792
    :type input: LayerOutput
    :param size: lstmemory group size.
    :type size: int
793
    :param name: name of lstmemory group.
L
luotao02 已提交
794
    :type name: basestring
795
    :param out_memory: output of previous time step.
796
    :type out_memory: LayerOutput | None
797
    :param reverse: process the input in a reverse order or not.
L
luotao02 已提交
798
    :type reverse: bool
799
    :param param_attr: parameter attribute, None means default attribute.
L
luotao02 已提交
800
    :type param_attr: ParameterAttribute
801
    :param act: last activiation type of lstm.
L
luotao02 已提交
802
    :type act: BaseActivation
803
    :param gate_act: gate activiation type of lstm.
L
luotao02 已提交
804
    :type gate_act: BaseActivation
805
    :param state_act: state activiation type of lstm.
L
luotao02 已提交
806 807 808
    :type state_act: BaseActivation
    :param lstm_bias_attr: bias parameter attribute of lstm layer.
                           False means no bias, None means default bias.
809 810
    :type lstm_bias_attr: ParameterAttribute|False|None
    :param input_proj_bias_attr: bias attribute for input to hidden projection.
811 812 813 814 815
                False means no bias, None means default bias.
    :type input_proj_bias_attr: ParameterAttribute|False|None
    :param input_proj_layer_attr: extra layer attribute for input to hidden
                projection of the LSTM unit, such as dropout, error clipping.
    :type input_proj_layer_attr: ExtraLayerAttribute
L
luotao02 已提交
816 817
    :param lstm_layer_attr: lstm layer's extra attribute.
    :type lstm_layer_attr: ExtraLayerAttribute
C
caoying03 已提交
818
    :return: the lstmemory group.
L
luotao02 已提交
819
    :rtype: LayerOutput
Z
zhangjinchao01 已提交
820 821 822
    """

    def __lstm_step__(ipt):
Q
qijun 已提交
823 824 825 826 827 828 829
        return lstmemory_unit(
            input=ipt,
            name=name,
            size=size,
            act=act,
            gate_act=gate_act,
            state_act=state_act,
830 831 832 833
            out_memory=out_memory,
            input_proj_bias_attr=input_proj_bias_attr,
            input_proj_layer_attr=input_proj_layer_attr,
            param_attr=param_attr,
Q
qijun 已提交
834
            lstm_layer_attr=lstm_layer_attr,
835
            lstm_bias_attr=lstm_bias_attr)
Q
qijun 已提交
836 837 838 839 840 841

    return recurrent_group(
        name='%s_recurrent_group' % name,
        step=__lstm_step__,
        reverse=reverse,
        input=input)
Z
zhangjinchao01 已提交
842 843 844 845


@wrap_name_default('gru_unit')
def gru_unit(input,
846
             memory_boot=None,
Z
zhangjinchao01 已提交
847 848 849
             size=None,
             name=None,
             gru_bias_attr=None,
W
wangyang59 已提交
850
             gru_param_attr=None,
Z
zhangjinchao01 已提交
851 852
             act=None,
             gate_act=None,
Y
Yu Yang 已提交
853 854
             gru_layer_attr=None,
             naive=False):
Z
zhangjinchao01 已提交
855
    """
856 857 858
    gru_unit defines the calculation process of a gated recurrent unit during a single 
    time step. This function is not a recurrent layer, so it can not be
    directly used to process sequence input. This function is always used in
C
caoying03 已提交
859 860
    the recurrent_group (see layers.py for more details) to implement attention
    mechanism.
Z
zhangjinchao01 已提交
861

C
caoying03 已提交
862 863
    Please see grumemory in layers.py for the details about the maths.

864
    :param input: input layer.
Z
zhangjinchao01 已提交
865
    :type input: LayerOutput
866 867
    :param memory_boot: the initialization state of the LSTM cell.
    :type memory_boot: LayerOutput | None
C
caoying03 已提交
868 869 870 871
    :param name: name of the gru group.
    :type name: basestring
    :param size: hidden size of the gru.
    :type size: int
872
    :param act: activation type of gru
C
caoying03 已提交
873
    :type act: BaseActivation
874
    :param gate_act: gate activation type or gru
C
caoying03 已提交
875
    :type gate_act: BaseActivation
876 877
    :param gru_layer_attr: Extra attribute of the gru layer.
    :type gru_layer_attr: ExtraLayerAttribute
C
caoying03 已提交
878 879
    :return: the gru output layer.
    :rtype: LayerOutput
Z
zhangjinchao01 已提交
880 881 882 883 884 885
    """

    assert input.size % 3 == 0
    if size is None:
        size = input.size / 3

886
    out_mem = memory(name=name, size=size, boot_layer=memory_boot)
Z
zhangjinchao01 已提交
887

Y
Yu Yang 已提交
888 889 890 891 892 893
    if naive:
        __step__ = gru_step_naive_layer
    else:
        __step__ = gru_step_layer

    gru_out = __step__(
Z
zhangjinchao01 已提交
894 895 896 897 898
        name=name,
        input=input,
        output_mem=out_mem,
        size=size,
        bias_attr=gru_bias_attr,
W
wangyang59 已提交
899
        param_attr=gru_param_attr,
Z
zhangjinchao01 已提交
900 901
        act=act,
        gate_act=gate_act,
Q
qijun 已提交
902
        layer_attr=gru_layer_attr)
Z
zhangjinchao01 已提交
903 904 905 906 907
    return gru_out


@wrap_name_default('gru_group')
def gru_group(input,
908
              memory_boot=None,
Z
zhangjinchao01 已提交
909 910 911 912
              size=None,
              name=None,
              reverse=False,
              gru_bias_attr=None,
W
wangyang59 已提交
913
              gru_param_attr=None,
Q
qijun 已提交
914 915
              act=None,
              gate_act=None,
Y
Yu Yang 已提交
916 917
              gru_layer_attr=None,
              naive=False):
C
caoying03 已提交
918
    """
919
    gru_group is a recurrent_group version of Gated Recurrent Unit. It
C
caoying03 已提交
920
    does exactly the same calculation as the grumemory layer does. A promising
921 922
    benefit is that gru hidden states are accessible to the user. This is
    especially useful in attention model. If you do not need to access
923
    any internal state and merely use the outputs of a GRU, it is recommended
C
caoying03 已提交
924 925 926 927 928 929 930 931
    to use the grumemory, which is relatively faster.

    Please see grumemory in layers.py for more detail about the maths.

    The example usage is:

    ..  code-block:: python

932
        gru = gru_group(input=[layer1],
C
caoying03 已提交
933 934 935 936
                        size=256,
                        act=TanhActivation(),
                        gate_act=SigmoidActivation())

937
    :param input: input layer.
C
caoying03 已提交
938
    :type input: LayerOutput
939 940
    :param memory_boot: the initialization state of the LSTM cell.
    :type memory_boot: LayerOutput | None
C
caoying03 已提交
941 942 943 944
    :param name: name of the gru group.
    :type name: basestring
    :param size: hidden size of the gru.
    :type size: int
945
    :param reverse: process the input in a reverse order or not.
C
caoying03 已提交
946
    :type reverse: bool
947
    :param act: activiation type of gru
C
caoying03 已提交
948
    :type act: BaseActivation
949
    :param gate_act: gate activiation type of gru
C
caoying03 已提交
950
    :type gate_act: BaseActivation
951 952 953 954 955
    :param gru_bias_attr: bias parameter attribute of gru layer,
                          False means no bias, None means default bias.
    :type gru_bias_attr: ParameterAttribute|False|None
    :param gru_layer_attr: Extra attribute of the gru layer.
    :type gru_layer_attr: ExtraLayerAttribute
C
caoying03 已提交
956 957 958 959
    :return: the gru group.
    :rtype: LayerOutput
    """

Z
zhangjinchao01 已提交
960 961 962
    def __gru_step__(ipt):
        return gru_unit(
            input=ipt,
963
            memory_boot=memory_boot,
Z
zhangjinchao01 已提交
964 965 966
            name=name,
            size=size,
            gru_bias_attr=gru_bias_attr,
W
wangyang59 已提交
967
            gru_param_attr=gru_param_attr,
Z
zhangjinchao01 已提交
968 969
            act=act,
            gate_act=gate_act,
Y
Yu Yang 已提交
970 971
            gru_layer_attr=gru_layer_attr,
            naive=naive)
Z
zhangjinchao01 已提交
972

Q
qijun 已提交
973 974 975 976 977
    return recurrent_group(
        name='%s_recurrent_group' % name,
        step=__gru_step__,
        reverse=reverse,
        input=input)
Z
zhangjinchao01 已提交
978 979 980 981 982 983 984 985 986 987 988


@wrap_name_default('simple_gru')
def simple_gru(input,
               size,
               name=None,
               reverse=False,
               mixed_param_attr=None,
               mixed_bias_param_attr=None,
               mixed_layer_attr=None,
               gru_bias_attr=None,
W
wangyang59 已提交
989
               gru_param_attr=None,
Z
zhangjinchao01 已提交
990 991
               act=None,
               gate_act=None,
Y
Yu Yang 已提交
992 993
               gru_layer_attr=None,
               naive=False):
C
caoying03 已提交
994
    """
995
    You may see gru_step_layer, grumemory in layers.py, gru_unit, gru_group,
996 997 998
    simple_gru in network.py. The reason why there are so many interfaces is
    that we have two ways to implement recurrent neural network. One way is to
    use one complete layer to implement rnn (including simple rnn, gru and lstm)
999
    with multiple time steps, such as recurrent_layer, lstmemory, grumemory. But 
1000
    the multiplication operation :math:`W x_t` is not computed in these layers.
1001
    See details in their interfaces in layers.py.
1002 1003 1004 1005 1006 1007
    The other implementation is to use an recurrent group which can ensemble a
    series of layers to compute rnn step by step. This way is flexible for
    attenion mechanism or other complex connections.

    - gru_step_layer: only compute rnn by one step. It needs an memory as input
      and can be used in recurrent group.
1008
    - gru_unit: a wrapper of gru_step_layer with memory.
1009 1010
    - gru_group: a GRU cell implemented by a combination of multiple layers in
      recurrent group.
1011
      But :math:`W x_t` is not done in group.
1012
    - gru_memory: a GRU cell implemented by one layer, which does same calculation
1013 1014
      with gru_group and is faster than gru_group.
    - simple_gru: a complete GRU implementation inlcuding :math:`W x_t` and
1015
      gru_group. :math:`W` contains :math:`W_r`, :math:`W_z` and :math:`W`, see
1016
      formula in grumemory.
1017

C
caoying03 已提交
1018 1019 1020 1021 1022 1023 1024
    The computational speed is that, grumemory is relatively better than
    gru_group, and gru_group is relatively better than simple_gru.

    The example usage is:

    ..  code-block:: python

1025
        gru = simple_gru(input=[layer1], size=256)
C
caoying03 已提交
1026

1027
    :param input: input layer.
C
caoying03 已提交
1028 1029 1030 1031 1032
    :type input: LayerOutput
    :param name: name of the gru group.
    :type name: basestring
    :param size: hidden size of the gru.
    :type size: int
1033
    :param reverse: process the input in a reverse order or not.
C
caoying03 已提交
1034
    :type reverse: bool
1035
    :param act: activiation type of gru
C
caoying03 已提交
1036
    :type act: BaseActivation
1037
    :param gate_act: gate activiation type of gru
C
caoying03 已提交
1038
    :type gate_act: BaseActivation
1039 1040 1041 1042 1043
    :param gru_bias_attr: bias parameter attribute of gru layer,
                          False means no bias, None means default bias.
    :type gru_bias_attr: ParameterAttribute|False|None
    :param gru_layer_attr: Extra attribute of the gru layer.
    :type gru_layer_attr: ExtraLayerAttribute
C
caoying03 已提交
1044 1045 1046
    :return: the gru group.
    :rtype: LayerOutput
    """
Q
qijun 已提交
1047 1048 1049 1050 1051
    with mixed_layer(
            name='%s_transform' % name,
            size=size * 3,
            bias_attr=mixed_bias_param_attr,
            layer_attr=mixed_layer_attr) as m:
Z
zhangjinchao01 已提交
1052 1053
        m += full_matrix_projection(input=input, param_attr=mixed_param_attr)

Q
qijun 已提交
1054 1055 1056 1057 1058 1059
    return gru_group(
        name=name,
        size=size,
        input=m,
        reverse=reverse,
        gru_bias_attr=gru_bias_attr,
W
wangyang59 已提交
1060
        gru_param_attr=gru_param_attr,
Q
qijun 已提交
1061 1062
        act=act,
        gate_act=gate_act,
Y
Yu Yang 已提交
1063 1064
        gru_layer_attr=gru_layer_attr,
        naive=naive)
Z
zhangjinchao01 已提交
1065 1066


1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078
@wrap_name_default('simple_gru2')
def simple_gru2(input,
                size,
                name=None,
                reverse=False,
                mixed_param_attr=None,
                mixed_bias_attr=None,
                gru_param_attr=None,
                gru_bias_attr=None,
                act=None,
                gate_act=None,
                mixed_layer_attr=None,
Q
qijun 已提交
1079
                gru_cell_attr=None):
1080
    """
1081 1082
    simple_gru2 is the same with simple_gru, but using grumemory instead.
    Please refer to grumemory in layers.py for more detail about the math.
1083 1084 1085 1086 1087 1088 1089 1090
    simple_gru2 is faster than simple_gru.

    The example usage is:

    ..  code-block:: python

        gru = simple_gru2(input=[layer1], size=256)

1091
    :param input: input layer.
1092 1093 1094 1095 1096
    :type input: LayerOutput
    :param name: name of the gru group.
    :type name: basestring
    :param size: hidden size of the gru.
    :type size: int
1097
    :param reverse: process the input in a reverse order or not.
1098
    :type reverse: bool
1099
    :param act: activiation type of gru
1100
    :type act: BaseActivation
1101
    :param gate_act: gate activiation type of gru
1102
    :type gate_act: BaseActivation
1103 1104 1105 1106 1107
    :param gru_bias_attr: bias parameter attribute of gru layer, 
                          False means no bias, None means default bias.
    :type gru_bias_attr: ParameterAttribute|False|None
    :param gru_layer_attr: Extra attribute of the gru layer.
    :type gru_layer_attr: ExtraLayerAttribute
1108 1109 1110
    :return: the gru group.
    :rtype: LayerOutput
    """
Q
qijun 已提交
1111 1112 1113 1114 1115
    with mixed_layer(
            name='%s_transform' % name,
            size=size * 3,
            bias_attr=mixed_bias_attr,
            layer_attr=mixed_layer_attr) as m:
1116 1117
        m += full_matrix_projection(input=input, param_attr=mixed_param_attr)

Q
qijun 已提交
1118 1119 1120 1121 1122 1123 1124 1125 1126
    return grumemory(
        name=name,
        input=m,
        reverse=reverse,
        bias_attr=gru_bias_attr,
        param_attr=gru_param_attr,
        act=act,
        gate_act=gate_act,
        layer_attr=gru_cell_attr)
1127 1128 1129


@wrap_name_default("bidirectional_gru")
Q
qijun 已提交
1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153
def bidirectional_gru(input,
                      size,
                      name=None,
                      return_seq=False,
                      fwd_mixed_param_attr=None,
                      fwd_mixed_bias_attr=None,
                      fwd_gru_param_attr=None,
                      fwd_gru_bias_attr=None,
                      fwd_act=None,
                      fwd_gate_act=None,
                      fwd_mixed_layer_attr=None,
                      fwd_gru_cell_attr=None,
                      bwd_mixed_param_attr=None,
                      bwd_mixed_bias_attr=None,
                      bwd_gru_param_attr=None,
                      bwd_gru_bias_attr=None,
                      bwd_act=None,
                      bwd_gate_act=None,
                      bwd_mixed_layer_attr=None,
                      bwd_gru_cell_attr=None,
                      last_seq_attr=None,
                      first_seq_attr=None,
                      concat_attr=None,
                      concat_act=None):
1154 1155
    """
    A bidirectional_gru is a recurrent unit that iterates over the input
1156
    sequence both in forward and backward orders, and then concatenate two
1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172
    outputs to form a final output. However, concatenation of two outputs
    is not the only way to form the final output, you can also, for example,
    just add them together.

    The example usage is:

    ..  code-block:: python

        bi_gru = bidirectional_gru(input=[input1], size=512)

    :param name: bidirectional gru layer name.
    :type name: basestring
    :param input: input layer.
    :type input: LayerOutput
    :param size: gru layer size.
    :type size: int
1173
    :param return_seq: If set False, the last time step of output are
1174
                       concatenated and returned.
1175 1176
                       If set True, the entire output sequences in forward 
                       and backward directions are concatenated and returned.
1177 1178 1179 1180 1181 1182
    :type return_seq: bool
    :return: LayerOutput object.
    :rtype: LayerOutput
    """
    args = locals()

Q
qijun 已提交
1183 1184 1185 1186 1187 1188
    fw = simple_gru2(
        name='%s_fw' % name,
        input=input,
        size=size,
        **dict((k[len('fwd_'):], v) for k, v in args.iteritems()
               if k.startswith('fwd_')))
1189

Q
qijun 已提交
1190 1191 1192 1193 1194 1195 1196
    bw = simple_gru2(
        name="%s_bw" % name,
        input=input,
        size=size,
        reverse=True,
        **dict((k[len('bwd_'):], v) for k, v in args.iteritems()
               if k.startswith('bwd_')))
1197 1198

    if return_seq:
Q
qijun 已提交
1199 1200
        return concat_layer(
            name=name, input=[fw, bw], layer_attr=concat_attr, act=concat_act)
1201
    else:
Q
qijun 已提交
1202 1203 1204 1205 1206 1207 1208 1209 1210
        fw_seq = last_seq(
            name="%s_fw_last" % name, input=fw, layer_attr=last_seq_attr)
        bw_seq = first_seq(
            name="%s_bw_last" % name, input=bw, layer_attr=first_seq_attr)
        return concat_layer(
            name=name,
            input=[fw_seq, bw_seq],
            layer_attr=concat_attr,
            act=concat_act)
1211 1212


Z
zhangjinchao01 已提交
1213
@wrap_name_default("bidirectional_lstm")
Q
qijun 已提交
1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237
def bidirectional_lstm(input,
                       size,
                       name=None,
                       return_seq=False,
                       fwd_mat_param_attr=None,
                       fwd_bias_param_attr=None,
                       fwd_inner_param_attr=None,
                       fwd_act=None,
                       fwd_gate_act=None,
                       fwd_state_act=None,
                       fwd_mixed_layer_attr=None,
                       fwd_lstm_cell_attr=None,
                       bwd_mat_param_attr=None,
                       bwd_bias_param_attr=None,
                       bwd_inner_param_attr=None,
                       bwd_act=None,
                       bwd_gate_act=None,
                       bwd_state_act=None,
                       bwd_mixed_layer_attr=None,
                       bwd_lstm_cell_attr=None,
                       last_seq_attr=None,
                       first_seq_attr=None,
                       concat_attr=None,
                       concat_act=None):
Z
zhangjinchao01 已提交
1238
    """
C
caoying03 已提交
1239
    A bidirectional_lstm is a recurrent unit that iterates over the input
1240 1241
    sequence both in forward and backward orders, and then concatenate two
    outputs to form a final output. However, concatenation of two outputs
C
caoying03 已提交
1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253
    is not the only way to form the final output, you can also, for example,
    just add them together.

    Please refer to  **Neural Machine Translation by Jointly Learning to Align
    and Translate** for more details about the bidirectional lstm.
    The link goes as follows:
    .. _Link: https://arxiv.org/pdf/1409.0473v3.pdf

    The example usage is:

    ..  code-block:: python

1254
        bi_lstm = bidirectional_lstm(input=[input1], size=512)
Z
zhangjinchao01 已提交
1255 1256 1257 1258 1259 1260 1261

    :param name: bidirectional lstm layer name.
    :type name: basestring
    :param input: input layer.
    :type input: LayerOutput
    :param size: lstm layer size.
    :type size: int
1262
    :param return_seq: If set False, the last time step of output are
C
caoying03 已提交
1263
                       concatenated and returned.
1264 1265
                       If set True, the entire output sequences in forward 
                       and backward directions are concatenated and returned.
Z
zhangjinchao01 已提交
1266
    :type return_seq: bool
1267
    :return: LayerOutput object.
Z
zhangjinchao01 已提交
1268 1269 1270 1271
    :rtype: LayerOutput
    """
    args = locals()

Q
qijun 已提交
1272 1273 1274 1275 1276 1277
    fw = simple_lstm(
        name='%s_fw' % name,
        input=input,
        size=size,
        **dict((k[len('fwd_'):], v) for k, v in args.iteritems()
               if k.startswith('fwd_')))
Z
zhangjinchao01 已提交
1278

Q
qijun 已提交
1279 1280 1281 1282 1283 1284 1285
    bw = simple_lstm(
        name="%s_bw" % name,
        input=input,
        size=size,
        reverse=True,
        **dict((k[len('bwd_'):], v) for k, v in args.iteritems()
               if k.startswith('bwd_')))
Z
zhangjinchao01 已提交
1286 1287

    if return_seq:
Q
qijun 已提交
1288 1289
        return concat_layer(
            name=name, input=[fw, bw], layer_attr=concat_attr, act=concat_act)
Z
zhangjinchao01 已提交
1290
    else:
Q
qijun 已提交
1291 1292 1293 1294 1295 1296 1297 1298 1299
        fw_seq = last_seq(
            name="%s_fw_last" % name, input=fw, layer_attr=last_seq_attr)
        bw_seq = first_seq(
            name="%s_bw_last" % name, input=bw, layer_attr=first_seq_attr)
        return concat_layer(
            name=name,
            input=[fw_seq, bw_seq],
            layer_attr=concat_attr,
            act=concat_act)
Z
zhangjinchao01 已提交
1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311


@wrap_name_default()
@wrap_act_default(param_names=['weight_act'], act=TanhActivation())
def simple_attention(encoded_sequence,
                     encoded_proj,
                     decoder_state,
                     transform_param_attr=None,
                     softmax_param_attr=None,
                     weight_act=None,
                     name=None):
    """
1312
    Calculate and return a context vector with attention mechanism.
1313
    Size of the context vector equals to size of the encoded_sequence.
Z
zhangjinchao01 已提交
1314 1315

    ..  math::
L
luotao02 已提交
1316 1317 1318 1319 1320

        a(s_{i-1},h_{j}) & = v_{a}f(W_{a}s_{t-1} + U_{a}h_{j})

        e_{i,j} & = a(s_{i-1}, h_{j})

1321
        a_{i,j} & = \\frac{exp(e_{i,j})}{\\sum_{k=1}^{T_x}{exp(e_{i,k})}}
L
luotao02 已提交
1322 1323

        c_{i} & = \\sum_{j=1}^{T_{x}}a_{i,j}h_{j}
Z
zhangjinchao01 已提交
1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334

    where :math:`h_{j}` is the jth element of encoded_sequence,
    :math:`U_{a}h_{j}` is the jth element of encoded_proj
    :math:`s_{i-1}` is decoder_state
    :math:`f` is weight_act, and is set to tanh by default.

    Please refer to **Neural Machine Translation by Jointly Learning to
    Align and Translate** for more details. The link is as follows:
    https://arxiv.org/abs/1409.0473.

    The example usage is:
L
luotao02 已提交
1335

Z
zhangjinchao01 已提交
1336 1337 1338 1339 1340 1341 1342 1343 1344
    ..  code-block:: python

        context = simple_attention(encoded_sequence=enc_seq,
                                   encoded_proj=enc_proj,
                                   decoder_state=decoder_prev,)

    :param name: name of the attention model.
    :type name: basestring
    :param softmax_param_attr: parameter attribute of sequence softmax
1345
                               that is used to produce attention weight.
Z
zhangjinchao01 已提交
1346
    :type softmax_param_attr: ParameterAttribute
1347 1348
    :param weight_act: activation of the attention model.
    :type weight_act: BaseActivation
Z
zhangjinchao01 已提交
1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364
    :param encoded_sequence: output of the encoder
    :type encoded_sequence: LayerOutput
    :param encoded_proj: attention weight is computed by a feed forward neural
                         network which has two inputs : decoder's hidden state
                         of previous time step and encoder's output.
                         encoded_proj is output of the feed-forward network for
                         encoder's output. Here we pre-compute it outside
                         simple_attention for speed consideration.
    :type encoded_proj: LayerOutput
    :param decoder_state: hidden state of decoder in previous time step
    :type decoder_state: LayerOutput
    :param transform_param_attr: parameter attribute of the feed-forward
                                network that takes decoder_state as inputs to
                                compute attention weight.
    :type transform_param_attr: ParameterAttribute
    :return: a context vector
R
ranqiu 已提交
1365
    :rtype: LayerOutput
Z
zhangjinchao01 已提交
1366 1367 1368 1369 1370
    """
    assert encoded_proj.size == decoder_state.size
    proj_size = encoded_proj.size

    with mixed_layer(size=proj_size, name="%s_transform" % name) as m:
Q
qijun 已提交
1371 1372
        m += full_matrix_projection(
            decoder_state, param_attr=transform_param_attr)
Z
zhangjinchao01 已提交
1373

Q
qijun 已提交
1374 1375
    expanded = expand_layer(
        input=m, expand_as=encoded_sequence, name='%s_expand' % name)
Z
zhangjinchao01 已提交
1376

Q
qijun 已提交
1377 1378
    with mixed_layer(
            size=proj_size, act=weight_act, name="%s_combine" % name) as m:
Z
zhangjinchao01 已提交
1379 1380 1381 1382 1383
        m += identity_projection(expanded)
        m += identity_projection(encoded_proj)

    # sequence softmax is used to normalize similarities between decoder state
    # and encoder outputs into a distribution
Q
qijun 已提交
1384 1385 1386 1387 1388 1389 1390
    attention_weight = fc_layer(
        input=m,
        size=1,
        act=SequenceSoftmaxActivation(),
        param_attr=softmax_param_attr,
        name="%s_softmax" % name,
        bias_attr=False)
Z
zhangjinchao01 已提交
1391

Q
qijun 已提交
1392 1393 1394 1395
    scaled = scaling_layer(
        weight=attention_weight,
        input=encoded_sequence,
        name='%s_scaling' % name)
Z
zhangjinchao01 已提交
1396

Q
qijun 已提交
1397 1398
    return pooling_layer(
        input=scaled, pooling_type=SumPooling(), name="%s_pooling" % name)
Z
zhangjinchao01 已提交
1399 1400


R
ranqiu 已提交
1401 1402
@wrap_name_default()
def dot_product_attention(encoded_sequence,
1403
                          attended_sequence,
R
ranqiu 已提交
1404 1405 1406 1407 1408
                          transformed_state,
                          softmax_param_attr=None,
                          name=None):
    """
    Calculate and return a context vector with dot-product attention mechanism.
1409
    The dimension of the context vector equals to that of the attended_sequence.
R
ranqiu 已提交
1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421

    ..  math::

        a(s_{i-1},h_{j}) & = s_{i-1}^\mathrm{T} h_{j}

        e_{i,j} & = a(s_{i-1}, h_{j})

        a_{i,j} & = \\frac{exp(e_{i,j})}{\\sum_{k=1}^{T_x}{exp(e_{i,k})}}

        c_{i} & = \\sum_{j=1}^{T_{x}}a_{i,j}z_{j}

    where :math:`h_{j}` is the jth element of encoded_sequence,
1422 1423
    :math:`z_{j}` is the jth element of attended_sequence,
    :math:`s_{i-1}` is transformed_state.
R
ranqiu 已提交
1424 1425 1426 1427 1428 1429

    The example usage is:

    ..  code-block:: python

        context = dot_product_attention(encoded_sequence=enc_seq,
1430
                                        attended_sequence=att_seq,
R
ranqiu 已提交
1431 1432
                                        transformed_state=state,)

1433 1434
    :param name: A prefix attached to the name of each layer that defined inside
                 the dot_product_attention.
R
ranqiu 已提交
1435
    :type name: basestring
1436
    :param softmax_param_attr: The parameter attribute of sequence softmax
R
ranqiu 已提交
1437 1438
                               that is used to produce attention weight.
    :type softmax_param_attr: ParameterAttribute
1439
    :param encoded_sequence: The output hidden vectors of the encoder.
R
ranqiu 已提交
1440
    :type encoded_sequence: LayerOutput
1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
    :param attended_sequence: The attention weight is computed by a feed forward neural
                              network which has two inputs : decoder's transformed hidden
                              state of previous time step and encoder's output.
                              attended_sequence is the sequence to be attended.
    :type attended_sequence: LayerOutput
    :param transformed_state: The transformed hidden state of decoder in previous time step.
                              Since the dot-product operation will be performed on it and the
                              encoded_sequence, their dimensions must be equal. For flexibility,
                              we suppose transformations of the decoder's hidden state have been
                              done outside dot_product_attention and no more will be performed
                              inside. Then users can use either the original or transformed one.
R
ranqiu 已提交
1452
    :type transformed_state: LayerOutput
1453
    :return: The context vector.
R
ranqiu 已提交
1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475
    :rtype: LayerOutput
    """
    assert transformed_state.size == encoded_sequence.size

    expanded = expand_layer(
        input=transformed_state,
        expanded_as=encoded_sequence,
        name='%s_expand' % name)

    m = linear_comb_layer(
        weights=expanded, vectors=encoded_sequence, name='%s_dot-product')

    attention_weight = fc_layer(
        input=m,
        size=1,
        act=SequenceSoftmaxActivation(),
        param_attr=softmax_param_attr,
        name="%s_softmax" % name,
        bias_attr=False)

    scaled = scaling_layer(
        weight=attention_weight,
1476
        input=attended_sequence,
R
ranqiu 已提交
1477 1478 1479 1480 1481 1482
        name='%s_scaling' % name)

    return pooling_layer(
        input=scaled, pooling_type=SumPooling(), name="%s_pooling" % name)


R
ranqiu 已提交
1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531
@wrap_name_default()
def multi_head_attention(query,
                         key,
                         value,
                         key_proj_size,
                         value_proj_size,
                         head_num,
                         attention_type,
                         softmax_param_attr=None,
                         name=None):
    """
    Calculate and return a context vector with dot-product attention mechanism.
    The dimension of the context vector equals to value_proj_size * head_num.

    Please refer to **Attention Is All You Need** for more details. The link is
    as follows:
    https://arxiv.org/abs/1706.03762.

    The example usage is:

    ..  code-block:: python

        context = multi_head_attention(query=decoder_state,
                                       key=enc_seq,
                                       value=enc_seq,
                                       key_proj_size=64,
                                       value_pro_size=64,
                                       head_num=8,
                                       attention_type='dot-product attention')

    :param name: A prefix attached to the name of each layer that defined inside
                 the multi_head_attention.
    :type name: basestring
    :param softmax_param_attr: The parameter attribute of sequence softmax
                               that is used to produce attention weight.
    :type softmax_param_attr: ParameterAttribute
    :param query: query is used to calculate attention weights over values at current step.
    :type query: LayerOutput
    :param key: key is used to calculate the attention weight of the corresponding value.
    :type key: LayerOutput
    :param value: value is the sequence to be attended.
    :type value: LayerOutput
    :param key_proj_size: The dimension of the linear projection performed on key and query.
    :type key_proj_size: int
    :param value_proj_size: The dimension of the linear projection performed on value.
    :type value_proj_size: int
    :param head_num: The number of attention heads.
    :type head_num: int
    :param attention_type: The type of the attention mechanism used in each attention
R
ranqiu 已提交
1532
                           heads. Now, we only support scaled dot-product attention and
R
ranqiu 已提交
1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559
                           additive attention.
    :type attention_type: basestring
    :return: The context vector.
    :rtype: LayerOutput
    """
    assert attention_type in ['dot-product attention', 'additive attention']

    with mixed_layer(
            size=key_proj_size * head_num,
            name='%s_query_proj' % name) as query_proj:
        query_proj += full_matrix_projection(query)
    query_proj = expand_layer(input=query_proj, expand_as=key)

    with mixed_layer(
            size=key_proj_size * head_num,
            name='%s_key_proj' % name) as key_proj:
        key_proj += full_matrix_projection(key)

    with mixed_layer(
            size=value_proj_size * head_num,
            name='%s_value_proj' % name) as value_proj:
        value_proj += full_matrix_projection(value)

    head_list = []
    for i in range(head_num):
        with mixed_layer(size=key_proj_size) as sub_query_proj:
            sub_query_proj += identity_projection(
R
ranqiu 已提交
1560
                query_proj, offset=key_proj_size * i, size=key_proj_size)
R
ranqiu 已提交
1561 1562 1563

        with mixed_layer(size=key_proj_size) as sub_key_proj:
            sub_key_proj += identity_projection(
R
ranqiu 已提交
1564
                key_proj, offset=key_proj_size * i, size=key_proj_size)
R
ranqiu 已提交
1565 1566 1567

        with mixed_layer(size=value_proj_size) as sub_value_proj:
            sub_value_proj += identity_projection(
R
ranqiu 已提交
1568
                value_proj, offset=value_proj_size * i, size=value_proj_size)
R
ranqiu 已提交
1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605

        if attention_type == 'dot-product attention':
            m = linear_comb_layer(
                weights=sub_query_proj,
                vectors=sub_key_proj,
                name='%s_dot-product_%d' % (name, i))
            m = slope_intercept_layer(
                input=m,
                slope=math.sqrt(1.0 / key_proj_size),
                name='%s_dot-product_scaling_%d' % (name, i))
        else:
            with mixed_layer(
                    size=key_proj_size,
                    act=TanhActivation(),
                    name='%s_combine_%d' % (name, i)) as m:
                m += identity_projection(sub_query_proj)
                m += identity_projection(sub_key_proj)

        attention_weight = fc_layer(
            input=m,
            size=1,
            act=SequenceSoftmaxActivation(),
            param_attr=softmax_param_attr,
            name="%s_softmax_%d" % (name, i),
            bias_attr=False)

        scaled = scaling_layer(
            weight=attention_weight,
            input=sub_value_proj,
            name='%s_scaling_%d' % (name, i))
        head = pooling_layer(
            input=scaled,
            pooling_type=SumPooling(),
            name="%s_pooling_%d" % (name, i))

        head_list.append(head)

R
ranqiu 已提交
1606
    attended = concat_layer(head_list)
R
ranqiu 已提交
1607 1608 1609 1610

    return attended


1611 1612 1613 1614 1615 1616 1617 1618
def inputs(layers, *args):
    """
    Declare the inputs of network. The order of input should be as same as
    the data provider's return order.

    :param layers: Input Layers.
    :type layers: list|tuple|LayerOutput.
    :return:
Z
zhangjinchao01 已提交
1619 1620
    """

1621 1622 1623 1624
    if isinstance(layers, LayerOutput) or isinstance(layers, basestring):
        layers = [layers]
    if len(args) != 0:
        layers.extend(args)
Z
zhangjinchao01 已提交
1625

Z
Zhaolong Xing 已提交
1626
    Inputs(*[l.name for l in layers])
1627 1628 1629 1630


def outputs(layers, *args):
    """
1631
    Declare the outputs of network. If user has not defined the inputs of
1632 1633 1634
    network, this method will calculate the input order by dfs travel.

    :param layers: Output layers.
Z
zhangjinchao01 已提交
1635 1636 1637 1638
    :type layers: list|tuple|LayerOutput
    :return:
    """

1639 1640
    traveled = set()

Z
zhangjinchao01 已提交
1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651
    def __dfs_travel__(layer,
                       predicate=lambda x: x.layer_type == LayerType.DATA):
        """
        DFS LRV Travel for output layer.

        The return order is define order for data_layer in this leaf node.

        :param layer:
        :type layer: LayerOutput
        :return:
        """
1652 1653 1654 1655 1656
        if layer in traveled:
            return []
        else:
            traveled.add(layer)

Z
zhangjinchao01 已提交
1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669
        assert isinstance(layer, LayerOutput), "layer is %s" % (layer)
        retv = []
        if layer.parents is not None:
            for p in layer.parents:
                retv.extend(__dfs_travel__(p, predicate))

        if predicate(layer):
            retv.append(layer)
        return retv

    if isinstance(layers, LayerOutput):
        layers = [layers]

1670 1671 1672
    if len(args) != 0:
        layers.extend(args)

Z
zhangjinchao01 已提交
1673
    assert len(layers) > 0
1674 1675

    if HasInputsSet():  # input already set
Z
Zhaolong Xing 已提交
1676
        Outputs(*[l.name for l in layers])
1677 1678
        return  # just return outputs.

Z
zhangjinchao01 已提交
1679
    if len(layers) != 1:
1680
        logger.warning("`outputs` routine try to calculate network's"
Z
zhangjinchao01 已提交
1681 1682 1683 1684 1685 1686 1687
                       " inputs and outputs order. It might not work well."
                       "Please see follow log carefully.")
    inputs = []
    outputs_ = []
    for each_layer in layers:
        assert isinstance(each_layer, LayerOutput)
        inputs.extend(__dfs_travel__(each_layer))
Q
qijun 已提交
1688 1689 1690
        outputs_.extend(
            __dfs_travel__(each_layer,
                           lambda x: x.layer_type == LayerType.COST))
Z
zhangjinchao01 已提交
1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707

    # Currently, we got each leaf node's inputs order, output order.
    # We merge them together.

    final_inputs = []
    final_outputs = []

    for each_input in inputs:
        assert isinstance(each_input, LayerOutput)
        if each_input.name not in final_inputs:
            final_inputs.append(each_input.name)

    for each_output in outputs_:
        assert isinstance(each_output, LayerOutput)
        if each_output.name not in final_outputs:
            final_outputs.append(each_output.name)

Q
qijun 已提交
1708
    logger.info("".join(["The input order is [", ", ".join(final_inputs), "]"]))
1709 1710 1711 1712

    if len(final_outputs) == 0:
        final_outputs = map(lambda x: x.name, layers)

Q
qijun 已提交
1713 1714
    logger.info("".join(
        ["The output order is [", ", ".join(final_outputs), "]"]))
Z
zhangjinchao01 已提交
1715 1716

    Inputs(*final_inputs)
1717
    Outputs(*final_outputs)