networks.py 44.2 KB
Newer Older
Z
zhangjinchao01 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442
# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""

"""
# from activations import *
from activations import LinearActivation, ReluActivation, SoftmaxActivation, \
    IdentityActivation, TanhActivation, SequenceSoftmaxActivation
from attrs import ExtraAttr
from default_decorators import wrap_name_default, wrap_act_default, \
    wrap_param_default
from layers import *  # There are too many layers used in network, so import *
from poolings import MaxPooling, SumPooling
from paddle.trainer.config_parser import *

__all__ = ['sequence_conv_pool', 'simple_lstm', "simple_img_conv_pool",
           "img_conv_bn_pool", 'dropout_layer', 'lstmemory_group',
           'lstmemory_unit', 'small_vgg', 'img_conv_group', 'vgg_16_network',
           'gru_unit', 'gru_group', 'simple_gru', 'simple_attention',
           'text_conv_pool',
           'bidirectional_lstm', 'outputs']


######################################################
#                     Text CNN                       #
######################################################

@wrap_name_default("sequence_conv_pooling")
def sequence_conv_pool(input,
                       context_len, hidden_size,
                       name=None,
                       context_start=None,
                       pool_type=None, context_proj_layer_name=None,
                       context_proj_param_attr=False,
                       fc_layer_name=None,
                       fc_param_attr=None,
                       fc_bias_attr=None, fc_act=None,
                       pool_bias_attr=None,
                       fc_attr=None,
                       context_attr=None,
                       pool_attr=None):
    """
    Text convolution pooling layers helper.

    Text input => Context Projection => FC Layer => Pooling => Output.

    :param name: name of output layer(pooling layer name)
    :type name: basestring
    :param input: name of input layer
    :type input: LayerOutput
    :param context_len: context projection length. See
                        context_projection's document.
    :type context_len: int
    :param hidden_size: FC Layer size.
    :type hidden_size: int
    :param context_start: context projection length. See
                          context_projection's context_start.
    :type context_start: int or None
    :param pool_type: pooling layer type. See pooling_layer's document.
    :type pool_type: BasePoolingType.
    :param context_proj_layer_name: context projection layer name.
                                    None if user don't care.
    :type context_proj_layer_name: basestring
    :param context_proj_param_attr: context projection parameter attribute.
                                    None if user don't care.
    :type context_proj_param_attr: ParameterAttribute or None.
    :param fc_layer_name: fc layer name. None if user don't care.
    :type fc_layer_name: basestring
    :param fc_param_attr: fc layer parameter attribute. None if user don't care.
    :type fc_param_attr: ParameterAttribute or None
    :param fc_bias_attr: fc bias parameter attribute. False if no bias,
                         None if user don't care.
    :type fc_bias_attr: ParameterAttribute or None
    :param fc_act: fc layer activation type. None means tanh
    :type fc_act: BaseActivation
    :param pool_bias_attr: pooling layer bias attr. None if don't care.
                           False if no bias.
    :type pool_bias_attr: ParameterAttribute or None.
    :param fc_attr: fc layer extra attribute.
    :type fc_attr: ExtraLayerAttribute
    :param context_attr: context projection layer extra attribute.
    :type context_attr: ExtraLayerAttribute
    :param pool_attr: pooling layer extra attribute.
    :type pool_attr: ExtraLayerAttribute
    :return: output layer name.
    :rtype: LayerOutput
    """
    # Set Default Value to param
    context_proj_layer_name = "%s_conv_proj" % name \
        if context_proj_layer_name is None else context_proj_layer_name

    with mixed_layer(name=context_proj_layer_name,
                     size=input.size * context_len,
                     act=LinearActivation(),
                     layer_attr=context_attr) as m:
        m += context_projection(input, context_len=context_len,
                                context_start=context_start,
                                padding_attr=context_proj_param_attr)

    fc_layer_name = "%s_conv_fc" % name \
        if fc_layer_name is None else fc_layer_name
    fl = fc_layer(name=fc_layer_name, input=m, size=hidden_size,
                  act=fc_act, layer_attr=fc_attr,
                  param_attr=fc_param_attr, bias_attr=fc_bias_attr)

    return pooling_layer(name=name, input=fl,
                         pooling_type=pool_type,
                         bias_attr=pool_bias_attr,
                         layer_attr=pool_attr)


text_conv_pool = sequence_conv_pool


############################################################################
#                       Images                                             #
############################################################################

@wrap_name_default("conv_pool")
def simple_img_conv_pool(input, filter_size, num_filters, pool_size, name=None,
                         pool_type=None, act=None, groups=1, conv_stride=1,
                         conv_padding=0, bias_attr=None, num_channel=None,
                         param_attr=None, shared_bias=True,
                         conv_layer_attr=None, pool_stride=1, pool_start=None,
                         pool_padding=0, pool_layer_attr=None):
    """
    Simple image convolution and pooling group.

    Input => conv => pooling

    :param name: group name
    :type name: basestring
    :param input: input layer name.
    :type input: LayerOutput
    :param filter_size: see img_conv_layer for details
    :type filter_size: int
    :param num_filters: see img_conv_layer for details
    :type num_filters: int
    :param pool_size: see img_pool_layer for details
    :type pool_size: int
    :param pool_type: see img_pool_layer for details
    :type pool_type: BasePoolingType
    :param act: see img_conv_layer for details
    :type act: BaseActivation
    :param groups: see img_conv_layer for details
    :type groups: int
    :param conv_stride: see img_conv_layer for details
    :type conv_stride: int
    :param conv_padding: see img_conv_layer for details
    :type conv_padding: int
    :param bias_attr: see img_conv_layer for details
    :type bias_attr: ParameterAttribute
    :param num_channel: see img_conv_layer for details
    :type num_channel: int
    :param param_attr: see img_conv_layer for details
    :type param_attr: ParameterAttribute
    :param shared_bias: see img_conv_layer for details
    :type shared_bias: bool
    :param conv_layer_attr: see img_conv_layer for details
    :type conv_layer_attr: ExtraLayerAttribute
    :param pool_stride: see img_conv_layer for details
    :type pool_stride: int
    :param pool_start: see img_conv_layer for details
    :type pool_start: int
    :param pool_padding: see img_conv_layer for details
    :type pool_padding: int
    :param pool_layer_attr: see img_conv_layer for details
    :type pool_layer_attr: ExtraLayerAttribute
    :return: Layer's output
    :rtype: LayerOutput
    """
    _conv_ = img_conv_layer(name="%s_conv" % name, input=input,
                            filter_size=filter_size,
                            num_filters=num_filters, num_channels=num_channel,
                            act=act, groups=groups,
                            stride=conv_stride,
                            padding=conv_padding, bias_attr=bias_attr,
                            param_attr=param_attr, shared_biases=shared_bias,
                            layer_attr=conv_layer_attr)
    return img_pool_layer(name="%s_pool" % name, input=_conv_,
                          pool_size=pool_size,
                          pool_type=pool_type, stride=pool_stride,
                          start=pool_start, padding=pool_padding,
                          layer_attr=pool_layer_attr)


@wrap_name_default("conv_bn_pool")
def img_conv_bn_pool(input, filter_size, num_filters, pool_size, name=None,
                     pool_type=None, act=None, groups=1, conv_stride=1,
                     conv_padding=0, conv_bias_attr=None, num_channel=None,
                     conv_param_attr=None, shared_bias=True,
                     conv_layer_attr=None, bn_param_attr=None,
                     bn_bias_attr=None, bn_layer_attr=None, pool_stride=1,
                     pool_start=None, pool_padding=0, pool_layer_attr=None):
    """
    Convolution, batch normalization, pooling group.

    :param name: group name
    :type name: basestring
    :param input: layer's input
    :type input: LayerOutput
    :param filter_size: see img_conv_layer's document
    :type filter_size: int
    :param num_filters: see img_conv_layer's document
    :type num_filters: int
    :param pool_size: see img_pool_layer's document.
    :type pool_size: int
    :param pool_type: see img_pool_layer's document.
    :type pool_type: BasePoolingType
    :param act: see batch_norm_layer's document.
    :type act: BaseActivation
    :param groups: see img_conv_layer's document
    :type groups: int
    :param conv_stride: see img_conv_layer's document.
    :type conv_stride: int
    :param conv_padding: see img_conv_layer's document.
    :type conv_padding: int
    :param conv_bias_attr: see img_conv_layer's document.
    :type conv_bias_attr: ParameterAttribute
    :param num_channel: see img_conv_layer's document.
    :type num_channel: int
    :param conv_param_attr: see img_conv_layer's document.
    :type conv_param_attr: ParameterAttribute
    :param shared_bias: see img_conv_layer's document.
    :type shared_bias: bool
    :param conv_layer_attr: see img_conv_layer's document.
    :type conv_layer_attr: ExtraLayerOutput
    :param bn_param_attr: see batch_norm_layer's document.
    :type bn_param_attr: ParameterAttribute.
    :param bn_bias_attr: see batch_norm_layer's document.
    :param bn_layer_attr: ParameterAttribute.
    :param pool_stride: see img_pool_layer's document.
    :type pool_stride: int
    :param pool_start: see img_pool_layer's document.
    :type pool_start: int
    :param pool_padding: see img_pool_layer's document.
    :type pool_padding: int
    :param pool_layer_attr: see img_pool_layer's document.
    :type pool_layer_attr: ExtraLayerAttribute
    :return: Layer groups output
    :rtype: LayerOutput
    """
    __conv__ = img_conv_layer(name="%s_conv" % name,
                              input=input, filter_size=filter_size,
                              num_filters=num_filters, num_channels=num_channel,
                              act=LinearActivation(), groups=groups,
                              stride=conv_stride, padding=conv_padding,
                              bias_attr=conv_bias_attr,
                              param_attr=conv_param_attr,
                              shared_biases=shared_bias,
                              layer_attr=conv_layer_attr)
    __bn__ = batch_norm_layer(name="%s_bn" % name,
                              input=__conv__, act=act,
                              bias_attr=bn_bias_attr, param_attr=bn_param_attr,
                              layer_attr=bn_layer_attr)
    return img_pool_layer(name="%s_pool" % name,
                          input=__bn__, pool_type=pool_type,
                          pool_size=pool_size, stride=pool_stride,
                          start=pool_start, padding=pool_padding,
                          layer_attr=pool_layer_attr)


@wrap_act_default(param_names=['conv_act'],
                  act=ReluActivation())
@wrap_param_default(param_names=['pool_type'],
                    default_factory=lambda _: MaxPooling())
def img_conv_group(input, conv_num_filter,
                   pool_size,
                   num_channels=None,
                   conv_padding=1,
                   conv_filter_size=3,
                   conv_act=None,
                   conv_with_batchnorm=False,
                   conv_batchnorm_drop_rate=0,
                   pool_stride=1,
                   pool_type=None):
    """
    Image Convolution Group, Used for vgg net.

    TODO(yuyang18): Complete docs

    :param conv_batchnorm_drop_rate:
    :param input:
    :param conv_num_filter:
    :param pool_size:
    :param num_channels:
    :param conv_padding:
    :param conv_filter_size:
    :param conv_act:
    :param conv_with_batchnorm:
    :param pool_stride:
    :param pool_type:
    :return:
    """
    tmp = input

    # Type checks
    assert isinstance(tmp, LayerOutput)
    assert isinstance(conv_num_filter, list) or isinstance(conv_num_filter,
                                                           tuple)
    for each_num_filter in conv_num_filter:
        assert isinstance(each_num_filter, int)

    assert isinstance(pool_size, int)

    def __extend_list__(obj):
        if not hasattr(obj, '__len__'):
            return [obj] * len(conv_num_filter)
        else:
            return obj

    conv_padding = __extend_list__(conv_padding)
    conv_filter_size = __extend_list__(conv_filter_size)
    conv_act = __extend_list__(conv_act)
    conv_with_batchnorm = __extend_list__(conv_with_batchnorm)
    conv_batchnorm_drop_rate = __extend_list__(conv_batchnorm_drop_rate)

    for i in xrange(len(conv_num_filter)):
        extra_kwargs = dict()
        if num_channels is not None:
            extra_kwargs['num_channels'] = num_channels
            num_channels = None
        if conv_with_batchnorm[i]:
            extra_kwargs['act'] = LinearActivation()
        else:
            extra_kwargs['act'] = conv_act[i]

        tmp = img_conv_layer(input=tmp, padding=conv_padding[i],
                             filter_size=conv_filter_size[i],
                             num_filters=conv_num_filter[i],
                             **extra_kwargs)

        # logger.debug("tmp.num_filters = %d" % tmp.num_filters)

        if conv_with_batchnorm[i]:
            dropout = conv_batchnorm_drop_rate[i]
            if dropout == 0 or abs(dropout) < 1e-5:  # dropout not set
                tmp = batch_norm_layer(input=tmp, act=conv_act[i])
            else:
                tmp = batch_norm_layer(input=tmp, act=conv_act[i],
                                       layer_attr=ExtraAttr(drop_rate=dropout))

    return img_pool_layer(input=tmp, stride=pool_stride, pool_size=pool_size,
                          pool_type=pool_type)


def small_vgg(input_image, num_channels, num_classes):
    def __vgg__(ipt, num_filter, times, dropouts, num_channels_=None):
        return img_conv_group(input=ipt, num_channels=num_channels_,
                              pool_size=2,
                              pool_stride=2,
                              conv_num_filter=[num_filter] * times,
                              conv_filter_size=3,
                              conv_act=ReluActivation(),
                              conv_with_batchnorm=True,
                              conv_batchnorm_drop_rate=dropouts,
                              pool_type=MaxPooling())

    tmp = __vgg__(input_image, 64, 2, [0.3, 0], num_channels)
    tmp = __vgg__(tmp, 128, 2, [0.4, 0])
    tmp = __vgg__(tmp, 256, 3, [0.4, 0.4, 0])
    tmp = __vgg__(tmp, 512, 3, [0.4, 0.4, 0])
    tmp = img_pool_layer(input = tmp, stride = 2,
                         pool_size = 2, pool_type = MaxPooling())
    tmp = dropout_layer(input=tmp, dropout_rate=0.5)
    tmp = fc_layer(input=tmp, size=512, layer_attr=ExtraAttr(drop_rate=0.5),
                   act=LinearActivation())
    tmp = batch_norm_layer(input=tmp, act=ReluActivation())
    return fc_layer(input=tmp, size=num_classes, act=SoftmaxActivation())


def vgg_16_network(input_image, num_channels, num_classes=1000):
    """
    Same model from https://gist.github.com/ksimonyan/211839e770f7b538e2d8

    :param num_classes:
    :param input_image:
    :type input_image: LayerOutput
    :param num_channels:
    :type num_channels: int
    :return:
    """

    tmp = img_conv_group(input=input_image, num_channels=num_channels,
                         conv_padding=1, conv_num_filter=[64, 64],
                         conv_filter_size=3,
                         conv_act=ReluActivation(), pool_size=2,
                         pool_stride=2,
                         pool_type=MaxPooling())

    tmp = img_conv_group(input=tmp, conv_num_filter=[128, 128], conv_padding=1,
                         conv_filter_size=3, conv_act=ReluActivation(),
                         pool_stride=2, pool_type=MaxPooling(),
                         pool_size=2)

    tmp = img_conv_group(input=tmp, conv_num_filter=[256, 256, 256],
                         conv_padding=1,
                         conv_filter_size=3, conv_act=ReluActivation(),
                         pool_stride=2, pool_type=MaxPooling(), pool_size=2)

    tmp = img_conv_group(input=tmp, conv_num_filter=[512, 512, 512],
                         conv_padding=1,
                         conv_filter_size=3, conv_act=ReluActivation(),
                         pool_stride=2, pool_type=MaxPooling(), pool_size=2)
    tmp = img_conv_group(input=tmp, conv_num_filter=[512, 512, 512],
                         conv_padding=1,
                         conv_filter_size=3, conv_act=ReluActivation(),
                         pool_stride=2, pool_type=MaxPooling(), pool_size=2)

    tmp = fc_layer(input=tmp, size=4096, act=ReluActivation(),
                   layer_attr=ExtraAttr(drop_rate=0.5))

    tmp = fc_layer(input=tmp, size=4096, act=ReluActivation(),
                   layer_attr=ExtraAttr(drop_rate=0.5))

    return fc_layer(input=tmp, size=num_classes, act=SoftmaxActivation())


############################################################################
#                       Recurrent                                          #
############################################################################

@wrap_name_default("lstm")
def simple_lstm(input, size, name=None, reverse=False, mat_param_attr=None,
                bias_param_attr=None, inner_param_attr=None, act=None,
                gate_act=None, state_act=None, mixed_layer_attr=None,
                lstm_cell_attr=None):
    """
    Simple LSTM Cell.

L
luotao02 已提交
443
    It just combine a mixed layer with fully_matrix_projection and a lstmemory
Z
zhangjinchao01 已提交
444 445 446 447
    layer. The simple lstm cell was implemented as follow equations.

    ..  math::

L
luotao02 已提交
448
        i_t & = \\sigma(W_{xi}x_{t} + W_{hi}h_{t-1} + W_{ci}c_{t-1} + b_i)
Z
zhangjinchao01 已提交
449

L
luotao02 已提交
450
        f_t & = \\sigma(W_{xf}x_{t} + W_{hf}h_{t-1} + W_{cf}c_{t-1} + b_f)
Z
zhangjinchao01 已提交
451

L
luotao02 已提交
452
        c_t & = f_tc_{t-1} + i_t tanh (W_{xc}x_t+W_{hc}h_{t-1} + b_c)
Z
zhangjinchao01 已提交
453

L
luotao02 已提交
454
        o_t & = \\sigma(W_{xo}x_{t} + W_{ho}h_{t-1} + W_{co}c_t + b_o)
Z
zhangjinchao01 已提交
455

L
luotao02 已提交
456
        h_t & = o_t tanh(c_t)
Z
zhangjinchao01 已提交
457 458 459 460 461 462 463 464 465 466 467 468

    Please refer **Generating Sequences With Recurrent Neural Networks** if you
    want to know what lstm is. Link_ is here.

    .. _Link: http://arxiv.org/abs/1308.0850

    :param name: lstm layer name.
    :type name: basestring
    :param input: input layer name.
    :type input: LayerOutput
    :param size: lstm layer size.
    :type size: int
C
caoying03 已提交
469
    :param reverse: whether to process the input data in a reverse order
Z
zhangjinchao01 已提交
470 471 472 473 474 475 476 477
    :type reverse: bool
    :param mat_param_attr: mixed layer's matrix projection parameter attribute.
    :type mat_param_attr: ParameterAttribute
    :param bias_param_attr: bias parameter attribute. False means no bias, None
                            means default bias.
    :type bias_param_attr: ParameterAttribute|False
    :param inner_param_attr: lstm cell parameter attribute.
    :type inner_param_attr: ParameterAttribute
C
caoying03 已提交
478
    :param act: lstm final activiation type
Z
zhangjinchao01 已提交
479
    :type act: BaseActivation
C
caoying03 已提交
480
    :param gate_act: lstm gate activiation type
Z
zhangjinchao01 已提交
481
    :type gate_act: BaseActivation
C
caoying03 已提交
482
    :param state_act: lstm state activiation type.
Z
zhangjinchao01 已提交
483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
    :type state_act: BaseActivation
    :param mixed_layer_attr: mixed layer's extra attribute.
    :type mixed_layer_attr: ExtraLayerAttribute
    :param lstm_cell_attr: lstm layer's extra attribute.
    :type lstm_cell_attr: ExtraLayerAttribute
    :return: lstm layer name.
    :rtype: LayerOutput
    """
    fc_name = 'lstm_transform_%s' % name
    with mixed_layer(name=fc_name, size=size * 4,
                     act=IdentityActivation(),
                     layer_attr=mixed_layer_attr, bias_attr=False) as m:
        m += full_matrix_projection(input, param_attr=mat_param_attr)

    return lstmemory(name=name, input=m, reverse=reverse,
                     bias_attr=bias_param_attr,
                     param_attr=inner_param_attr, act=act,
                     gate_act=gate_act, state_act=state_act,
                     layer_attr=lstm_cell_attr)


@wrap_name_default('lstm_unit')
L
luotao02 已提交
505
def lstmemory_unit(input, name=None, size=None, param_attr=None,
C
caoying03 已提交
506
                   act=None, gate_act=None, state_act=None,
L
luotao02 已提交
507 508
                   mixed_bias_attr=None, lstm_bias_attr=None,
                   mixed_layer_attr=None,lstm_layer_attr=None,
Z
zhangjinchao01 已提交
509 510
                   get_output_layer_attr=None):
    """
C
caoying03 已提交
511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542
    Define calculations that a LSTM unit performs in a single time step.
    This function itself is not a recurrent layer, so that it can not be
    directly applied to sequence input. This function is always used in
    recurrent_group (see layers.py for more details) to implement attention
    mechanism.

    Please refer to  **Generating Sequences With Recurrent Neural Networks**
    for more details about LSTM. The link goes as follows:
    .. _Link: https://arxiv.org/abs/1308.0850

    ..  math::

        i_t & = \\sigma(W_{xi}x_{t} + W_{hi}h_{t-1} + W_{ci}c_{t-1} + b_i)

        f_t & = \\sigma(W_{xf}x_{t} + W_{hf}h_{t-1} + W_{cf}c_{t-1} + b_f)

        c_t & = f_tc_{t-1} + i_t tanh (W_{xc}x_t+W_{hc}h_{t-1} + b_c)

        o_t & = \\sigma(W_{xo}x_{t} + W_{ho}h_{t-1} + W_{co}c_t + b_o)

        h_t & = o_t tanh(c_t)

    The example usage is:

    ..  code-block:: python

        lstm_step = lstmemory_unit(input=[layer1],
                                   size=256,
                                   act=TanhActivation(),
                                   gate_act=SigmoidActivation(),
                                   state_act=TanhActivation())

Z
zhangjinchao01 已提交
543

L
luotao02 已提交
544 545 546 547 548 549 550 551
    :param input: input layer name.
    :type input: LayerOutput
    :param name: lstmemory unit name.
    :type name: basestring
    :param size: lstmemory unit size.
    :type size: int
    :param param_attr: Parameter config, None if use default.
    :type param_attr: ParameterAttribute
C
caoying03 已提交
552
    :param act: lstm final activiation type
L
luotao02 已提交
553
    :type act: BaseActivation
C
caoying03 已提交
554
    :param gate_act: lstm gate activiation type
L
luotao02 已提交
555
    :type gate_act: BaseActivation
C
caoying03 已提交
556
    :param state_act: lstm state activiation type.
L
luotao02 已提交
557 558 559 560 561 562 563 564 565 566 567 568 569 570 571
    :type state_act: BaseActivation
    :param mixed_bias_attr: bias parameter attribute of mixed layer. 
                            False means no bias, None means default bias.
    :type mixed_bias_attr: ParameterAttribute|False
    :param lstm_bias_attr: bias parameter attribute of lstm layer.
                           False means no bias, None means default bias.
    :type lstm_bias_attr: ParameterAttribute|False
    :param mixed_layer_attr: mixed layer's extra attribute.
    :type mixed_layer_attr: ExtraLayerAttribute
    :param lstm_layer_attr: lstm layer's extra attribute.
    :type lstm_layer_attr: ExtraLayerAttribute
    :param get_output_layer_attr: get output layer's extra attribute.
    :type get_output_layer_attr: ExtraLayerAttribute
    :return: lstmemory unit name.
    :rtype: LayerOutput
Z
zhangjinchao01 已提交
572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608
    """
    if size is None:
        assert input.size % 4 == 0
        size = input.size / 4
    out_mem = memory(name=name, size=size)
    state_mem = memory(name="%s_state" % name, size=size)

    with mixed_layer(name="%s_input_recurrent" % name,
                     size=size * 4, bias_attr=mixed_bias_attr,
                     layer_attr=mixed_layer_attr,
                     act=IdentityActivation()) as m:
        m += identity_projection(input=input)
        m += full_matrix_projection(input=out_mem, param_attr=param_attr)

    lstm_out = lstm_step_layer(
        name=name,
        input=m,
        state=state_mem,
        size=size,
        bias_attr=lstm_bias_attr,
        act=act,
        gate_act=gate_act,
        state_act=state_act,
        layer_attr=lstm_layer_attr
    )
    get_output_layer(name='%s_state' % name,
                     input=lstm_out,
                     arg_name='state',
                     layer_attr=get_output_layer_attr)

    return lstm_out


@wrap_name_default('lstm_group')
def lstmemory_group(input, size=None, name=None,
                    reverse=False, param_attr=None,
                    act=None, gate_act=None, state_act=None,
L
luotao02 已提交
609
                    mixed_bias_attr=None, lstm_bias_attr=None,
Z
zhangjinchao01 已提交
610 611 612
                    mixed_layer_attr=None, lstm_layer_attr=None,
                    get_output_layer_attr=None):
    """
C
caoying03 已提交
613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637
    lstm_group is a recurrent layer group version Long Short Term Memory. It
    does exactly the same calculation as the lstmemory layer (see lstmemory in
    layers.py for the maths) does. A promising benefit is that LSTM memory
    cell states, or hidden states in every time step are accessible to for the
    user. This is especially useful in attention model. If you do not need to
    access to the internal states of the lstm, but merely use its outputs,
    it is recommanded to use the lstmemory, which is relatively faster than
    lstmemory_group.

    NOTE: In PaddlePaddle's implementation, the following input-to-hidden
    multiplications:
    :math:`W_{xi}x_{t}` , :math:`W_{xf}x_{t}`,
    :math:`W_{xc}x_t`, :math:`W_{xo}x_{t}` are not done in lstmemory_unit to
    speed up the calculations. Consequently, an additional mixed_layer with
    full_matrix_projection must be included before lstmemory_unit is called.

    The example usage is:

    ..  code-block:: python

        lstm_step = lstmemory_group(input=[layer1],
                                    size=256,
                                    act=TanhActivation(),
                                    gate_act=SigmoidActivation(),
                                    state_act=TanhActivation())
Z
zhangjinchao01 已提交
638

L
luotao02 已提交
639 640 641 642 643 644 645 646 647 648
    :param input: input layer name.
    :type input: LayerOutput
    :param name: lstmemory group name.
    :type name: basestring
    :param size: lstmemory group size.
    :type size: int
    :param reverse: is lstm reversed
    :type reverse: bool
    :param param_attr: Parameter config, None if use default.
    :type param_attr: ParameterAttribute
C
caoying03 已提交
649
    :param act: lstm final activiation type
L
luotao02 已提交
650
    :type act: BaseActivation
C
caoying03 已提交
651
    :param gate_act: lstm gate activiation type
L
luotao02 已提交
652
    :type gate_act: BaseActivation
C
caoying03 已提交
653
    :param state_act: lstm state activiation type.
L
luotao02 已提交
654
    :type state_act: BaseActivation
C
caoying03 已提交
655
    :param mixed_bias_attr: bias parameter attribute of mixed layer.
L
luotao02 已提交
656 657 658 659 660 661 662 663 664 665 666
                            False means no bias, None means default bias.
    :type mixed_bias_attr: ParameterAttribute|False
    :param lstm_bias_attr: bias parameter attribute of lstm layer.
                           False means no bias, None means default bias.
    :type lstm_bias_attr: ParameterAttribute|False
    :param mixed_layer_attr: mixed layer's extra attribute.
    :type mixed_layer_attr: ExtraLayerAttribute
    :param lstm_layer_attr: lstm layer's extra attribute.
    :type lstm_layer_attr: ExtraLayerAttribute
    :param get_output_layer_attr: get output layer's extra attribute.
    :type get_output_layer_attr: ExtraLayerAttribute
C
caoying03 已提交
667
    :return: the lstmemory group.
L
luotao02 已提交
668
    :rtype: LayerOutput
Z
zhangjinchao01 已提交
669 670 671 672
    """

    def __lstm_step__(ipt):
        return lstmemory_unit(input=ipt, name=name,
L
luotao02 已提交
673
                              size=size, mixed_bias_attr=mixed_bias_attr,
Z
zhangjinchao01 已提交
674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696
                              mixed_layer_attr=mixed_layer_attr,
                              param_attr=param_attr,
                              lstm_bias_attr=lstm_bias_attr,
                              act=act, gate_act=gate_act,
                              state_act=state_act,
                              lstm_layer_attr=lstm_layer_attr,
                              get_output_layer_attr=get_output_layer_attr)

    return recurrent_group(name='%s_recurrent_group' % name,
                           step=__lstm_step__,
                           reverse=reverse,
                           input=input)


@wrap_name_default('gru_unit')
def gru_unit(input,
             size=None,
             name=None,
             gru_bias_attr=None,
             act=None,
             gate_act=None,
             gru_layer_attr=None):
    """
C
caoying03 已提交
697 698 699 700 701
    Define calculations that a gated recurrent unit performs in a single time
    step. This function itself is not a recurrent layer, so that it can not be
    directly applied to sequence input. This function is almost always used in
    the recurrent_group (see layers.py for more details) to implement attention
    mechanism.
Z
zhangjinchao01 已提交
702

C
caoying03 已提交
703 704 705
    Please see grumemory in layers.py for the details about the maths.

    :param input: input layer name.
Z
zhangjinchao01 已提交
706
    :type input: LayerOutput
C
caoying03 已提交
707 708 709 710 711 712 713 714 715 716 717 718
    :param name: name of the gru group.
    :type name: basestring
    :param size: hidden size of the gru.
    :type size: int
    :param act: type of the activation
    :type act: BaseActivation
    :param gate_act: type of the gate activation
    :type gate_act: BaseActivation
    :param gru_layer_attr: Extra parameter attribute of the gru layer.
    :type gru_layer_attr: ParameterAttribute|False
    :return: the gru output layer.
    :rtype: LayerOutput
Z
zhangjinchao01 已提交
719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
    """

    assert input.size % 3 == 0
    if size is None:
        size = input.size / 3

    out_mem = memory(name=name, size=size)

    gru_out = gru_step_layer(
        name=name,
        input=input,
        output_mem=out_mem,
        size=size,
        bias_attr=gru_bias_attr,
        act=act,
        gate_act=gate_act,
        layer_attr=gru_layer_attr
    )
    return gru_out


@wrap_name_default('gru_group')
def gru_group(input,
              size=None,
              name=None,
              reverse=False,
              gru_bias_attr=None,
              act=None, gate_act=None,
              gru_layer_attr=None):
C
caoying03 已提交
748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787

    """
    gru_group is a recurrent layer group version Gated Recurrent Unit. It
    does exactly the same calculation as the grumemory layer does. A promising
    benefit is that gru hidden sates are accessible to for the user. This is
    especially useful in attention model. If you do not need to access to
    any internal state, but merely use the outputs of a GRU, it is recommanded
    to use the grumemory, which is relatively faster.

    Please see grumemory in layers.py for more detail about the maths.

    The example usage is:

    ..  code-block:: python

        gru = gur_group(input=[layer1],
                        size=256,
                        act=TanhActivation(),
                        gate_act=SigmoidActivation())

    :param input: input layer name.
    :type input: LayerOutput
    :param name: name of the gru group.
    :type name: basestring
    :param size: hidden size of the gru.
    :type size: int
    :param reverse: whether to process the input data in a reverse order
    :type reverse: bool
    :param act: type of the activiation
    :type act: BaseActivation
    :param gate_act: type of the gate activiation
    :type gate_act: BaseActivation
    :param gru_bias_attr: bias. False means no bias, None means default bias.
    :type gru_bias_attr: ParameterAttribute|False
    :param gru_layer_attr: Extra parameter attribute of the gru layer.
    :type gru_layer_attr: ParameterAttribute|False
    :return: the gru group.
    :rtype: LayerOutput
    """

Z
zhangjinchao01 已提交
788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817
    def __gru_step__(ipt):
        return gru_unit(
            input=ipt,
            name=name,
            size=size,
            gru_bias_attr=gru_bias_attr,
            act=act,
            gate_act=gate_act,
            gru_layer_attr=gru_layer_attr
        )

    return recurrent_group(name='%s_recurrent_group' % name,
                           step=__gru_step__,
                           reverse=reverse,
                           input=input)


@wrap_name_default('simple_gru')
def simple_gru(input,
               size,
               name=None,
               reverse=False,
               mixed_param_attr=None,
               mixed_bias_param_attr=None,
               mixed_layer_attr=None,
               gru_bias_attr=None,
               act=None,
               gate_act=None,
               gru_layer_attr=None
               ):
C
caoying03 已提交
818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854
    """
    simple_gru is also a recurrent layer group version Gated Recurrent Unit as
    gru_group. The difference only lies in implemention details.
    The computational speed is that, grumemory is relatively better than
    gru_group, and gru_group is relatively better than simple_gru.

    simple_gru does exactly the same calculation as the grumemory layer does.
    Please see grumemory in layers.py for more detail about the maths.

    The example usage is:

    ..  code-block:: python

        gru = gur_group(input=[layer1],
                        size=256,
                        act=TanhActivation(),
                        gate_act=SigmoidActivation())

    :param input: input layer name.
    :type input: LayerOutput
    :param name: name of the gru group.
    :type name: basestring
    :param size: hidden size of the gru.
    :type size: int
    :param reverse: whether to process the input data in a reverse order
    :type reverse: bool
    :param act: type of the activiation
    :type act: BaseActivation
    :param gate_act: type of the gate activiation
    :type gate_act: BaseActivation
    :param gru_bias_attr: bias. False means no bias, None means default bias.
    :type gru_bias_attr: ParameterAttribute|False
    :param gru_layer_attr: Extra parameter attribute of the gru layer.
    :type gru_layer_attr: ParameterAttribute|False
    :return: the gru group.
    :rtype: LayerOutput
    """
Z
zhangjinchao01 已提交
855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885
    with mixed_layer(name='%s_transform' % name,
                     size=size * 3,
                     bias_attr=mixed_bias_param_attr,
                     layer_attr=mixed_layer_attr) as m:
        m += full_matrix_projection(input=input, param_attr=mixed_param_attr)

    return gru_group(name=name,
                     size=size,
                     input=m,
                     reverse=reverse,
                     gru_bias_attr=gru_bias_attr,
                     act=act,
                     gate_act=gate_act,
                     gru_layer_attr=gru_layer_attr)


@wrap_name_default("bidirectional_lstm")
def bidirectional_lstm(input, size, name=None, return_seq=False,
                       fwd_mat_param_attr=None, fwd_bias_param_attr=None,
                       fwd_inner_param_attr=None, fwd_act=None,
                       fwd_gate_act=None, fwd_state_act=None,
                       fwd_mixed_layer_attr=None, fwd_lstm_cell_attr=None,

                       bwd_mat_param_attr=None, bwd_bias_param_attr=None,
                       bwd_inner_param_attr=None, bwd_act=None,
                       bwd_gate_act=None, bwd_state_act=None,
                       bwd_mixed_layer_attr=None, bwd_lstm_cell_attr=None,

                       last_seq_attr=None, first_seq_attr=None,
                       concat_attr=None, concat_act=None):
    """
C
caoying03 已提交
886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901
    A bidirectional_lstm is a recurrent unit that iterates over the input
    sequence both in forward and bardward orders, and then concatenate two
    outputs form a final output. However, concatenation of two outputs
    is not the only way to form the final output, you can also, for example,
    just add them together.

    Please refer to  **Neural Machine Translation by Jointly Learning to Align
    and Translate** for more details about the bidirectional lstm.
    The link goes as follows:
    .. _Link: https://arxiv.org/pdf/1409.0473v3.pdf

    The example usage is:

    ..  code-block:: python

        lstm_step = bidirectional_lstm(input=[input1], size=512)
Z
zhangjinchao01 已提交
902 903 904 905 906 907 908

    :param name: bidirectional lstm layer name.
    :type name: basestring
    :param input: input layer.
    :type input: LayerOutput
    :param size: lstm layer size.
    :type size: int
C
caoying03 已提交
909 910 911 912 913
    :param return_seq: If set False, outputs of the last time step are
                       concatenated and returned.
                       If set True, the entire output sequences that are
                       processed in forward and backward directions are
                       concatenated and returned.
Z
zhangjinchao01 已提交
914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951
    :type return_seq: bool
    :return: lstm layer name.
    :rtype: LayerOutput
    """
    args = locals()

    fw = simple_lstm(name='%s_fw' % name, input=input, size=size,
                     **dict((k[len('fwd_'):], v) for k, v in args.iteritems()
                        if k.startswith('fwd_')))

    bw = simple_lstm(name="%s_bw" % name, input=input, size=size,
                     reverse=True,
                     **dict((k[len('bwd_'):], v) for k, v in args.iteritems()
                        if k.startswith('bwd_')))

    if return_seq:
        return concat_layer(name=name, input=[fw, bw], layer_attr=concat_attr,
                            act=concat_act)
    else:
        fw_seq = last_seq(name="%s_fw_last" % name, input=fw,
                          layer_attr=last_seq_attr)
        bw_seq = first_seq(name="%s_bw_last" % name, input=bw,
                           layer_attr=first_seq_attr)
        return concat_layer(name=name, input=[fw_seq, bw_seq],
                            layer_attr=concat_attr, act=concat_act)


@wrap_name_default()
@wrap_act_default(param_names=['weight_act'], act=TanhActivation())
def simple_attention(encoded_sequence,
                     encoded_proj,
                     decoder_state,
                     transform_param_attr=None,
                     softmax_param_attr=None,
                     weight_act=None,
                     name=None):
    """
    Calculate and then return a context vector by attention machanism.
952
    Size of the context vector equals to size of the encoded_sequence.
Z
zhangjinchao01 已提交
953 954

    ..  math::
L
luotao02 已提交
955 956 957 958 959

        a(s_{i-1},h_{j}) & = v_{a}f(W_{a}s_{t-1} + U_{a}h_{j})

        e_{i,j} & = a(s_{i-1}, h_{j})

960
        a_{i,j} & = \\frac{exp(e_{i,j})}{\\sum_{k=1}^{T_x}{exp(e_{i,k})}}
L
luotao02 已提交
961 962

        c_{i} & = \\sum_{j=1}^{T_{x}}a_{i,j}h_{j}
Z
zhangjinchao01 已提交
963 964 965 966 967 968 969 970 971 972 973

    where :math:`h_{j}` is the jth element of encoded_sequence,
    :math:`U_{a}h_{j}` is the jth element of encoded_proj
    :math:`s_{i-1}` is decoder_state
    :math:`f` is weight_act, and is set to tanh by default.

    Please refer to **Neural Machine Translation by Jointly Learning to
    Align and Translate** for more details. The link is as follows:
    https://arxiv.org/abs/1409.0473.

    The example usage is:
L
luotao02 已提交
974

Z
zhangjinchao01 已提交
975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132
    ..  code-block:: python

        context = simple_attention(encoded_sequence=enc_seq,
                                   encoded_proj=enc_proj,
                                   decoder_state=decoder_prev,)

    :param name: name of the attention model.
    :type name: basestring
    :param softmax_param_attr: parameter attribute of sequence softmax
                               that is used to produce attention weight
    :type softmax_param_attr: ParameterAttribute
    :param weight_act: activation of the attention model
    :type weight_act: Activation
    :param encoded_sequence: output of the encoder
    :type encoded_sequence: LayerOutput
    :param encoded_proj: attention weight is computed by a feed forward neural
                         network which has two inputs : decoder's hidden state
                         of previous time step and encoder's output.
                         encoded_proj is output of the feed-forward network for
                         encoder's output. Here we pre-compute it outside
                         simple_attention for speed consideration.
    :type encoded_proj: LayerOutput
    :param decoder_state: hidden state of decoder in previous time step
    :type decoder_state: LayerOutput
    :param transform_param_attr: parameter attribute of the feed-forward
                                network that takes decoder_state as inputs to
                                compute attention weight.
    :type transform_param_attr: ParameterAttribute
    :return: a context vector
    """
    assert encoded_proj.size == decoder_state.size
    proj_size = encoded_proj.size

    with mixed_layer(size=proj_size, name="%s_transform" % name) as m:
        m += full_matrix_projection(decoder_state,
                                    param_attr=transform_param_attr)

    expanded = expand_layer(input=m, expand_as=encoded_sequence,
                            name='%s_expand' % name)

    with mixed_layer(size=proj_size, act=weight_act,
                     name="%s_combine" % name) as m:
        m += identity_projection(expanded)
        m += identity_projection(encoded_proj)

    # sequence softmax is used to normalize similarities between decoder state
    # and encoder outputs into a distribution
    attention_weight = fc_layer(input=m,
                                size=1,
                                act=SequenceSoftmaxActivation(),
                                param_attr=softmax_param_attr,
                                name="%s_softmax" % name,
                                bias_attr=False)

    scaled = scaling_layer(weight=attention_weight, input=encoded_sequence,
                           name='%s_scaling' % name)

    return pooling_layer(input=scaled, pooling_type=SumPooling(),
                         name="%s_pooling" % name)


############################################################################
#                         Miscs                                            #
############################################################################

@wrap_name_default("dropout")
def dropout_layer(input, dropout_rate, name=None):
    """
    @TODO(yuyang18): Add comments.

    :param name:
    :param input:
    :param dropout_rate:
    :return:
    """
    return addto_layer(name=name, input=input, act=LinearActivation(),
                       bias_attr=False,
                       layer_attr=ExtraAttr(drop_rate=dropout_rate))


def outputs(layers):
    """
    Declare the end of network. Currently it will only calculate the
    input/output order of network. It will calculate the predict network or
    train network's output automatically.


    :param layers:
    :type layers: list|tuple|LayerOutput
    :return:
    """

    def __dfs_travel__(layer,
                       predicate=lambda x: x.layer_type == LayerType.DATA):

        """
        DFS LRV Travel for output layer.

        The return order is define order for data_layer in this leaf node.

        :param layer:
        :type layer: LayerOutput
        :return:
        """
        assert isinstance(layer, LayerOutput), "layer is %s" % (layer)
        retv = []
        if layer.parents is not None:
            for p in layer.parents:
                retv.extend(__dfs_travel__(p, predicate))

        if predicate(layer):
            retv.append(layer)
        return retv

    if isinstance(layers, LayerOutput):
        layers = [layers]

    assert len(layers) > 0
    if len(layers) != 1:
        logger.warning("EndOfNetwork routine try to calculate network's"
                       " inputs and outputs order. It might not work well."
                       "Please see follow log carefully.")
    inputs = []
    outputs_ = []
    for each_layer in layers:
        assert isinstance(each_layer, LayerOutput)
        inputs.extend(__dfs_travel__(each_layer))
        outputs_.extend(__dfs_travel__(
            each_layer, lambda x: x.layer_type == LayerType.COST))

    # Currently, we got each leaf node's inputs order, output order.
    # We merge them together.

    final_inputs = []
    final_outputs = []

    for each_input in inputs:
        assert isinstance(each_input, LayerOutput)
        if each_input.name not in final_inputs:
            final_inputs.append(each_input.name)

    for each_output in outputs_:
        assert isinstance(each_output, LayerOutput)
        if each_output.name not in final_outputs:
            final_outputs.append(each_output.name)

    logger.info(
        "".join(["The input order is [", ", ".join(final_inputs), "]"])
    )
    logger.info(
        "".join(["The output order is [", ", ".join(final_outputs), "]"
                 ]))

    Inputs(*final_inputs)
    if len(final_outputs) != 0:
        Outputs(*final_outputs)
    else:
        Outputs(*map(lambda x: x.name, layers))