activation.py 32.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
# TODO: define activation functions of neural network
16

17
__all__ = [
18 19
    'ELU',
    'GELU',
20
    'Hardshrink',
21
    'Hardswish',
W
WangXi 已提交
22
    'Tanh',
23 24
    'Hardtanh',
    'PReLU',
25
    'ReLU',
26 27
    'ReLU6',
    'SELU',
C
ceci3 已提交
28
    'LeakyReLU',
29
    'Sigmoid',
30
    'Hardsigmoid',
31
    'Softmax',
32 33 34
    'Softplus',
    'Softshrink',
    'Softsign',
35
    'Swish',
36
    'Tanhshrink',
37
    'ThresholdedReLU',
38
    'LogSigmoid',
39
    'LogSoftmax',
40
    'Maxout',
41 42
]

43 44 45
from ...fluid.dygraph import layers
from ...fluid import core
from ...fluid.framework import in_dygraph_mode
46 47
from ...fluid.param_attr import ParamAttr
from ...fluid.initializer import Constant
Q
Qi Li 已提交
48
from paddle.framework import get_default_dtype
49
from .. import functional as F
50 51


52
class ELU(layers.Layer):
53
    r"""
54 55
    ELU Activation.

56
    .. math::
57

58 59 60 61 62 63
        ELU(x) = max(0, x) + min(0, \\alpha * (e^{x}-1))

    Parameters:
        alpha (float, optional): The 'alpha' value of the ELU formulation. Default is 1.0.
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.
64

65 66 67
    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.
68

69 70 71
    Examples:
        .. code-block:: python

72
            import paddle
73

Z
zhupengyang 已提交
74
            x = paddle.to_tensor([[-1. ,6.], [1., 15.6]])
75 76 77 78
            m = paddle.nn.ELU(0.2)
            out = m(x)
            # [[-0.12642411  6.        ]
            #  [ 1.          15.6      ]]
79 80 81 82 83 84 85 86 87 88 89 90
    """

    def __init__(self, alpha=1.0, name=None):
        super(ELU, self).__init__()
        self._alpha = alpha
        self._name = name

    def forward(self, x):
        return F.elu(x, self._alpha, self._name)


class GELU(layers.Layer):
91
    r"""
92 93 94 95
    GELU Activation.

    If approximate is True

96
    .. math::
97 98 99 100 101

        GELU(x) = 0.5 * x * (1 + tanh(\\sqrt{\\frac{2}{\\pi}} * (x + 0.044715x^{3})))

    else

102
    .. math::
103 104 105 106 107 108 109

        GELU(x) = 0.5 * x * (1 + erf(\\frac{x}{\\sqrt{2}}))

    Parameters:
        approximate (bool, optional): Wether to enable approximation. Default is False.
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.
110

111 112 113
    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.
114

115 116 117
    Examples:
        .. code-block:: python

118 119
            import paddle
            import numpy as np
120

121
            x = paddle.to_tensor(np.array([[-1, 0.5],[1, 1.5]]))
122

123 124
            m = paddle.nn.GELU()
            out = m(x) # [-0.158655 0.345731 0.841345 1.39979]
125

126 127
            m = paddle.nn.GELU(True)
            out = m(x) # [-0.158808 0.345714 0.841192 1.39957]
128 129 130 131 132 133 134 135 136 137 138
    """

    def __init__(self, approximate=False, name=None):
        super(GELU, self).__init__()
        self._approximate = approximate
        self._name = name

    def forward(self, x):
        return F.gelu(x, self._approximate, self._name)


139
class Hardshrink(layers.Layer):
140
    r"""
141 142 143 144 145
    Hardshrink Activation

    .. math::

        hardshrink(x)=
146 147 148 149 150 151 152
            \\left\\{
            \\begin{aligned}
            &x, & & if \\ x > threshold \\\\
            &x, & & if \\ x < -threshold \\\\
            &0, & & if \\ others
            \\end{aligned}
            \\right.
153 154 155 156 157 158 159 160 161 162 163 164 165 166

    Parameters:
        threshold (float, optional): The value of threshold for hardthrink. Default is 0.5
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.

    Examples:

        .. code-block:: python

167
            import paddle
168

Z
zhupengyang 已提交
169
            x = paddle.to_tensor([-1, 0.3, 2.5])
170 171
            m = paddle.nn.Hardshrink()
            out = m(x) # [-1., 0., 2.5]
172 173 174 175 176 177 178 179
    """

    def __init__(self, threshold=0.5, name=None):
        super(Hardshrink, self).__init__()
        self._threshold = threshold
        self._name = name

    def forward(self, x):
180
        return F.hardshrink(x, self._threshold, self._name)
181 182


183
class Hardswish(layers.Layer):
184
    r"""
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228
    Hardswish activation

    Hardswish is proposed in MobileNetV3, and performs better in computational stability
    and efficiency compared to swish function. For more details please refer
    to: https://arxiv.org/pdf/1905.02244.pdf

    .. math::

        Hardswish(x)=
            \\left\\{
            \\begin{aligned}
            &0, & & \\text{if } x \\leq -3 \\\\
            &x, & & \\text{if } x \\geq 3 \\\\
            &\\frac{x(x+3)}{6}, & & \\text{otherwise}
            \\end{aligned}
            \\right.

    Parameters:
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.

    Examples:

        .. code-block:: python

            import paddle

            x = paddle.to_tensor([-4., 5., 1.])
            m = paddle.nn.Hardswish()
            out = m(x) # [0., 5., 0.666667]
    """

    def __init__(self, name=None):
        super(Hardswish, self).__init__()
        self._name = name

    def forward(self, x):
        return F.hardswish(x, self._name)


W
WangXi 已提交
229
class Tanh(layers.Layer):
230
    r"""
W
WangXi 已提交
231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
    Tanh Activation.

    .. math::
        Tanh(x) = \\frac{e^{x} - e^{-x}}{e^{x} + e^{-x}}

    Parameters:
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.

    Examples:

        .. code-block:: python

            import paddle
            import numpy as np

            x = paddle.to_tensor(np.array([-0.4, -0.2, 0.1, 0.3]))
            m = paddle.nn.Tanh()
            out = m(x)
W
WangXi 已提交
254
            print(out)
W
WangXi 已提交
255 256 257 258 259 260 261 262 263 264 265
            # [-0.37994896 -0.19737532  0.09966799  0.29131261]
    """

    def __init__(self, name=None):
        super(Tanh, self).__init__()
        self._name = name

    def forward(self, x):
        return F.tanh(x, self._name)


266
class Hardtanh(layers.Layer):
267
    r"""
268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
    Hardtanh Activation

    .. math::

        Hardtanh(x)= \\begin{cases}
                        max, \\text{if } x > max \\\\
                        min, \\text{if } x < min \\\\
                        x,  \\text{otherwise}
                      \\end{cases}

    Parameters:
        min (float, optional): The value of min for Hardtanh. Default is -1.
        max (float, optional): The value of max for Hardtanh. Default is 1.
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.
283

284 285 286
    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.
287

288 289 290 291 292
    Examples:
        .. code-block:: python

            import paddle

Z
zhupengyang 已提交
293
            x = paddle.to_tensor([-1.5, 0.3, 2.5])
294
            m = paddle.nn.Hardtanh()
Z
zhupengyang 已提交
295
            out = m(x) # [-1., 0.3, 1.]
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
    """

    def __init__(self, min=-1.0, max=1.0, name=None):
        super(Hardtanh, self).__init__()
        self._min = min
        self._max = max
        self._name = name

    def forward(self, x):
        return F.hardtanh(x, self._min, self._max, self._name)


class PReLU(layers.Layer):
    """
    PReLU Activation.

    .. math::

        PReLU(x) = max(0, x) + weight * min(0, x)

    Parameters:
        num_parameters (int, optional): Number of `weight` to learn. The supported values are:
318
            1 - a single parameter `alpha` is used for all input channels;
319 320 321
            Number of channels - a seperate `alpha` is used for each input channel.
            Default is 1.
        init (float, optional): Init value of learnable `weight`. Default is 0.25.
322
        weight_attr(ParamAttr, optional): The parameter attribute for the learnable `weight`.
323
            Default is None. For more information, please refer to :ref:`api_paddle_ParamAttr`.
324 325
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.
326

327
    Shape:
Q
Qi Li 已提交
328
        - input: Tensor with any shape. Default dtype is float32.
329
        - output: Tensor with the same shape as input.
330

331 332 333 334 335 336
    Examples:
        .. code-block:: python

            import paddle
            import numpy as np

Q
Qi Li 已提交
337
            paddle.set_default_dtype("float64")
338 339 340 341 342 343

            data = np.array([[[[-2.0,  3.0, -4.0,  5.0],
                            [ 3.0, -4.0,  5.0, -6.0],
                            [-7.0, -8.0,  8.0,  9.0]],
                            [[ 1.0, -2.0, -3.0,  4.0],
                            [-5.0,  6.0,  7.0, -8.0],
Q
Qi Li 已提交
344
                            [ 6.0,  7.0,  8.0,  9.0]]]], 'float64')
345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
            x = paddle.to_tensor(data)
            m = paddle.nn.PReLU(1, 0.25)
            out = m(x)
            # [[[[-0.5 ,  3.  , -1.  ,  5.  ],
            #    [ 3.  , -1.  ,  5.  , -1.5 ],
            #    [-1.75, -2.  ,  8.  ,  9.  ]],
            #   [[ 1.  , -0.5 , -0.75,  4.  ],
            #    [-1.25,  6.  ,  7.  , -2.  ],
            #    [ 6.  ,  7.  ,  8.  ,  9.  ]]]]
    """

    def __init__(self, num_parameters=1, init=0.25, weight_attr=None,
                 name=None):
        super(PReLU, self).__init__()
        self._num_parameters = num_parameters
        self._init = init
        self._weight_attr = weight_attr
        self._name = name

        self._weight = self.create_parameter(
            attr=self._weight_attr,
Q
Qi Li 已提交
366 367
            shape=[self._num_parameters],
            dtype=get_default_dtype(),
368
            is_bias=False,
Q
Qi Li 已提交
369
            default_initializer=Constant(self._init))
370 371 372 373 374

    def forward(self, x):
        return F.prelu(x, self._weight)


375 376 377 378
class ReLU(layers.Layer):
    """
    ReLU Activation.

379
    .. math::
380

381
        ReLU(x) = max(x, 0)
382 383

    Parameters:
384 385 386 387 388 389
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.
390

391 392 393
    Examples:
        .. code-block:: python

394
            import paddle
395

Z
zhupengyang 已提交
396
            x = paddle.to_tensor([-2., 0., 1.])
397 398
            m = paddle.nn.ReLU()
            out = m(x) # [0., 0., 1.]
399 400
    """

401
    def __init__(self, name=None):
402
        super(ReLU, self).__init__()
403
        self._name = name
404

405 406
    def forward(self, x):
        return F.relu(x, self._name)
407 408


409 410 411 412 413 414
class ReLU6(layers.Layer):
    """
    ReLU6 Activation

    .. math::

415
        ReLU6(x) = min(max(0,x), 6)
416 417 418 419 420 421 422 423 424 425 426 427

    Parameters:
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.

    Examples:
        .. code-block:: python

428 429
            import paddle
            import numpy as np
430

431 432 433
            x = paddle.to_tensor(np.array([-1, 0.3, 6.5]))
            m = paddle.nn.ReLU6()
            out = m(x) # [0, 0.3, 6]
434 435 436 437 438 439 440 441 442 443 444
    """

    def __init__(self, name=None):
        super(ReLU6, self).__init__()
        self._name = name

    def forward(self, x):
        return F.relu6(x, self._name)


class SELU(layers.Layer):
445
    r"""
446 447 448 449
    SELU Activation

    .. math::

450 451 452 453 454
        SELU(x)= scale *
                 \\begin{cases}
                   x, \\text{if } x > 0 \\\\
                   alpha * e^{x} - alpha, \\text{if } x <= 0
                 \\end{cases}
455 456

    Parameters:
457 458
        scale (float, optional): The value of scale(must be greater than 1.0) for SELU. Default is 1.0507009873554804934193349852946
        alpha (float, optional): The value of alpha(must be no less than zero) for SELU. Default is 1.6732632423543772848170429916717
459 460 461 462 463 464 465 466 467 468
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.

    Examples:
        .. code-block:: python

469 470
            import paddle
            import numpy as np
471

472
            x = paddle.to_tensor(np.array([[0.0, 1.0],[2.0, 3.0]]))
473 474
            m = paddle.nn.SELU()
            out = m(x) # [[0, 1.050701],[2.101402, 3.152103]]
475 476 477 478 479 480 481 482 483 484 485 486 487 488 489
    """

    def __init__(self,
                 scale=1.0507009873554804934193349852946,
                 alpha=1.6732632423543772848170429916717,
                 name=None):
        super(SELU, self).__init__()
        self._scale = scale
        self._alpha = alpha
        self._name = name

    def forward(self, x):
        return F.selu(x, self._scale, self._alpha, self._name)


C
ceci3 已提交
490
class LeakyReLU(layers.Layer):
491
    r"""
C
ceci3 已提交
492 493
    Leaky ReLU Activation.

494
    .. math::
C
ceci3 已提交
495

496
        LeakyReLU(x)=
497 498 499 500 501 502
            \\left\\{
            \\begin{aligned}
            &x, & & if \\ x >= 0 \\\\
            &negative\_slope * x, & & otherwise \\\\
            \\end{aligned}
            \\right. \\\\
C
ceci3 已提交
503 504

    Parameters:
505 506
        negative_slope (float, optional): Slope of the activation function at
            :math:`x < 0` . Default is 0.01.
507 508
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.
509

510 511 512
    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.
513

C
ceci3 已提交
514 515 516
    Examples:
        .. code-block:: python

517
            import paddle
C
Chen Long 已提交
518
            import numpy as np
519

520
            m = paddle.nn.LeakyReLU()
Z
zhupengyang 已提交
521
            x = paddle.to_tensor(np.array([-2, 0, 1], 'float32'))
522
            out = m(x)  # [-0.02, 0., 1.]
C
ceci3 已提交
523 524
    """

525
    def __init__(self, negative_slope=0.01, name=None):
C
ceci3 已提交
526
        super(LeakyReLU, self).__init__()
527
        self._negative_slope = negative_slope
528
        self._name = name
C
ceci3 已提交
529

530
    def forward(self, x):
531
        return F.leaky_relu(x, self._negative_slope, self._name)
C
ceci3 已提交
532 533


534 535
class Sigmoid(layers.Layer):
    """
536
    this interface is used to construct a callable object of the ``Sigmoid`` class. This layer calcluate the `sigmoid` of input x.
537

538
    .. math::
S
swtkiwi 已提交
539

540
        Sigmoid(x) = \\frac{1}{1 + e^{-x}}
541

542 543
    Parameters:
        name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
544

545 546
    Shape:
        x: N-D tensor, available dtype is float16, float32, float64.
547 548

    Returns:
549
        A callable object of Sigmoid.
550

551
    Examples:
552

553 554
        .. code-block:: python

555 556 557
          import paddle

          m = paddle.nn.Sigmoid()
558 559
          x = paddle.to_tensor([1.0, 2.0, 3.0, 4.0])
          out = m(x) # [0.7310586, 0.880797, 0.95257413, 0.98201376]
560 561
    """

562
    def __init__(self, name=None):
563
        super(Sigmoid, self).__init__()
564
        self.name = name
565

566 567
    def forward(self, x):
        return F.sigmoid(x, self.name)
568 569


570
class Hardsigmoid(layers.Layer):
571
    r"""
572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
    This interface is used to construct a callable object of the ``Hardsigmoid`` class.
    This layer calcluate the `hardsigmoid` of input x.

    A 3-part piecewise linear approximation of sigmoid(https://arxiv.org/abs/1603.00391),
    which is much faster than sigmoid.

    .. math::

        Hardsigmoid(x)=
            \\left\\{
            \\begin{aligned}
            &0, & & \\text{if } x \\leq -3 \\\\
            &1, & & \\text{if } x \\geq 3 \\\\
            &x/6 + 1/2, & & \\text{otherwise}
            \\end{aligned}
            \\right.

    Parameters:
        name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        x: N-D tensor, available dtype is float32, float64.

    Returns:
        A callable object of Hardsigmoid.

    Examples:

        .. code-block:: python

          import paddle

Z
zhupengyang 已提交
604
          m = paddle.nn.Hardsigmoid()
605 606 607 608 609 610 611 612 613 614 615 616
          x = paddle.to_tensor([-4., 5., 1.])
          out = m(x) # [0., 1, 0.666667]
    """

    def __init__(self, name=None):
        super(Hardsigmoid, self).__init__()
        self.name = name

    def forward(self, x):
        return F.hardsigmoid(x, self.name)


617
class Softplus(layers.Layer):
618
    r"""
619 620 621 622
    Softplus Activation

    .. math::

623 624
        Softplus(x) = \\frac{1}{beta} * \\log(1 + e^{beta * x}) \\\\
        \\text{For numerical stability, the implementation reverts to the linear function when: beta * x > threshold.}
625 626

    Parameters:
627 628
        beta (float, optional): The value of beta for Softplus. Default is 1
        threshold (float, optional): The value of threshold for Softplus. Default is 20
629 630 631 632 633 634 635 636 637 638
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.

    Examples:
        .. code-block:: python

639 640
            import paddle
            import numpy as np
641

642 643 644
            x = paddle.to_tensor(np.array([-0.4, -0.2, 0.1, 0.3]))
            m = paddle.nn.Softplus()
            out = m(x) # [0.513015, 0.598139, 0.744397, 0.854355]
645 646 647 648 649 650 651 652 653 654 655 656 657
    """

    def __init__(self, beta=1, threshold=20, name=None):
        super(Softplus, self).__init__()
        self._beta = beta
        self._threshold = threshold
        self._name = name

    def forward(self, x):
        return F.softplus(x, self._beta, self._threshold, self._name)


class Softshrink(layers.Layer):
658
    r"""
659 660 661 662
    Softshrink Activation

    .. math::

663 664 665 666 667
        Softshrink(x)= \\begin{cases}
                        x - threshold, \\text{if } x > threshold \\\\
                        x + threshold, \\text{if } x < -threshold \\\\
                        0,  \\text{otherwise}
                      \\end{cases}
668 669

    Parameters:
670
        threshold (float, optional): The value of threshold(must be no less than zero) for softplus. Default is 0.5
671 672 673 674 675 676 677 678 679 680
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.

    Examples:
        .. code-block:: python

681 682
            import paddle
            import numpy as np
683

684 685 686
            x = paddle.to_tensor(np.array([-0.9, -0.2, 0.1, 0.8]))
            m = paddle.nn.Softshrink()
            out = m(x) # [-0.4, 0, 0, 0.3]
687 688 689 690 691 692 693 694 695 696 697 698
    """

    def __init__(self, threshold=0.5, name=None):
        super(Softshrink, self).__init__()
        self._threshold = threshold
        self._name = name

    def forward(self, x):
        return F.softshrink(x, self._threshold, self._name)


class Softsign(layers.Layer):
699
    r"""
700 701 702 703
    Softsign Activation

    .. math::

704
        Softsign(x) = \\frac{x}{1 + |x|}
705 706 707 708 709 710 711 712 713 714 715 716

    Parameters:
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.

    Examples:
        .. code-block:: python

717 718
            import paddle
            import numpy as np
719

720 721 722
            x = paddle.to_tensor(np.array([-0.4, -0.2, 0.1, 0.3]))
            m = paddle.nn.Softsign()
            out = m(x) # [-0.285714, -0.166667, 0.0909091, 0.230769]
723 724 725 726 727 728 729 730 731 732
    """

    def __init__(self, name=None):
        super(Softsign, self).__init__()
        self._name = name

    def forward(self, x):
        return F.softsign(x, self._name)


733
class Swish(layers.Layer):
734
    r"""
735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767
    Swish Activation.

    .. math::

        Swish(x) = \\frac{x}{1 + e^{-x}}

    Parameters:
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.

    Examples:
        .. code-block:: python

            import paddle
            import numpy as np

            x = paddle.to_tensor(np.array([-2., 0., 1.]))
            m = paddle.nn.Swish()
            out = m(x) # [-0.238406, 0., 0.731059]
    """

    def __init__(self, name=None):
        super(Swish, self).__init__()
        self._name = name

    def forward(self, x):
        return F.swish(x, self._name)


768 769 770 771 772 773
class Tanhshrink(layers.Layer):
    """
    Tanhshrink Activation

    .. math::

774
        Tanhshrink(x) = x - tanh(x)
775 776 777 778 779 780 781 782 783 784 785 786

    Parameters:
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.

    Examples:
        .. code-block:: python

787 788
            import paddle
            import numpy as np
789

790 791 792
            x = paddle.to_tensor(np.array([-0.4, -0.2, 0.1, 0.3]))
            m = paddle.nn.Tanhshrink()
            out = m(x) # [-0.020051, -0.00262468, 0.000332005, 0.00868739]
793 794 795 796 797 798 799 800 801 802
    """

    def __init__(self, name=None):
        super(Tanhshrink, self).__init__()
        self._name = name

    def forward(self, x):
        return F.tanhshrink(x, self._name)


803
class ThresholdedReLU(layers.Layer):
804
    r"""
805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842
    Thresholded ReLU Activation

    .. math::

        ThresholdedReLU(x) = \\begin{cases}
                               x, \\text{if } x > threshold \\\\
                               0, \\text{otherwise}
                              \\end{cases}

    Parameters:
        threshold (float, optional): The value of threshold for ThresholdedReLU. Default is 1.0
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.

    Examples:
        .. code-block:: python

            import paddle
            import numpy as np

            x = paddle.to_tensor(np.array([2., 0., 1.]))
            m = paddle.nn.ThresholdedReLU()
            out = m(x) # [2., 0., 0.]
    """

    def __init__(self, threshold=1.0, name=None):
        super(ThresholdedReLU, self).__init__()
        self._threshold = threshold
        self._name = name

    def forward(self, x):
        return F.thresholded_relu(x, self._threshold, self._name)


843
class LogSigmoid(layers.Layer):
844
    r"""
845
    LogSigmoid Activation.
846

847
    .. math::
848

849
        LogSigmoid(x) = log \\frac{1}{1 + e^{-x}}
850 851 852 853 854

    Parameters:
        x (Tensor): The input Tensor with data type float32, or float64.
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.
855

856 857 858
    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.
859

860 861 862
    Examples:
        .. code-block:: python

863
            import paddle
864

865
            x = paddle.to_tensor([1.0, 2.0, 3.0, 4.0])
866 867
            m = paddle.nn.LogSigmoid()
            out = m(x) # [-0.313262 -0.126928 -0.0485874 -0.0181499]
868 869 870 871 872 873 874
    """

    def __init__(self, name=None):
        super(LogSigmoid, self).__init__()
        self._name = name

    def forward(self, x):
875
        return F.log_sigmoid(x, self._name)
876 877


878
class Softmax(layers.Layer):
879
    r"""
880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000
    Softmax Activation.

    This operator implements the softmax layer. The calculation process is as follows:

    1. The dimension :attr:`axis` of ``x`` will be permuted to the last.

    2. Then ``x`` will be logically flattened to a 2-D matrix. The matrix's second
    dimension(row length) is the same as the dimension :attr:`axis` of ``x``,
    and the first dimension(column length) is the product of all other dimensions
    of ``x``. For each row of the matrix, the softmax operator squashes the
    K-dimensional(K is the width of the matrix, which is also the size of ``x``'s
    dimension :attr:`axis`) vector of arbitrary real values to a K-dimensional
    vector of real values in the range [0, 1] that add up to 1.

    3. After the softmax operation is completed, the inverse operations of steps 1 and 2
    are performed to restore the two-dimensional matrix to the same dimension as the ``x`` .

    It computes the exponential of the given dimension and the sum of exponential
    values of all the other dimensions in the K-dimensional vector input.
    Then the ratio of the exponential of the given dimension and the sum of
    exponential values of all the other dimensions is the output of the softmax
    operator.

    For each row :math:`i` and each column :math:`j` in the matrix, we have:

    .. math::

        Softmax[i, j] = \\frac{\\exp(x[i, j])}{\\sum_j(exp(x[i, j])}

    Example:

    .. code-block:: text

        Case 1:
          Input:
            x.shape = [2, 3, 4]
            x.data = [[[2.0, 3.0, 4.0, 5.0],
                       [3.0, 4.0, 5.0, 6.0],
                       [7.0, 8.0, 8.0, 9.0]],
                      [[1.0, 2.0, 3.0, 4.0],
                       [5.0, 6.0, 7.0, 8.0],
                       [6.0, 7.0, 8.0, 9.0]]]

          Attrs:
            axis = -1

          Output:
            out.shape = [2, 3, 4]
            out.data = [[[0.0320586 , 0.08714432, 0.23688282, 0.64391426],
                         [0.0320586 , 0.08714432, 0.23688282, 0.64391426],
                         [0.07232949, 0.19661193, 0.19661193, 0.53444665]],
                        [[0.0320586 , 0.08714432, 0.23688282, 0.64391426],
                         [0.0320586 , 0.08714432, 0.23688282, 0.64391426],
                         [0.0320586 , 0.08714432, 0.23688282, 0.64391426]]]

        Case 2:
          Input:
            x.shape = [2, 3, 4]
            x.data = [[[2.0, 3.0, 4.0, 5.0],
                       [3.0, 4.0, 5.0, 6.0],
                       [7.0, 8.0, 8.0, 9.0]],
                      [[1.0, 2.0, 3.0, 4.0],
                       [5.0, 6.0, 7.0, 8.0],
                       [6.0, 7.0, 8.0, 9.0]]]
          Attrs:
            axis = 1

          Output:
            out.shape = [2, 3, 4]
            out.data = [[[0.00657326, 0.00657326, 0.01714783, 0.01714783],
                         [0.01786798, 0.01786798, 0.04661262, 0.04661262],
                         [0.97555875, 0.97555875, 0.93623955, 0.93623955]],
                        [[0.00490169, 0.00490169, 0.00490169, 0.00490169],
                         [0.26762315, 0.26762315, 0.26762315, 0.26762315],
                         [0.72747516, 0.72747516, 0.72747516, 0.72747516]]]

    Parameters:
        axis (int, optional): The axis along which to perform log_softmax
            calculations. It should be in range [-D, D), where D is the
            dimensions of ``x`` . If ``axis`` < 0, it works the same way as
            :math:`axis + D` . Default is -1.
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.

    Examples:
        .. code-block:: python

            import paddle
            import numpy as np

            x = np.array([[[2.0, 3.0, 4.0, 5.0],
                        [3.0, 4.0, 5.0, 6.0],
                        [7.0, 8.0, 8.0, 9.0]],
                        [[1.0, 2.0, 3.0, 4.0],
                        [5.0, 6.0, 7.0, 8.0],
                        [6.0, 7.0, 8.0, 9.0]]], 'float32')
            x = paddle.to_tensor(x)
            m = paddle.nn.Softmax()
            out = m(x)
            # [[[0.0320586 , 0.08714432, 0.23688282, 0.64391426],
            #   [0.0320586 , 0.08714432, 0.23688282, 0.64391426],
            #   [0.07232949, 0.19661193, 0.19661193, 0.53444665]],
            # [[0.0320586 , 0.08714432, 0.23688282, 0.64391426],
            #   [0.0320586 , 0.08714432, 0.23688282, 0.64391426],
            #   [0.0320586 , 0.08714432, 0.23688282, 0.64391426]]]
    """

    def __init__(self, axis=-1, name=None):
        super(Softmax, self).__init__()
        self._axis = axis
        self._dtype = None
        self._name = name

    def forward(self, x):
        return F.softmax(x, self._axis, self._dtype, self._name)


1001
class LogSoftmax(layers.Layer):
1002
    r"""
1003 1004 1005 1006
    This operator implements the log_softmax layer. The calculation process is as follows:

    .. math::

Z
zhupengyang 已提交
1007 1008 1009 1010
        \\begin{aligned} 
        Out[i, j] &= log(softmax(x)) \\\\
        &= log(\\frac{\\exp(X[i, j])}{\\sum_j(\\exp(X[i, j])})
        \\end{aligned}
1011 1012

    Parameters:
1013 1014 1015 1016 1017 1018
        axis (int, optional): The axis along which to perform log_softmax
            calculations. It should be in range [-D, D), where D is the
            dimensions of the input Tensor . If ``axis`` < 0, it works the
            same way as :math:`axis + D` . Default is -1.
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.
1019

1020 1021 1022
    Shape:
        - input: Tensor with any shape.
        - output: Tensor with the same shape as input.
1023 1024 1025 1026

    Examples:
        .. code-block:: python

1027 1028
            import paddle

Z
zhupengyang 已提交
1029 1030 1031 1032 1033 1034
            x = [[[-2.0, 3.0, -4.0, 5.0],
                  [3.0, -4.0, 5.0, -6.0],
                  [-7.0, -8.0, 8.0, 9.0]],
                 [[1.0, -2.0, -3.0, 4.0],
                  [-5.0, 6.0, 7.0, -8.0],
                  [6.0, 7.0, 8.0, 9.0]]]
1035 1036 1037 1038 1039 1040 1041 1042 1043
            m = paddle.nn.LogSoftmax()
            x = paddle.to_tensor(x)
            out = m(x)
            # [[[ -7.1278396   -2.1278396   -9.127839    -0.12783948]
            #   [ -2.1270514   -9.127051    -0.12705144 -11.127051  ]
            #   [-16.313261   -17.313261    -1.3132617   -0.31326184]]
            #  [[ -3.0518122   -6.051812    -7.051812    -0.051812  ]
            #   [-12.313267    -1.3132664   -0.3132665  -15.313267  ]
            #   [ -3.4401896   -2.4401896   -1.4401896   -0.44018966]]]
1044 1045
    """

1046
    def __init__(self, axis=-1, name=None):
1047 1048
        super(LogSoftmax, self).__init__()
        self._axis = axis
1049
        self._name = name
1050

1051 1052
    def forward(self, x):
        return F.log_softmax(x, self._axis)
1053 1054 1055


class Maxout(layers.Layer):
1056
    r"""
1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
    Maxout Activation.

    Assumed the input shape is (N, Ci, H, W).
    The output shape is (N, Co, H, W).
    Then Co = Ci/groups and the operator formula is as follows:

    .. math::

        &out_{si+j} = \max_{k} x_{gsi + sk + j} \\\\
        &g = groups \\\\
        &s = \\frac{input.size}{num\\_channels} \\\\
        &0 \\le i < \\frac{num\\_channels}{groups} \\\\
        &0 \\le j < s \\\\
        &0 \\le k < groups

    Parameters:
        groups (int, optional): The groups number of maxout. `groups` specifies the
            index of channel dimension where maxout will be performed. This must be
            a factor of number of features. Default is 1.
        axis (int, optional): The axis along which to perform maxout calculations.
            It should be 1 when data format is NCHW, be -1 or 3 when data format
            is NHWC. If ``axis`` < 0, it works the same way as :math:`axis + D` ,
            where D is the dimensions of ``x`` . Default is 1.
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Shape:
        - input: :math:`(N, C_{in}, H_{in}, W_{in})`
        - output: :math:`(N, C_{out}, H_{out}, W_{out})`

    Examples:
        .. code-block:: python

            import paddle

            x = paddle.rand([1, 2, 3, 4])
            # [[[[0.5002636  0.22272532 0.17402348 0.2874594 ]
            #    [0.95313174 0.6228939  0.7129065  0.7087491 ]
            #    [0.02879342 0.88725346 0.61093384 0.38833922]]
            #   [[0.5231306  0.03807496 0.91661984 0.15602879]
            #    [0.666127   0.616567   0.30741522 0.24044901]
            #    [0.7142536  0.7351477  0.31588817 0.23782359]]]]
            m = paddle.nn.Maxout(groups=2)
            out = m(x)
            # [[[[0.5231306  0.22272532 0.91661984 0.2874594 ]
            #    [0.95313174 0.6228939  0.7129065  0.7087491 ]
            #    [0.7142536  0.88725346 0.61093384 0.38833922]]]]
    """

    def __init__(self, groups, axis=1, name=None):
        super(Maxout, self).__init__()
        self._groups = groups
        self._axis = axis
        self._name = name

    def forward(self, x):
        return F.maxout(x, self._groups, self._axis, self._name)