initializer.py 49.7 KB
Newer Older
1
#   Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
import math
16
import functools
17
from . import framework
18
from . import core
19 20 21 22 23 24 25
from .framework import (
    _non_static_mode,
    in_dygraph_mode,
    _in_legacy_dygraph,
    default_main_program,
    _current_expected_place,
)
26
from .lazy_init import lazy_init_helper
27
from .framework import program_guard
28
import numpy as np
29
from .core import VarDesc
W
Wu Yi 已提交
30
from . import unique_name
31
from .data_feeder import check_variable_and_dtype, check_type, check_dtype
32
from paddle import _C_ops, _legacy_C_ops
33
import paddle
34

35
__all__ = [
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
    'Constant',
    'Uniform',
    'Normal',
    'TruncatedNormal',
    'Xavier',
    'Bilinear',
    'MSRA',
    'ConstantInitializer',
    'UniformInitializer',
    'NormalInitializer',
    'TruncatedNormalInitializer',
    'XavierInitializer',
    'BilinearInitializer',
    'MSRAInitializer',
    'NumpyArrayInitializer',
    'set_global_initializer',
52
]
53

54 55 56
_global_weight_initializer_ = None
_global_bias_initializer_ = None

57 58 59 60 61 62 63 64 65 66

class Initializer(object):
    """Base class for variable initializers

    Defines the common interface of variable initializers.
    They add operations to the init program that are used
    to initialize variables. Users should not use this class
    directly, but need to use one of its implementations.
    """

W
whs 已提交
67
    def __init__(self):
68 69
        pass

70
    def __call__(self, param, block=None):
71
        if not lazy_init_helper().state:
72 73 74 75 76
            return self.forward(param, block)

        return self._lazy_init(param, block)

    def forward(self, param, block=None):
77
        """Add corresponding initialization operations to the network"""
78 79
        raise NotImplementedError()

80
    def _lazy_init(self, param, block=None):
81 82 83
        """
        Apply lazy initialization
        """
84 85
        assert in_dygraph_mode()

86 87 88 89 90 91
        def init_op_creator(forward, param, block):
            new_var = param._to_static_var(True, block=block)
            # Record initializer operator
            with lazy_init_helper():
                forward(new_var, block)

92
        # Add hook function for initializing param in dygraph mode
93
        param.set_init_func(functools.partial(self.forward, param, block))
94 95 96
        param._init_op_creator = functools.partial(
            init_op_creator, self.forward, param
        )
97 98 99

        return param

100 101
    def _check_block(self, block):
        if block is None:
102
            block = default_main_program().global_block()
103 104 105

        return block

106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
    def _compute_fans(self, var):
        """Compute the fan_in and the fan_out for layers

        This method computes the fan_in and the fan_out
        for neural network layers, if not specified. It is
        not possible to perfectly estimate fan_in and fan_out.
        This method will estimate it correctly for matrix multiply and
        convolutions.

        Args:
            var: variable for which fan_in and fan_out have to be computed

        Returns:
            tuple of two integers (fan_in, fan_out)
        """
        shape = var.shape
        if not shape or len(shape) == 0:
            fan_in = fan_out = 1
        elif len(shape) == 1:
            fan_in = fan_out = shape[0]
        elif len(shape) == 2:
            # This is the case for simple matrix multiply
            fan_in = shape[0]
            fan_out = shape[1]
        else:
            # Assume this to be a convolutional kernel
            # In PaddlePaddle, the shape of the kernel is like:
            # [num_filters, num_filter_channels, ...] where the remaining
            # dimensions are the filter_size
            receptive_field_size = np.prod(shape[2:])
            fan_in = shape[1] * receptive_field_size
            fan_out = shape[0] * receptive_field_size

        return (fan_in, fan_out)

141 142 143

class ConstantInitializer(Initializer):
    """Implements the constant initializer
144 145

    Args:
146
        value (float32): constant value to initialize the variable
147 148 149 150

    Examples:
        .. code-block:: python

151 152 153
            import paddle
            import paddle.fluid as fluid
            paddle.enable_static()
D
Double_V 已提交
154
            x = fluid.data(name="data", shape=[8, 32, 32], dtype="float32")
155 156 157 158
            fc = fluid.layers.fc(
                input=x,
                size=10,
                param_attr=fluid.initializer.Constant(value=2.0))
159

160 161
    """

162
    def __init__(self, value=0.0, force_cpu=False):
163 164 165
        assert value is not None
        super(ConstantInitializer, self).__init__()
        self._value = value
166
        self._force_cpu = force_cpu
167

168
    def forward(self, var, block=None):
169
        """Initialize the input tensor with constant.
170 171

        Args:
172 173 174
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
175 176

        Returns:
177
            The initialization op
178
        """
179 180
        block = self._check_block(block)

181 182 183
        assert isinstance(var, framework.Variable) or isinstance(
            var, framework.EagerParamBase
        )
184
        assert isinstance(block, framework.Block)
185

186 187
        if in_dygraph_mode():
            place = _current_expected_place()
188 189
            if self._force_cpu:
                place = core.CPUPlace()
190 191 192
            _C_ops.full_(
                var, var.shape, str(float(self._value)), var.dtype, place
            )
193 194
            return None
        elif _in_legacy_dygraph():
195 196 197 198 199 200 201 202 203 204 205 206 207
            _legacy_C_ops.fill_constant(
                var,
                'value',
                float(self._value),
                'force_cpu',
                self._force_cpu,
                'dtype',
                int(var.dtype),
                'str_value',
                str(float(self._value)),
                'shape',
                var.shape,
            )
208 209
            return None
        else:
210 211 212 213 214 215 216 217 218 219 220 221
            op = block.append_op(
                type="fill_constant",
                outputs={"Out": var},
                attrs={
                    "shape": var.shape,
                    "dtype": int(var.dtype),
                    "value": float(self._value),
                    'str_value': str(float(self._value)),
                    'force_cpu': self._force_cpu,
                },
                stop_gradient=True,
            )
222

223
            var.op = op
224
            return op
225 226 227


class UniformInitializer(Initializer):
228
    """Implements the random uniform distribution initializer
229 230 231 232 233

    Args:
        low (float): lower boundary of the uniform distribution
        high (float): upper boundary of the uniform distribution
        seed (int): random seed
234 235 236 237 238 239
        diag_num (int): the number of diagonal elements to initialize.
            If set to 0, diagonal initialization will be not performed.
        diag_step (int): Step size between two diagonal elements,
            which is generally the width of the square matrix.
        diag_val (float): the value of the diagonal element to be initialized,
            default 1.0. It takes effect only if the diag_num is greater than 0.
240 241 242 243

    Examples:
        .. code-block:: python

X
xiaoting 已提交
244
            import paddle.fluid as fluid
245
            x = fluid.data(name='x', shape=[None, 1], dtype='float32')
246
            fc = fluid.layers.fc(input=x, size=10,
247
                param_attr=fluid.initializer.Uniform(low=-0.5, high=0.5))
248 249
    """

250 251 252
    def __init__(
        self, low=-1.0, high=1.0, seed=0, diag_num=0, diag_step=0, diag_val=1.0
    ):
253 254
        assert low is not None
        assert high is not None
255
        assert high >= low
256
        assert seed is not None
257 258 259 260
        assert diag_num is not None
        assert diag_step is not None
        assert diag_val is not None
        if diag_num > 0 or diag_step > 0:
261
            assert diag_num > 0 and diag_step > 0
262 263 264 265
        super(UniformInitializer, self).__init__()
        self._low = low
        self._high = high
        self._seed = seed
266 267 268
        self._diag_num = diag_num
        self._diag_step = diag_step
        self._diag_val = diag_val
269

270
    def forward(self, var, block=None):
271
        """Initialize the input tensor with Uniform distribution.
272 273

        Args:
274 275 276
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
277 278

        Returns:
279
            The initialization op
280
        """
281 282
        block = self._check_block(block)

283
        assert isinstance(block, framework.Block)
284 285 286 287 288 289
        check_variable_and_dtype(
            var,
            "Out",
            ["uint16", "float16", "float32", "float64"],
            "uniform_random",
        )
290

D
dzhwinter 已提交
291 292
        if self._seed == 0:
            self._seed = block.program.random_seed
W
Wu Yi 已提交
293

X
polish  
Xin Pan 已提交
294
        # to be compatible of fp16 initializers
295
        if var.dtype == VarDesc.VarType.FP16:
W
Wu Yi 已提交
296
            out_dtype = VarDesc.VarType.FP32
297 298 299 300 301 302 303 304 305
            out_var = block.create_var(
                name=unique_name.generate(
                    ".".join(['uniform_random', var.name, 'tmp'])
                ),
                shape=var.shape,
                dtype=out_dtype,
                type=VarDesc.VarType.LOD_TENSOR,
                persistable=False,
            )
W
Wu Yi 已提交
306 307 308 309
        else:
            out_dtype = var.dtype
            out_var = var

J
Jiabin Yang 已提交
310
        if framework._non_static_mode():
311
            if in_dygraph_mode():
312
                out_var = _C_ops.uniform(
313 314 315 316 317 318 319
                    var.shape,
                    out_dtype,
                    self._low,
                    self._high,
                    self._seed,
                    _current_expected_place(),
                )
320 321
            elif _in_legacy_dygraph():
                out_var = _legacy_C_ops.uniform_random(
322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338
                    'shape',
                    var.shape,
                    'min',
                    self._low,
                    'max',
                    self._high,
                    'seed',
                    self._seed,
                    'dtype',
                    out_dtype,
                    'diag_num',
                    self._diag_num,
                    'diag_step',
                    self._diag_step,
                    'diag_val',
                    self._diag_val,
                )
339
            if var.dtype == VarDesc.VarType.FP16:
340 341 342
                if in_dygraph_mode():
                    var_tmp = _C_ops.cast(out_var, var.dtype)
                elif _in_legacy_dygraph():
343 344 345 346 347 348 349
                    var_tmp = _legacy_C_ops.cast(
                        out_var,
                        'in_dtype',
                        out_var.dtype,
                        'out_dtype',
                        var.dtype,
                    )
350
                var_tmp._share_underline_tensor_to(var)
351
            else:
352
                out_var._share_underline_tensor_to(var)
353 354
            return None
        else:
355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
            op = block.append_op(
                type="uniform_random",
                inputs={},
                outputs={"Out": out_var},
                attrs={
                    "shape": var.shape,
                    "dtype": out_dtype,
                    "min": self._low,
                    "max": self._high,
                    "seed": self._seed,
                    "diag_num": self._diag_num,
                    "diag_step": self._diag_step,
                    "diag_val": self._diag_val,
                },
                stop_gradient=True,
            )
371 372

            if var.dtype == VarDesc.VarType.FP16:
373 374 375 376 377 378
                block.append_op(
                    type="cast",
                    inputs={"X": out_var},
                    outputs={"Out": var},
                    attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype},
                )
W
Wu Yi 已提交
379

380
            var.op = op
381
            return op
382 383 384


class NormalInitializer(Initializer):
385 386 387 388 389 390 391 392 393 394
    """Implements the Random Normal(Gaussian) distribution initializer

    Args:
        loc (float): mean of the normal distribution
        scale (float): standard deviation of the normal distribution
        seed (int): random seed

    Examples:
        .. code-block:: python

X
xsrobin 已提交
395
            import paddle.fluid as fluid
396
            x = fluid.data(name="data", shape=[None, 32, 32], dtype="float32")
X
xsrobin 已提交
397 398
            fc = fluid.layers.fc(input=x, size=10,
                param_attr=fluid.initializer.Normal(loc=0.0, scale=2.0))
399

400 401 402 403 404 405 406 407 408 409 410
    """

    def __init__(self, loc=0.0, scale=1.0, seed=0):
        assert loc is not None
        assert scale is not None
        assert seed is not None
        super(NormalInitializer, self).__init__()
        self._mean = loc
        self._std_dev = scale
        self._seed = seed

411
    def forward(self, var, block=None):
412
        """Initialize the input tensor with Normal distribution.
413 414

        Args:
415 416 417
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
418 419

        Returns:
420
            The initialization op
421
        """
422 423
        block = self._check_block(block)

424
        assert isinstance(block, framework.Block)
425

426 427 428 429 430 431
        check_variable_and_dtype(
            var,
            "Out",
            ["uint16", "float16", "float32", "float64"],
            "guassian_random",
        )
432

D
dzhwinter 已提交
433 434
        if self._seed == 0:
            self._seed = block.program.random_seed
W
Wu Yi 已提交
435

436 437
        if in_dygraph_mode():
            place = _current_expected_place()
438
            out_var = _C_ops.gaussian(
439 440 441 442 443 444 445
                var.shape,
                self._mean,
                self._std_dev,
                self._seed,
                var.dtype,
                place,
            )
446
            out_var._share_underline_tensor_to(var)
447 448 449
            return None

        if _in_legacy_dygraph():
450
            out_var = _legacy_C_ops.gaussian_random(
451 452 453 454 455 456 457 458 459 460 461 462 463
                'shape',
                var.shape,
                'dtype',
                var.dtype,
                'mean',
                self._mean,
                'std',
                self._std_dev,
                'seed',
                self._seed,
                'use_mkldnn',
                False,
            )
464

465
            out_var._share_underline_tensor_to(var)
466 467
            return None
        else:
468 469 470 471 472 473 474 475 476 477 478 479 480
            op = block.append_op(
                type="gaussian_random",
                outputs={"Out": var},
                attrs={
                    "shape": var.shape,
                    "dtype": var.dtype,
                    "mean": self._mean,
                    "std": self._std_dev,
                    "seed": self._seed,
                    "use_mkldnn": False,
                },
                stop_gradient=True,
            )
481
            var.op = op
482
            return op
483 484


485 486 487 488 489 490 491 492 493 494 495
class TruncatedNormalInitializer(Initializer):
    """Implements the Random TruncatedNormal(Gaussian) distribution initializer

    Args:
        loc (float): mean of the normal distribution
        scale (float): standard deviation of the normal distribution
        seed (int): random seed

    Examples:
        .. code-block:: python

X
xiaoting 已提交
496
            import paddle.fluid as fluid
497
            x = fluid.data(name='x', shape=[None, 1], dtype='float32')
498 499 500 501 502 503 504 505
            fc = fluid.layers.fc(input=x, size=10,
                param_attr=fluid.initializer.TruncatedNormal(loc=0.0, scale=2.0))
    """

    def __init__(self, loc=0.0, scale=1.0, seed=0):
        assert loc is not None
        assert scale is not None
        assert seed is not None
W
whs 已提交
506
        super(TruncatedNormalInitializer, self).__init__()
507 508 509 510
        self._mean = loc
        self._std_dev = scale
        self._seed = seed

511
    def forward(self, var, block=None):
512
        """Initialize the input tensor with TruncatedNormal distribution.
513 514

        Args:
515 516 517
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
518 519

        Returns:
520
            The initialization op
521
        """
522 523
        block = self._check_block(block)

524 525
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
526

527 528
        if self._seed == 0:
            self._seed = block.program.random_seed
529 530

        # to be compatible of fp16 initalizers
531
        if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
532
            out_dtype = VarDesc.VarType.FP32
533 534 535 536 537 538 539 540 541
            out_var = block.create_var(
                name=unique_name.generate(
                    ".".join(['truncated_gaussian_random', var.name, 'tmp'])
                ),
                shape=var.shape,
                dtype=out_dtype,
                type=VarDesc.VarType.LOD_TENSOR,
                persistable=False,
            )
542 543 544 545
        else:
            out_dtype = var.dtype
            out_var = var

546
        if in_dygraph_mode():
547
            out_var = _C_ops.truncated_gaussian_random(
548 549 550 551 552 553 554
                var.shape,
                self._mean,
                self._std_dev,
                self._seed,
                out_dtype,
                _current_expected_place(),
            )
555
            if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
556
                var_tmp = _C_ops.cast(out_var, var.dtype)
557 558 559 560 561 562
                var_tmp._share_underline_tensor_to(var)
            else:
                out_var._share_underline_tensor_to(var)
            return None

        if _in_legacy_dygraph():
563
            out_var = _legacy_C_ops.truncated_gaussian_random(
564 565 566 567 568 569 570 571 572 573 574
                'shape',
                var.shape,
                'dtype',
                out_dtype,
                'mean',
                self._mean,
                'std',
                self._std_dev,
                'seed',
                self._seed,
            )
575
            if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
576 577 578
                var_tmp = _legacy_C_ops.cast(
                    out_var, 'in_dtype', out_var.dtype, 'out_dtype', var.dtype
                )
579
                var_tmp._share_underline_tensor_to(var)
580
            else:
581
                out_var._share_underline_tensor_to(var)
582 583
            return None
        else:
584 585 586 587 588 589 590 591 592 593 594 595
            op = block.append_op(
                type="truncated_gaussian_random",
                outputs={"Out": out_var},
                attrs={
                    "shape": var.shape,
                    "dtype": out_dtype,
                    "mean": self._mean,
                    "std": self._std_dev,
                    "seed": self._seed,
                },
                stop_gradient=True,
            )
596

597
            if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
598 599 600 601 602 603
                block.append_op(
                    type="cast",
                    inputs={"X": out_var},
                    outputs={"Out": var},
                    attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype},
                )
604
            var.op = op
605
            return op
606 607


608
class XavierInitializer(Initializer):
609
    r"""
610
    This class implements the Xavier weight initializer from the paper
Q
qiaolongfei 已提交
611 612 613
    `Understanding the difficulty of training deep feedforward neural
    networks <http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf>`_
    by Xavier Glorot and Yoshua Bengio.
614 615 616

    This initializer is designed to keep the scale of the gradients
    approximately same in all the layers. In case of Uniform distribution,
Q
qiaolongfei 已提交
617 618 619 620 621 622
    the range is [-x, x], where

    .. math::

        x = \sqrt{\\frac{6.0}{fan\_in + fan\_out}}

623
    In case of Normal distribution, the mean is 0 and the standard deviation
Q
qiaolongfei 已提交
624
    is
625

Q
qiaolongfei 已提交
626
    .. math::
627

Q
qiaolongfei 已提交
628
        \sqrt{\\frac{2.0}{fan\_in + fan\_out}}
629 630


Q
qiaolongfei 已提交
631
    Args:
X
xiaoting 已提交
632 633
        uniform (bool,default True): whether to use uniform ,if False use normal distribution
        fan_in (float,default None): fan_in for Xavier initialization. If None, it is
Q
qiaolongfei 已提交
634
                inferred from the variable.
X
xiaoting 已提交
635
        fan_out (float,default None): fan_out for Xavier initialization. If None, it is
Q
qiaolongfei 已提交
636 637 638 639 640 641 642 643 644
                 inferred from the variable.
        seed (int): random seed

    Note:
        It is recommended to set fan_in and fan_out to None for most cases.

    Examples:
        .. code-block:: python

X
xiaoting 已提交
645
            import paddle.fluid as fluid
X
xiaoting 已提交
646
            queries = fluid.data(name='x', shape=[None,1], dtype='float32')
Q
qiaolongfei 已提交
647 648 649 650 651 652 653
            fc = fluid.layers.fc(
                input=queries, size=10,
                param_attr=fluid.initializer.Xavier(uniform=False))

    """

    def __init__(self, uniform=True, fan_in=None, fan_out=None, seed=0):
654 655 656 657 658 659 660 661
        assert uniform is not None
        assert seed is not None
        super(XavierInitializer, self).__init__()
        self._uniform = uniform
        self._fan_in = fan_in
        self._fan_out = fan_out
        self._seed = seed

662
    def forward(self, var, block=None):
663
        """Initialize the input tensor with Xavier initialization.
664 665

        Args:
666 667 668
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
669 670

        Returns:
671
            The initialization op
672
        """
673 674
        block = self._check_block(block)

675
        assert isinstance(block, framework.Block)
676 677 678 679 680 681
        check_variable_and_dtype(
            var,
            "Out",
            ["uint16", "float16", "float32", "float64"],
            "xavier_init",
        )
682

683 684 685 686 687 688
        f_in, f_out = self._compute_fans(var)

        # If fan_in and fan_out are passed, use them
        fan_in = f_in if self._fan_in is None else self._fan_in
        fan_out = f_out if self._fan_out is None else self._fan_out

D
dzhwinter 已提交
689 690 691
        if self._seed == 0:
            self._seed = block.program.random_seed

692
        # to be compatible of fp16 initalizers
693
        if var.dtype == VarDesc.VarType.FP16 or (
694 695
            var.dtype == VarDesc.VarType.BF16 and not self._uniform
        ):
696
            out_dtype = VarDesc.VarType.FP32
697 698 699 700 701 702 703 704 705
            out_var = block.create_var(
                name=unique_name.generate(
                    ".".join(['xavier_init', var.name, 'tmp'])
                ),
                shape=var.shape,
                dtype=out_dtype,
                type=VarDesc.VarType.LOD_TENSOR,
                persistable=False,
            )
706 707 708 709
        else:
            out_dtype = var.dtype
            out_var = var

J
Jiabin Yang 已提交
710
        if framework._non_static_mode():
711
            if self._uniform:
712
                limit = math.sqrt(6.0 / float(fan_in + fan_out))
713
                if in_dygraph_mode():
714
                    out_var = _C_ops.uniform(
715 716 717 718 719 720 721
                        out_var.shape,
                        out_dtype,
                        -limit,
                        limit,
                        self._seed,
                        _current_expected_place(),
                    )
722
                elif _in_legacy_dygraph():
723
                    out_var = _legacy_C_ops.uniform_random(
724 725 726 727 728 729 730 731 732 733 734
                        'shape',
                        out_var.shape,
                        'min',
                        -limit,
                        'max',
                        limit,
                        'seed',
                        self._seed,
                        'dtype',
                        out_dtype,
                    )
735
            else:
736
                std = math.sqrt(2.0 / float(fan_in + fan_out))
737 738 739

                if in_dygraph_mode():
                    place = _current_expected_place()
740
                    out_var = _C_ops.gaussian(
741 742
                        out_var.shape, 0.0, std, self._seed, out_dtype, place
                    )
743
                else:
744
                    out_var = _legacy_C_ops.gaussian_random(
745 746 747 748 749 750 751 752 753 754 755
                        'shape',
                        out_var.shape,
                        'dtype',
                        out_dtype,
                        'mean',
                        0.0,
                        'std',
                        std,
                        'seed',
                        self._seed,
                    )
756 757

            if var.dtype == VarDesc.VarType.FP16 or (
758 759
                var.dtype == VarDesc.VarType.BF16 and not self._uniform
            ):
760
                if in_dygraph_mode():
761
                    var_tmp = _C_ops.cast(out_var, var.dtype)
762
                elif _in_legacy_dygraph():
763 764 765 766 767 768 769
                    var_tmp = _legacy_C_ops.cast(
                        out_var,
                        'in_dtype',
                        out_var.dtype,
                        'out_dtype',
                        var.dtype,
                    )
770
                var_tmp._share_underline_tensor_to(var)
771
            else:
772
                out_var._share_underline_tensor_to(var)
773
            return None
774
        else:
775
            if self._uniform:
776
                limit = math.sqrt(6.0 / float(fan_in + fan_out))
777 778 779 780 781 782 783 784 785 786 787 788 789
                op = block.append_op(
                    type="uniform_random",
                    inputs={},
                    outputs={"Out": out_var},
                    attrs={
                        "shape": out_var.shape,
                        "dtype": out_dtype,
                        "min": -limit,
                        "max": limit,
                        "seed": self._seed,
                    },
                    stop_gradient=True,
                )
790
            else:
791
                std = math.sqrt(2.0 / float(fan_in + fan_out))
792 793 794 795 796 797 798 799 800 801 802 803
                op = block.append_op(
                    type="gaussian_random",
                    outputs={"Out": out_var},
                    attrs={
                        "shape": out_var.shape,
                        "dtype": out_var.dtype,
                        "mean": 0.0,
                        "std": std,
                        "seed": self._seed,
                    },
                    stop_gradient=True,
                )
804 805

            if var.dtype == VarDesc.VarType.FP16 or (
806 807 808 809 810 811 812 813
                var.dtype == VarDesc.VarType.BF16 and not self._uniform
            ):
                block.append_op(
                    type="cast",
                    inputs={"X": out_var},
                    outputs={"Out": var},
                    attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype},
                )
814

815
            var.op = op
816
            return op
817 818 819


class MSRAInitializer(Initializer):
820
    r"""Implements the MSRA initializer a.k.a. Kaiming Initializer
821 822

    This class implements the weight initialization from the paper
823 824 825 826 827 828 829 830
    `Delving Deep into Rectifiers: Surpassing Human-Level Performance on
    ImageNet Classification <https://arxiv.org/abs/1502.01852>`_
    by Kaiming He, Xiangyu Zhang, Shaoqing Ren and Jian Sun. This is a
    robust initialization method that particularly considers the rectifier
    nonlinearities. In case of Uniform distribution, the range is [-x, x], where

    .. math::

831
        x = gain \times \sqrt{\frac{3}{fan\_in}}
832 833 834 835 836 837

    In case of Normal distribution, the mean is 0 and the standard deviation
    is

    .. math::

838
        \frac{gain}{\sqrt{{fan\_in}}}
839 840

    Args:
841 842 843
        uniform (bool, optional): whether to use uniform or normal distribution
        fan_in (float32|None, optional): fan_in (in_features) of trainable Tensor, If None, it will be infered automaticly. If you don't want to use in_features of the Tensor, you can set the value of 'fan_in' smartly by yourself. default is None.
        seed (int32, optional): random seed.
844 845
        negative_slope (float, optional): negative_slope (only used with leaky_relu). default is 0.0.
        nonlinearity(str, optional): the non-linear function. default is relu.
846 847 848 849 850 851

    Note:
        It is recommended to set fan_in to None for most cases.

    Examples:
        .. code-block:: python
X
xsrobin 已提交
852

853
            import paddle
X
xsrobin 已提交
854
            import paddle.fluid as fluid
855
            paddle.enable_static()
D
Double_V 已提交
856
            x = fluid.data(name="data", shape=[8, 32, 32], dtype="float32")
X
xsrobin 已提交
857 858
            fc = fluid.layers.fc(input=x, size=10,
                param_attr=fluid.initializer.MSRA(uniform=False))
859

860 861
    """

862 863 864 865 866 867 868 869 870
    def __init__(
        self,
        uniform=True,
        fan_in=None,
        seed=0,
        negative_slope=0,
        nonlinearity='relu',
    ):
        """Constructor for MSRAInitializer"""
871 872 873 874 875 876
        assert uniform is not None
        assert seed is not None
        super(MSRAInitializer, self).__init__()
        self._uniform = uniform
        self._fan_in = fan_in
        self._seed = seed
877 878
        self._negative_slope = negative_slope
        self._nonlinearity = nonlinearity
879

880
    def forward(self, var, block=None):
881
        """Initialize the input tensor with MSRA initialization.
882 883

        Args:
884 885 886
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
887 888

        Returns:
889
            The initialization op
890
        """
891 892
        block = self._check_block(block)

893 894 895 896 897 898 899
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        f_in, f_out = self._compute_fans(var)

        # If fan_in is passed, use it
        fan_in = f_in if self._fan_in is None else self._fan_in

D
dzhwinter 已提交
900 901 902
        if self._seed == 0:
            self._seed = block.program.random_seed

903
        # to be compatible of fp16 initalizers
904
        if var.dtype == VarDesc.VarType.FP16 or (
905 906
            var.dtype == VarDesc.VarType.BF16 and not self._uniform
        ):
907
            out_dtype = VarDesc.VarType.FP32
908 909 910 911 912 913 914 915 916
            out_var = block.create_var(
                name=unique_name.generate(
                    ".".join(['masra_init', var.name, 'tmp'])
                ),
                shape=var.shape,
                dtype=out_dtype,
                type=VarDesc.VarType.LOD_TENSOR,
                persistable=False,
            )
917 918 919 920
        else:
            out_dtype = var.dtype
            out_var = var

J
Jiabin Yang 已提交
921
        if framework._non_static_mode():
922
            if self._uniform:
923 924
                gain = calculate_gain(self._nonlinearity, self._negative_slope)
                limit = gain * math.sqrt(3.0 / float(fan_in))
925
                if in_dygraph_mode():
926
                    out_var = _C_ops.uniform(
927 928 929 930 931 932 933
                        var.shape,
                        out_dtype,
                        -limit,
                        limit,
                        self._seed,
                        _current_expected_place(),
                    )
934 935
                else:
                    out_var = _legacy_C_ops.uniform_random(
936 937 938 939 940 941 942 943 944 945 946
                        'shape',
                        out_var.shape,
                        'min',
                        -limit,
                        'max',
                        limit,
                        'seed',
                        self._seed,
                        'dtype',
                        int(out_dtype),
                    )
947
            else:
948 949
                gain = calculate_gain(self._nonlinearity, self._negative_slope)
                std = gain / math.sqrt(float(fan_in))
950 951
                if in_dygraph_mode():
                    place = _current_expected_place()
952
                    out_var = _C_ops.gaussian(
953 954
                        out_var.shape, 0.0, std, self._seed, out_dtype, place
                    )
955
                else:
956
                    out_var = _legacy_C_ops.gaussian_random(
957 958 959 960 961 962 963 964 965 966 967
                        'shape',
                        out_var.shape,
                        'dtype',
                        int(out_dtype),
                        'mean',
                        0.0,
                        'std',
                        std,
                        'seed',
                        self._seed,
                    )
968 969

            if var.dtype == VarDesc.VarType.FP16 or (
970 971
                var.dtype == VarDesc.VarType.BF16 and not self._uniform
            ):
972 973 974
                if in_dygraph_mode():
                    var_tmp = _C_ops.cast(out_var, var.dtype)
                elif _in_legacy_dygraph():
975 976 977 978 979 980 981
                    var_tmp = _legacy_C_ops.cast(
                        out_var,
                        'in_dtype',
                        out_var.dtype,
                        'out_dtype',
                        var.dtype,
                    )
982
                var_tmp._share_underline_tensor_to(var)
983
            else:
984
                out_var._share_underline_tensor_to(var)
985
            return None
986
        else:
987
            if self._uniform:
988 989
                gain = calculate_gain(self._nonlinearity, self._negative_slope)
                limit = gain * math.sqrt(3.0 / float(fan_in))
990 991 992 993 994 995 996 997 998 999 1000 1001 1002
                op = block.append_op(
                    type="uniform_random",
                    inputs={},
                    outputs={"Out": out_var},
                    attrs={
                        "shape": out_var.shape,
                        "dtype": int(out_dtype),
                        "min": -limit,
                        "max": limit,
                        "seed": self._seed,
                    },
                    stop_gradient=True,
                )
1003 1004

            else:
1005 1006
                gain = calculate_gain(self._nonlinearity, self._negative_slope)
                std = gain / math.sqrt(float(fan_in))
1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
                op = block.append_op(
                    type="gaussian_random",
                    outputs={"Out": out_var},
                    attrs={
                        "shape": out_var.shape,
                        "dtype": int(out_dtype),
                        "mean": 0.0,
                        "std": std,
                        "seed": self._seed,
                    },
                    stop_gradient=True,
                )
1019 1020

            if var.dtype == VarDesc.VarType.FP16 or (
1021 1022 1023 1024 1025 1026 1027 1028
                var.dtype == VarDesc.VarType.BF16 and not self._uniform
            ):
                block.append_op(
                    type="cast",
                    inputs={"X": out_var},
                    outputs={"Out": var},
                    attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype},
                )
1029

1030
            var.op = op
1031
            return op
1032 1033


1034
class BilinearInitializer(Initializer):
1035
    """
1036 1037 1038
    This initializer can be used in transposed convolution operator to
    act as upsampling. Users can upsample a feature map with shape of
    (B, C, H, W) by any integer factor. The usage is:
1039 1040 1041 1042 1043

    Examples:

        .. code-block:: python

1044
            import math
1045 1046 1047 1048 1049

            import paddle
            import paddle.nn as nn
            from paddle.regularizer import L2Decay

X
xsrobin 已提交
1050 1051
            factor = 2
            C = 2
D
Double_V 已提交
1052 1053
            B = 8
            H = W = 32
1054 1055 1056 1057
            w_attr = paddle.ParamAttr(learning_rate=0.,
                                      regularizer=L2Decay(0.),
                                      initializer=nn.initializer.Bilinear())
            data = paddle.rand([B, 3, H, W], dtype='float32')
C
cnn 已提交
1058
            conv_up = nn.Conv2DTranspose(3,
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069
                                         out_channels=C,
                                         kernel_size=2 * factor - factor % 2,
                                         padding=int(
                                             math.ceil((factor - 1) / 2.)),
                                         stride=factor,
                                         weight_attr=w_attr,
                                         bias_attr=False)
            x = conv_up(data)

    Where, `out_channels=C` and `groups=C` means this is channel-wise transposed
    convolution. The filter shape will be (C, 1, K, K) where K is `kernel_size`,
1070 1071 1072 1073
    This initializer will set a (K, K) interpolation kernel for every channel
    of the filter identically. The resulting shape of the output feature map
    will be (B, C, factor * H, factor * W). Note that the learning rate and the
    weight decay are set to 0 in order to keep coefficient values of bilinear
1074 1075
    interpolation unchanged during training.

1076 1077 1078
    """

    def __init__(self):
1079
        """Constructor for BilinearInitializer."""
1080 1081
        super(BilinearInitializer, self).__init__()

1082
    def forward(self, var, block=None):
1083
        """Initialize the input tensor with Bilinear initialization.
1084 1085

        Args:
1086 1087 1088
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
1089 1090

        Returns:
1091
            The initialization op
1092
        """
1093 1094
        block = self._check_block(block)

1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
        if not isinstance(var, framework.Variable):
            raise ValueError("var must be framework.Variable.")

        if not isinstance(block, framework.Block):
            raise ValueError("block must be framework.Block.")

        shape = var.shape
        if len(shape) != 4:
            raise ValueError("the length of shape must be 4.")
        if shape[2] != shape[3]:
            raise ValueError("shape[2] must be equal to shape[3].")

        weight = np.zeros(np.prod(var.shape), dtype='float32')
        size = shape[3]
        # factor
1110
        f = np.ceil(size / 2.0)
1111
        # center
1112
        c = (2 * f - 1 - f % 2) / (2.0 * f)
1113 1114 1115 1116 1117 1118
        for i in range(np.prod(shape)):
            x = i % size
            y = (i / size) % size
            weight[i] = (1 - abs(x / f - c)) * (1 - abs(y / f - c))
        weight = np.reshape(weight, shape)

1119
        # to be compatible of fp16 initalizers
1120
        if var.dtype in [
1121 1122 1123
            VarDesc.VarType.FP16,
            VarDesc.VarType.BF16,
            VarDesc.VarType.FP64,
1124
        ]:
1125
            out_dtype = VarDesc.VarType.FP32
1126 1127 1128 1129 1130 1131 1132 1133 1134
            out_var = block.create_var(
                name=unique_name.generate(
                    ".".join(['bilinear_init', var.name, 'tmp'])
                ),
                shape=var.shape,
                dtype=out_dtype,
                type=VarDesc.VarType.LOD_TENSOR,
                persistable=False,
            )
1135 1136 1137 1138 1139
        else:
            out_dtype = var.dtype
            out_var = var

        if out_dtype == VarDesc.VarType.FP32:
1140 1141 1142
            value_name = "fp32_values"
            values = [float(v) for v in weight.flat]
        else:
1143 1144
            raise TypeError("Unsupported dtype %s", var.dtype)

1145 1146
        if np.prod(shape) > 1024 * 1024:
            raise ValueError("The size of input is too big. ")
1147

J
Jiabin Yang 已提交
1148
        if framework._non_static_mode():
1149
            if in_dygraph_mode():
1150 1151 1152 1153 1154 1155 1156
                _C_ops.assign_value_(
                    out_var,
                    list(shape),
                    out_dtype,
                    values,
                    _current_expected_place(),
                )
1157
            elif _in_legacy_dygraph():
1158 1159 1160 1161 1162 1163 1164 1165 1166
                _legacy_C_ops.assign_value(
                    out_var,
                    'shape',
                    list(shape),
                    'dtype',
                    out_dtype,
                    value_name,
                    values,
                )
1167
            if var.dtype in [
1168 1169 1170
                VarDesc.VarType.FP16,
                VarDesc.VarType.BF16,
                VarDesc.VarType.FP64,
1171
            ]:
1172 1173 1174
                if in_dygraph_mode():
                    var_tmp = _C_ops.cast(out_var, var.dtype)
                elif _in_legacy_dygraph():
1175 1176 1177 1178 1179 1180 1181
                    var_tmp = _legacy_C_ops.cast(
                        out_var,
                        'in_dtype',
                        out_var.dtype,
                        'out_dtype',
                        var.dtype,
                    )
1182
                var_tmp._share_underline_tensor_to(var)
1183
            else:
1184
                out_var._share_underline_tensor_to(var)
1185 1186
            return None
        else:
1187 1188 1189 1190 1191 1192 1193 1194 1195
            op = block.append_op(
                type='assign_value',
                outputs={'Out': [out_var]},
                attrs={
                    'dtype': out_dtype,
                    'shape': list(shape),
                    value_name: values,
                },
            )
1196 1197

            if var.dtype in [
1198 1199 1200
                VarDesc.VarType.FP16,
                VarDesc.VarType.BF16,
                VarDesc.VarType.FP64,
1201
            ]:
1202 1203 1204 1205 1206 1207
                block.append_op(
                    type="cast",
                    inputs={"X": out_var},
                    outputs={"Out": var},
                    attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype},
                )
1208

1209
            var.op = op
1210
            return op
1211 1212


1213 1214
class NumpyArrayInitializer(Initializer):
    """Init an parameter with an numpy array
1215
    This op initialize the variable by numpy array.
1216 1217 1218 1219

    Args:
        value (numpy): numpy array to initialize the variable

1220 1221 1222
    Returns:
        A Tensor variable initialized by numpy.

1223 1224 1225
    Examples:
        .. code-block:: python

1226
            import paddle.fluid as fluid
1227 1228
            import numpy
            x = fluid.data(name="x", shape=[2, 1], dtype='float32')
1229 1230 1231 1232 1233 1234
            fc = fluid.layers.fc(input=x, size=10,
                param_attr=fluid.initializer.NumpyArrayInitializer(numpy.array([1,2])))
    """

    def __init__(self, value):
        import numpy
1235

1236 1237 1238 1239
        assert isinstance(value, numpy.ndarray)
        super(NumpyArrayInitializer, self).__init__()
        self._value = value

1240
    def forward(self, var, block=None):
1241
        """Initialize the input tensor with Numpy array.
1242 1243

        Args:
1244 1245 1246
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
1247 1248

        Returns:
1249
            The initialization op
1250
        """
1251 1252
        block = self._check_block(block)

1253 1254
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
1255 1256

        # to be compatible of fp16 initalizers
1257
        if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
1258 1259
            out_dtype = VarDesc.VarType.FP32
            np_value = self._value.astype("float32")
1260 1261 1262 1263 1264 1265 1266 1267 1268
            out_var = block.create_var(
                name=unique_name.generate(
                    ".".join(['numpy_array_init', var.name, 'tmp'])
                ),
                shape=var.shape,
                dtype=out_dtype,
                type=VarDesc.VarType.LOD_TENSOR,
                persistable=False,
            )
1269 1270 1271 1272 1273 1274
        else:
            out_var = var
            out_dtype = var.dtype
            np_value = self._value

        if out_dtype == VarDesc.VarType.FP32:
1275
            value_name = "fp32_values"
1276 1277
            values = [float(v) for v in np_value.flat]
        elif out_dtype == VarDesc.VarType.INT32:
1278
            value_name = "int32_values"
1279
            values = [int(v) for v in np_value.flat]
1280 1281
        else:
            raise ValueError("Unsupported dtype %s", self._value.dtype)
X
Xin Pan 已提交
1282
        if self._value.size > 1024 * 1024 * 1024:
1283 1284 1285 1286
            raise ValueError(
                "The size of input is too big. Please consider "
                "saving it to file and 'load_op' to load it"
            )
1287

J
Jiabin Yang 已提交
1288
        if framework._non_static_mode():
1289
            if in_dygraph_mode():
1290 1291 1292 1293 1294 1295 1296
                _C_ops.assign_value_(
                    out_var,
                    list(self._value.shape),
                    out_dtype,
                    values,
                    _current_expected_place(),
                )
1297
            elif _in_legacy_dygraph():
1298 1299 1300 1301 1302 1303 1304 1305 1306
                _legacy_C_ops.assign_value(
                    out_var,
                    'shape',
                    list(self._value.shape),
                    'dtype',
                    out_dtype,
                    value_name,
                    values,
                )
1307
            if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
1308 1309 1310
                if in_dygraph_mode():
                    var_tmp = _C_ops.cast(out_var, var.dtype)
                elif _in_legacy_dygraph():
1311 1312 1313 1314 1315 1316 1317
                    var_tmp = _legacy_C_ops.cast(
                        out_var,
                        'in_dtype',
                        out_var.dtype,
                        'out_dtype',
                        var.dtype,
                    )
1318
                var_tmp._share_underline_tensor_to(var)
1319
            else:
1320
                out_var._share_underline_tensor_to(var)
1321 1322
            return None
        else:
1323 1324 1325 1326 1327 1328 1329 1330 1331 1332
            op = block.append_op(
                type='assign_value',
                outputs={'Out': out_var},
                attrs={
                    'dtype': out_dtype,
                    'shape': list(self._value.shape),
                    value_name: values,
                },
                stop_gradient=True,
            )
1333 1334

            if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
1335 1336 1337 1338 1339 1340
                block.append_op(
                    type="cast",
                    inputs={"X": out_var},
                    outputs={"Out": var},
                    attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype},
                )
1341

1342
            var.op = op
1343
            return op
1344 1345


1346 1347 1348 1349 1350 1351
def set_global_initializer(weight_init, bias_init=None):
    """
    This API is used to set up global model parameter initializer in framework.

    After this API is invoked, the global initializer will takes effect in subsequent code.

1352
    The model parameters include ``weight`` and ``bias`` . In the framework, they correspond
1353
    to ``paddle.ParamAttr`` , which is inherited from ``paddle.Tensor`` , and is a persistable Variable.
1354
    This API only takes effect for model parameters, not for variables created through apis such as
1355
    :ref:`api_fluid_layers_create_global_var` , :ref:`api_fluid_layers_create_tensor`.
1356

1357 1358 1359 1360 1361 1362 1363
    If the initializer is also set up by ``param_attr`` or ``bias_attr`` when creating a network layer,
    the global initializer setting here will not take effect because it has a lower priority.

    If you want to cancel the global initializer in framework, please set global initializer to ``None`` .

    Args:
        weight_init (Initializer): set the global initializer for ``weight`` of model parameters.
1364
        bias_init (Initializer, optional): set the global initializer for ``bias`` of model parameters.
1365 1366 1367 1368 1369 1370 1371 1372
            Default: None.

    Returns:
        None

    Examples:
        .. code-block:: python

1373 1374 1375 1376 1377
            import paddle
            import paddle.nn as nn

            nn.initializer.set_global_initializer(nn.initializer.Uniform(), nn.initializer.Constant())
            x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.)
1378 1379 1380

            # The weight of conv1 is initialized by Uniform
            # The bias of conv1 is initialized by Constant
1381 1382
            conv1 = nn.Conv2D(4, 6, (3, 3))
            y_var1 = conv1(x_var)
1383 1384 1385 1386

            # If set param_attr/bias_attr too, global initializer will not take effect
            # The weight of conv2 is initialized by Xavier
            # The bias of conv2 is initialized by Normal
1387
            conv2 = nn.Conv2D(4, 6, (3, 3),
1388 1389 1390
                weight_attr=nn.initializer.XavierUniform(),
                bias_attr=nn.initializer.Normal())
            y_var2 = conv2(x_var)
1391 1392

            # Cancel the global initializer in framework, it will takes effect in subsequent code
1393
            nn.initializer.set_global_initializer(None)
1394
    """
1395

1396 1397 1398 1399 1400 1401
    check_type(
        weight_init,
        'weight_init',
        (Initializer, type(None)),
        'set_global_initializer',
    )
1402 1403 1404
    global _global_weight_initializer_
    _global_weight_initializer_ = weight_init

1405 1406 1407 1408 1409 1410
    check_type(
        bias_init,
        'bias_init',
        (Initializer, type(None)),
        'set_global_initializer',
    )
1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428
    global _global_bias_initializer_
    _global_bias_initializer_ = bias_init


def _global_weight_initializer():
    """
    Return the global weight initializer, The user doesn't need to use it.
    """
    return _global_weight_initializer_


def _global_bias_initializer():
    """
    Return the global weight initializer, The user doesn't need to use it.
    """
    return _global_bias_initializer_


1429 1430
def calculate_gain(nonlinearity, param=None):
    """
1431
    Get the recommended ``gain`` value of some nonlinearity function. ``gain`` value can be used in some
1432
    ``paddle.nn.initializer`` api to adjust the initialization value.
1433 1434

    Args:
1435
        nonlinearity(str): name of nonlinearity activation function. If it is a linear function, such as:
1436
            `linear/conv1d/conv2d/conv3d/conv1d_transpose/conv2d_transpose/conv3d_transpose` , 1.0 will be returned.
1437
        param(bool|int|float, optional): optional parameter for somme nonlinearity function. Now, it only applies to
1438
            'leaky_relu'. Default: None, it will be calculated as 0.01 in the formula.
1439 1440

    Returns:
1441
        A float value, which is the recommended gain for this nonlinearity function.
1442 1443 1444

    Examples:
        .. code-block:: python
1445

1446 1447 1448
            import paddle
            gain = paddle.nn.initializer.calculate_gain('tanh') # 5.0 / 3
            gain = paddle.nn.initializer.calculate_gain('leaky_relu', param=1.0) # 1.0 = math.sqrt(2.0 / (1+param^2))
1449
            initializer = paddle.nn.initializer.Orthogonal(gain)
1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462

    """
    if param is None:
        param = 0.01
    else:
        assert isinstance(param, (bool, int, float))
        param = float(param)
    recommended_gain = {
        'sigmoid': 1,
        'linear': 1,
        'conv1d': 1,
        'conv2d': 1,
        'conv3d': 1,
1463 1464 1465
        'conv1d_transpose': 1,
        'conv2d_transpose': 1,
        'conv3d_transpose': 1,
1466 1467 1468
        'tanh': 5.0 / 3,
        'relu': math.sqrt(2.0),
        'leaky_relu': math.sqrt(2.0 / (1 + param**2)),
1469
        'selu': 3.0 / 4,
1470 1471 1472 1473
    }
    if nonlinearity in recommended_gain.keys():
        return recommended_gain[nonlinearity]
    else:
1474 1475
        raise ValueError(
            "nonlinearity function {} is not suppported now.".format(
1476 1477 1478
                nonlinearity
            )
        )
1479 1480


1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492
# We short the class name, since users will use the initializer with the package
# name. The sample code:
#
# import paddle.fluid as fluid
#
# hidden = fluid.layers.fc(...,
#                          param_attr=ParamAttr(fluid.initializer.Xavier()))
#
# It is no need to add an `Initializer` as the class suffix
Constant = ConstantInitializer
Uniform = UniformInitializer
Normal = NormalInitializer
1493
TruncatedNormal = TruncatedNormalInitializer
1494 1495
Xavier = XavierInitializer
MSRA = MSRAInitializer
1496
Bilinear = BilinearInitializer