initializer.py 48.6 KB
Newer Older
1
#   Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15 16
from __future__ import print_function

17
import math
18
from . import framework
19
from . import core
20
from .framework import _non_static_mode, in_dygraph_mode, _in_legacy_dygraph, default_main_program, _current_expected_place
21
import numpy as np
22
from .core import VarDesc
W
Wu Yi 已提交
23
from . import unique_name
24
from .data_feeder import check_variable_and_dtype, check_type, check_dtype
25
from paddle import _C_ops
26
import paddle
27

28
__all__ = [
29
    'Constant', 'Uniform', 'Normal', 'TruncatedNormal', 'Xavier', 'Bilinear',
30 31
    'MSRA', 'ConstantInitializer', 'UniformInitializer', 'NormalInitializer',
    'TruncatedNormalInitializer', 'XavierInitializer', 'BilinearInitializer',
32
    'MSRAInitializer', 'NumpyArrayInitializer', 'set_global_initializer'
33
]
34

35 36 37
_global_weight_initializer_ = None
_global_bias_initializer_ = None

38 39 40 41 42 43 44 45 46 47

class Initializer(object):
    """Base class for variable initializers

    Defines the common interface of variable initializers.
    They add operations to the init program that are used
    to initialize variables. Users should not use this class
    directly, but need to use one of its implementations.
    """

W
whs 已提交
48
    def __init__(self):
49 50
        pass

51
    def __call__(self, param, block=None):
52 53 54 55
        """Add corresponding initialization operations to the network
        """
        raise NotImplementedError()

56 57
    def _check_block(self, block):
        if block is None:
58
            block = default_main_program().global_block()
59 60 61

        return block

62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
    def _compute_fans(self, var):
        """Compute the fan_in and the fan_out for layers

        This method computes the fan_in and the fan_out
        for neural network layers, if not specified. It is
        not possible to perfectly estimate fan_in and fan_out.
        This method will estimate it correctly for matrix multiply and
        convolutions.

        Args:
            var: variable for which fan_in and fan_out have to be computed

        Returns:
            tuple of two integers (fan_in, fan_out)
        """
        shape = var.shape
        if not shape or len(shape) == 0:
            fan_in = fan_out = 1
        elif len(shape) == 1:
            fan_in = fan_out = shape[0]
        elif len(shape) == 2:
            # This is the case for simple matrix multiply
            fan_in = shape[0]
            fan_out = shape[1]
        else:
            # Assume this to be a convolutional kernel
            # In PaddlePaddle, the shape of the kernel is like:
            # [num_filters, num_filter_channels, ...] where the remaining
            # dimensions are the filter_size
            receptive_field_size = np.prod(shape[2:])
            fan_in = shape[1] * receptive_field_size
            fan_out = shape[0] * receptive_field_size

        return (fan_in, fan_out)

97 98 99

class ConstantInitializer(Initializer):
    """Implements the constant initializer
100 101

    Args:
D
Double_V 已提交
102
        value (float32): constant value to initialize the variable 
103 104 105 106

    Examples:
        .. code-block:: python

107 108 109
            import paddle
            import paddle.fluid as fluid
            paddle.enable_static()
D
Double_V 已提交
110
            x = fluid.data(name="data", shape=[8, 32, 32], dtype="float32")
111 112 113 114
            fc = fluid.layers.fc(
                input=x,
                size=10,
                param_attr=fluid.initializer.Constant(value=2.0))
115

116 117
    """

118
    def __init__(self, value=0.0, force_cpu=False):
119 120 121
        assert value is not None
        super(ConstantInitializer, self).__init__()
        self._value = value
122
        self._force_cpu = force_cpu
123

124 125
    def __call__(self, var, block=None):
        """Initialize the input tensor with constant.
126 127

        Args:
128 129 130
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
131 132

        Returns:
133
            The initialization op
134
        """
135 136
        block = self._check_block(block)

137 138
        assert (isinstance(var, framework.Variable)
                or isinstance(var, framework.EagerParamBase))
139
        assert isinstance(block, framework.Block)
140

141 142
        if in_dygraph_mode():
            place = _current_expected_place()
143 144
            if self._force_cpu:
                place = core.CPUPlace()
145 146 147 148
            _C_ops.final_state_full_(var, var.shape, str(float(self._value)),
                                     var.dtype, place)
            return None
        elif _in_legacy_dygraph():
149 150
            _C_ops.fill_constant(var, 'value', float(self._value),
                                 'force_cpu', self._force_cpu, 'dtype',
151 152
                                 int(var.dtype), 'str_value',
                                 str(float(self._value)), 'shape', var.shape)
153 154
            return None
        else:
155 156 157 158 159 160 161 162 163 164
            op = block.append_op(type="fill_constant",
                                 outputs={"Out": var},
                                 attrs={
                                     "shape": var.shape,
                                     "dtype": int(var.dtype),
                                     "value": float(self._value),
                                     'str_value': str(float(self._value)),
                                     'force_cpu': self._force_cpu
                                 },
                                 stop_gradient=True)
165

166
            var.op = op
167
            return op
168 169 170


class UniformInitializer(Initializer):
171
    """Implements the random uniform distribution initializer
172 173 174 175 176

    Args:
        low (float): lower boundary of the uniform distribution
        high (float): upper boundary of the uniform distribution
        seed (int): random seed
177 178 179 180 181 182
        diag_num (int): the number of diagonal elements to initialize.
            If set to 0, diagonal initialization will be not performed.
        diag_step (int): Step size between two diagonal elements,
            which is generally the width of the square matrix.
        diag_val (float): the value of the diagonal element to be initialized,
            default 1.0. It takes effect only if the diag_num is greater than 0.
183 184 185 186

    Examples:
        .. code-block:: python

X
xiaoting 已提交
187
            import paddle.fluid as fluid
188
            x = fluid.data(name='x', shape=[None, 1], dtype='float32')
189
            fc = fluid.layers.fc(input=x, size=10,
190
    		param_attr=fluid.initializer.Uniform(low=-0.5, high=0.5))
191 192
    """

193 194 195 196 197 198 199
    def __init__(self,
                 low=-1.0,
                 high=1.0,
                 seed=0,
                 diag_num=0,
                 diag_step=0,
                 diag_val=1.0):
200 201
        assert low is not None
        assert high is not None
202
        assert high >= low
203
        assert seed is not None
204 205 206 207 208
        assert diag_num is not None
        assert diag_step is not None
        assert diag_val is not None
        if diag_num > 0 or diag_step > 0:
            assert (diag_num > 0 and diag_step > 0)
209 210 211 212
        super(UniformInitializer, self).__init__()
        self._low = low
        self._high = high
        self._seed = seed
213 214 215
        self._diag_num = diag_num
        self._diag_step = diag_step
        self._diag_val = diag_val
216

217 218
    def __call__(self, var, block=None):
        """Initialize the input tensor with Uniform distribution.
219 220

        Args:
221 222 223
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
224 225

        Returns:
226
            The initialization op
227
        """
228 229
        block = self._check_block(block)

230
        assert isinstance(block, framework.Block)
231 232
        check_variable_and_dtype(var, "Out",
                                 ["uint16", "float16", "float32", "float64"],
233 234
                                 "uniform_random")

D
dzhwinter 已提交
235 236
        if self._seed == 0:
            self._seed = block.program.random_seed
W
Wu Yi 已提交
237

X
polish  
Xin Pan 已提交
238
        # to be compatible of fp16 initializers
239
        if var.dtype == VarDesc.VarType.FP16:
W
Wu Yi 已提交
240
            out_dtype = VarDesc.VarType.FP32
241 242 243 244 245 246
            out_var = block.create_var(name=unique_name.generate(".".join(
                ['uniform_random', var.name, 'tmp'])),
                                       shape=var.shape,
                                       dtype=out_dtype,
                                       type=VarDesc.VarType.LOD_TENSOR,
                                       persistable=False)
W
Wu Yi 已提交
247 248 249 250
        else:
            out_dtype = var.dtype
            out_var = var

J
Jiabin Yang 已提交
251
        if framework._non_static_mode():
252 253 254 255 256 257 258
            out_var = _C_ops.uniform_random(
                'shape', var.shape, 'min', self._low, 'max', self._high, 'seed',
                self._seed, 'dtype', out_dtype, 'diag_num', self._diag_num,
                'diag_step', self._diag_step, 'diag_val', self._diag_val)
            if var.dtype == VarDesc.VarType.FP16:
                var_tmp = _C_ops.cast(out_var, 'in_dtype', out_var.dtype,
                                      'out_dtype', var.dtype)
259
                var_tmp._share_underline_tensor_to(var)
260
            else:
261
                out_var._share_underline_tensor_to(var)
262 263
            return None
        else:
264 265 266 267 268 269 270 271 272 273 274 275 276 277
            op = block.append_op(type="uniform_random",
                                 inputs={},
                                 outputs={"Out": out_var},
                                 attrs={
                                     "shape": var.shape,
                                     "dtype": out_dtype,
                                     "min": self._low,
                                     "max": self._high,
                                     "seed": self._seed,
                                     "diag_num": self._diag_num,
                                     "diag_step": self._diag_step,
                                     "diag_val": self._diag_val
                                 },
                                 stop_gradient=True)
278 279

            if var.dtype == VarDesc.VarType.FP16:
280 281 282 283 284 285 286
                block.append_op(type="cast",
                                inputs={"X": out_var},
                                outputs={"Out": var},
                                attrs={
                                    "in_dtype": out_var.dtype,
                                    "out_dtype": var.dtype
                                })
W
Wu Yi 已提交
287

288
            var.op = op
289
            return op
290 291 292


class NormalInitializer(Initializer):
293 294 295 296 297 298 299 300 301 302
    """Implements the Random Normal(Gaussian) distribution initializer

    Args:
        loc (float): mean of the normal distribution
        scale (float): standard deviation of the normal distribution
        seed (int): random seed

    Examples:
        .. code-block:: python

X
xsrobin 已提交
303
            import paddle.fluid as fluid
304
            x = fluid.data(name="data", shape=[None, 32, 32], dtype="float32")
X
xsrobin 已提交
305 306
            fc = fluid.layers.fc(input=x, size=10,
                param_attr=fluid.initializer.Normal(loc=0.0, scale=2.0))
307

308 309 310 311 312 313 314 315 316 317 318
    """

    def __init__(self, loc=0.0, scale=1.0, seed=0):
        assert loc is not None
        assert scale is not None
        assert seed is not None
        super(NormalInitializer, self).__init__()
        self._mean = loc
        self._std_dev = scale
        self._seed = seed

319 320
    def __call__(self, var, block=None):
        """Initialize the input tensor with Normal distribution.
321 322

        Args:
323 324 325
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
326 327

        Returns:
328
            The initialization op
329
        """
330 331
        block = self._check_block(block)

332
        assert isinstance(block, framework.Block)
333

334 335
        check_variable_and_dtype(var, "Out",
                                 ["uint16", "float16", "float32", "float64"],
336
                                 "guassian_random")
337

338 339 340
        # to be compatible of fp16 initalizers
        if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
            out_dtype = VarDesc.VarType.FP32
341 342 343 344 345 346
            out_var = block.create_var(name=unique_name.generate(".".join(
                ['normal_init', var.name, 'tmp'])),
                                       shape=var.shape,
                                       dtype=out_dtype,
                                       type=VarDesc.VarType.LOD_TENSOR,
                                       persistable=False)
347 348 349 350
        else:
            out_dtype = var.dtype
            out_var = var

D
dzhwinter 已提交
351 352
        if self._seed == 0:
            self._seed = block.program.random_seed
W
Wu Yi 已提交
353

354 355 356 357 358 359 360 361 362 363 364 365 366 367
        if in_dygraph_mode():
            place = _current_expected_place()
            out_var = _C_ops.final_state_gaussian_random(
                var.shape, self._mean, self._std_dev, self._seed, out_dtype,
                place)

            if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
                var_tmp = _C_ops.final_state_cast(out_var, var.dtype)
                var_tmp._share_underline_tensor_to(var)
            else:
                out_var._share_underline_tensor_to(var)
            return None

        if _in_legacy_dygraph():
368 369 370 371
            out_var = _C_ops.gaussian_random('shape', var.shape, 'dtype',
                                             out_dtype, 'mean', self._mean,
                                             'std', self._std_dev, 'seed',
                                             self._seed, 'use_mkldnn', False)
372 373 374 375 376 377 378

            if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
                var_tmp = _C_ops.cast(out_var, 'in_dtype', out_var.dtype,
                                      'out_dtype', var.dtype)
                var_tmp._share_underline_tensor_to(var)
            else:
                out_var._share_underline_tensor_to(var)
379 380
            return None
        else:
381 382 383 384 385 386 387 388 389 390 391
            op = block.append_op(type="gaussian_random",
                                 outputs={"Out": out_var},
                                 attrs={
                                     "shape": var.shape,
                                     "dtype": out_dtype,
                                     "mean": self._mean,
                                     "std": self._std_dev,
                                     "seed": self._seed,
                                     "use_mkldnn": False
                                 },
                                 stop_gradient=True)
392

393
            if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
394 395 396 397 398 399 400
                block.append_op(type="cast",
                                inputs={"X": out_var},
                                outputs={"Out": var},
                                attrs={
                                    "in_dtype": out_var.dtype,
                                    "out_dtype": var.dtype
                                })
401
            var.op = op
402
            return op
403 404


405 406 407 408 409 410 411 412 413 414 415
class TruncatedNormalInitializer(Initializer):
    """Implements the Random TruncatedNormal(Gaussian) distribution initializer

    Args:
        loc (float): mean of the normal distribution
        scale (float): standard deviation of the normal distribution
        seed (int): random seed

    Examples:
        .. code-block:: python

X
xiaoting 已提交
416
            import paddle.fluid as fluid
417
            x = fluid.data(name='x', shape=[None, 1], dtype='float32')
418 419 420 421 422 423 424 425
            fc = fluid.layers.fc(input=x, size=10,
                param_attr=fluid.initializer.TruncatedNormal(loc=0.0, scale=2.0))
    """

    def __init__(self, loc=0.0, scale=1.0, seed=0):
        assert loc is not None
        assert scale is not None
        assert seed is not None
W
whs 已提交
426
        super(TruncatedNormalInitializer, self).__init__()
427 428 429 430
        self._mean = loc
        self._std_dev = scale
        self._seed = seed

431 432
    def __call__(self, var, block=None):
        """Initialize the input tensor with TruncatedNormal distribution.
433 434

        Args:
435 436 437
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
438 439

        Returns:
440
            The initialization op
441
        """
442 443
        block = self._check_block(block)

444 445
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
446

447 448
        if self._seed == 0:
            self._seed = block.program.random_seed
449 450

        # to be compatible of fp16 initalizers
451
        if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
452
            out_dtype = VarDesc.VarType.FP32
453 454 455 456 457 458
            out_var = block.create_var(name=unique_name.generate(".".join(
                ['truncated_gaussian_random', var.name, 'tmp'])),
                                       shape=var.shape,
                                       dtype=out_dtype,
                                       type=VarDesc.VarType.LOD_TENSOR,
                                       persistable=False)
459 460 461 462
        else:
            out_dtype = var.dtype
            out_var = var

463 464 465 466 467 468 469 470 471 472 473 474
        if in_dygraph_mode():
            out_var = _C_ops.final_state_truncated_gaussian_random(
                var.shape, self._mean, self._std_dev, self._seed, out_dtype,
                _current_expected_place())
            if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
                var_tmp = _C_ops.final_state_cast(out_var, var.dtype)
                var_tmp._share_underline_tensor_to(var)
            else:
                out_var._share_underline_tensor_to(var)
            return None

        if _in_legacy_dygraph():
475 476 477 478 479
            out_var = _C_ops.truncated_gaussian_random('shape', var.shape,
                                                       'dtype', out_dtype,
                                                       'mean', self._mean,
                                                       'std', self._std_dev,
                                                       'seed', self._seed)
480 481 482
            if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
                var_tmp = _C_ops.cast(out_var, 'in_dtype', out_var.dtype,
                                      'out_dtype', var.dtype)
483
                var_tmp._share_underline_tensor_to(var)
484
            else:
485
                out_var._share_underline_tensor_to(var)
486 487
            return None
        else:
488 489 490 491 492 493 494 495 496 497
            op = block.append_op(type="truncated_gaussian_random",
                                 outputs={"Out": out_var},
                                 attrs={
                                     "shape": var.shape,
                                     "dtype": out_dtype,
                                     "mean": self._mean,
                                     "std": self._std_dev,
                                     "seed": self._seed
                                 },
                                 stop_gradient=True)
498

499
            if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
500 501 502 503 504 505 506
                block.append_op(type="cast",
                                inputs={"X": out_var},
                                outputs={"Out": var},
                                attrs={
                                    "in_dtype": out_var.dtype,
                                    "out_dtype": var.dtype
                                })
507
            var.op = op
508
            return op
509 510


511
class XavierInitializer(Initializer):
512
    r"""
513
    This class implements the Xavier weight initializer from the paper
Q
qiaolongfei 已提交
514 515 516
    `Understanding the difficulty of training deep feedforward neural
    networks <http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf>`_
    by Xavier Glorot and Yoshua Bengio.
517 518 519

    This initializer is designed to keep the scale of the gradients
    approximately same in all the layers. In case of Uniform distribution,
Q
qiaolongfei 已提交
520 521 522 523 524 525
    the range is [-x, x], where

    .. math::

        x = \sqrt{\\frac{6.0}{fan\_in + fan\_out}}

526
    In case of Normal distribution, the mean is 0 and the standard deviation
Q
qiaolongfei 已提交
527
    is
528

Q
qiaolongfei 已提交
529
    .. math::
530

Q
qiaolongfei 已提交
531
        \sqrt{\\frac{2.0}{fan\_in + fan\_out}}
532 533


Q
qiaolongfei 已提交
534
    Args:
X
xiaoting 已提交
535 536
        uniform (bool,default True): whether to use uniform ,if False use normal distribution
        fan_in (float,default None): fan_in for Xavier initialization. If None, it is
Q
qiaolongfei 已提交
537
                inferred from the variable.
X
xiaoting 已提交
538
        fan_out (float,default None): fan_out for Xavier initialization. If None, it is
Q
qiaolongfei 已提交
539 540 541 542 543 544 545 546 547
                 inferred from the variable.
        seed (int): random seed

    Note:
        It is recommended to set fan_in and fan_out to None for most cases.

    Examples:
        .. code-block:: python

X
xiaoting 已提交
548
            import paddle.fluid as fluid
X
xiaoting 已提交
549
            queries = fluid.data(name='x', shape=[None,1], dtype='float32')
Q
qiaolongfei 已提交
550 551 552 553 554 555 556
            fc = fluid.layers.fc(
                input=queries, size=10,
                param_attr=fluid.initializer.Xavier(uniform=False))

    """

    def __init__(self, uniform=True, fan_in=None, fan_out=None, seed=0):
557 558 559 560 561 562 563 564
        assert uniform is not None
        assert seed is not None
        super(XavierInitializer, self).__init__()
        self._uniform = uniform
        self._fan_in = fan_in
        self._fan_out = fan_out
        self._seed = seed

565 566
    def __call__(self, var, block=None):
        """Initialize the input tensor with Xavier initialization.
567 568

        Args:
569 570 571
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
572 573

        Returns:
574
            The initialization op
575
        """
576 577
        block = self._check_block(block)

578
        assert isinstance(block, framework.Block)
579 580
        check_variable_and_dtype(var, "Out",
                                 ["uint16", "float16", "float32", "float64"],
581 582
                                 "xavier_init")

583 584 585 586 587 588
        f_in, f_out = self._compute_fans(var)

        # If fan_in and fan_out are passed, use them
        fan_in = f_in if self._fan_in is None else self._fan_in
        fan_out = f_out if self._fan_out is None else self._fan_out

D
dzhwinter 已提交
589 590 591
        if self._seed == 0:
            self._seed = block.program.random_seed

592
        # to be compatible of fp16 initalizers
593 594
        if var.dtype == VarDesc.VarType.FP16 or (
                var.dtype == VarDesc.VarType.BF16 and not self._uniform):
595
            out_dtype = VarDesc.VarType.FP32
596 597 598 599 600 601
            out_var = block.create_var(name=unique_name.generate(".".join(
                ['xavier_init', var.name, 'tmp'])),
                                       shape=var.shape,
                                       dtype=out_dtype,
                                       type=VarDesc.VarType.LOD_TENSOR,
                                       persistable=False)
602 603 604 605
        else:
            out_dtype = var.dtype
            out_var = var

J
Jiabin Yang 已提交
606
        if framework._non_static_mode():
607
            if self._uniform:
608
                limit = math.sqrt(6.0 / float(fan_in + fan_out))
609 610 611 612 613 614 615 616 617
                if in_dygraph_mode():
                    out_var = _C_ops.final_state_uniform_random(
                        out_var.shape, out_dtype, -limit, limit, self._seed,
                        _current_expected_place())
                elif _in_legacy_dygraph():
                    out_var = _C_ops.uniform_random('shape', out_var.shape,
                                                    'min', -limit, 'max', limit,
                                                    'seed', self._seed, 'dtype',
                                                    out_dtype)
618
            else:
619
                std = math.sqrt(2.0 / float(fan_in + fan_out))
620 621 622 623 624 625

                if in_dygraph_mode():
                    place = _current_expected_place()
                    out_var = _C_ops.final_state_gaussian_random(
                        out_var.shape, 0.0, std, self._seed, out_dtype, place)
                else:
626 627 628 629
                    out_var = _C_ops.gaussian_random('shape', out_var.shape,
                                                     'dtype', out_dtype, 'mean',
                                                     0.0, 'std', std, 'seed',
                                                     self._seed)
630 631 632

            if var.dtype == VarDesc.VarType.FP16 or (
                    var.dtype == VarDesc.VarType.BF16 and not self._uniform):
633 634 635 636 637
                if in_dygraph_mode():
                    var_tmp = _C_ops.final_state_cast(out_var, var.dtype)
                elif _in_legacy_dygraph():
                    var_tmp = _C_ops.cast(out_var, 'in_dtype', out_var.dtype,
                                          'out_dtype', var.dtype)
638
                var_tmp._share_underline_tensor_to(var)
639
            else:
640
                out_var._share_underline_tensor_to(var)
641
            return None
642
        else:
643
            if self._uniform:
644
                limit = math.sqrt(6.0 / float(fan_in + fan_out))
645 646 647 648 649 650 651 652 653 654 655
                op = block.append_op(type="uniform_random",
                                     inputs={},
                                     outputs={"Out": out_var},
                                     attrs={
                                         "shape": out_var.shape,
                                         "dtype": out_dtype,
                                         "min": -limit,
                                         "max": limit,
                                         "seed": self._seed
                                     },
                                     stop_gradient=True)
656
            else:
657
                std = math.sqrt(2.0 / float(fan_in + fan_out))
658 659 660 661 662 663 664 665 666 667
                op = block.append_op(type="gaussian_random",
                                     outputs={"Out": out_var},
                                     attrs={
                                         "shape": out_var.shape,
                                         "dtype": out_dtype,
                                         "mean": 0.0,
                                         "std": std,
                                         "seed": self._seed
                                     },
                                     stop_gradient=True)
668 669 670

            if var.dtype == VarDesc.VarType.FP16 or (
                    var.dtype == VarDesc.VarType.BF16 and not self._uniform):
671 672 673 674 675 676 677
                block.append_op(type="cast",
                                inputs={"X": out_var},
                                outputs={"Out": var},
                                attrs={
                                    "in_dtype": out_var.dtype,
                                    "out_dtype": var.dtype
                                })
678

679
            var.op = op
680
            return op
681 682 683


class MSRAInitializer(Initializer):
684
    r"""Implements the MSRA initializer a.k.a. Kaiming Initializer
685 686

    This class implements the weight initialization from the paper
687 688 689 690 691 692 693 694
    `Delving Deep into Rectifiers: Surpassing Human-Level Performance on
    ImageNet Classification <https://arxiv.org/abs/1502.01852>`_
    by Kaiming He, Xiangyu Zhang, Shaoqing Ren and Jian Sun. This is a
    robust initialization method that particularly considers the rectifier
    nonlinearities. In case of Uniform distribution, the range is [-x, x], where

    .. math::

695
        x = gain \times \sqrt{\frac{3}{fan\_in}}
696 697 698 699 700 701

    In case of Normal distribution, the mean is 0 and the standard deviation
    is

    .. math::

702
        \frac{gain}{\sqrt{{fan\_in}}}
703 704

    Args:
705 706 707
        uniform (bool, optional): whether to use uniform or normal distribution
        fan_in (float32|None, optional): fan_in (in_features) of trainable Tensor, If None, it will be infered automaticly. If you don't want to use in_features of the Tensor, you can set the value of 'fan_in' smartly by yourself. default is None.
        seed (int32, optional): random seed.
708 709
        negative_slope (float, optional): negative_slope (only used with leaky_relu). default is 0.0.
        nonlinearity(str, optional): the non-linear function. default is relu.
710 711 712 713 714 715

    Note:
        It is recommended to set fan_in to None for most cases.

    Examples:
        .. code-block:: python
X
xsrobin 已提交
716

717
            import paddle
X
xsrobin 已提交
718
            import paddle.fluid as fluid
719
            paddle.enable_static()
D
Double_V 已提交
720
            x = fluid.data(name="data", shape=[8, 32, 32], dtype="float32")
X
xsrobin 已提交
721 722
            fc = fluid.layers.fc(input=x, size=10,
                param_attr=fluid.initializer.MSRA(uniform=False))
723

724 725
    """

726 727 728 729 730 731
    def __init__(self,
                 uniform=True,
                 fan_in=None,
                 seed=0,
                 negative_slope=0,
                 nonlinearity='relu'):
732 733 734 735 736 737 738 739
        """Constructor for MSRAInitializer
        """
        assert uniform is not None
        assert seed is not None
        super(MSRAInitializer, self).__init__()
        self._uniform = uniform
        self._fan_in = fan_in
        self._seed = seed
740 741
        self._negative_slope = negative_slope
        self._nonlinearity = nonlinearity
742

743 744
    def __call__(self, var, block=None):
        """Initialize the input tensor with MSRA initialization.
745 746

        Args:
747 748 749
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
750 751

        Returns:
752
            The initialization op
753
        """
754 755
        block = self._check_block(block)

756 757 758 759 760 761 762
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        f_in, f_out = self._compute_fans(var)

        # If fan_in is passed, use it
        fan_in = f_in if self._fan_in is None else self._fan_in

D
dzhwinter 已提交
763 764 765
        if self._seed == 0:
            self._seed = block.program.random_seed

766
        # to be compatible of fp16 initalizers
767 768
        if var.dtype == VarDesc.VarType.FP16 or (
                var.dtype == VarDesc.VarType.BF16 and not self._uniform):
769
            out_dtype = VarDesc.VarType.FP32
770 771 772 773 774 775
            out_var = block.create_var(name=unique_name.generate(".".join(
                ['masra_init', var.name, 'tmp'])),
                                       shape=var.shape,
                                       dtype=out_dtype,
                                       type=VarDesc.VarType.LOD_TENSOR,
                                       persistable=False)
776 777 778 779
        else:
            out_dtype = var.dtype
            out_var = var

J
Jiabin Yang 已提交
780
        if framework._non_static_mode():
781
            if self._uniform:
782 783 784
                gain = calculate_gain(self._nonlinearity, self._negative_slope)
                limit = gain * math.sqrt(3.0 / float(fan_in))

785 786 787 788 789
                out_var = _C_ops.uniform_random('shape', out_var.shape, 'min',
                                                -limit, 'max', limit, 'seed',
                                                self._seed, 'dtype',
                                                int(out_dtype))
            else:
790 791
                gain = calculate_gain(self._nonlinearity, self._negative_slope)
                std = gain / math.sqrt(float(fan_in))
792 793 794 795 796
                if in_dygraph_mode():
                    place = _current_expected_place()
                    out_var = _C_ops.final_state_gaussian_random(
                        out_var.shape, 0.0, std, self._seed, out_dtype, place)
                else:
797 798 799 800 801
                    out_var = _C_ops.gaussian_random('shape',
                                                     out_var.shape, 'dtype',
                                                     int(out_dtype), 'mean',
                                                     0.0, 'std', std, 'seed',
                                                     self._seed)
802 803 804 805 806

            if var.dtype == VarDesc.VarType.FP16 or (
                    var.dtype == VarDesc.VarType.BF16 and not self._uniform):
                var_tmp = _C_ops.cast(out_var, 'in_dtype', out_var.dtype,
                                      'out_dtype', var.dtype)
807
                var_tmp._share_underline_tensor_to(var)
808
            else:
809
                out_var._share_underline_tensor_to(var)
810
            return None
811
        else:
812
            if self._uniform:
813 814
                gain = calculate_gain(self._nonlinearity, self._negative_slope)
                limit = gain * math.sqrt(3.0 / float(fan_in))
815 816 817 818 819 820 821 822 823 824 825
                op = block.append_op(type="uniform_random",
                                     inputs={},
                                     outputs={"Out": out_var},
                                     attrs={
                                         "shape": out_var.shape,
                                         "dtype": int(out_dtype),
                                         "min": -limit,
                                         "max": limit,
                                         "seed": self._seed
                                     },
                                     stop_gradient=True)
826 827

            else:
828 829
                gain = calculate_gain(self._nonlinearity, self._negative_slope)
                std = gain / math.sqrt(float(fan_in))
830 831 832 833 834 835 836 837 838 839
                op = block.append_op(type="gaussian_random",
                                     outputs={"Out": out_var},
                                     attrs={
                                         "shape": out_var.shape,
                                         "dtype": int(out_dtype),
                                         "mean": 0.0,
                                         "std": std,
                                         "seed": self._seed
                                     },
                                     stop_gradient=True)
840 841 842

            if var.dtype == VarDesc.VarType.FP16 or (
                    var.dtype == VarDesc.VarType.BF16 and not self._uniform):
843 844 845 846 847 848 849
                block.append_op(type="cast",
                                inputs={"X": out_var},
                                outputs={"Out": var},
                                attrs={
                                    "in_dtype": out_var.dtype,
                                    "out_dtype": var.dtype
                                })
850

851
            var.op = op
852
            return op
853 854


855
class BilinearInitializer(Initializer):
856
    """
857 858 859
    This initializer can be used in transposed convolution operator to
    act as upsampling. Users can upsample a feature map with shape of
    (B, C, H, W) by any integer factor. The usage is:
860 861 862 863 864

    Examples:

        .. code-block:: python

865
            import math
866 867 868 869 870

            import paddle
            import paddle.nn as nn
            from paddle.regularizer import L2Decay

X
xsrobin 已提交
871 872
            factor = 2
            C = 2
D
Double_V 已提交
873 874
            B = 8
            H = W = 32
875 876 877 878
            w_attr = paddle.ParamAttr(learning_rate=0.,
                                      regularizer=L2Decay(0.),
                                      initializer=nn.initializer.Bilinear())
            data = paddle.rand([B, 3, H, W], dtype='float32')
C
cnn 已提交
879
            conv_up = nn.Conv2DTranspose(3,
880 881 882 883 884 885 886 887 888 889 890
                                         out_channels=C,
                                         kernel_size=2 * factor - factor % 2,
                                         padding=int(
                                             math.ceil((factor - 1) / 2.)),
                                         stride=factor,
                                         weight_attr=w_attr,
                                         bias_attr=False)
            x = conv_up(data)

    Where, `out_channels=C` and `groups=C` means this is channel-wise transposed
    convolution. The filter shape will be (C, 1, K, K) where K is `kernel_size`,
891 892 893 894
    This initializer will set a (K, K) interpolation kernel for every channel
    of the filter identically. The resulting shape of the output feature map
    will be (B, C, factor * H, factor * W). Note that the learning rate and the
    weight decay are set to 0 in order to keep coefficient values of bilinear
895 896
    interpolation unchanged during training.

897 898 899 900 901 902 903
    """

    def __init__(self):
        """Constructor for BilinearInitializer.
        """
        super(BilinearInitializer, self).__init__()

904 905
    def __call__(self, var, block=None):
        """Initialize the input tensor with Bilinear initialization.
906 907

        Args:
908 909 910
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
911 912

        Returns:
913
            The initialization op
914
        """
915 916
        block = self._check_block(block)

917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940
        if not isinstance(var, framework.Variable):
            raise ValueError("var must be framework.Variable.")

        if not isinstance(block, framework.Block):
            raise ValueError("block must be framework.Block.")

        shape = var.shape
        if len(shape) != 4:
            raise ValueError("the length of shape must be 4.")
        if shape[2] != shape[3]:
            raise ValueError("shape[2] must be equal to shape[3].")

        weight = np.zeros(np.prod(var.shape), dtype='float32')
        size = shape[3]
        # factor
        f = np.ceil(size / 2.)
        # center
        c = (2 * f - 1 - f % 2) / (2. * f)
        for i in range(np.prod(shape)):
            x = i % size
            y = (i / size) % size
            weight[i] = (1 - abs(x / f - c)) * (1 - abs(y / f - c))
        weight = np.reshape(weight, shape)

941
        # to be compatible of fp16 initalizers
942 943 944
        if var.dtype in [
                VarDesc.VarType.FP16, VarDesc.VarType.BF16, VarDesc.VarType.FP64
        ]:
945
            out_dtype = VarDesc.VarType.FP32
946 947 948 949 950 951
            out_var = block.create_var(name=unique_name.generate(".".join(
                ['bilinear_init', var.name, 'tmp'])),
                                       shape=var.shape,
                                       dtype=out_dtype,
                                       type=VarDesc.VarType.LOD_TENSOR,
                                       persistable=False)
952 953 954 955 956
        else:
            out_dtype = var.dtype
            out_var = var

        if out_dtype == VarDesc.VarType.FP32:
957 958 959
            value_name = "fp32_values"
            values = [float(v) for v in weight.flat]
        else:
960 961
            raise TypeError("Unsupported dtype %s", var.dtype)

962 963
        if np.prod(shape) > 1024 * 1024:
            raise ValueError("The size of input is too big. ")
964

J
Jiabin Yang 已提交
965
        if framework._non_static_mode():
966 967
            _C_ops.assign_value(out_var, 'shape', list(shape), 'dtype',
                                out_dtype, value_name, values)
968 969 970 971 972 973
            if var.dtype in [
                    VarDesc.VarType.FP16, VarDesc.VarType.BF16,
                    VarDesc.VarType.FP64
            ]:
                var_tmp = _C_ops.cast(out_var, 'in_dtype', out_var.dtype,
                                      'out_dtype', var.dtype)
974
                var_tmp._share_underline_tensor_to(var)
975
            else:
976
                out_var._share_underline_tensor_to(var)
977 978
            return None
        else:
979 980 981 982 983 984 985
            op = block.append_op(type='assign_value',
                                 outputs={'Out': [out_var]},
                                 attrs={
                                     'dtype': out_dtype,
                                     'shape': list(shape),
                                     value_name: values
                                 })
986 987 988 989 990

            if var.dtype in [
                    VarDesc.VarType.FP16, VarDesc.VarType.BF16,
                    VarDesc.VarType.FP64
            ]:
991 992 993 994 995 996 997
                block.append_op(type="cast",
                                inputs={"X": out_var},
                                outputs={"Out": var},
                                attrs={
                                    "in_dtype": out_var.dtype,
                                    "out_dtype": var.dtype
                                })
998

999
            var.op = op
1000
            return op
1001 1002


1003 1004
class NumpyArrayInitializer(Initializer):
    """Init an parameter with an numpy array
1005
    This op initialize the variable by numpy array.
1006 1007 1008 1009

    Args:
        value (numpy): numpy array to initialize the variable

1010 1011 1012
    Returns:
        A Tensor variable initialized by numpy.

1013 1014 1015
    Examples:
        .. code-block:: python

1016
            import paddle.fluid as fluid
1017 1018
            import numpy
            x = fluid.data(name="x", shape=[2, 1], dtype='float32')
1019 1020 1021 1022 1023 1024 1025 1026 1027 1028
            fc = fluid.layers.fc(input=x, size=10,
                param_attr=fluid.initializer.NumpyArrayInitializer(numpy.array([1,2])))
    """

    def __init__(self, value):
        import numpy
        assert isinstance(value, numpy.ndarray)
        super(NumpyArrayInitializer, self).__init__()
        self._value = value

1029 1030
    def __call__(self, var, block=None):
        """Initialize the input tensor with Numpy array.
1031 1032

        Args:
1033 1034 1035
            var(Tensor): Tensor that needs to be initialized.
            block(Block, optional): The block in which initialization ops
                   should be added. Used in static graph only, default None.
1036 1037

        Returns:
1038
            The initialization op
1039
        """
1040 1041
        block = self._check_block(block)

1042 1043
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
1044 1045

        # to be compatible of fp16 initalizers
1046
        if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
1047 1048
            out_dtype = VarDesc.VarType.FP32
            np_value = self._value.astype("float32")
1049 1050 1051 1052 1053 1054
            out_var = block.create_var(name=unique_name.generate(".".join(
                ['numpy_array_init', var.name, 'tmp'])),
                                       shape=var.shape,
                                       dtype=out_dtype,
                                       type=VarDesc.VarType.LOD_TENSOR,
                                       persistable=False)
1055 1056 1057 1058 1059 1060
        else:
            out_var = var
            out_dtype = var.dtype
            np_value = self._value

        if out_dtype == VarDesc.VarType.FP32:
1061
            value_name = "fp32_values"
1062 1063
            values = [float(v) for v in np_value.flat]
        elif out_dtype == VarDesc.VarType.INT32:
1064
            value_name = "int32_values"
1065
            values = [int(v) for v in np_value.flat]
1066 1067
        else:
            raise ValueError("Unsupported dtype %s", self._value.dtype)
X
Xin Pan 已提交
1068
        if self._value.size > 1024 * 1024 * 1024:
1069 1070
            raise ValueError("The size of input is too big. Please consider "
                             "saving it to file and 'load_op' to load it")
1071

J
Jiabin Yang 已提交
1072
        if framework._non_static_mode():
1073 1074
            _C_ops.assign_value(out_var, 'shape', list(self._value.shape),
                                'dtype', out_dtype, value_name, values)
1075 1076 1077
            if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
                var_tmp = _C_ops.cast(out_var, 'in_dtype', out_var.dtype,
                                      'out_dtype', var.dtype)
1078
                var_tmp._share_underline_tensor_to(var)
1079
            else:
1080
                out_var._share_underline_tensor_to(var)
1081 1082
            return None
        else:
1083 1084 1085 1086 1087 1088 1089 1090
            op = block.append_op(type='assign_value',
                                 outputs={'Out': out_var},
                                 attrs={
                                     'dtype': out_dtype,
                                     'shape': list(self._value.shape),
                                     value_name: values
                                 },
                                 stop_gradient=True)
1091 1092

            if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]:
1093 1094 1095 1096 1097 1098 1099
                block.append_op(type="cast",
                                inputs={"X": out_var},
                                outputs={"Out": var},
                                attrs={
                                    "in_dtype": out_var.dtype,
                                    "out_dtype": var.dtype
                                })
1100

1101
            var.op = op
1102
            return op
1103 1104


1105 1106 1107 1108 1109 1110 1111
def set_global_initializer(weight_init, bias_init=None):
    """
    This API is used to set up global model parameter initializer in framework.

    After this API is invoked, the global initializer will takes effect in subsequent code.

    The model parameters include ``weight`` and ``bias`` . In the framework, they correspond 
1112
    to ``paddle.ParamAttr`` , which is inherited from ``paddle.Tensor`` , and is a persistable Variable.
1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
    This API only takes effect for model parameters, not for variables created through apis such as 
    :ref:`api_fluid_layers_create_global_var` , :ref:`api_fluid_layers_create_tensor`.
    
    If the initializer is also set up by ``param_attr`` or ``bias_attr`` when creating a network layer,
    the global initializer setting here will not take effect because it has a lower priority.

    If you want to cancel the global initializer in framework, please set global initializer to ``None`` .

    Args:
        weight_init (Initializer): set the global initializer for ``weight`` of model parameters.
        bias_init (Initializer, optional): set the global initializer for ``bias`` of model parameters. 
            Default: None.

    Returns:
        None

    Examples:
        .. code-block:: python

1132 1133 1134 1135 1136
            import paddle
            import paddle.nn as nn

            nn.initializer.set_global_initializer(nn.initializer.Uniform(), nn.initializer.Constant())
            x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.)
1137 1138 1139

            # The weight of conv1 is initialized by Uniform
            # The bias of conv1 is initialized by Constant
1140 1141
            conv1 = nn.Conv2D(4, 6, (3, 3))
            y_var1 = conv1(x_var)
1142 1143 1144 1145

            # If set param_attr/bias_attr too, global initializer will not take effect
            # The weight of conv2 is initialized by Xavier
            # The bias of conv2 is initialized by Normal
1146 1147 1148 1149
            conv2 = nn.Conv2D(4, 6, (3, 3), 
                weight_attr=nn.initializer.XavierUniform(),
                bias_attr=nn.initializer.Normal())
            y_var2 = conv2(x_var)
1150 1151

            # Cancel the global initializer in framework, it will takes effect in subsequent code
1152
            nn.initializer.set_global_initializer(None)
1153
    """
1154

1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179
    check_type(weight_init, 'weight_init', (Initializer, type(None)),
               'set_global_initializer')
    global _global_weight_initializer_
    _global_weight_initializer_ = weight_init

    check_type(bias_init, 'bias_init', (Initializer, type(None)),
               'set_global_initializer')
    global _global_bias_initializer_
    _global_bias_initializer_ = bias_init


def _global_weight_initializer():
    """
    Return the global weight initializer, The user doesn't need to use it.
    """
    return _global_weight_initializer_


def _global_bias_initializer():
    """
    Return the global weight initializer, The user doesn't need to use it.
    """
    return _global_bias_initializer_


1180 1181
def calculate_gain(nonlinearity, param=None):
    """
1182 1183
    Get the recommended ``gain`` value of some nonlinearity function. ``gain`` value can be used in some 
    ``paddle.nn.initializer`` api to adjust the initialization value.
1184 1185

    Args:
1186 1187
        nonlinearity(str): name of nonlinearity activation function. If it is a linear function, such as: 
            `linear/conv1d/conv2d/conv3d/conv1d_transpose/conv2d_transpose/conv3d_transpose` , 1.0 will be returned.
1188
        param(bool|int|float, optional): optional parameter for somme nonlinearity function. Now, it only applies to 
1189
            'leaky_relu'. Default: None, it will be calculated as 0.01 in the formula.
1190 1191

    Returns:
1192
        A float value, which is the recommended gain for this nonlinearity function.
1193 1194 1195

    Examples:
        .. code-block:: python
1196

1197 1198 1199
            import paddle
            gain = paddle.nn.initializer.calculate_gain('tanh') # 5.0 / 3
            gain = paddle.nn.initializer.calculate_gain('leaky_relu', param=1.0) # 1.0 = math.sqrt(2.0 / (1+param^2))
1200
            initializer = paddle.nn.initializer.Orthogonal(gain)
1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213

    """
    if param is None:
        param = 0.01
    else:
        assert isinstance(param, (bool, int, float))
        param = float(param)
    recommended_gain = {
        'sigmoid': 1,
        'linear': 1,
        'conv1d': 1,
        'conv2d': 1,
        'conv3d': 1,
1214 1215 1216
        'conv1d_transpose': 1,
        'conv2d_transpose': 1,
        'conv3d_transpose': 1,
1217 1218 1219 1220 1221 1222 1223 1224
        'tanh': 5.0 / 3,
        'relu': math.sqrt(2.0),
        'leaky_relu': math.sqrt(2.0 / (1 + param**2)),
        'selu': 3.0 / 4
    }
    if nonlinearity in recommended_gain.keys():
        return recommended_gain[nonlinearity]
    else:
1225 1226 1227
        raise ValueError(
            "nonlinearity function {} is not suppported now.".format(
                nonlinearity))
1228 1229


1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241
# We short the class name, since users will use the initializer with the package
# name. The sample code:
#
# import paddle.fluid as fluid
#
# hidden = fluid.layers.fc(...,
#                          param_attr=ParamAttr(fluid.initializer.Xavier()))
#
# It is no need to add an `Initializer` as the class suffix
Constant = ConstantInitializer
Uniform = UniformInitializer
Normal = NormalInitializer
1242
TruncatedNormal = TruncatedNormalInitializer
1243 1244
Xavier = XavierInitializer
MSRA = MSRAInitializer
1245
Bilinear = BilinearInitializer