initializer.py 31.5 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15 16
from __future__ import print_function

17
from . import framework
18
import numpy as np
S
rename  
sneaxiy 已提交
19
from .wrapped_decorator import signature_safe_contextmanager
20
from .core import VarDesc
W
Wu Yi 已提交
21
from . import unique_name
22

23
__all__ = [
24 25 26
    'Constant', 'Uniform', 'Normal', 'TruncatedNormal', 'Xavier', 'Bilinear',
    'MSRA', 'force_init_on_cpu', 'init_on_cpu', 'ConstantInitializer',
    'UniformInitializer', 'NormalInitializer', 'TruncatedNormalInitializer',
27 28
    'XavierInitializer', 'BilinearInitializer', 'MSRAInitializer',
    'NumpyArrayInitializer'
29
]
30

31 32 33 34
_force_init_on_cpu_ = False


def force_init_on_cpu():
Q
qiaolongfei 已提交
35 36 37
    """
    The flag of whether force to init variables on CPU.

Q
Qiao Longfei 已提交
38 39
    Returns:
        bool: the state if we should force init on CPU.
40

Q
qiaolongfei 已提交
41
    Examples:
Q
Qiao Longfei 已提交
42

Q
qiaolongfei 已提交
43 44
        .. code-block:: python

X
xsrobin 已提交
45 46 47 48
            import paddle.fluid as fluid
            if fluid.initializer.force_init_on_cpu():
                step = fluid.layers.create_global_var(
                    shape=[2,3], value=1.0, dtype='float32')
Q
qiaolongfei 已提交
49 50

    """
51 52 53
    return _force_init_on_cpu_


S
rename  
sneaxiy 已提交
54
@signature_safe_contextmanager
55 56
def init_on_cpu():
    """
Q
qiaolongfei 已提交
57
    Force the variable to be inited on CPU.
58 59

    Examples:
Q
qiaolongfei 已提交
60 61
        .. code-block:: python

X
xsrobin 已提交
62 63 64 65
            import paddle.fluid as fluid
            with fluid.initializer.init_on_cpu():
                step = fluid.layers.create_global_var(
                    shape=[2,3], value=1.0, dtype='float32')
66 67 68 69 70 71 72 73 74

    """
    global _force_init_on_cpu_

    pre_state = force_init_on_cpu()
    _force_init_on_cpu_ = True
    yield
    _force_init_on_cpu_ = pre_state

75 76 77 78 79 80 81 82 83 84

class Initializer(object):
    """Base class for variable initializers

    Defines the common interface of variable initializers.
    They add operations to the init program that are used
    to initialize variables. Users should not use this class
    directly, but need to use one of its implementations.
    """

W
whs 已提交
85
    def __init__(self):
86 87 88 89 90 91 92
        pass

    def __call__(self, param, block):
        """Add corresponding initialization operations to the network
        """
        raise NotImplementedError()

93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
    def _compute_fans(self, var):
        """Compute the fan_in and the fan_out for layers

        This method computes the fan_in and the fan_out
        for neural network layers, if not specified. It is
        not possible to perfectly estimate fan_in and fan_out.
        This method will estimate it correctly for matrix multiply and
        convolutions.

        Args:
            var: variable for which fan_in and fan_out have to be computed

        Returns:
            tuple of two integers (fan_in, fan_out)
        """
        shape = var.shape
        if not shape or len(shape) == 0:
            fan_in = fan_out = 1
        elif len(shape) == 1:
            fan_in = fan_out = shape[0]
        elif len(shape) == 2:
            # This is the case for simple matrix multiply
            fan_in = shape[0]
            fan_out = shape[1]
        else:
            # Assume this to be a convolutional kernel
            # In PaddlePaddle, the shape of the kernel is like:
            # [num_filters, num_filter_channels, ...] where the remaining
            # dimensions are the filter_size
            receptive_field_size = np.prod(shape[2:])
            fan_in = shape[1] * receptive_field_size
            fan_out = shape[0] * receptive_field_size

        return (fan_in, fan_out)

128 129 130

class ConstantInitializer(Initializer):
    """Implements the constant initializer
131 132

    Args:
D
Double_V 已提交
133
        value (float32): constant value to initialize the variable 
134 135 136 137

    Examples:
        .. code-block:: python

138
    	    import paddle.fluid as fluid
D
Double_V 已提交
139
            x = fluid.data(name="data", shape=[8, 32, 32], dtype="float32")
140 141 142
	    fc = fluid.layers.fc(input=x, size=10,
    		param_attr=fluid.initializer.Constant(value=2.0))

143 144
    """

145
    def __init__(self, value=0.0, force_cpu=False):
146 147 148
        assert value is not None
        super(ConstantInitializer, self).__init__()
        self._value = value
149
        self._force_cpu = force_cpu
150 151 152 153 154 155 156 157 158 159 160 161 162 163

    def __call__(self, var, block):
        """Add constant initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
164 165 166 167 168 169 170 171 172 173 174 175 176 177 178

        # to be compatible of fp16 initializers
        if var.dtype == VarDesc.VarType.FP16:
            out_dtype = VarDesc.VarType.FP32
            out_var = block.create_var(
                name=unique_name.generate(".".join(
                    ['constant_init', var.name, 'tmp'])),
                shape=var.shape,
                dtype=out_dtype,
                type=VarDesc.VarType.LOD_TENSOR,
                persistable=False)
        else:
            out_dtype = var.dtype
            out_var = var

179
        # Initialization Ops should be prepended and not appended
W
Wu Yi 已提交
180
        op = block._prepend_op(
181
            type="fill_constant",
182
            outputs={"Out": out_var},
183 184
            attrs={
                "shape": var.shape,
185
                "dtype": int(out_dtype),
186 187
                "value": float(self._value),
                'force_cpu': self._force_cpu or force_init_on_cpu()
M
minqiyang 已提交
188 189
            },
            stop_gradient=True)
190 191 192 193 194 195 196 197 198

        if var.dtype == VarDesc.VarType.FP16:
            block.append_op(
                type="cast",
                inputs={"X": out_var},
                outputs={"Out": var},
                attrs={"in_dtype": out_var.dtype,
                       "out_dtype": var.dtype})

L
lujun 已提交
199
        if not framework.in_dygraph_mode():
200
            var.op = op
201 202 203 204
        return op


class UniformInitializer(Initializer):
205
    """Implements the random uniform distribution initializer
206 207 208 209 210

    Args:
        low (float): lower boundary of the uniform distribution
        high (float): upper boundary of the uniform distribution
        seed (int): random seed
211 212 213 214 215 216
        diag_num (int): the number of diagonal elements to initialize.
            If set to 0, diagonal initialization will be not performed.
        diag_step (int): Step size between two diagonal elements,
            which is generally the width of the square matrix.
        diag_val (float): the value of the diagonal element to be initialized,
            default 1.0. It takes effect only if the diag_num is greater than 0.
217 218 219 220

    Examples:
        .. code-block:: python

X
xiaoting 已提交
221
            import paddle.fluid as fluid
222
            x = fluid.data(name='x', shape=[None, 1], dtype='float32')
223
            fc = fluid.layers.fc(input=x, size=10,
224
    		param_attr=fluid.initializer.Uniform(low=-0.5, high=0.5))
225 226
    """

227 228 229 230 231 232 233
    def __init__(self,
                 low=-1.0,
                 high=1.0,
                 seed=0,
                 diag_num=0,
                 diag_step=0,
                 diag_val=1.0):
234 235
        assert low is not None
        assert high is not None
236
        assert high >= low
237
        assert seed is not None
238 239 240 241 242
        assert diag_num is not None
        assert diag_step is not None
        assert diag_val is not None
        if diag_num > 0 or diag_step > 0:
            assert (diag_num > 0 and diag_step > 0)
243 244 245 246
        super(UniformInitializer, self).__init__()
        self._low = low
        self._high = high
        self._seed = seed
247 248 249
        self._diag_num = diag_num
        self._diag_step = diag_step
        self._diag_val = diag_val
250 251 252 253 254 255 256 257 258 259 260 261 262 263 264

    def __call__(self, var, block):
        """Add uniform distribution initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        # Initialization Ops should be prepended and not appended
D
dzhwinter 已提交
265 266
        if self._seed == 0:
            self._seed = block.program.random_seed
W
Wu Yi 已提交
267

X
polish  
Xin Pan 已提交
268
        # to be compatible of fp16 initializers
W
Wu Yi 已提交
269 270 271
        if var.dtype == VarDesc.VarType.FP16:
            out_dtype = VarDesc.VarType.FP32
            out_var = block.create_var(
272 273
                name=unique_name.generate(".".join(
                    ['uniform_random', var.name, 'tmp'])),
W
Wu Yi 已提交
274 275 276 277 278 279 280 281
                shape=var.shape,
                dtype=out_dtype,
                type=VarDesc.VarType.LOD_TENSOR,
                persistable=False)
        else:
            out_dtype = var.dtype
            out_var = var

W
Wu Yi 已提交
282
        op = block._prepend_op(
283
            type="uniform_random",
284
            inputs={},
W
Wu Yi 已提交
285
            outputs={"Out": out_var},
286 287
            attrs={
                "shape": var.shape,
W
Wu Yi 已提交
288
                "dtype": out_dtype,
289 290
                "min": self._low,
                "max": self._high,
291 292 293 294
                "seed": self._seed,
                "diag_num": self._diag_num,
                "diag_step": self._diag_step,
                "diag_val": self._diag_val
M
minqiyang 已提交
295 296
            },
            stop_gradient=True)
W
Wu Yi 已提交
297 298 299 300 301 302 303 304 305

        if var.dtype == VarDesc.VarType.FP16:
            block.append_op(
                type="cast",
                inputs={"X": out_var},
                outputs={"Out": var},
                attrs={"in_dtype": out_var.dtype,
                       "out_dtype": var.dtype})

L
lujun 已提交
306
        if not framework.in_dygraph_mode():
307
            var.op = op
308
        return op
309 310 311


class NormalInitializer(Initializer):
312 313 314 315 316 317 318 319 320 321
    """Implements the Random Normal(Gaussian) distribution initializer

    Args:
        loc (float): mean of the normal distribution
        scale (float): standard deviation of the normal distribution
        seed (int): random seed

    Examples:
        .. code-block:: python

X
xsrobin 已提交
322
            import paddle.fluid as fluid
323
            x = fluid.data(name="data", shape=[None, 32, 32], dtype="float32")
X
xsrobin 已提交
324 325
            fc = fluid.layers.fc(input=x, size=10,
                param_attr=fluid.initializer.Normal(loc=0.0, scale=2.0))
326

327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
    """

    def __init__(self, loc=0.0, scale=1.0, seed=0):
        assert loc is not None
        assert scale is not None
        assert seed is not None
        super(NormalInitializer, self).__init__()
        self._mean = loc
        self._std_dev = scale
        self._seed = seed

    def __call__(self, var, block):
        """Add normal distribution initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        # Initialization Ops should be prepended and not appended
D
dzhwinter 已提交
352 353
        if self._seed == 0:
            self._seed = block.program.random_seed
W
Wu Yi 已提交
354 355 356 357 358

        # to be compatible of fp16 initalizers
        if var.dtype == VarDesc.VarType.FP16:
            out_dtype = VarDesc.VarType.FP32
            out_var = block.create_var(
359 360
                name=unique_name.generate(".".join(
                    ['gaussian_random', var.name, 'tmp'])),
W
Wu Yi 已提交
361 362 363 364 365 366 367 368
                shape=var.shape,
                dtype=out_dtype,
                type=VarDesc.VarType.LOD_TENSOR,
                persistable=False)
        else:
            out_dtype = var.dtype
            out_var = var

W
Wu Yi 已提交
369
        op = block._prepend_op(
370
            type="gaussian_random",
W
Wu Yi 已提交
371
            outputs={"Out": out_var},
372 373
            attrs={
                "shape": var.shape,
W
Wu Yi 已提交
374
                "dtype": out_dtype,
375 376
                "mean": self._mean,
                "std": self._std_dev,
G
gongweibao 已提交
377 378
                "seed": self._seed,
                "use_mkldnn": False
M
minqiyang 已提交
379 380
            },
            stop_gradient=True)
W
Wu Yi 已提交
381 382 383 384 385 386 387 388

        if var.dtype == VarDesc.VarType.FP16:
            block.append_op(
                type="cast",
                inputs={"X": out_var},
                outputs={"Out": var},
                attrs={"in_dtype": out_var.dtype,
                       "out_dtype": var.dtype})
L
lujun 已提交
389
        if not framework.in_dygraph_mode():
390
            var.op = op
391
        return op
392 393


394 395 396 397 398 399 400 401 402 403 404
class TruncatedNormalInitializer(Initializer):
    """Implements the Random TruncatedNormal(Gaussian) distribution initializer

    Args:
        loc (float): mean of the normal distribution
        scale (float): standard deviation of the normal distribution
        seed (int): random seed

    Examples:
        .. code-block:: python

X
xiaoting 已提交
405
            import paddle.fluid as fluid
406
            x = fluid.data(name='x', shape=[None, 1], dtype='float32')
407 408 409 410 411 412 413 414
            fc = fluid.layers.fc(input=x, size=10,
                param_attr=fluid.initializer.TruncatedNormal(loc=0.0, scale=2.0))
    """

    def __init__(self, loc=0.0, scale=1.0, seed=0):
        assert loc is not None
        assert scale is not None
        assert seed is not None
W
whs 已提交
415
        super(TruncatedNormalInitializer, self).__init__()
416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
        self._mean = loc
        self._std_dev = scale
        self._seed = seed

    def __call__(self, var, block):
        """Add truncated normal distribution initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        # Initialization Ops should be prepended and not appended
        if self._seed == 0:
            self._seed = block.program.random_seed
436 437 438 439 440 441

        # to be compatible of fp16 initalizers
        if var.dtype == VarDesc.VarType.FP16:
            out_dtype = VarDesc.VarType.FP32
            out_var = block.create_var(
                name=unique_name.generate(".".join(
442
                    ['truncated_gaussian_random', var.name, 'tmp'])),
443 444 445 446 447 448 449 450
                shape=var.shape,
                dtype=out_dtype,
                type=VarDesc.VarType.LOD_TENSOR,
                persistable=False)
        else:
            out_dtype = var.dtype
            out_var = var

451 452
        op = block._prepend_op(
            type="truncated_gaussian_random",
453
            outputs={"Out": out_var},
454 455
            attrs={
                "shape": var.shape,
456
                "dtype": out_dtype,
457 458 459
                "mean": self._mean,
                "std": self._std_dev,
                "seed": self._seed
M
minqiyang 已提交
460 461
            },
            stop_gradient=True)
462 463 464 465 466 467 468 469

        if var.dtype == VarDesc.VarType.FP16:
            block.append_op(
                type="cast",
                inputs={"X": out_var},
                outputs={"Out": var},
                attrs={"in_dtype": out_var.dtype,
                       "out_dtype": var.dtype})
L
lujun 已提交
470
        if not framework.in_dygraph_mode():
471
            var.op = op
472 473 474
        return op


475
class XavierInitializer(Initializer):
Q
qiaolongfei 已提交
476
    """
477
    This class implements the Xavier weight initializer from the paper
Q
qiaolongfei 已提交
478 479 480
    `Understanding the difficulty of training deep feedforward neural
    networks <http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf>`_
    by Xavier Glorot and Yoshua Bengio.
481 482 483

    This initializer is designed to keep the scale of the gradients
    approximately same in all the layers. In case of Uniform distribution,
Q
qiaolongfei 已提交
484 485 486 487 488 489
    the range is [-x, x], where

    .. math::

        x = \sqrt{\\frac{6.0}{fan\_in + fan\_out}}

490
    In case of Normal distribution, the mean is 0 and the standard deviation
Q
qiaolongfei 已提交
491
    is
492

Q
qiaolongfei 已提交
493
    .. math::
494

Q
qiaolongfei 已提交
495
        \sqrt{\\frac{2.0}{fan\_in + fan\_out}}
496 497


Q
qiaolongfei 已提交
498
    Args:
X
xiaoting 已提交
499 500
        uniform (bool,default True): whether to use uniform ,if False use normal distribution
        fan_in (float,default None): fan_in for Xavier initialization. If None, it is
Q
qiaolongfei 已提交
501
                inferred from the variable.
X
xiaoting 已提交
502
        fan_out (float,default None): fan_out for Xavier initialization. If None, it is
Q
qiaolongfei 已提交
503 504 505 506 507 508 509 510 511
                 inferred from the variable.
        seed (int): random seed

    Note:
        It is recommended to set fan_in and fan_out to None for most cases.

    Examples:
        .. code-block:: python

X
xiaoting 已提交
512
            import paddle.fluid as fluid
X
xiaoting 已提交
513
            queries = fluid.data(name='x', shape=[None,1], dtype='float32')
Q
qiaolongfei 已提交
514 515 516 517 518 519 520
            fc = fluid.layers.fc(
                input=queries, size=10,
                param_attr=fluid.initializer.Xavier(uniform=False))

    """

    def __init__(self, uniform=True, fan_in=None, fan_out=None, seed=0):
521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547
        assert uniform is not None
        assert seed is not None
        super(XavierInitializer, self).__init__()
        self._uniform = uniform
        self._fan_in = fan_in
        self._fan_out = fan_out
        self._seed = seed

    def __call__(self, var, block):
        """Add xavier initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        f_in, f_out = self._compute_fans(var)

        # If fan_in and fan_out are passed, use them
        fan_in = f_in if self._fan_in is None else self._fan_in
        fan_out = f_out if self._fan_out is None else self._fan_out

D
dzhwinter 已提交
548 549 550
        if self._seed == 0:
            self._seed = block.program.random_seed

551 552 553 554 555 556 557 558 559 560 561 562 563 564
        # to be compatible of fp16 initalizers
        if var.dtype == VarDesc.VarType.FP16:
            out_dtype = VarDesc.VarType.FP32
            out_var = block.create_var(
                name=unique_name.generate(".".join(
                    ['xavier_init', var.name, 'tmp'])),
                shape=var.shape,
                dtype=out_dtype,
                type=VarDesc.VarType.LOD_TENSOR,
                persistable=False)
        else:
            out_dtype = var.dtype
            out_var = var

565 566
        if self._uniform:
            limit = np.sqrt(6.0 / float(fan_in + fan_out))
W
Wu Yi 已提交
567
            op = block._prepend_op(
568
                type="uniform_random",
569
                inputs={},
570
                outputs={"Out": out_var},
571
                attrs={
572 573
                    "shape": out_var.shape,
                    "dtype": out_dtype,
574 575 576
                    "min": -limit,
                    "max": limit,
                    "seed": self._seed
M
minqiyang 已提交
577 578
                },
                stop_gradient=True)
579 580 581

        else:
            std = np.sqrt(2.0 / float(fan_in + fan_out))
W
Wu Yi 已提交
582
            op = block._prepend_op(
583
                type="gaussian_random",
584
                outputs={"Out": out_var},
585
                attrs={
586 587
                    "shape": out_var.shape,
                    "dtype": out_dtype,
588 589 590
                    "mean": 0.0,
                    "std": std,
                    "seed": self._seed
M
minqiyang 已提交
591 592
                },
                stop_gradient=True)
593 594 595 596 597 598 599 600 601

        if var.dtype == VarDesc.VarType.FP16:
            block.append_op(
                type="cast",
                inputs={"X": out_var},
                outputs={"Out": var},
                attrs={"in_dtype": out_var.dtype,
                       "out_dtype": var.dtype})

L
lujun 已提交
602
        if not framework.in_dygraph_mode():
603
            var.op = op
604
        return op
605 606 607 608 609 610


class MSRAInitializer(Initializer):
    """Implements the MSRA initializer a.k.a. Kaiming Initializer

    This class implements the weight initialization from the paper
611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629
    `Delving Deep into Rectifiers: Surpassing Human-Level Performance on
    ImageNet Classification <https://arxiv.org/abs/1502.01852>`_
    by Kaiming He, Xiangyu Zhang, Shaoqing Ren and Jian Sun. This is a
    robust initialization method that particularly considers the rectifier
    nonlinearities. In case of Uniform distribution, the range is [-x, x], where

    .. math::

        x = \sqrt{\\frac{6.0}{fan\_in}}

    In case of Normal distribution, the mean is 0 and the standard deviation
    is

    .. math::

        \sqrt{\\frac{2.0}{fan\_in}}

    Args:
        uniform (bool): whether to use uniform or normal distribution
D
Double_V 已提交
630 631 632
        fan_in (float32|None): fan_in for MSRAInitializer. If None, it is\
        inferred from the variable. default is None.
        seed (int32): random seed
633 634 635 636 637 638

    Note:
        It is recommended to set fan_in to None for most cases.

    Examples:
        .. code-block:: python
X
xsrobin 已提交
639 640

            import paddle.fluid as fluid
D
Double_V 已提交
641
            x = fluid.data(name="data", shape=[8, 32, 32], dtype="float32")
X
xsrobin 已提交
642 643
            fc = fluid.layers.fc(input=x, size=10,
                param_attr=fluid.initializer.MSRA(uniform=False))
644

645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674
    """

    def __init__(self, uniform=True, fan_in=None, seed=0):
        """Constructor for MSRAInitializer
        """
        assert uniform is not None
        assert seed is not None
        super(MSRAInitializer, self).__init__()
        self._uniform = uniform
        self._fan_in = fan_in
        self._seed = seed

    def __call__(self, var, block):
        """Add MSRA initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
        f_in, f_out = self._compute_fans(var)

        # If fan_in is passed, use it
        fan_in = f_in if self._fan_in is None else self._fan_in

D
dzhwinter 已提交
675 676 677
        if self._seed == 0:
            self._seed = block.program.random_seed

678 679 680 681 682 683 684 685 686 687 688 689 690 691
        # to be compatible of fp16 initalizers
        if var.dtype == VarDesc.VarType.FP16:
            out_dtype = VarDesc.VarType.FP32
            out_var = block.create_var(
                name=unique_name.generate(".".join(
                    ['masra_init', var.name, 'tmp'])),
                shape=var.shape,
                dtype=out_dtype,
                type=VarDesc.VarType.LOD_TENSOR,
                persistable=False)
        else:
            out_dtype = var.dtype
            out_var = var

692 693
        if self._uniform:
            limit = np.sqrt(6.0 / float(fan_in))
W
Wu Yi 已提交
694
            op = block._prepend_op(
695
                type="uniform_random",
696
                inputs={},
697
                outputs={"Out": out_var},
698
                attrs={
699 700
                    "shape": out_var.shape,
                    "dtype": int(out_dtype),
701 702 703
                    "min": -limit,
                    "max": limit,
                    "seed": self._seed
M
minqiyang 已提交
704 705
                },
                stop_gradient=True)
706 707 708

        else:
            std = np.sqrt(2.0 / float(fan_in))
W
Wu Yi 已提交
709
            op = block._prepend_op(
710
                type="gaussian_random",
711
                outputs={"Out": out_var},
712
                attrs={
713 714
                    "shape": out_var.shape,
                    "dtype": int(out_dtype),
715 716 717
                    "mean": 0.0,
                    "std": std,
                    "seed": self._seed
M
minqiyang 已提交
718 719
                },
                stop_gradient=True)
720 721 722 723 724 725 726 727 728

        if var.dtype == VarDesc.VarType.FP16:
            block.append_op(
                type="cast",
                inputs={"X": out_var},
                outputs={"Out": var},
                attrs={"in_dtype": out_var.dtype,
                       "out_dtype": var.dtype})

L
lujun 已提交
729
        if not framework.in_dygraph_mode():
730
            var.op = op
731
        return op
732 733


734
class BilinearInitializer(Initializer):
735
    """
736 737 738
    This initializer can be used in transposed convolution operator to
    act as upsampling. Users can upsample a feature map with shape of
    (B, C, H, W) by any integer factor. The usage is:
739 740 741 742 743

    Examples:

        .. code-block:: python

X
xsrobin 已提交
744 745 746
            import paddle.fluid as fluid
            factor = 2
            C = 2
D
Double_V 已提交
747 748
            B = 8
            H = W = 32
X
xsrobin 已提交
749 750 751
            w_attr = fluid.param_attr.ParamAttr(
                learning_rate=0., 
                regularizer=fluid.regularizer.L2Decay(0.),
752
                initializer=fluid.initializer.Bilinear())
D
Double_V 已提交
753
            x = fluid.data(name="data", shape=[B, 3, H, W], 
X
xsrobin 已提交
754 755 756 757 758 759 760 761 762 763 764
                                  dtype="float32")
            conv_up = fluid.layers.conv2d_transpose(
                input=x,
                num_filters=C,
                output_size=None,
                filter_size=2 * factor - factor % 2,
                padding=int(math.ceil((factor - 1) / 2.)),
                stride=factor,
                groups=C,
                param_attr=w_attr,
                bias_attr=False)
765 766

    Where, `num_filters=C` and `groups=C` means this is channel-wise transposed
767 768 769 770 771
    convolution. The filter shape will be (C, 1, K, K) where K is `filer_size`,
    This initializer will set a (K, K) interpolation kernel for every channel
    of the filter identically. The resulting shape of the output feature map
    will be (B, C, factor * H, factor * W). Note that the learning rate and the
    weight decay are set to 0 in order to keep coefficient values of bilinear
772 773
    interpolation unchanged during training.

774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789
    """

    def __init__(self):
        """Constructor for BilinearInitializer.
        """
        super(BilinearInitializer, self).__init__()

    def __call__(self, var, block):
        """Add biliear initialization ops for a variable

        Args:
            var (Variable): Variable that needs to be initialized.
            block (Block): The block in which initialization ops should
                           be added.

        Returns:
790
            Operator: the initialization op
791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820

        Raises:
            ValueError: If type of `var` and `block` is not right.
                        If the shape of `var` size is not 4 and
                        var.shape[2] != var.shape[3].
        """
        if not isinstance(var, framework.Variable):
            raise ValueError("var must be framework.Variable.")

        if not isinstance(block, framework.Block):
            raise ValueError("block must be framework.Block.")

        shape = var.shape
        if len(shape) != 4:
            raise ValueError("the length of shape must be 4.")
        if shape[2] != shape[3]:
            raise ValueError("shape[2] must be equal to shape[3].")

        weight = np.zeros(np.prod(var.shape), dtype='float32')
        size = shape[3]
        # factor
        f = np.ceil(size / 2.)
        # center
        c = (2 * f - 1 - f % 2) / (2. * f)
        for i in range(np.prod(shape)):
            x = i % size
            y = (i / size) % size
            weight[i] = (1 - abs(x / f - c)) * (1 - abs(y / f - c))
        weight = np.reshape(weight, shape)

821 822 823 824 825 826 827 828 829 830 831 832 833 834 835
        # to be compatible of fp16 initalizers
        if var.dtype == VarDesc.VarType.FP16:
            out_dtype = VarDesc.VarType.FP32
            out_var = block.create_var(
                name=unique_name.generate(".".join(
                    ['bilinear_init', var.name, 'tmp'])),
                shape=var.shape,
                dtype=out_dtype,
                type=VarDesc.VarType.LOD_TENSOR,
                persistable=False)
        else:
            out_dtype = var.dtype
            out_var = var

        if out_dtype == VarDesc.VarType.FP32:
836 837 838 839 840 841 842 843
            value_name = "fp32_values"
            values = [float(v) for v in weight.flat]
        else:
            raise ValueError("Unsupported dtype %s", input.dtype)
        if np.prod(shape) > 1024 * 1024:
            raise ValueError("The size of input is too big. ")
        op = block.append_op(
            type='assign_value',
844
            outputs={'Out': [out_var]},
845
            attrs={
846
                'dtype': out_dtype,
847 848 849
                'shape': list(shape),
                value_name: values
            })
850 851 852 853 854 855 856 857 858

        if var.dtype == VarDesc.VarType.FP16:
            block.append_op(
                type="cast",
                inputs={"X": out_var},
                outputs={"Out": var},
                attrs={"in_dtype": out_var.dtype,
                       "out_dtype": var.dtype})

L
lujun 已提交
859
        if not framework.in_dygraph_mode():
860
            var.op = op
861 862 863
        return op


864 865
class NumpyArrayInitializer(Initializer):
    """Init an parameter with an numpy array
866
    This op initialize the variable by numpy array.
867 868 869 870

    Args:
        value (numpy): numpy array to initialize the variable

871 872 873
    Returns:
        A Tensor variable initialized by numpy.

874 875 876
    Examples:
        .. code-block:: python

877
            import paddle.fluid as fluid
878 879
            import numpy
            x = fluid.data(name="x", shape=[2, 1], dtype='float32')
880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902
            fc = fluid.layers.fc(input=x, size=10,
                param_attr=fluid.initializer.NumpyArrayInitializer(numpy.array([1,2])))
    """

    def __init__(self, value):
        import numpy
        assert isinstance(value, numpy.ndarray)
        super(NumpyArrayInitializer, self).__init__()
        self._value = value

    def __call__(self, var, block):
        """Add constant initialization ops for a variable

        Args:
            var: Variable that needs to be initialized
            block: The block in which initialization ops
                   should be added

        Returns:
            the initialization op
        """
        assert isinstance(var, framework.Variable)
        assert isinstance(block, framework.Block)
903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919

        # to be compatible of fp16 initalizers
        if var.dtype == VarDesc.VarType.FP16:
            out_dtype = VarDesc.VarType.FP32
            np_value = self._value.astype("float32")
            out_var = block.create_var(
                name=unique_name.generate(".".join(
                    ['numpy_array_init', var.name, 'tmp'])),
                shape=var.shape,
                dtype=out_dtype,
                type=VarDesc.VarType.LOD_TENSOR,
                persistable=False)
        else:
            out_var = var
            out_dtype = var.dtype
            np_value = self._value

920
        # Initialization Ops should be prepended and not appended
921
        if out_dtype == VarDesc.VarType.FP32:
922
            value_name = "fp32_values"
923 924
            values = [float(v) for v in np_value.flat]
        elif out_dtype == VarDesc.VarType.INT32:
925
            value_name = "int32_values"
926
            values = [int(v) for v in np_value.flat]
927 928
        else:
            raise ValueError("Unsupported dtype %s", self._value.dtype)
X
Xin Pan 已提交
929
        if self._value.size > 1024 * 1024 * 1024:
930 931 932 933
            raise ValueError("The size of input is too big. Please consider "
                             "saving it to file and 'load_op' to load it")
        op = block._prepend_op(
            type='assign_value',
934
            outputs={'Out': out_var},
935
            attrs={
936
                'dtype': out_dtype,
937
                'shape': list(self._value.shape),
938 939 940
                value_name: values
            },
            stop_gradient=True)
941 942 943 944 945 946 947 948 949

        if var.dtype == VarDesc.VarType.FP16:
            block.append_op(
                type="cast",
                inputs={"X": out_var},
                outputs={"Out": var},
                attrs={"in_dtype": out_var.dtype,
                       "out_dtype": var.dtype})

L
lujun 已提交
950
        if not framework.in_dygraph_mode():
951
            var.op = op
952 953 954
        return op


955 956 957 958 959 960 961 962 963 964 965 966
# We short the class name, since users will use the initializer with the package
# name. The sample code:
#
# import paddle.fluid as fluid
#
# hidden = fluid.layers.fc(...,
#                          param_attr=ParamAttr(fluid.initializer.Xavier()))
#
# It is no need to add an `Initializer` as the class suffix
Constant = ConstantInitializer
Uniform = UniformInitializer
Normal = NormalInitializer
967
TruncatedNormal = TruncatedNormalInitializer
968 969
Xavier = XavierInitializer
MSRA = MSRAInitializer
970
Bilinear = BilinearInitializer