param_attr.py 10.8 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
F
fengjiayi 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
F
fengjiayi 已提交
9 10 11 12 13
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
F
update  
fengjiayi 已提交
14

15 16
from __future__ import print_function

17
import six
18
import warnings
19

20 21
from .initializer import Initializer, Xavier, Constant
from .regularizer import WeightDecayRegularizer
Y
Yu Yang 已提交
22

23 24 25 26
__all__ = [
    'ParamAttr',
    'WeightNormParamAttr',
]
Y
Yu Yang 已提交
27

Y
Yu Yang 已提交
28 29

class ParamAttr(object):
C
chengduoZH 已提交
30
    """
Z
Zeng Jinle 已提交
31 32 33
    Create a object to represent the attribute of parameter. The attributes are:
    name, initializer, learning rate, regularizer, trainable, gradient clip,
    and model average.
34 35 36 37 38 39
    
    Note:
        ``gradient_clip`` of ``ParamAttr`` HAS BEEN DEPRECATED since 2.0. 
        It is recommended to use ``minimize(loss, grad_clip=clip)`` to clip gradient. 
        There are three clipping strategies: :ref:`api_fluid_clip_GradientClipByGlobalNorm` , 
        :ref:`api_fluid_clip_GradientClipByNorm` , :ref:`api_fluid_clip_GradientClipByValue` .
Z
Zeng Jinle 已提交
40 41 42 43 44 45 46 47 48 49

    Parameters:
        name (str, optional): The parameter's name. Default None, meaning that the name
                would be created automatically.
        initializer (Initializer, optional): The method to initial this parameter. Default
                None, meaning that the weight parameter is initialized by Xavier initializer,
                and the bias parameter is initialized by 0.
        learning_rate (float): The parameter's learning rate. The learning rate when
                optimize is the global learning rates times the parameter's learning rate times
                the factor of learning rate scheduler. Default 1.0.
50 51 52 53 54
        regularizer (WeightDecayRegularizer, optional): Regularization strategy. There are two method: 
                :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If 
                regularizer is also set in ``optimizer`` (such as :ref:`api_fluid_optimizer_SGDOptimizer` ), 
                that regularizer setting in optimizer will be ignored. Default None, meaning there is 
                no regularization.
Z
Zeng Jinle 已提交
55 56 57
        trainable (bool): Whether this parameter is trainable. Default True.
        do_model_average (bool): Whether this parameter should do model average
                when model average is enabled. Default False.
C
chengduoZH 已提交
58 59 60 61

    Examples:
        .. code-block:: python

Z
Zeng Jinle 已提交
62 63
            import paddle.fluid as fluid

C
chengduoZH 已提交
64 65
            w_param_attrs = fluid.ParamAttr(name="fc_weight",
                                            learning_rate=0.5,
T
Tink_Y 已提交
66
                                            regularizer=fluid.regularizer.L2Decay(1.0),
C
chengduoZH 已提交
67
                                            trainable=True)
Z
Zeng Jinle 已提交
68
            print(w_param_attrs.name) # "fc_weight"
69
            x = fluid.data(name='X', shape=[None, 1], dtype='float32')
C
chengduoZH 已提交
70 71 72
            y_predict = fluid.layers.fc(input=x, size=10, param_attr=w_param_attrs)
    """

Y
Yu Yang 已提交
73 74 75 76 77
    def __init__(self,
                 name=None,
                 initializer=None,
                 learning_rate=1.0,
                 regularizer=None,
Y
Yu Yang 已提交
78
                 trainable=True,
79
                 do_model_average=True):
Y
Yu Yang 已提交
80
        self.name = name
H
hong 已提交
81 82 83
        if isinstance(self.name, six.string_types) and self.name == "":
            raise ValueError("name of ParamAttr can not be empty str")

Y
Yu Yang 已提交
84 85 86 87
        self.initializer = initializer
        self.learning_rate = learning_rate
        self.regularizer = regularizer
        self.trainable = trainable
88
        self.do_model_average = do_model_average
Y
Yu Yang 已提交
89

Y
yuyang18 已提交
90
    def _set_default_initializer(self, initializer):
C
chengduoZH 已提交
91 92 93
        """
        Set the default initializer, the initializer should be Constant,
        Uniform, Normal, Xavier, MSRA.
C
chengduoZH 已提交
94 95 96 97 98 99

        Args:
            initializer(Initializer): the initializer to set.

        Returns:
            None
C
chengduoZH 已提交
100
        """
Y
Yu Yang 已提交
101 102 103 104 105 106 107 108 109 110
        if initializer is None:
            if self.initializer is None:
                raise ValueError("ParamAttr.initializer is not set")
            return

        if self.initializer is not None:
            return

        self.initializer = initializer

Y
yuyang18 已提交
111
    def _set_default_param_initializer(self):
C
chengduoZH 已提交
112 113
        """
        Set the default initializer for the parameter with Xavier.
C
chengduoZH 已提交
114 115 116 117 118 119

        Args:
            None.

        Returns:
            None.
C
chengduoZH 已提交
120
        """
Y
yuyang18 已提交
121
        self._set_default_initializer(Xavier())
Y
Yu Yang 已提交
122

Y
yuyang18 已提交
123
    def _set_default_bias_initializer(self):
C
chengduoZH 已提交
124 125
        """
        Set the default initializer for the bias with Constant(0.0).
C
chengduoZH 已提交
126 127 128 129 130 131

        Args:
            None.

        Returns:
            None.
C
chengduoZH 已提交
132
        """
Y
yuyang18 已提交
133
        self._set_default_initializer(Constant(0.0))
Y
Yu Yang 已提交
134 135

    @staticmethod
Y
yuyang18 已提交
136
    def _to_attr(arg):
C
chengduoZH 已提交
137 138 139 140 141 142 143 144 145 146 147 148 149 150
        """
        Create ParamAttr[s].

        Args:
            arg: Arguments to initialize ParamAttr[s]. arg's type can be
                str, Initializer, float, WeightDecayRegularizer, BaseGradientClipAttr,
                bool, ParamAttr, or a list of above type.

        Returns:
            ParamAttr[s]: ParamAttr[s] initialized with arg.

        Raises:
            arg can not initialize a ParamAttr.
        """
Y
Yu Yang 已提交
151 152
        if arg is None:
            return ParamAttr()
153
        elif isinstance(arg, list) or isinstance(arg, tuple):
Y
yuyang18 已提交
154
            return [ParamAttr._to_attr(a) for a in arg]
Y
Yu Yang 已提交
155 156
        elif isinstance(arg, ParamAttr):
            return arg
157
        elif isinstance(arg, six.string_types):
Y
Yu Yang 已提交
158 159 160 161 162 163
            return ParamAttr(name=arg)
        elif isinstance(arg, Initializer):
            return ParamAttr(initializer=arg)
        elif isinstance(arg, WeightDecayRegularizer):
            return ParamAttr(regularizer=arg)
        elif isinstance(arg, bool):
Y
yuyang18 已提交
164
            return ParamAttr._to_attr(None) if arg else False
Y
Yu Yang 已提交
165 166 167
        else:
            raise TypeError("{0} cast to ParamAttr".format(type(arg)))

Y
yuyang18 已提交
168
    def _to_kwargs(self, with_initializer=False):
C
chengduoZH 已提交
169 170 171 172 173 174 175 176 177
        """
        Returns the attributes of this parameter.

        Args:
            with_initializer(bool): Whether to add initializer attr.

        Returns:
            Parameter attributes(map): The attributes of this parameter.
        """
Y
Yu Yang 已提交
178 179
        kwargs = {
            'name': self.name,
G
guosheng 已提交
180 181 182
            'optimize_attr': {
                'learning_rate': self.learning_rate
            },
Y
Yu Yang 已提交
183
            'regularizer': self.regularizer,
Y
Yu Yang 已提交
184
            'trainable': self.trainable,
185
            'do_model_average': self.do_model_average
Y
Yu Yang 已提交
186 187 188 189
        }
        if with_initializer:
            kwargs['initializer'] = self.initializer
        return kwargs
G
guosheng 已提交
190 191 192 193


class WeightNormParamAttr(ParamAttr):
    """
194
    Parameter of weight Norm. Weight Norm is a reparameterization of the weight vectors
195
    in a neural network that decouples the magnitude of those weight vectors from
C
chengduoZH 已提交
196 197 198 199
    their direction. Weight Norm has been implemented as discussed in this
    paper: `Weight Normalization: A Simple Reparameterization to Accelerate
    Training of Deep Neural Networks
    <https://arxiv.org/pdf/1602.07868.pdf>`_.
200 201 202 203 204 205
      
    Note:
        ``gradient_clip`` of ``WeightNormParamAttr`` HAS BEEN DEPRECATED since 2.0. 
        It is recommended to use ``minimize(loss, grad_clip=clip)`` to clip gradient. 
        There are three clipping strategies: :ref:`api_fluid_clip_GradientClipByGlobalNorm` , 
        :ref:`api_fluid_clip_GradientClipByNorm` , :ref:`api_fluid_clip_GradientClipByValue` .
C
chengduoZH 已提交
206 207

    Args:
208 209
        dim(int): Dimension over which to compute the norm. Dim is a non-negative
            number which is less than the rank of weight Tensor. For Example, dim can
T
tianshuo78520a 已提交
210
            be chosen from 0, 1, 2, 3 for convolution whose weight shape is [cout, cin, kh, kw]
211 212 213 214 215 216 217 218 219
            and rank is 4. Default None, meaning that all elements will be normalized.
        name(str, optional): The parameter's name. Default None, meaning that the name would
            be created automatically. Please refer to :ref:`api_guide_Name` for more details.
        initializer(Initializer): The method to initialize this parameter, such as
            ``initializer = fluid.initializer.ConstantInitializer(1.0)``. Default None,
            meaning that the weight parameter is initialized by Xavier initializer, and
            the bias parameter is initialized by 0.
        learning_rate(float32): The parameter's learning rate when
            optimizer is :math:`global\_lr * parameter\_lr * scheduler\_factor`.
X
Xin Pan 已提交
220
            Default 1.0.
221 222 223 224
        regularizer (WeightDecayRegularizer, optional): Regularization strategy. There are two method: 
            :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If regularizer 
            is also set in ``optimizer`` (such as :ref:`api_fluid_optimizer_SGDOptimizer` ), that regularizer 
            setting in optimizer will be ignored. Default None, meaning there is no regularization.
225 226
        trainable(bool, optional): Whether this parameter is trainable. Default True.
        do_model_average(bool, optional): Whether this parameter should do model average.
X
Xin Pan 已提交
227
            Default False.
C
chengduoZH 已提交
228 229 230

    Examples:
        .. code-block:: python
231 232
            
            import paddle.fluid as fluid
C
chengduoZH 已提交
233 234 235
            data = fluid.layers.data(name="data", shape=[3, 32, 32], dtype="float32")
            fc = fluid.layers.fc(input=data,
                                 size=1000,
236
                                 param_attr=fluid.WeightNormParamAttr(
237 238 239 240 241 242 243
                                          dim=None,
                                          name='weight_norm_param',
                                          initializer=fluid.initializer.ConstantInitializer(1.0),
                                          learning_rate=1.0,
                                          regularizer=fluid.regularizer.L2DecayRegularizer(regularization_coeff=0.1),
                                          trainable=True,
                                          do_model_average=False))
C
chengduoZH 已提交
244

G
guosheng 已提交
245 246 247
    """
    # List to record the parameters reparameterized by weight normalization.
    # If these parameters are treated as Variable rather than Parameter,
248
    # it can be used to discriminate these parameters and help to serialize
G
guosheng 已提交
249 250 251
    # these paramters for inference.
    params_with_weight_norm = []

X
Xin Pan 已提交
252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
    def __init__(self,
                 dim=None,
                 name=None,
                 initializer=None,
                 learning_rate=1.0,
                 regularizer=None,
                 trainable=True,
                 do_model_average=False):
        super(WeightNormParamAttr, self).__init__(
            name=name,
            initializer=initializer,
            learning_rate=learning_rate,
            regularizer=regularizer,
            trainable=trainable,
            do_model_average=do_model_average)
G
guosheng 已提交
267
        self.dim = dim