regularizer.py 10.6 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from __future__ import print_function
16
import logging
17

18
from . import framework
J
Jiabin Yang 已提交
19
from .framework import _non_static_mode, _varbase_creator
C
chengduoZH 已提交
20
from . import core
W
wanghuancoder 已提交
21
from paddle import _C_ops
22

Y
yuyang18 已提交
23
__all__ = ['L1Decay', 'L2Decay', 'L1DecayRegularizer', 'L2DecayRegularizer']
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39


class WeightDecayRegularizer(object):
    """Base class for weight decay regularizers

    Defines the common interface of weight-decay regularizers.
    Weight-decay regularizers are added only during the backward
    pass for faster regularization. They add operations to the network
    that correspond to gradient of the regularization function.
    Users should not use this class directly, but need to use one
    of its implementations
    """

    def __init__(self):
        pass

C
chengduoZH 已提交
40
    def __call__(self, param, grad, block):
41 42 43 44
        """Add corresponding weight decay operations to the network
        """
        raise NotImplementedError()

F
fengjiayi 已提交
45 46 47 48 49
    def __str__(self):
        """Debug string
        """
        raise NotImplementedError()

50 51

class L2DecayRegularizer(WeightDecayRegularizer):
52
    r""" 
53
    Implement the L2 Weight Decay Regularization, which helps to prevent the model over-fitting.
54

55 56 57 58 59
    It can be set in :ref:`api_fluid_ParamAttr` or ``optimizer`` (such as :ref:`api_fluid_optimizer_SGDOptimizer` ). 
    When set in ``ParamAttr`` , it only takes effect for trainable parameters in this layer. When set in 
    ``optimizer`` , it takes effect for all trainable parameters. When set together, ``ParamAttr`` has 
    higher priority than ``optimizer`` .
    
60
    In the implementation, the formula of L2 Weight Decay Regularization is as follows:
61 62 63 64 65 66

    .. math::

        L2WeightDecay = reg\_coeff * parameter

    Args:
67
        regularization_coeff(float, optional): regularization coeff. Default:0.0
68 69 70 71

    Examples:
        .. code-block:: python

72
            # Example1: set Regularizer in optimizer
73
            import paddle.fluid as fluid
74

75 76 77 78 79 80 81 82 83
            main_prog = fluid.Program()
            startup_prog = fluid.Program()
            with fluid.program_guard(main_prog, startup_prog):
                data = fluid.layers.data(name='image', shape=[3, 28, 28], dtype='float32')
                label = fluid.layers.data(name='label', shape=[1], dtype='int64')
                hidden = fluid.layers.fc(input=data, size=128, act='relu')
                prediction = fluid.layers.fc(input=hidden, size=10, act='softmax')
                loss = fluid.layers.cross_entropy(input=prediction, label=label)
                avg_loss = fluid.layers.mean(loss)
84 85
            optimizer = fluid.optimizer.Adagrad(
                learning_rate=1e-4,
86
                regularization=fluid.regularizer.L2Decay(
87
                    regularization_coeff=0.1))
88
            optimizer.minimize(avg_loss)
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112


            # Example2: set Regularizer both in ParamAttr and optimizer
            import paddle.fluid as fluid

            l1 = fluid.regularizer.L1Decay(regularization_coeff=0.1)
            l2 = fluid.regularizer.L2Decay(regularization_coeff=0.1)
            x = fluid.layers.uniform_random([3,4])
            
            # set L1 regularization in fluid.ParamAttr
            w_param = fluid.ParamAttr(regularizer=l1)
            hidden1 = fluid.layers.fc(x, 8, param_attr=w_param)  # fc_0.w_0(L1), fc_0.b_0
            hidden2 = fluid.layers.fc(hidden1, 16, param_attr=w_param)   # fc_1.w_0(L1), fc_1.b_0
            predict = fluid.layers.fc(hidden2, 32)    # fc_3.w_0, fc_3.b_0
            avg_loss = fluid.layers.mean(predict)

            # set L2 regularization in optimizer
            optimizer = fluid.optimizer.SGD(learning_rate=1e-4, regularization=l2)
            optimizer.minimize(avg_loss)
            
            # it will Print Message:
            # Regularization of [fc_0.w_0, fc_1.w_0] have been set by ParamAttr or WeightNormParamAttr already. 
            # So, the Regularization of Optimizer will not take effect for these parameters!

113 114 115 116 117 118 119
    """

    def __init__(self, regularization_coeff=0.0):
        assert regularization_coeff is not None
        super(L2DecayRegularizer, self).__init__()
        self._regularization_coeff = regularization_coeff

C
chengduoZH 已提交
120
    def __call__(self, param, grad, block):
121 122 123 124 125 126 127 128 129 130 131 132
        """Add L2 weight decay ops to network

        Adds L2 weight decay ops.
        L2WeightDecay = reg_coeff * parameter

        Args:
            param: parameter variable for which regularization is applied
            block: block in which variable is to be created

        Returns:
            new variable for weight decay
        """
133
        assert isinstance(param, framework.Variable)
134
        assert isinstance(block, framework.Block)
C
chengduoZH 已提交
135

J
Jiabin Yang 已提交
136
        if framework._non_static_mode():
W
wanghuancoder 已提交
137
            return _C_ops.scale(param, "scale", self._regularization_coeff)
H
Hongyu Liu 已提交
138 139 140
        else:
            decay = block.create_var(
                dtype=param.dtype, shape=param.shape, lod_level=param.lod_level)
C
chengduoZH 已提交
141

142 143 144 145 146 147
            # Append Op to calculate decay
            block.append_op(
                type='scale',
                inputs={"X": param},
                outputs={"Out": decay},
                attrs={"scale": self._regularization_coeff})
148

149
            return decay
150

F
fengjiayi 已提交
151 152 153
    def __str__(self):
        return "L2Decay, regularization_coeff=%f" % self._regularization_coeff

154 155

class L1DecayRegularizer(WeightDecayRegularizer):
156
    r"""
157 158
    Implement the L1 Weight Decay Regularization, which encourages the weights to be sparse.
    
159 160 161 162 163
    It can be set in :ref:`api_fluid_ParamAttr` or ``optimizer`` (such as :ref:`api_fluid_optimizer_SGDOptimizer` ). 
    When set in ``ParamAttr`` , it only takes effect for trainable parameters in this layer. When set in 
    ``optimizer`` , it takes effect for all trainable parameters. When set together, ``ParamAttr`` has 
    higher priority than ``optimizer`` .
    
164 165
    In the implementation, the formula of L1 Weight Decay Regularization is as follows:
	
166 167 168 169 170
    .. math::

        L1WeightDecay = reg\_coeff * sign(parameter)

    Args:
171
        regularization_coeff(float, optional): regularization coeff. Default:0.0.
172
	
173 174 175
    Examples:
        .. code-block:: python

176
            # Example1: set Regularizer in optimizer
177
            import paddle.fluid as fluid
178

179 180 181 182 183 184 185 186 187
            main_prog = fluid.Program()
            startup_prog = fluid.Program()
            with fluid.program_guard(main_prog, startup_prog):
                data = fluid.layers.data(name='image', shape=[3, 28, 28], dtype='float32')
                label = fluid.layers.data(name='label', shape=[1], dtype='int64')
                hidden = fluid.layers.fc(input=data, size=128, act='relu')
                prediction = fluid.layers.fc(input=hidden, size=10, act='softmax')
                loss = fluid.layers.cross_entropy(input=prediction, label=label)
                avg_loss = fluid.layers.mean(loss)
X
Xin Pan 已提交
188 189 190 191
            optimizer = fluid.optimizer.Adagrad(
                learning_rate=1e-4,
                regularization=fluid.regularizer.L1DecayRegularizer(
                    regularization_coeff=0.1))
192
            optimizer.minimize(avg_loss)
193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
 

            # Example2: set Regularizer both in ParamAttr and optimizer
            import paddle.fluid as fluid

            l1 = fluid.regularizer.L1Decay(regularization_coeff=0.1)
            l2 = fluid.regularizer.L2Decay(regularization_coeff=0.1)
            x = fluid.layers.uniform_random([3,4])
            
            # set L1 regularization in fluid.ParamAttr
            w_param = fluid.ParamAttr(regularizer=l1)
            hidden1 = fluid.layers.fc(x, 8, param_attr=w_param)  # fc_0.w_0(L1), fc_0.b_0
            hidden2 = fluid.layers.fc(hidden1, 16, param_attr=w_param)  # fc_1.w_0(L1), fc_1.b_0
            predict = fluid.layers.fc(hidden2, 32)   # fc_3.w_0, fc_3.b_0
            avg_loss = fluid.layers.mean(predict)

            # set L2 regularization in optimizer
            optimizer = fluid.optimizer.SGD(learning_rate=1e-4, regularization=l2)
            optimizer.minimize(avg_loss)
            
            # it will Print Message:
            # Regularization of [fc_0.w_0, fc_1.w_0] have been set by ParamAttr or WeightNormParamAttr already. 
            # So, the Regularization of Optimizer will not take effect for these parameters!

217 218 219 220 221 222 223
    """

    def __init__(self, regularization_coeff=0.0):
        assert regularization_coeff is not None
        super(L1DecayRegularizer, self).__init__()
        self._regularization_coeff = regularization_coeff

C
chengduoZH 已提交
224
    def __call__(self, param, grad, block):
225 226 227 228 229 230 231 232 233 234 235 236
        """Add L1 weight decay ops to network

        Adds L1 weight decay ops.
        L1WeightDecay = reg_coeff * sign(parameter)

        Args:
            param: parameter variable for which regularization is applied
            block: block in which variable is to be created

        Returns:
            new variable for weight decay
        """
237
        assert isinstance(param, framework.Variable)
238
        assert isinstance(block, framework.Block)
C
chengduo 已提交
239

J
Jiabin Yang 已提交
240
        if framework._non_static_mode():
241
            sign = block.create_var(dtype=param.dtype, shape=param.shape)
H
Hongyu Liu 已提交
242 243
            decay = block.create_var(dtype=param.dtype, shape=param.shape)
        else:
244 245
            sign = block.create_var(
                dtype=param.dtype, shape=param.shape, lod_level=param.lod_level)
H
Hongyu Liu 已提交
246 247
            decay = block.create_var(
                dtype=param.dtype, shape=param.shape, lod_level=param.lod_level)
C
chengduoZH 已提交
248

249
        # Append sign op
250
        block.append_op(type='sign', inputs={"X": param}, outputs={"Out": sign})
251 252 253 254

        # Append scale op to the output of sign op
        block.append_op(
            type='scale',
255
            inputs={"X": sign},
256 257 258 259
            outputs={"Out": decay},
            attrs={"scale": self._regularization_coeff})

        return decay
260

F
fengjiayi 已提交
261 262 263
    def __str__(self):
        return "L1Decay, regularization_coeff=%f" % self._regularization_coeff

264 265 266 267 268 269 270

# We short the class name, since users will use the regulaizer with the package
# name. The sample code:
#
# import paddle.fluid as fluid
#
# hidden = fluid.layers.fc(...,
Y
Yu Yang 已提交
271
#                          param_attr=fluid.regularizer.Xavier())
272 273 274 275
#
# It is no need to add a `Regularizer` as the class suffix
L1Decay = L1DecayRegularizer
L2Decay = L2DecayRegularizer