regularizer.py 11.2 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
import logging
16

17
from . import framework
H
hong 已提交
18
from .framework import _non_static_mode, _varbase_creator, in_dygraph_mode
C
chengduoZH 已提交
19
from . import core
20
from paddle import _C_ops, _legacy_C_ops
21

Y
yuyang18 已提交
22
__all__ = ['L1Decay', 'L2Decay', 'L1DecayRegularizer', 'L2DecayRegularizer']
23 24


25
class WeightDecayRegularizer:
26 27 28 29 30 31 32 33 34 35 36 37 38
    """Base class for weight decay regularizers

    Defines the common interface of weight-decay regularizers.
    Weight-decay regularizers are added only during the backward
    pass for faster regularization. They add operations to the network
    that correspond to gradient of the regularization function.
    Users should not use this class directly, but need to use one
    of its implementations
    """

    def __init__(self):
        pass

C
chengduoZH 已提交
39
    def __call__(self, param, grad, block):
40
        """Add corresponding weight decay operations to the network"""
41 42
        raise NotImplementedError()

F
fengjiayi 已提交
43
    def __str__(self):
44
        """Debug string"""
F
fengjiayi 已提交
45 46
        raise NotImplementedError()

47 48

class L2DecayRegularizer(WeightDecayRegularizer):
49
    r"""
50
    Implement the L2 Weight Decay Regularization, which helps to prevent the model over-fitting.
51

52 53 54
    It can be set in :ref:`api_fluid_ParamAttr` or ``optimizer`` (such as :ref:`api_fluid_optimizer_SGDOptimizer` ).
    When set in ``ParamAttr`` , it only takes effect for trainable parameters in this layer. When set in
    ``optimizer`` , it takes effect for all trainable parameters. When set together, ``ParamAttr`` has
55
    higher priority than ``optimizer`` .
56

57
    In the implementation, the formula of L2 Weight Decay Regularization is as follows:
58 59 60 61 62 63

    .. math::

        L2WeightDecay = reg\_coeff * parameter

    Args:
64
        regularization_coeff(float, optional): regularization coeff. Default:0.0
65 66 67 68

    Examples:
        .. code-block:: python

69
            # Example1: set Regularizer in optimizer
70
            import paddle.fluid as fluid
2
201716010711 已提交
71 72
            import paddle
            paddle.enable_static()
73

74 75 76 77 78 79 80
            main_prog = fluid.Program()
            startup_prog = fluid.Program()
            with fluid.program_guard(main_prog, startup_prog):
                data = fluid.layers.data(name='image', shape=[3, 28, 28], dtype='float32')
                label = fluid.layers.data(name='label', shape=[1], dtype='int64')
                hidden = fluid.layers.fc(input=data, size=128, act='relu')
                prediction = fluid.layers.fc(input=hidden, size=10, act='softmax')
81 82 83 84
                loss = paddle.nn.functional.cross_entropy(
                    input=prediction, label=label,
                    reduction='none', use_softmax=False
                )
2
201716010711 已提交
85
                avg_loss = paddle.mean(loss)
86 87
            optimizer = fluid.optimizer.Adagrad(
                learning_rate=1e-4,
88
                regularization=fluid.regularizer.L2Decay(
89
                    regularization_coeff=0.1))
90
            optimizer.minimize(avg_loss)
91 92 93 94


            # Example2: set Regularizer both in ParamAttr and optimizer
            import paddle.fluid as fluid
2
201716010711 已提交
95 96
            import paddle
            paddle.enable_static()
97 98 99

            l1 = fluid.regularizer.L1Decay(regularization_coeff=0.1)
            l2 = fluid.regularizer.L2Decay(regularization_coeff=0.1)
100
            x = paddle.uniform([3,4])
101

102 103 104 105 106
            # set L1 regularization in fluid.ParamAttr
            w_param = fluid.ParamAttr(regularizer=l1)
            hidden1 = fluid.layers.fc(x, 8, param_attr=w_param)  # fc_0.w_0(L1), fc_0.b_0
            hidden2 = fluid.layers.fc(hidden1, 16, param_attr=w_param)   # fc_1.w_0(L1), fc_1.b_0
            predict = fluid.layers.fc(hidden2, 32)    # fc_3.w_0, fc_3.b_0
2
201716010711 已提交
107
            avg_loss = paddle.mean(predict)
108 109 110 111

            # set L2 regularization in optimizer
            optimizer = fluid.optimizer.SGD(learning_rate=1e-4, regularization=l2)
            optimizer.minimize(avg_loss)
112

113
            # it will Print Message:
114
            # Regularization of [fc_0.w_0, fc_1.w_0] have been set by ParamAttr or WeightNormParamAttr already.
115 116
            # So, the Regularization of Optimizer will not take effect for these parameters!

117 118 119 120
    """

    def __init__(self, regularization_coeff=0.0):
        assert regularization_coeff is not None
121
        super().__init__()
122 123
        self._regularization_coeff = regularization_coeff

C
chengduoZH 已提交
124
    def __call__(self, param, grad, block):
125 126 127 128 129 130 131 132 133 134 135 136
        """Add L2 weight decay ops to network

        Adds L2 weight decay ops.
        L2WeightDecay = reg_coeff * parameter

        Args:
            param: parameter variable for which regularization is applied
            block: block in which variable is to be created

        Returns:
            new variable for weight decay
        """
137
        assert isinstance(param, framework.Variable)
138
        assert isinstance(block, framework.Block)
C
chengduoZH 已提交
139

J
Jiabin Yang 已提交
140
        if framework._non_static_mode():
141
            if framework.in_dygraph_mode():
142 143 144
                return _C_ops.scale(
                    param, self._regularization_coeff, 0.0, True
                )
145
            else:
146 147 148
                return _legacy_C_ops.scale(
                    param, "scale", self._regularization_coeff
                )
H
Hongyu Liu 已提交
149
        else:
150 151 152
            decay = block.create_var(
                dtype=param.dtype, shape=param.shape, lod_level=param.lod_level
            )
C
chengduoZH 已提交
153

154
            # Append Op to calculate decay
155 156 157 158 159 160
            block.append_op(
                type='scale',
                inputs={"X": param},
                outputs={"Out": decay},
                attrs={"scale": self._regularization_coeff},
            )
161

162
            return decay
163

F
fengjiayi 已提交
164 165 166
    def __str__(self):
        return "L2Decay, regularization_coeff=%f" % self._regularization_coeff

167 168

class L1DecayRegularizer(WeightDecayRegularizer):
169
    r"""
170
    Implement the L1 Weight Decay Regularization, which encourages the weights to be sparse.
171 172 173 174

    It can be set in :ref:`api_fluid_ParamAttr` or ``optimizer`` (such as :ref:`api_fluid_optimizer_SGDOptimizer` ).
    When set in ``ParamAttr`` , it only takes effect for trainable parameters in this layer. When set in
    ``optimizer`` , it takes effect for all trainable parameters. When set together, ``ParamAttr`` has
175
    higher priority than ``optimizer`` .
176

177
    In the implementation, the formula of L1 Weight Decay Regularization is as follows:
178

179 180 181 182 183
    .. math::

        L1WeightDecay = reg\_coeff * sign(parameter)

    Args:
184
        regularization_coeff(float, optional): regularization coeff. Default:0.0.
185

186 187 188
    Examples:
        .. code-block:: python

189
            # Example1: set Regularizer in optimizer
190
            import paddle.fluid as fluid
2
201716010711 已提交
191 192
            import paddle
            paddle.enable_static()
193 194 195 196 197 198 199
            main_prog = fluid.Program()
            startup_prog = fluid.Program()
            with fluid.program_guard(main_prog, startup_prog):
                data = fluid.layers.data(name='image', shape=[3, 28, 28], dtype='float32')
                label = fluid.layers.data(name='label', shape=[1], dtype='int64')
                hidden = fluid.layers.fc(input=data, size=128, act='relu')
                prediction = fluid.layers.fc(input=hidden, size=10, act='softmax')
200 201 202 203
                loss = paddle.nn.functional.cross_entropy(
                    input=prediction, label=label,
                    reduction='none', use_softmax=False
                )
2
201716010711 已提交
204
                avg_loss = paddle.mean(loss)
X
Xin Pan 已提交
205 206 207 208
            optimizer = fluid.optimizer.Adagrad(
                learning_rate=1e-4,
                regularization=fluid.regularizer.L1DecayRegularizer(
                    regularization_coeff=0.1))
209
            optimizer.minimize(avg_loss)
210

211 212 213

            # Example2: set Regularizer both in ParamAttr and optimizer
            import paddle.fluid as fluid
2
201716010711 已提交
214 215
            import paddle
            paddle.enable_static()
216 217
            l1 = fluid.regularizer.L1Decay(regularization_coeff=0.1)
            l2 = fluid.regularizer.L2Decay(regularization_coeff=0.1)
218
            x = paddle.uniform([3,4])
219

220 221 222 223 224
            # set L1 regularization in fluid.ParamAttr
            w_param = fluid.ParamAttr(regularizer=l1)
            hidden1 = fluid.layers.fc(x, 8, param_attr=w_param)  # fc_0.w_0(L1), fc_0.b_0
            hidden2 = fluid.layers.fc(hidden1, 16, param_attr=w_param)  # fc_1.w_0(L1), fc_1.b_0
            predict = fluid.layers.fc(hidden2, 32)   # fc_3.w_0, fc_3.b_0
2
201716010711 已提交
225
            avg_loss = paddle.mean(predict)
226 227 228 229

            # set L2 regularization in optimizer
            optimizer = fluid.optimizer.SGD(learning_rate=1e-4, regularization=l2)
            optimizer.minimize(avg_loss)
230

231
            # it will Print Message:
232
            # Regularization of [fc_0.w_0, fc_1.w_0] have been set by ParamAttr or WeightNormParamAttr already.
233 234
            # So, the Regularization of Optimizer will not take effect for these parameters!

235 236 237 238
    """

    def __init__(self, regularization_coeff=0.0):
        assert regularization_coeff is not None
239
        super().__init__()
240 241
        self._regularization_coeff = regularization_coeff

C
chengduoZH 已提交
242
    def __call__(self, param, grad, block):
243 244 245 246 247 248 249 250 251 252 253 254
        """Add L1 weight decay ops to network

        Adds L1 weight decay ops.
        L1WeightDecay = reg_coeff * sign(parameter)

        Args:
            param: parameter variable for which regularization is applied
            block: block in which variable is to be created

        Returns:
            new variable for weight decay
        """
255
        assert isinstance(param, framework.Variable)
256
        assert isinstance(block, framework.Block)
C
chengduo 已提交
257

J
Jiabin Yang 已提交
258
        if framework._non_static_mode():
259
            sign = block.create_var(dtype=param.dtype, shape=param.shape)
H
Hongyu Liu 已提交
260 261
            decay = block.create_var(dtype=param.dtype, shape=param.shape)
        else:
262 263 264 265 266 267
            sign = block.create_var(
                dtype=param.dtype, shape=param.shape, lod_level=param.lod_level
            )
            decay = block.create_var(
                dtype=param.dtype, shape=param.shape, lod_level=param.lod_level
            )
H
hong 已提交
268
        if in_dygraph_mode():
269 270
            sign = _C_ops.sign(param)
            return _C_ops.scale(sign, self._regularization_coeff, 0.0, True)
C
chengduoZH 已提交
271

272
        # Append sign op
273
        block.append_op(type='sign', inputs={"X": param}, outputs={"Out": sign})
274 275

        # Append scale op to the output of sign op
276 277 278 279 280 281
        block.append_op(
            type='scale',
            inputs={"X": sign},
            outputs={"Out": decay},
            attrs={"scale": self._regularization_coeff},
        )
282 283

        return decay
284

F
fengjiayi 已提交
285 286 287
    def __str__(self):
        return "L1Decay, regularization_coeff=%f" % self._regularization_coeff

288 289 290 291 292 293 294

# We short the class name, since users will use the regulaizer with the package
# name. The sample code:
#
# import paddle.fluid as fluid
#
# hidden = fluid.layers.fc(...,
Y
Yu Yang 已提交
295
#                          param_attr=fluid.regularizer.Xavier())
296 297 298 299
#
# It is no need to add a `Regularizer` as the class suffix
L1Decay = L1DecayRegularizer
L2Decay = L2DecayRegularizer