clip.py 11.8 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
F
fengjiayi 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
F
fengjiayi 已提交
9 10 11 12 13
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
F
update  
fengjiayi 已提交
14

15 16
from __future__ import print_function

F
fengjiayi 已提交
17
import copy
18
import six
F
fengjiayi 已提交
19

Y
Yu Yang 已提交
20
import functools
21 22
from . import layers
from . import framework
F
fengjiayi 已提交
23
from . import core
Y
Yu Yang 已提交
24

F
fengjiayi 已提交
25
__all__ = [
26
    'ErrorClipByValue',
F
fengjiayi 已提交
27 28 29
    'GradientClipByValue',
    'GradientClipByNorm',
    'GradientClipByGlobalNorm',
F
fengjiayi 已提交
30
]
Y
Yu Yang 已提交
31 32


F
fengjiayi 已提交
33
class BaseErrorClipAttr(object):
F
fengjiayi 已提交
34 35 36
    def __str__(self):
        raise NotImplementedError()

Y
yuyang18 已提交
37
    def _append_clip_op(self, block, grad_name):
F
fengjiayi 已提交
38 39 40 41
        raise NotImplementedError()


class ErrorClipByValue(BaseErrorClipAttr):
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
    """
    Clips tensor values to the range [min, max].

    Given a tensor t, this operation clips its value to min and max inplace.

    - Any values less than min are set to min.
    - Any values greater than max are set to max.

    Args:
        max (float): The maximum value to clip by.
        min (float, optional): The minimum value to clip by. if not set by user, \
        will be set to -max by framework.

    Examples:
        .. code-block:: python

            var = fluid.framework.Variable(..., error_clip=ErrorClipByValue(max=5.0), ...)
    """

F
fengjiayi 已提交
61 62 63 64 65 66 67 68 69
    def __init__(self, max, min=None):
        max = float(max)
        if min is None:
            min = -max
        else:
            min = float(min)
        self.max = max
        self.min = min

F
fengjiayi 已提交
70 71 72
    def __str__(self):
        return "ByValue, min=%f, max=%f" % (self.min, self.max)

Y
yuyang18 已提交
73
    def _append_clip_op(self, block, grad_name):
74 75 76 77
        clip_op_desc = block.desc.append_op()
        clip_op_desc.set_type("clip")
        clip_op_desc.set_input("X", [grad_name])
        clip_op_desc.set_output("Out", [grad_name])
W
Wu Yi 已提交
78 79
        clip_op_desc._set_attr("min", self.min)
        clip_op_desc._set_attr("max", self.max)
F
fengjiayi 已提交
80 81 82 83 84 85


def error_clip_callback(block, context):
    # the context is a grad_to_var map
    grad_to_var = context
    op_desc = block.desc.op(block.desc.op_size() - 1)
86
    for grad_n in [n for n in op_desc.output_arg_names() if n in grad_to_var]:
W
Wu Yi 已提交
87
        fwd_var = block._var_recursive(grad_to_var[grad_n])
F
fengjiayi 已提交
88
        error_clip = getattr(fwd_var, "error_clip", None)
F
fengjiayi 已提交
89 90 91 92 93
        if not (error_clip is None or isinstance(error_clip,
                                                 BaseErrorClipAttr)):
            raise TypeError(
                "Variable's error_clip should be an instance of BaseErrorClipAttr or None."
            )
F
fengjiayi 已提交
94
        if error_clip is not None:
Y
yuyang18 已提交
95
            error_clip._append_clip_op(block, grad_n)
F
fengjiayi 已提交
96 97


Y
Yu Yang 已提交
98
class BaseGradientClipAttr(object):
F
fengjiayi 已提交
99 100 101
    def __str__(self):
        raise NotImplementedError()

Y
yuyang18 已提交
102
    def _process_context(self, context, param, grad):
Y
Yu Yang 已提交
103 104
        raise NotImplementedError()

Y
yuyang18 已提交
105
    def _create_operators(self, param, grad):
Y
Yu Yang 已提交
106 107 108 109
        raise NotImplementedError()


class NullGradientClipAttr(BaseGradientClipAttr):
F
fengjiayi 已提交
110 111 112
    def __str__(self):
        return "Null"

Y
yuyang18 已提交
113
    def _process_context(self, context, param, grad):
Y
Yu Yang 已提交
114 115
        pass

Y
yuyang18 已提交
116
    def _create_operators(self, param, grad):
Y
Yu Yang 已提交
117 118 119 120
        return param, grad


class GradientClipByValue(BaseGradientClipAttr):
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
    """
    Clips gradient values to the range [min, max].

    Given a tensor t, this operation clips its value to min and max inplace.

    - Any values less than min are set to min.
    - Any values greater than max are set to max.

    Args:
        max (float): The maximum value to clip by.
        min (float, optional): The minimum value to clip by. if not set by user, \
        will be set to -max by framework.

    Examples:
        .. code-block:: python

            w_param_attrs = ParamAttr(name=None,
              initializer=UniformInitializer(low=-1.0, high=1.0, seed=0),
              learning_rate=1.0,
              regularizer=L1Decay(1.0),
              trainable=True,
              clip=GradientClipByValue(-1.0, 1.0))
            y_predict = fluid.layers.fc(input=x, size=1, param_attr=w_param_attrs)
    """

Y
Yu Yang 已提交
146 147 148 149 150 151 152 153 154
    def __init__(self, max, min=None):
        max = float(max)
        if min is None:
            min = -max
        else:
            min = float(min)
        self.max = max
        self.min = min

F
fengjiayi 已提交
155 156 157
    def __str__(self):
        return "ByValue, min=%f, max=%f" % (self.min, self.max)

Y
yuyang18 已提交
158
    def _process_context(self, context, param, grad):
Y
Yu Yang 已提交
159 160
        pass

Y
yuyang18 已提交
161
    def _create_operators(self, param, grad):
Y
Yu Yang 已提交
162 163 164 165
        new_grad = layers.clip(x=grad, min=self.min, max=self.max)
        return param, new_grad


F
fengjiayi 已提交
166
class GradientClipByNorm(BaseGradientClipAttr):
167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
    """
    Clips tensor values to a maximum L2-norm.

    This operator limits the L2 norm of the input :math:`X` within :math:`max\_norm`.
    If the L2 norm of :math:`X` is less than or equal to :math:`max\_norm`, :math:`Out`
    will be the same as :math:`X`. If the L2 norm of :math:`X` is greater than
    :math:`max\_norm`, :math:`X` will be linearly scaled to make the L2 norm of
    :math:`Out` equal to :math:`max\_norm`, as shown in the following formula:

    .. math::

        Out = \\frac{max\_norm * X}{norm(X)},

    where :math:`norm(X)` represents the L2 norm of :math:`X`.

    Args:
        clip_norm (float): The maximum norm value

    Examples:
        .. code-block:: python

            w_param_attrs = ParamAttr(name=None,
              initializer=UniformInitializer(low=-1.0, high=1.0, seed=0),
              learning_rate=1.0,
              regularizer=L1Decay(1.0),
              trainable=True,
              clip=GradientClipByNorm(clip_norm=2.0))
            y_predict = fluid.layers.fc(input=x, size=1, param_attr=w_param_attrs)

    """

F
fengjiayi 已提交
198 199 200
    def __init__(self, clip_norm):
        self.clip_norm = clip_norm

F
fengjiayi 已提交
201 202 203
    def __str__(self):
        return "ByNorm, clip_norm=%f" % self.clip_norm

Y
yuyang18 已提交
204
    def _process_context(self, context, param, grad):
F
fengjiayi 已提交
205 206
        pass

Y
yuyang18 已提交
207
    def _create_operators(self, param, grad):
F
fengjiayi 已提交
208 209 210 211
        new_grad = layers.clip_by_norm(x=grad, max_norm=self.clip_norm)
        return param, new_grad


F
fengjiayi 已提交
212
class GradientClipByGlobalNorm(BaseGradientClipAttr):
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250
    """
    Clips values of multiple tensors by the ratio of the sum of their norms.

    Given a list of tensors t_list, and a clipping ratio clip_norm, this
    operation returns a list of clipped tensors list_clipped and the global
    norm (global_norm) of all tensors in t_list.

    To perform the clipping, the values :math:`t\_list[i]` are set to:

    .. math::

        t\_list[i] = t\_list[i] * \\frac{clip\_norm}{\max(global\_norm, clip\_norm)}

    where:

    .. math::

        global\_norm = \sqrt{\sum_{i=0}^{N-1}(l2norm(t\_list[i]))^2}

    If :math:`clip\_norm > global\_norm` then the entries in t_list remain as they are,
    otherwise they're all shrunk by the global ratio.

    Args:
        clip_norm (float): The maximum norm value
        group_name (str, optional): The group name for this clip.

    Examples:
        .. code-block:: python

            p_g_clip = fluid.backward.append_backward(loss=avg_cost_clip)

            with fluid.program_guard(main_program=prog_clip):
                fluid.clip.set_gradient_clip(
                    fluid.clip.GradientClipByGlobalNorm(clip_norm=2.0))
                p_g_clip = fluid.clip.append_gradient_clip_ops(p_g_clip)

    """

F
update  
fengjiayi 已提交
251
    def __init__(self, clip_norm, group_name="default_group"):
252 253
        if not isinstance(group_name, six.string_types):
            raise TypeError("'group_name' must be a %s." % (six.string_types))
F
update  
fengjiayi 已提交
254 255 256

        self.clip_norm = clip_norm
        self.group_name = group_name
257

F
fengjiayi 已提交
258 259 260 261
    def __str__(self):
        return "ByGlobalNorm, group_name=%s, clip_norm=%f" % (self.group_name,
                                                              self.clip_norm)

Y
yuyang18 已提交
262
    def _process_context(self, context, param, grad):
F
update  
fengjiayi 已提交
263 264 265 266 267 268 269 270 271 272
        if self.group_name not in context:
            context[self.group_name] = []
            context[self.group_name + "_clip_value"] = self.clip_norm
            context[self.group_name + "_clip"] = layers.fill_constant(
                shape=[1], dtype="float32", value=self.clip_norm)
        else:
            if not self.clip_norm == context[self.group_name + "_clip_value"]:
                raise ValueError(
                    "All parameters' 'clip_norm' of a same group should be the same"
                )
F
fengjiayi 已提交
273

F
update  
fengjiayi 已提交
274 275
        local_norm_var = layers.reduce_sum(input=layers.pow(x=grad, factor=2.0))
        context[self.group_name].append(local_norm_var)
F
fengjiayi 已提交
276

F
update  
fengjiayi 已提交
277
        self.context = context
278

Y
yuyang18 已提交
279
    def _create_operators(self, param, grad):
F
update  
fengjiayi 已提交
280 281 282
        group_scale_name = self.group_name + "_scale"
        if group_scale_name not in self.context:
            group_norm_var = layers.sums(input=self.context[self.group_name])
T
tensor-tang 已提交
283
            group_norm_var = layers.sqrt(x=group_norm_var)
F
update  
fengjiayi 已提交
284 285 286
            clip_var = self.context[self.group_name + "_clip"]
            group_scale_var = layers.elementwise_div(
                x=clip_var,
F
fengjiayi 已提交
287
                y=layers.elementwise_max(
F
update  
fengjiayi 已提交
288
                    x=clip_var, y=group_norm_var))
289
            assert group_scale_var.shape == (1, )
F
update  
fengjiayi 已提交
290
            self.context[group_scale_name] = group_scale_var
F
fengjiayi 已提交
291

F
update  
fengjiayi 已提交
292 293
        new_grad = layers.elementwise_mul(
            x=grad, y=self.context[group_scale_name])
294
        return param, new_grad
F
fengjiayi 已提交
295 296


F
fengjiayi 已提交
297
def set_gradient_clip(clip, param_list=None, program=None):
F
fengjiayi 已提交
298
    """
299 300 301 302 303 304 305 306 307 308
    To specify parameters that require gradient clip.

    Args:
        clip(BaseGradientClipAttr): An instance of some derived class of BaseGradientClipAttr,
                which describes the type and detailed attributes of required gradient clip.
        param_list(list(Variable)): Parameters that require gradient clip.
                It can be a list of parameter or a list of parameter's name.
                When it's None, all parameters in the program will be included.
        program(Program): The program where parameters are.
                Will be the default main program when assigned with None.
F
fengjiayi 已提交
309
    """
F
fengjiayi 已提交
310 311 312 313
    if not isinstance(clip, BaseGradientClipAttr):
        raise TypeError(
            "'clip' should be an instance of BaseGradientClipAttr's derived class"
        )
F
fengjiayi 已提交
314 315 316 317
    if program is None:
        program = framework.default_main_program()
    if param_list is None:
        param_list = program.block(0).all_parameters()
318
    if all(isinstance(elem, six.string_types) for elem in param_list):
F
fengjiayi 已提交
319 320 321 322 323 324 325
        param_list = [program.block(0).var(elem) for elem in param_list]
    if not all(isinstance(elem, framework.Parameter) for elem in param_list):
        raise TypeError(
            "'param_list' should be a list of Parameter or basestring(parameter's name)."
        )

    for param in param_list:
F
fengjiayi 已提交
326
        param.gradient_clip_attr = copy.deepcopy(clip)
F
fengjiayi 已提交
327 328


329
def append_gradient_clip_ops(param_grads):
Y
Yu Yang 已提交
330
    context = dict()
331 332 333
    for p, g in param_grads:
        if g is None:
            continue
W
Wu Yi 已提交
334
        with p.block.program._optimized_guard([p, g]):
Y
yuyang18 已提交
335 336 337 338 339 340 341
            clip_attr = getattr(p, 'gradient_clip_attr', NullGradientClipAttr())
            if clip_attr is None:
                clip_attr = NullGradientClipAttr()
            if not isinstance(clip_attr, BaseGradientClipAttr):
                raise TypeError(
                    "clip attribute should be an instance of BaseGradientClipAttr"
                )
Y
Yu Yang 已提交
342

Y
yuyang18 已提交
343
            clip_attr._process_context(context=context, param=p, grad=g)
Y
yuyang18 已提交
344 345

    res = []
346 347 348
    for p, g in param_grads:
        if g is None:
            continue
W
Wu Yi 已提交
349
        with p.block.program._optimized_guard([p, g]):
Y
yuyang18 已提交
350
            res.append(clip_attr._create_operators(param=p, grad=g))
Y
Yu Yang 已提交
351

Y
yuyang18 已提交
352
    return res
Y
Yu Yang 已提交
353 354 355


ClipByValue = GradientClipByValue
F
fengjiayi 已提交
356 357
ClipByNorm = GradientClipByNorm
ClipByGlobalNorm = GradientClipByGlobalNorm