提交 8623e48b 编写于 作者: A Abhinav Arora 提交者: GitHub

Add python API for backward regularization ops (#5135)

* Add regularizer code
* Fix code
上级 be00b0c4
...@@ -505,6 +505,8 @@ class Parameter(Variable): ...@@ -505,6 +505,8 @@ class Parameter(Variable):
self.optimize_attr = kwargs.get('optimize_attr', {'learning_rate': 1.0}) self.optimize_attr = kwargs.get('optimize_attr', {'learning_rate': 1.0})
self.regularizer = kwargs.get('regularizer', None)
# program is a global instance. # program is a global instance.
g_program = Program() g_program = Program()
......
...@@ -2,6 +2,7 @@ from collections import defaultdict ...@@ -2,6 +2,7 @@ from collections import defaultdict
import paddle.v2.framework.framework as framework import paddle.v2.framework.framework as framework
from paddle.v2.framework.backward import append_backward_ops from paddle.v2.framework.backward import append_backward_ops
from paddle.v2.framework.regularizer import append_regularization_ops
__all__ = [ __all__ = [
'SGDOptimizer', 'MomentumOptimizer', 'AdagradOptimizer', 'AdamOptimizer', 'SGDOptimizer', 'MomentumOptimizer', 'AdagradOptimizer', 'AdamOptimizer',
...@@ -161,6 +162,8 @@ class Optimizer(object): ...@@ -161,6 +162,8 @@ class Optimizer(object):
""" """
params_grads = append_backward_ops(loss, parameter_list, no_grad_set or params_grads = append_backward_ops(loss, parameter_list, no_grad_set or
set()) set())
# Add regularization if any
params_grads = append_regularization_ops(params_grads)
optimize_ops = self.create_optimization_pass(params_grads, loss) optimize_ops = self.create_optimization_pass(params_grads, loss)
return optimize_ops return optimize_ops
......
import paddle.v2.framework.framework as framework
__all__ = ['append_regularization_ops', 'L2DecayRegularizer']
def append_regularization_ops(parameters_and_grads):
"""Create and add backward regularization Operators
Creates and adds backward regularization operators in the BlockDesc.
This will add gradients of the regularizer function to the gradients
of the parameters and return these modified gradients. This is the
same as implementing weight decay in optimizers for regularization.
Args:
parameters_and_grads: A list of (parameters, gradients) pairs
that need to be regularized.
Returns:
list of (parameters, gradients) pair with the regularized gradient
Raises:
Exception: Unknown regularization type
"""
params_and_grads = []
for param, grad in parameters_and_grads:
# If no gradient or no regularization specified,
# then we don't need to do anything
if grad is None or param.regularizer is None:
params_and_grads.append((param, grad))
continue
# Add variable for regularization term in grad block
regularization_term = param.regularizer(param, grad.block)
assert grad.shape == regularization_term.shape
grad.block.append_op(
type='elementwise_add',
inputs={"X": grad,
"Y": regularization_term},
outputs={"Out": grad})
params_and_grads.append((param, grad))
return params_and_grads
class WeightDecayRegularizer(object):
"""Base class for weight decay regularizers
Defines the common interface of weight-decay regularizers.
Weight-decay regularizers are added only during the backward
pass for faster regularization. They add operations to the network
that correspond to gradient of the regularization function.
Users should not use this class directly, but need to use one
of its implementations
"""
def __init__(self):
pass
def __call__(self, param, block):
"""Add corresponding weight decay operations to the network
"""
raise NotImplementedError()
class L2DecayRegularizer(WeightDecayRegularizer):
"""Implements the L2 Weight Decay Regularization
"""
def __init__(self, regularization_coeff=0.0):
assert regularization_coeff is not None
super(L2DecayRegularizer, self).__init__()
self._regularization_coeff = regularization_coeff
def __call__(self, param, block):
"""Add L2 weight decay ops to network
Adds L2 weight decay ops.
L2WeightDecay = reg_coeff * parameter
Args:
param: parameter variable for which regularization is applied
block: block in which variable is to be created
Returns:
new variable for weight decay
"""
assert isinstance(param, framework.Parameter)
assert isinstance(block, framework.Block)
decay = block.create_var(
dtype="float32", shape=param.shape, lod_level=param.lod_level)
# Append Op to calculate decay
block.append_op(
type='scale',
inputs={"X": param},
outputs={"Out": decay},
attrs={"scale": self._regularization_coeff})
return decay
import unittest
import paddle.v2.framework.framework as framework
import paddle.v2.framework.optimizer as optimizer
import paddle.v2.framework.regularizer as regularizer
from paddle.v2.framework.backward import append_backward_ops
class TestL2DecayRegularizer(unittest.TestCase):
def test_l2decay_regularizer(self):
program = framework.Program()
block = program.global_block()
mul_x = block.create_parameter(
dtype="float32",
shape=[5, 10],
lod_level=0,
name="mul.x",
regularizer=regularizer.L2DecayRegularizer(0.5))
self.assertTrue(mul_x.regularizer is not None)
self.assertTrue(
isinstance(mul_x.regularizer, regularizer.L2DecayRegularizer))
mul_y = block.create_var(
dtype="float32", shape=[10, 8], lod_level=0, name="mul.y")
mul_out = block.create_var(
dtype="float32", shape=[5, 8], lod_level=0, name="mul.out")
block.append_op(
type="mul",
inputs={"X": mul_x,
"Y": mul_y},
outputs={"Out": mul_out},
attrs={"x_num_col_dims": 1})
params_grads = append_backward_ops(mul_out)
self.assertEqual(len(params_grads), 1)
count_ops = len(block.ops)
params_grads = optimizer.append_regularization_ops(params_grads)
self.assertEqual(len(params_grads), 1)
self.assertEqual(len(block.ops), count_ops + 2)
self.assertEqual(block.ops[-1].type, 'elementwise_add')
self.assertEqual(block.ops[-2].type, 'scale')
if __name__ == '__main__':
unittest.main()
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册