未验证 提交 93ad3394 编写于 作者: Z Zhou Wei 提交者: GitHub

add new API:LambdaDecay,test=develop (#24801)

add new API:LambdaDecay
上级 2d028389
...@@ -23,7 +23,7 @@ from ..data_feeder import check_type ...@@ -23,7 +23,7 @@ from ..data_feeder import check_type
__all__ = [ __all__ = [
'NoamDecay', 'PiecewiseDecay', 'NaturalExpDecay', 'ExponentialDecay', 'NoamDecay', 'PiecewiseDecay', 'NaturalExpDecay', 'ExponentialDecay',
'InverseTimeDecay', 'PolynomialDecay', 'CosineDecay', 'LinearLrWarmup', 'InverseTimeDecay', 'PolynomialDecay', 'CosineDecay', 'LinearLrWarmup',
'ReduceLROnPlateau', 'StepDecay', 'MultiStepDecay' 'ReduceLROnPlateau', 'StepDecay', 'MultiStepDecay', 'LambdaDecay'
] ]
...@@ -1087,3 +1087,70 @@ class MultiStepDecay(_LearningRateEpochDecay): ...@@ -1087,3 +1087,70 @@ class MultiStepDecay(_LearningRateEpochDecay):
return self.base_lr * (decay_rate**i) return self.base_lr * (decay_rate**i)
return self.base_lr * (decay_rate**len(self.milestones)) return self.base_lr * (decay_rate**len(self.milestones))
class LambdaDecay(_LearningRateEpochDecay):
"""
:api_attr: imperative
Sets the learning rate of ``optimizer`` to the initial lr times a multiplicative factor, and this multiplicative
factor is computed by function ``lr_lambda`` . ``lr_lambda`` is funciton which receives ``epoch`` .
The algorithm can be described as the code below.
.. code-block:: text
learning_rate = 0.5 # init learning_rate
lr_lambda = lambda epoch: 0.95 ** epoch
learning_rate = 0.5 # epoch 0
learning_rate = 0.475 # epoch 1
learning_rate = 0.45125 # epoch 2
Parameters:
learning_rate (float|int): The initial learning rate. It can be set to python float or int number.
lr_lambda (function): A function which computes a multiplicative factor given an integer parameter ``epoch`` , and
then multiply the initial learning rate by this multiplicative factor.
Returns:
None.
Examples:
.. code-block:: python
import paddle.fluid as fluid
import numpy as np
with fluid.dygraph.guard():
x = np.random.uniform(-1, 1, [10, 10]).astype("float32")
linear = fluid.dygraph.Linear(10, 10)
input = fluid.dygraph.to_variable(x)
scheduler = fluid.dygraph.LambdaDecay(0.5, lr_lambda=lambda x: 0.95**x)
adam = fluid.optimizer.Adam(learning_rate = scheduler, parameter_list = linear.parameters())
for epoch in range(6):
for batch_id in range(5):
out = linear(input)
loss = fluid.layers.reduce_mean(out)
adam.minimize(loss)
scheduler.epoch()
print("epoch:%d, current lr is %f" .format(epoch, adam.current_step_lr()))
# epoch:0, current lr is 0.5
# epoch:1, current lr is 0.475
# epoch:2, current lr is 0.45125
"""
def __init__(self, learning_rate, lr_lambda):
if not callable(lr_lambda):
raise TypeError(
"The type of 'lr_lambda' in 'LambdaDecay' must be 'function', but received %s."
% type(lr_lambda))
self.lr_lambda = lr_lambda
super(LambdaDecay, self).__init__(learning_rate)
def get_lr(self):
base_lr = self.create_lr_var(self.base_lr)
return self.base_lr * self.lr_lambda(self.epoch_num)
...@@ -116,6 +116,10 @@ def step_decay(global_step, learning_rate, step_size, decay_rate=0.1): ...@@ -116,6 +116,10 @@ def step_decay(global_step, learning_rate, step_size, decay_rate=0.1):
return learning_rate * math.pow(decay_rate, global_step // step_size) return learning_rate * math.pow(decay_rate, global_step // step_size)
def lambda_decay(global_step, learning_rate, lr_lambda):
return learning_rate * lr_lambda(global_step)
class TestLearningRateDecayDygraph(unittest.TestCase): class TestLearningRateDecayDygraph(unittest.TestCase):
def test_NoamDecay(self): def test_NoamDecay(self):
with fluid.dygraph.guard(): with fluid.dygraph.guard():
...@@ -217,6 +221,29 @@ class TestLearningRateDecayDygraph(unittest.TestCase): ...@@ -217,6 +221,29 @@ class TestLearningRateDecayDygraph(unittest.TestCase):
lr = fluid.dygraph.MultiStepDecay(learning_rate, [20, 30, 50], lr = fluid.dygraph.MultiStepDecay(learning_rate, [20, 30, 50],
1) 1)
def test_LambdaDecay(self):
with fluid.dygraph.guard():
learning_rate = 0.5
lr_lambda = lambda x: 0.95**x
scheduler = fluid.dygraph.LambdaDecay(learning_rate, lr_lambda)
linear = fluid.dygraph.nn.Linear(10, 10)
adam = fluid.optimizer.Adam(
scheduler, parameter_list=linear.parameters())
for epoch in range(30):
right_result = lambda_decay(epoch, learning_rate, lr_lambda)
fluid_result = scheduler().numpy()[0]
scheduler.epoch()
self.assertAlmostEqual(
right_result,
fluid_result,
msg='Failed lr scheduler in epoch {0}, Python result is {1}, Fluid result is {2}'.
format(epoch, right_result, fluid_result))
with self.assertRaises(TypeError):
lr = fluid.dygraph.LambdaDecay(learning_rate, "test")
class TestLearningRateDecay(unittest.TestCase): class TestLearningRateDecay(unittest.TestCase):
def check_decay(self, python_decay_fn, fluid_decay_fn, kwargs): def check_decay(self, python_decay_fn, fluid_decay_fn, kwargs):
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册