Skip to content

P
Paddle

PaddlePaddle / Paddle
大约 1 年 前同步成功

通知 2298
Star 20931
Fork 5422
P
Paddle
未验证 提交 94365855 编写于 作者: L LoneRanger 提交者： GitHub
浏览文件
操作

replace the AdagradOptimizer... 

replace the AdagradOptimizer 、adamaxOptimizer、AdadeltaOptimizer、RMSPropOptimizer、LambOptimizer and Momentum (#54152)

* replace the AdadeltaOptimizer with Adadelta

* replace the RMSPropOptimizer with RMSProp

* replace the LambOptimizer with lamb

* replace the momentum in contrib/optimizer.py with Momentum in python/paddle/optimizer/momentum.py

* fix bug

* fix bug

* fix bug

* fix bug of Lamp

* fix bug of Lamp

* fix bug of import

* replace the AdamaxOptimizer with Admax and change the optimizer base for AdagradOptimizer

* fix bug

* fix bug

* Update optimizer.py

* fix bug

* fix bug
上级 a1396a80
展开全部 显示空白变更内容
内联 并排
Showing with 155 addition and 2111 deletion
python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py
浏览文件 @ 94365855
......@@ -13,8 +13,8 @@
import logging
import paddle
from paddle.fluid.optimizer import AdamOptimizer
from paddle.fluid.optimizer import LambOptimizer as LAMB
from .meta_optimizer_base import MetaOptimizerBase
......@@ -55,14 +55,13 @@ class LambOptimizer(MetaOptimizerBase):
_exclude_from_weight_decay_fn = exclude_fn
self.lamb_opt = LAMB(
self.lamb_opt = paddle.optimizer.Lamb(
learning_rate=opt._learning_rate,
lamb_weight_decay=configs['lamb_weight_decay'],
beta1=opt._beta1,
beta2=opt._beta2,
epsilon=opt._epsilon,
parameter_list=opt._parameter_list,
regularization=opt.regularization,
parameters=opt._parameter_list,
grad_clip=opt._grad_clip,
exclude_from_weight_decay_fn=_exclude_from_weight_decay_fn,
name=opt._name,
......@@ -111,7 +110,7 @@ class LambOptimizer(MetaOptimizerBase):
return self.lamb_opt.apply_gradients(params_grads=params_grads)
def apply_optimize(self, loss, startup_program, params_grads):
return self.lamb_opt.apply_optimize(
return self.lamb_opt._apply_optimize(
loss, startup_program=startup_program, params_grads=params_grads
)
......
python/paddle/fluid/__init__.py
浏览文件 @ 94365855
......@@ -53,7 +53,6 @@ from . import initializer
from .initializer import set_global_initializer
from . import layers
from . import dygraph
from . import contrib
from . import optimizer
from . import backward
from .backward import gradients
......@@ -105,7 +104,6 @@ __all__ = (
'io',
'initializer',
'layers',
'contrib',
'dygraph',
'enable_dygraph',
'disable_dygraph',
......
python/paddle/fluid/contrib/__init__.py 已删除 100644 → 0
浏览文件 @ a1396a80
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve.
# Copyright (c) 2021 NVIDIA Corporation. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import optimizer
from .optimizer import *
__all__ = []
__all__ += optimizer.__all__
python/paddle/fluid/contrib/optimizer.py 已删除 100644 → 0
浏览文件 @ a1396a80
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from paddle.fluid.optimizer import Optimizer
from paddle.regularizer import L1Decay
from paddle.regularizer import L2Decay
from paddle.fluid import core
from paddle.fluid import framework
from paddle.fluid.framework import program_guard
from paddle.fluid import unique_name
from paddle.fluid import layers
from paddle.fluid.layer_helper import LayerHelper
import warnings
from paddle import _C_ops, _legacy_C_ops
__all__ = ['Momentum']
class Momentum(Optimizer):
r"""
Simple Momentum optimizer with velocity state
This optimizer has a flag for Nestrov Momentum.
The update equations are as follows:
.. math::
& velocity = mu * velocity + gradient
& if (use\_nesterov):
&\quad param = param - (gradient + mu * velocity) * learning\_rate
& else:
&\quad param = param - learning\_rate * velocity
Parameters:
learning_rate (float|Variable): The learning rate used to update parameters. \
Can be a float value or a Variable with one float value as data element.
momentum (float): Momentum factor
parameter_list (Iterable, optional): Iterable of ``Variable`` names to update to minimize ``loss``. \
This parameter is required in dygraph mode. \
The default value is None in static graph mode, at this time all parameters will be updated.
use_nesterov (bool, optional): Enables Nesterov momentum, default is false.
regularization (WeightDecayRegularizer, optional): The strategy of regularization. There are two method: \
:ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If a parameter has set \
regularizer using :ref:`api_fluid_ParamAttr` already, the regularization setting here in optimizer will be \
ignored for this parameter. Otherwise, the regularization setting here in optimizer will take effect. \
Default None, meaning there is no regularization.
grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of
some derived class of ``GradientClipBase`` . There are three cliping strategies
( :ref:`api_fluid_clip_GradientClipByGlobalNorm` , :ref:`api_fluid_clip_GradientClipByNorm` ,
:ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping.
multi_precision (bool, optional): Whether to use multi-precision during weight updating. Default is false.
rescale_grad (float, optional): Multiply the gradient with `rescale_grad` before updating. \
Often choose to be ``1.0/batch_size``.
name (str, optional): This parameter is used by developers to print debugging information. \
For details, please refer to :ref:`api_guide_Name`. Default is None.
Examples:
.. code-block:: python
import paddle
import paddle.fluid as fluid
import numpy as np
paddle.enable_static()
place = fluid.CPUPlace()
main = fluid.Program()
with fluid.program_guard(main):
x = paddle.static.data(name='x', shape=[1, 13], dtype='float32')
y = paddle.static.data(name='y', shape=[1], dtype='float32')
linear = paddle.nn.Linear(13, 1)
y_predict = linear(x)
cost = paddle.nn.functional.square_error_cost(input=y_predict, label=y)
avg_cost = paddle.mean(cost)
moment_optimizer = fluid.contrib.optimizer.Momentum(learning_rate=0.001, momentum=0.9)
moment_optimizer.minimize(avg_cost)
fetch_list = [avg_cost]
train_reader = paddle.batch(
paddle.dataset.uci_housing.train(), batch_size=1)
feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
exe = fluid.Executor(place)
exe.run(paddle.static.default_startup_program())
for data in train_reader():
exe.run(main, feed=feeder.feed(data), fetch_list=fetch_list)
"""
_velocity_acc_str = "velocity"
def __init__(
self,
learning_rate,
momentum,
parameter_list=None,
use_nesterov=False,
regularization=None,
grad_clip=None,
multi_precision=False,
rescale_grad=1.0,
name=None,
):
assert learning_rate is not None
assert momentum is not None
predicate = lambda regular: isinstance(regular, L2Decay)
py_regular = None if predicate(regularization) else regularization
super().__init__(
learning_rate=learning_rate,
parameter_list=parameter_list,
regularization=py_regular,
grad_clip=grad_clip,
name=name,
)
self.type = "momentum"
self._momentum = momentum
self._use_nesterov = bool(use_nesterov)
self._regularization_method = ""
self._regularization_coeff = 0
if isinstance(regularization, L2Decay):
self._regularization_method = "l2_decay"
self._regularization_coeff = regularization._coeff
self._multi_precision = multi_precision
self._rescale_grad = rescale_grad
self._master_weights = {}
def _create_master_weight(self, param):
assert isinstance(self.helper, LayerHelper)
var_name = param.name + "_fp32_master"
var_name = unique_name.generate(var_name)
var = paddle.static.create_global_var(
name=var_name,
shape=param.shape,
value=0,
dtype='float32',
persistable=True,
)
block = self.helper.startup_program.global_block()
block.append_op(
type="cast",
inputs={"X": [param]},
outputs={"Out": [var]},
attrs={
"in_dtype": param.dtype,
"out_dtype": core.VarDesc.VarType.FP32,
},
)
self._master_weights[param.name] = var
return var
def _get_accumulator(self, name, param):
"""Utility function to fetch an accumulator for a parameter
Args:
name: name of the accumulator
param: parameter variable for which accumulator is to be fetched
Returns:
accumulator variable for the parameter
"""
if self._name is not None:
name = self._name + "_" + name
find_master = (
self._multi_precision and param.dtype == core.VarDesc.VarType.FP16
)
target_param = (
self._master_weights[param.name] if find_master else param
)
target_name = target_param.name
if (
name not in self._accumulators
or target_name not in self._accumulators[name]
):
raise Exception(
"Accumulator {} does not exist for parameter {}".format(
name, target_name
)
)
return self._accumulators[name][target_name]
def _create_accumulators(self, block, parameters):
assert isinstance(block, framework.Block)
for p in parameters:
if self._multi_precision and p.dtype == core.VarDesc.VarType.FP16:
master_p = self._create_master_weight(p)
self._add_accumulator(self._velocity_acc_str, master_p)
continue
if (
p.dtype == core.VarDesc.VarType.FP16
and not self._multi_precision
):
warnings.warn(
"Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence."
"Consider using multi_precision=True option of the Momentum optimizer."
)
self._add_accumulator(self._velocity_acc_str, p)
def _append_optimize_op(self, block, param_and_grad):
assert isinstance(block, framework.Block)
velocity_acc = self._get_accumulator(
self._velocity_acc_str, param_and_grad[0]
)
lr = self._create_param_lr(param_and_grad)
find_master = (
self._multi_precision
and param_and_grad[0].dtype == core.VarDesc.VarType.FP16
)
master_weight = (
self._master_weights[param_and_grad[0].name]
if find_master
else None
)
if framework.in_dygraph_mode():
_, _, _ = _legacy_C_ops.momentum(
param_and_grad[0],
param_and_grad[1],
velocity_acc,
lr,
master_weight,
param_and_grad[0],
velocity_acc,
master_weight,
'mu',
self._momentum,
'use_nesterov',
self._use_nesterov,
'regularization_method',
self._regularization_method,
'regularization_coeff',
self._regularization_coeff,
'multi_precision',
find_master,
)
return None
attrs = {
"mu": self._momentum,
"use_nesterov": self._use_nesterov,
"regularization_method": self._regularization_method,
"regularization_coeff": self._regularization_coeff,
"multi_precision": find_master,
"rescale_grad": self._rescale_grad,
}
inputs = {
"Param": [param_and_grad[0]],
"Grad": [param_and_grad[1]],
"Velocity": [velocity_acc],
"LearningRate": [lr],
}
outputs = {
"ParamOut": [param_and_grad[0]],
"VelocityOut": [velocity_acc],
}
if find_master:
inputs["MasterParam"] = master_weight
outputs["MasterParamOut"] = master_weight
# create the momentum optimize op
momentum_op = block.append_op(
type=self.type,
inputs=inputs,
outputs=outputs,
attrs=attrs,
stop_gradient=True,
)
return momentum_op
python/paddle/fluid/optimizer.py
浏览文件 @ 94365855
此差异已折叠。
python/setup.py.in
浏览文件 @ 94365855
......@@ -426,7 +426,6 @@ packages=['paddle',
'paddle.fluid.proto',
'paddle.fluid.proto.profiler',
'paddle.fluid.layers',
'paddle.fluid.contrib',
'paddle.fluid.incubate',
'paddle.incubate.distributed.fleet',
'paddle.fluid.incubate.checkpoint',
......
setup.py
浏览文件 @ 94365855
......@@ -1430,7 +1430,6 @@ def get_setup_parameters():
'paddle.fluid.proto',
'paddle.fluid.proto.profiler',
'paddle.fluid.layers',
'paddle.fluid.contrib',
'paddle.fluid.incubate',
'paddle.incubate.distributed.fleet',
'paddle.fluid.incubate.checkpoint',
......
test/amp/test_amp_api.py
浏览文件 @ 94365855
......@@ -70,7 +70,7 @@ class TestStaticDecorate(AmpTestBase):
)
out = model(x)
loss = paddle.mean(out)
optimizer = paddle.fluid.optimizer.Adadelta(learning_rate=0.001)
optimizer = paddle.optimizer.Adadelta(learning_rate=0.001)
optimizer = paddle.static.amp.decorate(
optimizer,
init_loss_scaling=128.0,
......
test/book/notest_understand_sentiment.py
浏览文件 @ 94365855
......@@ -84,7 +84,7 @@ def train(
else:
raise NotImplementedError()
adagrad = fluid.optimizer.Adagrad(learning_rate=0.002)
adagrad = paddle.optimizer.Adagrad(learning_rate=0.002)
adagrad.minimize(cost)
train_data = paddle.batch(
......
test/contrib/test_image_classification_fp16.py
浏览文件 @ 94365855
......@@ -139,7 +139,7 @@ def train(net_type, use_cuda, save_dirname, is_local):
# Test program
test_program = train_program.clone(for_test=True)
optimizer = fluid.optimizer.Lamb(learning_rate=0.001)
optimizer = paddle.optimizer.Lamb(learning_rate=0.001)
amp_lists = paddle.static.amp.AutoMixedPrecisionLists(
custom_black_varnames={"loss", "conv2d_0.w_0"}
......@@ -513,7 +513,7 @@ class TestAmpWithNonIterableDataLoader(unittest.TestCase):
)
avg_cost = paddle.mean(cost)
optimizer = fluid.optimizer.Lamb(learning_rate=0.001)
optimizer = paddle.optimizer.Lamb(learning_rate=0.001)
amp_lists = paddle.static.amp.AutoMixedPrecisionLists(
custom_black_varnames={"loss", "conv2d_0.w_0"}
)
......
test/dygraph_to_static/test_reinforcement_learning.py
浏览文件 @ 94365855
......@@ -73,8 +73,8 @@ def train(args, place, to_static):
policy = Policy()
eps = np.finfo(np.float32).eps.item()
optimizer = fluid.optimizer.AdamaxOptimizer(
learning_rate=1e-2, parameter_list=policy.parameters()
optimizer = paddle.optimizer.Adamax(
learning_rate=1e-2, parameters=policy.parameters()
)
def get_mean_and_std(values=[]):
......
test/dygraph_to_static/test_sentiment.py
浏览文件 @ 94365855
......@@ -328,8 +328,8 @@ def train(args, to_static):
model = GRU(args.vocab_size, args.batch_size, args.padding_size)
elif args.model_type == 'bigru_net':
model = BiGRU(args.vocab_size, args.batch_size, args.padding_size)
sgd_optimizer = fluid.optimizer.Adagrad(
learning_rate=args.lr, parameter_list=model.parameters()
sgd_optimizer = paddle.optimizer.Adagrad(
learning_rate=args.lr, parameters=model.parameters()
)
loss_data = []
......
test/legacy_test/fleet_ps_training.py
浏览文件 @ 94365855
......@@ -30,7 +30,7 @@ with fluid.device_guard("gpu"):
input_y = paddle.cast(input_y, dtype="int64")
cost = mlp(input_x, input_y)
optimizer = fluid.optimizer.Adagrad(learning_rate=0.01)
optimizer = paddle.optimizer.Adagrad(learning_rate=0.01)
role = role_maker.PaddleCloudRoleMaker()
fleet.init(role)
......
test/legacy_test/test_adadelta_op.py
浏览文件 @ 94365855
......@@ -403,118 +403,5 @@ class TestAdadeltaMultiPrecision2_0(unittest.TestCase):
)
class TestAdadeltaMultiPrecision1_0(unittest.TestCase):
def dygraph_adadelta_mp(self, use_amp, mp):
paddle.disable_static()
paddle.seed(10)
paddle.set_device('gpu')
input = paddle.randn((2, 2))
model = paddle.nn.Linear(2, 2)
optimizer = paddle.fluid.optimizer.Adadelta(
learning_rate=0.001,
parameter_list=model.parameters(),
)
optimizer._multi_precision = mp
if use_amp:
model = paddle.amp.decorate(models=model, level='O2')
scaler = paddle.amp.GradScaler(init_loss_scaling=1024)
for idx in range(5):
if use_amp:
with paddle.amp.auto_cast(level='O2'):
output = model(input)
loss = paddle.mean(output)
scaled = scaler.scale(loss)
scaled.backward()
scaler.minimize(optimizer, scaled)
optimizer.clear_gradients()
else:
output = model(input)
loss = paddle.mean(output)
optimizer.minimize(loss)
optimizer.clear_gradients()
return output, model.parameters()
def static_adadelta_mp(self, use_amp, mp):
paddle.enable_static()
paddle.seed(100)
np.random.seed(100)
exe = paddle.static.Executor('gpu')
train_program = paddle.static.Program()
startup_program = paddle.static.Program()
optimizer = paddle.fluid.optimizer.Adadelta(learning_rate=0.001)
optimizer._multi_precision = mp
if use_amp:
optimizer = paddle.static.amp.decorate(
optimizer,
init_loss_scaling=128.0,
use_dynamic_loss_scaling=True,
use_pure_fp16=True,
use_fp16_guard=False,
)
with paddle.static.program_guard(train_program, startup_program):
if use_amp:
data = paddle.static.data(
shape=[2, 2], name='X', dtype='float16'
)
else:
data = paddle.static.data(
shape=[2, 2], name='X', dtype='float32'
)
hidden = paddle.static.nn.fc(x=data, size=10)
loss = paddle.mean(hidden)
optimizer.minimize(loss)
exe.run(startup_program)
if use_amp:
optimizer.amp_init(
place=paddle.CUDAPlace(0), scope=paddle.static.global_scope()
)
x = np.random.random(size=(2, 2)).astype('float16')
else:
x = np.random.random(size=(2, 2)).astype('float32')
out = []
for idx in range(5):
(loss_data,) = exe.run(
train_program, feed={"X": x}, fetch_list=[loss.name]
)
out.append(loss_data)
return out
def test_main(self):
if not paddle.is_compiled_with_cuda():
return
"Test dygraph mode"
output1_dy, params1_dy = self.dygraph_adadelta_mp(use_amp=True, mp=True)
output2_dy, params2_dy = self.dygraph_adadelta_mp(
use_amp=False, mp=False
)
np.testing.assert_allclose(
output1_dy.astype('float32').numpy(),
output2_dy.astype('float32').numpy(),
rtol=1e-05,
atol=0.1,
)
for idx in range(len(params1_dy)):
np.testing.assert_allclose(
params1_dy[idx].astype('float32').numpy(),
params2_dy[idx].astype('float32').numpy(),
rtol=1e-05,
atol=0.1,
)
"Test static mode"
output1_st = self.static_adadelta_mp(use_amp=True, mp=True)
output2_st = self.static_adadelta_mp(use_amp=False, mp=False)
for idx in range(len(output1_st)):
np.testing.assert_allclose(
output1_st[idx].astype('float32'),
output2_st[idx].astype('float32'),
rtol=1e-05,
atol=0.1,
)
if __name__ == "__main__":
unittest.main()
test/legacy_test/test_adagrad_op.py
浏览文件 @ 94365855
......@@ -369,117 +369,6 @@ class TestAdagradMultiPrecision2_0(unittest.TestCase):
)
class TestAdagradMultiPrecision1_0(unittest.TestCase):
def dygraph_adagrad_mp(self, use_amp, mp):
paddle.disable_static()
paddle.seed(10)
paddle.set_device('gpu')
input = paddle.randn((2, 2))
model = paddle.nn.Linear(2, 2)
optimizer = paddle.fluid.optimizer.Adagrad(
learning_rate=0.001, parameter_list=model.parameters()
)
optimizer._multi_precision = mp
if use_amp:
model = paddle.amp.decorate(models=model, level='O2')
scaler = paddle.amp.GradScaler(init_loss_scaling=1024)
for idx in range(5):
if use_amp:
with paddle.amp.auto_cast(level='O2'):
output = model(input)
loss = paddle.mean(output)
scaled = scaler.scale(loss)
scaled.backward()
scaler.minimize(optimizer, scaled)
optimizer.clear_gradients()
else:
output = model(input)
loss = paddle.mean(output)
optimizer.minimize(loss)
optimizer.clear_gradients()
return output, model.parameters()
def static_adagrad_mp(self, use_amp, mp):
paddle.enable_static()
paddle.seed(100)
np.random.seed(100)
exe = paddle.static.Executor('gpu')
train_program = paddle.static.Program()
startup_program = paddle.static.Program()
optimizer = paddle.fluid.optimizer.Adagrad(learning_rate=0.001)
optimizer._multi_precision = mp
if use_amp:
optimizer = paddle.static.amp.decorate(
optimizer,
init_loss_scaling=128.0,
use_dynamic_loss_scaling=True,
use_pure_fp16=True,
use_fp16_guard=False,
)
with paddle.static.program_guard(train_program, startup_program):
if use_amp:
data = paddle.static.data(
shape=[2, 2], name='X', dtype='float16'
)
else:
data = paddle.static.data(
shape=[2, 2], name='X', dtype='float32'
)
hidden = paddle.static.nn.fc(x=data, size=10)
loss = paddle.mean(hidden)
optimizer.minimize(loss)
exe.run(startup_program)
if use_amp:
optimizer.amp_init(
place=paddle.CUDAPlace(0), scope=paddle.static.global_scope()
)
x = np.random.random(size=(2, 2)).astype('float16')
else:
x = np.random.random(size=(2, 2)).astype('float32')
out = []
for idx in range(5):
(loss_data,) = exe.run(
train_program, feed={"X": x}, fetch_list=[loss.name]
)
out.append(loss_data)
return out
def test_main(self):
if not paddle.is_compiled_with_cuda():
return
"Test dygraph mode"
output1_dy, params1_dy = self.dygraph_adagrad_mp(use_amp=True, mp=True)
output2_dy, params2_dy = self.dygraph_adagrad_mp(
use_amp=False, mp=False
)
np.testing.assert_allclose(
output1_dy.astype('float32').numpy(),
output2_dy.astype('float32').numpy(),
rtol=1e-05,
atol=0.1,
)
for idx in range(len(params1_dy)):
np.testing.assert_allclose(
params1_dy[idx].astype('float32').numpy(),
params2_dy[idx].astype('float32').numpy(),
rtol=1e-05,
atol=0.1,
)
"Test static mode"
output1_st = self.static_adagrad_mp(use_amp=True, mp=True)
output2_st = self.static_adagrad_mp(use_amp=False, mp=False)
for idx in range(len(output1_st)):
np.testing.assert_allclose(
output1_st[idx].astype('float32'),
output2_st[idx].astype('float32'),
rtol=1e-05,
atol=0.1,
)
if __name__ == "__main__":
paddle.enable_static()
unittest.main()
test/legacy_test/test_adamax_op.py
浏览文件 @ 94365855
......@@ -397,114 +397,5 @@ class TestAdamaxMultiPrecision2_0(unittest.TestCase):
)
class TestAdamaxMultiPrecision1_0(unittest.TestCase):
def dygraph_adamax_mp(self, use_amp, mp):
paddle.disable_static()
paddle.seed(10)
paddle.set_device('gpu')
input = paddle.randn((2, 2))
model = paddle.nn.Linear(2, 2)
optimizer = paddle.fluid.optimizer.Adamax(
learning_rate=0.001, parameter_list=model.parameters()
)
optimizer._multi_precision = mp
if use_amp:
model = paddle.amp.decorate(models=model, level='O2')
scaler = paddle.amp.GradScaler(init_loss_scaling=1024)
for idx in range(5):
if use_amp:
with paddle.amp.auto_cast(level='O2'):
output = model(input)
loss = paddle.mean(output)
scaled = scaler.scale(loss)
scaled.backward()
scaler.minimize(optimizer, scaled)
optimizer.clear_gradients()
else:
output = model(input)
loss = paddle.mean(output)
optimizer.minimize(loss)
optimizer.clear_gradients()
return output, model.parameters()
def static_adamax_mp(self, use_amp, mp):
paddle.enable_static()
paddle.seed(100)
np.random.seed(100)
exe = paddle.static.Executor('gpu')
train_program = paddle.static.Program()
startup_program = paddle.static.Program()
optimizer = paddle.fluid.optimizer.Adamax(learning_rate=0.001)
optimizer._multi_precision = mp
if use_amp:
optimizer = paddle.static.amp.decorate(
optimizer,
init_loss_scaling=128.0,
use_dynamic_loss_scaling=True,
use_pure_fp16=True,
use_fp16_guard=False,
)
with paddle.static.program_guard(train_program, startup_program):
if use_amp:
data = paddle.static.data(
shape=[2, 2], name='X', dtype='float16'
)
else:
data = paddle.static.data(
shape=[2, 2], name='X', dtype='float32'
)
hidden = paddle.static.nn.fc(x=data, size=10)
loss = paddle.mean(hidden)
optimizer.minimize(loss)
exe.run(startup_program)
if use_amp:
optimizer.amp_init(
place=paddle.CUDAPlace(0), scope=paddle.static.global_scope()
)
x = np.random.random(size=(2, 2)).astype('float16')
else:
x = np.random.random(size=(2, 2)).astype('float32')
out = []
for idx in range(5):
(loss_data,) = exe.run(
train_program, feed={"X": x}, fetch_list=[loss.name]
)
out.append(loss_data)
return out
def test_main(self):
if not paddle.is_compiled_with_cuda():
return
"Test dygraph mode"
output1_dy, params1_dy = self.dygraph_adamax_mp(use_amp=True, mp=True)
output2_dy, params2_dy = self.dygraph_adamax_mp(use_amp=False, mp=False)
np.testing.assert_allclose(
output1_dy.astype('float32').numpy(),
output2_dy.astype('float32').numpy(),
rtol=1e-05,
atol=0.1,
)
for idx in range(len(params1_dy)):
np.testing.assert_allclose(
params1_dy[idx].astype('float32').numpy(),
params2_dy[idx].astype('float32').numpy(),
rtol=1e-05,
atol=0.1,
)
"Test static mode"
output1_st = self.static_adamax_mp(use_amp=True, mp=True)
output2_st = self.static_adamax_mp(use_amp=False, mp=False)
for idx in range(len(output1_st)):
np.testing.assert_allclose(
output1_st[idx].astype('float32'),
output2_st[idx].astype('float32'),
rtol=1e-05,
atol=0.1,
)
if __name__ == "__main__":
unittest.main()
test/legacy_test/test_case.py
浏览文件 @ 94365855
......@@ -614,7 +614,7 @@ class TestMutiTask(unittest.TestCase):
one = paddle.tensor.fill_constant(shape=[1], dtype='int32', value=1)
adam = optimizer.Adam(learning_rate=0.001)
adagrad = optimizer.Adagrad(learning_rate=0.001)
adagrad = paddle.optimizer.Adagrad(learning_rate=0.001)
def fn_1():
sum = paddle.multiply(x, y)
......
test/legacy_test/test_eager_deletion_dynamic_rnn_base.py
浏览文件 @ 94365855
......@@ -42,7 +42,7 @@ def train(network, use_cuda, batch_size=32, pass_num=2):
cost = network(data, label, word_dict_size)
cost.persistable = True
optimizer = fluid.optimizer.Adagrad(learning_rate=0.2)
optimizer = paddle.optimizer.Adagrad(learning_rate=0.2)
optimizer.minimize(cost)
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
......
test/legacy_test/test_imperative_optimizer.py
浏览文件 @ 94365855
......@@ -23,22 +23,17 @@ from paddle import fluid
from paddle.distributed.fleet.meta_optimizers import DGCMomentumOptimizer
from paddle.fluid import core
from paddle.fluid.optimizer import (
AdadeltaOptimizer,
AdagradOptimizer,
Adam,
AdamaxOptimizer,
DecayedAdagradOptimizer,
DpsgdOptimizer,
ExponentialMovingAverage,
FtrlOptimizer,
LambOptimizer,
LarsMomentumOptimizer,
LookaheadOptimizer,
ModelAverage,
MomentumOptimizer,
PipelineOptimizer,
RecomputeOptimizer,
RMSPropOptimizer,
SGDOptimizer,
)
......@@ -593,13 +588,13 @@ class TestImperativeLarsMomentumOptimizer(TestImperativeOptimizerBase):
class TestImperativeAdagradOptimizer(TestImperativeOptimizerBase):
def get_optimizer_dygraph(self, parameter_list):
optimizer = AdagradOptimizer(
learning_rate=0.2, parameter_list=parameter_list
optimizer = paddle.optimizer.Adagrad(
learning_rate=0.2, parameters=parameter_list
)
return optimizer
def get_optimizer(self):
optimizer = AdagradOptimizer(learning_rate=0.2)
optimizer = paddle.optimizer.Adagrad(learning_rate=0.2)
return optimizer
def test_adagrad(self):
......@@ -608,13 +603,13 @@ class TestImperativeAdagradOptimizer(TestImperativeOptimizerBase):
class TestImperativeAdamaxOptimizer(TestImperativeOptimizerBase):
def get_optimizer_dygraph(self, parameter_list):
optimizer = AdamaxOptimizer(
learning_rate=0.2, parameter_list=parameter_list
optimizer = paddle.optimizer.Adamax(
learning_rate=0.2, parameters=parameter_list
)
return optimizer
def get_optimizer(self):
optimizer = AdamaxOptimizer(learning_rate=0.2)
optimizer = paddle.optimizer.Adamax(learning_rate=0.2)
return optimizer
def test_adamax(self):
......@@ -661,16 +656,16 @@ class TestImperativeDecayedAdagradOptimizer(TestImperativeOptimizerBase):
class TestImperativeAdadeltaOptimizer(TestImperativeOptimizerBase):
def get_optimizer_dygraph(self, parameter_list):
optimizer = AdadeltaOptimizer(
optimizer = paddle.optimizer.Adadelta(
learning_rate=0.0003,
epsilon=1.0e-6,
rho=0.95,
parameter_list=parameter_list,
parameters=parameter_list,
)
return optimizer
def get_optimizer(self):
optimizer = AdadeltaOptimizer(
optimizer = paddle.optimizer.Adadelta(
learning_rate=0.0003, epsilon=1.0e-6, rho=0.95
)
return optimizer
......@@ -681,13 +676,13 @@ class TestImperativeAdadeltaOptimizer(TestImperativeOptimizerBase):
class TestImperativeRMSPropOptimizer(TestImperativeOptimizerBase):
def get_optimizer_dygraph(self, parameter_list):
optimizer = RMSPropOptimizer(
learning_rate=0.1, parameter_list=parameter_list
optimizer = paddle.optimizer.RMSProp(
learning_rate=0.1, parameters=parameter_list
)
return optimizer
def get_optimizer(self):
optimizer = RMSPropOptimizer(learning_rate=0.1)
optimizer = paddle.optimizer.RMSProp(learning_rate=0.1)
return optimizer
def test_rmsprop(self):
......@@ -715,15 +710,15 @@ def exclude_fn(param):
class TestImperativeLambOptimizer(TestImperativeOptimizerBase):
def get_optimizer_dygraph(self, parameter_list):
optimizer = LambOptimizer(
optimizer = paddle.optimizer.Lamb(
learning_rate=0.002,
exclude_from_weight_decay_fn=exclude_fn,
parameter_list=parameter_list,
parameters=parameter_list,
)
return optimizer
def get_optimizer(self):
optimizer = LambOptimizer(
optimizer = paddle.optimizer.Lamb(
learning_rate=0.002, exclude_from_weight_decay_fn=exclude_fn
)
return optimizer
......
test/legacy_test/test_imperative_optimizer_v2.py
浏览文件 @ 94365855
......@@ -23,9 +23,6 @@ from paddle import fluid
from paddle.distributed.fleet.meta_optimizers import DGCMomentumOptimizer
from paddle.fluid import core
from paddle.fluid.optimizer import (
AdadeltaOptimizer,
AdagradOptimizer,
AdamaxOptimizer,
DecayedAdagradOptimizer,
DpsgdOptimizer,
ExponentialMovingAverage,
......@@ -36,7 +33,6 @@ from paddle.fluid.optimizer import (
MomentumOptimizer,
PipelineOptimizer,
RecomputeOptimizer,
RMSPropOptimizer,
)
# Note(wangzhongpu)
......@@ -721,13 +717,13 @@ class TestImperativeLarsMomentumOptimizer(TestImperativeOptimizerBase):
class TestImperativeAdagradOptimizer(TestImperativeOptimizerBase):
def get_optimizer_dygraph(self, parameter_list):
optimizer = AdagradOptimizer(
learning_rate=0.2, parameter_list=parameter_list
optimizer = paddle.optimizer.Adagrad(
learning_rate=0.2, parameters=parameter_list
)
return optimizer
def get_optimizer(self):
optimizer = AdagradOptimizer(learning_rate=0.2)
optimizer = paddle.optimizer.Adagrad(learning_rate=0.2)
return optimizer
def test_adagrad(self):
......@@ -736,13 +732,13 @@ class TestImperativeAdagradOptimizer(TestImperativeOptimizerBase):
class TestImperativeAdamaxOptimizer(TestImperativeOptimizerBase):
def get_optimizer_dygraph(self, parameter_list):
optimizer = AdamaxOptimizer(
learning_rate=0.2, parameter_list=parameter_list
optimizer = paddle.optimizer.Adamax(
learning_rate=0.2, parameters=parameter_list
)
return optimizer
def get_optimizer(self):
optimizer = AdamaxOptimizer(learning_rate=0.2)
optimizer = paddle.optimizer.Adamax(learning_rate=0.2)
return optimizer
def test_adamax(self):
......@@ -789,16 +785,16 @@ class TestImperativeDecayedAdagradOptimizer(TestImperativeOptimizerBase):
class TestImperativeAdadeltaOptimizer(TestImperativeOptimizerBase):
def get_optimizer_dygraph(self, parameter_list):
optimizer = AdadeltaOptimizer(
optimizer = paddle.optimizer.Adadelta(
learning_rate=0.0003,
epsilon=1.0e-6,
rho=0.95,
parameter_list=parameter_list,
parameters=parameter_list,
)
return optimizer
def get_optimizer(self):
optimizer = AdadeltaOptimizer(
optimizer = paddle.optimizer.Adadelta(
learning_rate=0.0003, epsilon=1.0e-6, rho=0.95
)
return optimizer
......@@ -809,13 +805,13 @@ class TestImperativeAdadeltaOptimizer(TestImperativeOptimizerBase):
class TestImperativeRMSPropOptimizer(TestImperativeOptimizerBase):
def get_optimizer_dygraph(self, parameter_list):
optimizer = RMSPropOptimizer(
learning_rate=0.1, parameter_list=parameter_list
optimizer = paddle.optimizer.RMSProp(
learning_rate=0.1, parameters=parameter_list
)
return optimizer
def get_optimizer(self):
optimizer = RMSPropOptimizer(learning_rate=0.1)
optimizer = paddle.optimizer.RMSProp(learning_rate=0.1)
return optimizer
def test_rmsprop(self):
......
test/legacy_test/test_momentum_op.py
浏览文件 @ 94365855
......@@ -677,11 +677,11 @@ class TestMomentumOpWithDecayAPI(unittest.TestCase):
out = linear(inp)
loss = paddle.mean(out)
# This can be any optimizer supported by dygraph.
momentum = paddle.fluid.contrib.optimizer.Momentum(
momentum = paddle.optimizer.Momentum(
learning_rate=0.01,
momentum=0.9,
parameter_list=linear.parameters(),
regularization=regularization,
parameters=linear.parameters(),
weight_decay=regularization,
)
momentum.minimize(loss)
......@@ -703,7 +703,7 @@ class TestMomentumOpWithDecayAPI(unittest.TestCase):
)
avg_cost = paddle.mean(cost)
momentum_optimizer = paddle.fluid.contrib.optimizer.Momentum(
momentum_optimizer = paddle.optimizer.Momentum(
learning_rate=0.1, momentum=0.9
)
momentum_optimizer.minimize(avg_cost)
......@@ -833,11 +833,11 @@ class TestMomentumOpVsMomentumOpWithDecayAPI(unittest.TestCase):
weight_attr=paddle.nn.initializer.Constant(value=2.0),
bias_attr=paddle.nn.initializer.Constant(value=2.0),
)
momentum_new = paddle.fluid.contrib.optimizer.Momentum(
momentum_new = paddle.optimizer.Momentum(
learning_rate=0.01,
momentum=0.9,
parameter_list=linear_new.parameters(),
regularization=paddle.regularizer.L2Decay(coeff=0.1),
parameters=linear_new.parameters(),
weight_decay=paddle.regularizer.L2Decay(coeff=0.1),
)
self.__update_params(momentum=momentum_new, linear=linear_new)
......
test/legacy_test/test_optimizer.py
浏览文件 @ 94365855
......@@ -248,72 +248,6 @@ class TestMomentumOptimizer(unittest.TestCase):
self.assertAlmostEqual(init_ops[0].attr('value'), 0.0)
class TestAdagradOptimizer(unittest.TestCase):
class MockAdagrad(optimizer.AdagradOptimizer):
def get_accumulators(self):
return self._accumulators
def get_moment_str(self):
return self._moment_acc_str
def test_adagrad_optimizer(self):
init_program = framework.Program()
program = framework.Program()
block = program.global_block()
mul_x = block.create_parameter(
dtype="float32",
shape=[5, 10],
lod_level=0,
name="mul.x",
optimize_attr={'learning_rate': 1.1},
)
mul_y = block.create_var(
dtype="float32", shape=[10, 8], lod_level=0, name="mul.y"
)
mul_out = block.create_var(
dtype="float32", shape=[5, 8], lod_level=0, name="mul.out"
)
block.append_op(
type="mul",
inputs={"X": mul_x, "Y": mul_y},
outputs={"Out": mul_out},
attrs={"x_num_col_dims": 1},
)
mean_out = block.create_var(
dtype="float32", shape=[1], lod_level=0, name="mean.out"
)
block.append_op(
type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out}
)
learning_rate = 0.01
adagrad_optimizer = self.MockAdagrad(
learning_rate=learning_rate, epsilon=1.0e-6
)
params_grads = append_backward(mean_out)
self.assertEqual(len(params_grads), 1)
self.assertEqual(len(adagrad_optimizer.get_accumulators()), 0)
with framework.program_guard(program, init_program):
opts = adagrad_optimizer.apply_gradients(params_grads)
self.assertEqual(len(opts), 2)
self.assertEqual([op.type for op in opts], ["scale", "adagrad"])
# Check accumulators
accumulators = adagrad_optimizer.get_accumulators()
self.assertEqual(len(accumulators), 1)
self.assertTrue(adagrad_optimizer.get_moment_str() in accumulators)
moment_acc = accumulators[adagrad_optimizer.get_moment_str()]
self.assertEqual(len(moment_acc), 1)
self.assertTrue(mul_x.name in moment_acc)
# Check init_program
init_ops = init_program.global_block().ops
self.assertEqual(len(init_ops), 2)
self.assertEqual(init_ops[1].type, "fill_constant")
self.assertAlmostEqual(init_ops[1].attr('value'), learning_rate)
self.assertEqual(init_ops[0].type, "fill_constant")
self.assertAlmostEqual(init_ops[0].attr('value'), 0.0)
class TestAdamOptimizer(unittest.TestCase):
class MockAdam(optimizer.AdamOptimizer):
def get_accumulators(self):
......@@ -385,77 +319,6 @@ class TestAdamOptimizer(unittest.TestCase):
self.assertAlmostEqual(init_ops[-1].attr('value'), learning_rate)
class TestAdamaxOptimizer(unittest.TestCase):
class MockAdamax(optimizer.AdamaxOptimizer):
def get_accumulators(self):
return self._accumulators
def get_moment_str(self):
return self._moment_acc_str
def get_inf_norm_str(self):
return self._inf_norm_acc_str
def test_adamax_optimizer(self):
init_program = framework.Program()
program = framework.Program()
block = program.global_block()
mul_x = block.create_parameter(
dtype="float32",
shape=[5, 10],
lod_level=0,
name="mul.x",
optimize_attr={'learning_rate': 1.1},
)
mul_y = block.create_var(
dtype="float32", shape=[10, 8], lod_level=0, name="mul.y"
)
mul_out = block.create_var(
dtype="float32", shape=[5, 8], lod_level=0, name="mul.out"
)
block.append_op(
type="mul",
inputs={"X": mul_x, "Y": mul_y},
outputs={"Out": mul_out},
attrs={"x_num_col_dims": 1},
)
mean_out = block.create_var(
dtype="float32", shape=[1], lod_level=0, name="mean.out"
)
block.append_op(
type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out}
)
learning_rate = 0.01
adamax_optimizer = self.MockAdamax(
learning_rate=learning_rate, beta1=0.9, beta2=0.999
)
params_grads = append_backward(mean_out)
self.assertEqual(len(params_grads), 1)
self.assertEqual(len(adamax_optimizer.get_accumulators()), 0)
with framework.program_guard(program, init_program):
opts = adamax_optimizer.apply_gradients(params_grads)
self.assertEqual(len(opts), 3)
self.assertEqual([op.type for op in opts], ["scale", "adamax", "scale"])
# Check accumulators
accumulators = adamax_optimizer.get_accumulators()
self.assertEqual(len(accumulators), 3)
self.assertTrue(adamax_optimizer.get_moment_str() in accumulators)
self.assertTrue(adamax_optimizer.get_inf_norm_str() in accumulators)
moment_acc = accumulators[adamax_optimizer.get_moment_str()]
inf_norm_acc = accumulators[adamax_optimizer.get_inf_norm_str()]
self.assertEqual(len(moment_acc), 1)
self.assertEqual(len(inf_norm_acc), 1)
self.assertTrue(mul_x.name in moment_acc)
self.assertTrue(mul_x.name in inf_norm_acc)
# Check init_program
init_ops = init_program.global_block().ops
self.assertEqual(len(init_ops), 4)
self.assertEqual(init_ops[-1].type, "fill_constant")
self.assertAlmostEqual(init_ops[-1].attr('value'), learning_rate)
class TestDpsgdOptimizer(unittest.TestCase):
def test_dpsgd_optimizer(self):
def check_dpsgd_optimizer(optimizer_attr):
......
test/legacy_test/test_regularizer.py
浏览文件 @ 94365855
......@@ -203,9 +203,9 @@ class TestRegularizer(unittest.TestCase):
avg_cost = model(data, label, self.word_len)
optimizer = fluid.optimizer.Adagrad(
optimizer = paddle.optimizer.Adagrad(
learning_rate=0.1,
regularization=paddle.regularizer.L2Decay(1.0),
weight_decay=paddle.regularizer.L2Decay(1.0),
)
optimizer.minimize(avg_cost)
param_sum = self.run_program(place, [data, label])
......@@ -236,7 +236,7 @@ class TestRegularizer(unittest.TestCase):
para_sum.append(paddle.sum(para_mul))
avg_cost_l2 += paddle.add_n(para_sum) * 0.5
optimizer = fluid.optimizer.Adagrad(learning_rate=0.1)
optimizer = paddle.optimizer.Adagrad(learning_rate=0.1)
optimizer.minimize(avg_cost_l2)
param_sum = self.run_program(place, [data, label])
return param_sum
......
test/legacy_test/test_regularizer_api.py
浏览文件 @ 94365855
......@@ -116,9 +116,9 @@ class TestRegularizer(unittest.TestCase):
avg_cost = model(data, label, self.word_len)
optimizer = fluid.optimizer.Adagrad(
optimizer = paddle.optimizer.Adagrad(
learning_rate=0.1,
regularization=paddle.regularizer.L2Decay(1.0),
weight_decay=paddle.regularizer.L2Decay(1.0),
)
optimizer.minimize(avg_cost)
param_sum = self.run_program(place, [data, label])
......@@ -149,7 +149,7 @@ class TestRegularizer(unittest.TestCase):
para_sum.append(paddle.sum(para_mul))
avg_cost_l2 += paddle.add_n(para_sum) * 0.5
optimizer = fluid.optimizer.Adagrad(learning_rate=0.1)
optimizer = paddle.optimizer.Adagrad(learning_rate=0.1)
optimizer.minimize(avg_cost_l2)
param_sum = self.run_program(place, [data, label])
return param_sum
......
test/legacy_test/test_rmsprop_op.py
浏览文件 @ 94365855
......@@ -521,119 +521,6 @@ class TestRMSPropMultiPrecision2_0(unittest.TestCase):
)
class TestRMSPropMultiPrecision1_0(unittest.TestCase):
def dygraph_rmsprop_mp(self, use_amp, mp):
paddle.disable_static()
paddle.seed(10)
paddle.set_device('gpu')
input = paddle.randn((2, 2))
model = paddle.nn.Linear(2, 2)
optimizer = paddle.fluid.optimizer.RMSProp(
learning_rate=0.001,
parameter_list=model.parameters(),
)
optimizer._multi_precision = mp
if use_amp:
model = paddle.amp.decorate(models=model, level='O2')
scaler = paddle.amp.GradScaler(init_loss_scaling=1024)
for idx in range(5):
if use_amp:
with paddle.amp.auto_cast(level='O2'):
output = model(input)
loss = paddle.mean(output)
scaled = scaler.scale(loss)
scaled.backward()
scaler.minimize(optimizer, scaled)
optimizer.clear_gradients()
else:
output = model(input)
loss = paddle.mean(output)
optimizer.minimize(loss)
optimizer.clear_gradients()
return output, model.parameters()
def static_rmsprop_mp(self, use_amp, mp):
paddle.enable_static()
paddle.seed(100)
np.random.seed(100)
exe = paddle.static.Executor('gpu')
train_program = paddle.static.Program()
startup_program = paddle.static.Program()
optimizer = paddle.fluid.optimizer.RMSProp(learning_rate=0.001)
optimizer._multi_precision = mp
if use_amp:
optimizer = paddle.static.amp.decorate(
optimizer,
init_loss_scaling=128.0,
use_dynamic_loss_scaling=True,
use_pure_fp16=True,
use_fp16_guard=False,
)
with paddle.static.program_guard(train_program, startup_program):
if use_amp:
data = paddle.static.data(
shape=[2, 2], name='X', dtype='float16'
)
else:
data = paddle.static.data(
shape=[2, 2], name='X', dtype='float32'
)
hidden = paddle.static.nn.fc(x=data, size=10)
loss = paddle.mean(hidden)
optimizer.minimize(loss)
exe.run(startup_program)
if use_amp:
optimizer.amp_init(
place=paddle.CUDAPlace(0), scope=paddle.static.global_scope()
)
x = np.random.random(size=(2, 2)).astype('float16')
else:
x = np.random.random(size=(2, 2)).astype('float32')
out = []
for idx in range(5):
(loss_data,) = exe.run(
train_program, feed={"X": x}, fetch_list=[loss.name]
)
out.append(loss_data)
return out
def test_main(self):
if not paddle.is_compiled_with_cuda():
return
"Test dygraph mode"
output1_dy, params1_dy = self.dygraph_rmsprop_mp(use_amp=True, mp=True)
output2_dy, params2_dy = self.dygraph_rmsprop_mp(
use_amp=False, mp=False
)
np.testing.assert_allclose(
output1_dy.astype('float32').numpy(),
output2_dy.astype('float32').numpy(),
rtol=1e-05,
atol=0.1,
)
for idx in range(len(params1_dy)):
np.testing.assert_allclose(
params1_dy[idx].astype('float32').numpy(),
params2_dy[idx].astype('float32').numpy(),
rtol=1e-05,
atol=0.1,
)
"Test static mode"
output1_st = self.static_rmsprop_mp(use_amp=True, mp=True)
output2_st = self.static_rmsprop_mp(use_amp=False, mp=False)
for idx in range(len(output1_st)):
np.testing.assert_allclose(
output1_st[idx].astype('float32'),
output2_st[idx].astype('float32'),
rtol=1e-05,
atol=0.1,
)
if __name__ == "__main__":
paddle.enable_static()
unittest.main()
test/legacy_test/test_trainable.py
浏览文件 @ 94365855
......@@ -73,8 +73,8 @@ class TestTrainable(unittest.TestCase):
self.check_trainable(
test_trainable,
feed_dict,
op_count={'adamax': 1, 'scale': 1, 'mul_grad': 0},
optimizer=fluid.optimizer.Adamax(learning_rate=0.2),
op_count={'adamax': 1, 'scale': 1, 'mul_grad': 1},
optimizer=paddle.optimizer.Adamax(learning_rate=0.2),
)
......
test/legacy_test/test_weight_decay.py
浏览文件 @ 94365855
......@@ -157,7 +157,7 @@ class TestWeightDecay(unittest.TestCase):
for var in main_prog.block(0).all_parameters()
]
optimizer = fluid.optimizer.Adagrad(
optimizer = paddle.optimizer.Adagrad(
learning_rate=self.learning_rate
)
optimizer.minimize(avg_cost)
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或