未验证 提交 7a58431c 编写于 作者: Z zhang wenhui 提交者: GitHub

fix norm api doc, test=develop (#27652)

* fix norm api doc, test=develop

* fix error message, test=develop

* fix api norm, test=develop

* add adagrad, test=develop

* fix bug, test=develop

* fix bug, test=develop

* add spetral_norm, test=develop

* fix adagrad, test=develop

* merge , test=develop
上级 3eb106da
develop 1.8.5 2.0.1-rocm-post 2.4.1 Ligoml-patch-1 OliverLPH-patch-1 OliverLPH-patch-2 PaddlePM-patch-1 PaddlePM-patch-2 ZHUI-patch-1 add_default_att add_kylinv10 add_model_benchmark_ci add_some_yaml_config addfile all_new_design_exec ascendrc ascendrelease bugfix-eval-frame-leakgae cherry-pick-fix-customOP-random-fail cherry_undefined_var compile_windows cp_2.4_fix_numpy delete_2.0.1-rocm-post delete_add_default_att delete_all_new_design_exec delete_ascendrc delete_compile_windows delete_delete_addfile delete_disable_iterable_dataset_unittest delete_fix_dataloader_memory_leak delete_fix_imperative_dygraph_error delete_fix_retry_ci delete_fix_undefined_var delete_improve_sccache delete_paralleltest delete_prv-disable-more-cache delete_revert-31068-fix_conv3d_windows delete_revert-31562-mean delete_revert-33630-bug-fix delete_revert-34159-add_npu_bce_logical_dev delete_revert-34910-spinlocks_for_allocator delete_revert-35069-revert-34910-spinlocks_for_allocator delete_revert-36057-dev/read_flags_in_ut dingjiaweiww-patch-1 disable_iterable_dataset_unittest dy2static enable_eager_model_test final_state_gen_python_c final_state_intermediate fix-numpy-issue fix-run-program-grad-node-mem fix_check fix_concat_slice fix_custom_device_copy_sync fix_dataloader_memory_leak fix_dlpack_for fix_imperative_dygraph_error fix_newexe_gc fix_npu_ci fix_op_flops fix_retry_ci fix_rnn_docs fix_tensor_type fix_undefined_var fix_var_stop_gradient_error fixiscan fixiscan1 fixiscan2 fixiscan3 hack_event improve_sccache incuabte/new_frl incubate/frl_train_eval incubate/infrt incubate/new_frl incubate/new_frl_rc incubate/stride inplace_addto layer_norm make_flag_adding_easier matmul_double_grad move_embedding_to_phi move_histogram_to_pten move_sgd_to_phi move_slice_to_pten move_temporal_shift_to_phi move_yolo_box_to_phi npu_fix_alloc operator_opt paralleltest pass-compile-eval-frame preln_ernie prv-disable-more-cache prv-md-even-more prv-onednn-2.5 prv-reshape-mkldnn-ut2 pten_tensor_refactor release-deleted/2.5 release-rc/2.5 release/2.0 release/2.0-rc release/2.0-rc1 release/2.1 release/2.2 release/2.3 release/2.3-fc-ernie-fix release/2.4 release/2.5 release/llm_2.5 revert-31068-fix_conv3d_windows revert-31562-mean revert-32290-develop-hardlabel revert-33037-forci revert-33475-fix_cifar_label_dimension revert-33630-bug-fix revert-34159-add_npu_bce_logical_dev revert-34406-add_copy_from_tensor revert-34910-spinlocks_for_allocator revert-35069-revert-34910-spinlocks_for_allocator revert-36057-dev/read_flags_in_ut revert-36201-refine_fast_threaded_ssa_graph_executor revert-36985-add_license revert-37318-refactor_dygraph_to_eager revert-37926-eager_coreops_500 revert-37956-revert-37727-pylayer_support_tuple revert-38100-mingdong revert-38301-allocation_rearrange_pr revert-38703-numpy_bf16_package_reupload revert-38732-remove_useless_header_in_elementwise_mul_grad revert-38959-Reduce_Grad revert-39143-adjust_empty revert-39227-move_trace_op_to_pten revert-39268-dev/remove_concat_fluid_kernel revert-40170-support_partial_grad revert-41056-revert-40727-move_some_activaion_to_phi revert-41065-revert-40993-mv_ele_floordiv_pow revert-41068-revert-40790-phi_new revert-41944-smaller_inference_api_test revert-42149-do-not-reset-default-stream-for-stream-safe-cuda-allocator revert-43155-fix_ut_tempfile revert-43882-revert-41944-smaller_inference_api_test revert-45808-phi/simplify_size_op revert-46827-deform_comment revert-47325-remove_cudnn_hardcode revert-47645-add_npu_storage_dims revert-48815-set_free_when_no_cache_hit_default_value_true revert-49499-test_ninja_on_ci revert-49654-prim_api_gen revert-49673-modify_get_single_cov revert-49763-fix_static_composite_gen revert-50158-fix_found_inf_bug_for_custom_optimizer revert-50188-refine_optimizer_create_accumulators revert-50335-fix_optminizer_set_auxiliary_var_bug revert-51676-flag_delete revert-51850-fix_softmaxce_dev revert-52175-dev_peak_memory revert-52186-deve revert-52523-test_py38 revert-52912-develop revert-53248-set_cmake_policy revert-54029-fix_windows_compile_bug revert-54068-support_translating_op_attribute revert-54214-modify_cmake_dependencies revert-54370-offline_pslib revert-54391-fix_cmake_md5error revert-54411-fix_cpp17_compile revert-54466-offline_pslib revert-54480-cmake-rocksdb revert-55568-fix_BF16_bug1 revert-56328-new_ir_support_vector_type_place_transfer revert-56366-fix_openssl_bug revert-56545-revert-56366-fix_openssl_bug revert-56620-fix_new_ir_ocr_bug revert-56925-check_inputs_grad_semantic revert-57005-refine_stride_flag rocm_dev_0217 sd_conv_linear_autocast semi-auto/rule-base support-0D-sort support_weight_transpose test_benchmark_ci test_for_Filtetfiles test_model_benchmark test_model_benchmark_ci zhiqiu-patch-1 v2.5.1 v2.5.0 v2.5.0-rc1 v2.5.0-rc0 v2.4.2 v2.4.1 v2.4.0 v2.4.0-rc0 v2.3.2 v2.3.1 v2.3.0 v2.3.0-rc0 v2.2.2 v2.2.1 v2.2.0 v2.2.0-rc0 v2.2.0-bak0 v2.1.3 v2.1.2 v2.1.1 v2.1.0 v2.1.0-rc0 v2.0.2 v2.0.1 v2.0.0 v2.0.0-rc1 v2.0.0-rc0
无相关合并请求
...@@ -381,7 +381,8 @@ class BatchNormKernel<platform::CPUDeviceContext, T> ...@@ -381,7 +381,8 @@ class BatchNormKernel<platform::CPUDeviceContext, T>
break; break;
} }
default: default:
PADDLE_THROW("Unknown storage order: %s", data_layout_str); PADDLE_THROW(platform::errors::InvalidArgument(
"Unknown storage order: %s", data_layout_str));
} }
// if MomentumTensor is set, use MomentumTensor value, momentum // if MomentumTensor is set, use MomentumTensor value, momentum
...@@ -446,7 +447,8 @@ class BatchNormKernel<platform::CPUDeviceContext, T> ...@@ -446,7 +447,8 @@ class BatchNormKernel<platform::CPUDeviceContext, T>
break; break;
} }
default: default:
PADDLE_THROW("Unknown storage order: %d", data_layout); PADDLE_THROW(platform::errors::InvalidArgument(
"Unknown storage order: %d", data_layout));
} }
} }
}; };
...@@ -799,7 +801,8 @@ class BatchNormGradKernel<platform::CPUDeviceContext, T> ...@@ -799,7 +801,8 @@ class BatchNormGradKernel<platform::CPUDeviceContext, T>
break; break;
} }
default: default:
PADDLE_THROW("Unknown storage order: %s", data_layout_str); PADDLE_THROW(platform::errors::InvalidArgument(
"Unknown storage order: %s", data_layout_str));
} }
} }
}; };
......
...@@ -108,7 +108,8 @@ void FlListenAndServOp::RunSyncLoop(framework::Executor *executor, ...@@ -108,7 +108,8 @@ void FlListenAndServOp::RunSyncLoop(framework::Executor *executor,
auto optimize_blocks = auto optimize_blocks =
Attr<std::vector<framework::BlockDesc *>>(kOptimizeBlocks); Attr<std::vector<framework::BlockDesc *>>(kOptimizeBlocks);
PADDLE_ENFORCE_GE(num_blocks, 2, PADDLE_ENFORCE_GE(num_blocks, 2,
"server program should have at least 2 blocks"); platform::errors::InvalidArgument(
"server program should have at least 2 blocks"));
// Prepare all the server block // Prepare all the server block
std::vector<int> optimize_blocks_list; std::vector<int> optimize_blocks_list;
...@@ -192,7 +193,8 @@ void FlListenAndServOp::RunImpl(const framework::Scope &scope, ...@@ -192,7 +193,8 @@ void FlListenAndServOp::RunImpl(const framework::Scope &scope,
auto fan_in = Attr<int>("Fanin"); auto fan_in = Attr<int>("Fanin");
auto inputs = Inputs("X"); auto inputs = Inputs("X");
PADDLE_ENFORCE_EQ(!rpc_service_, true, "rpc_service_ must null"); PADDLE_ENFORCE_EQ(!rpc_service_, true, platform::errors::InvalidArgument(
"rpc_service_ must null"));
std::string endpoint = Attr<std::string>("endpoint"); std::string endpoint = Attr<std::string>("endpoint");
VLOG(4) << "sync_mode:" << sync_mode << ", fan_in:" << fan_in VLOG(4) << "sync_mode:" << sync_mode << ", fan_in:" << fan_in
...@@ -215,7 +217,8 @@ void FlListenAndServOp::RunImpl(const framework::Scope &scope, ...@@ -215,7 +217,8 @@ void FlListenAndServOp::RunImpl(const framework::Scope &scope,
Attr<std::vector<framework::BlockDesc *>>(kOptimizeBlocks); Attr<std::vector<framework::BlockDesc *>>(kOptimizeBlocks);
PADDLE_ENFORCE_GE( PADDLE_ENFORCE_GE(
optimize_blocks.size(), 1, optimize_blocks.size(), 1,
"optimize blocks should be 1 at least on the pserver side."); platform::errors::InvalidArgument(
"optimize blocks should be 1 at least on the pserver side."));
auto *program = optimize_blocks[0]->Program(); auto *program = optimize_blocks[0]->Program();
framework::Executor executor(dev_place); framework::Executor executor(dev_place);
......
...@@ -3674,10 +3674,11 @@ def spectral_norm(weight, dim=0, power_iters=1, eps=1e-12, name=None): ...@@ -3674,10 +3674,11 @@ def spectral_norm(weight, dim=0, power_iters=1, eps=1e-12, name=None):
Examples: Examples:
.. code-block:: python .. code-block:: python
import paddle.fluid as fluid import paddle
weight = fluid.data(name='weight', shape=[2, 8, 32, 32], dtype='float32') paddle.enable_static()
x = fluid.layers.spectral_norm(weight=weight, dim=1, power_iters=2) weight = paddle.data(name='weight', shape=[2, 8, 32, 32], dtype='float32')
x = paddle.static.nn.spectral_norm(weight=weight, dim=1, power_iters=2)
""" """
helper = LayerHelper('spectral_norm', **locals()) helper = LayerHelper('spectral_norm', **locals())
check_variable_and_dtype(weight, 'weight', ['float32', 'float64'], check_variable_and_dtype(weight, 'weight', ['float32', 'float64'],
......
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import unittest
import numpy as np
import paddle
import paddle.fluid.core as core
from paddle.fluid.op import Operator
from op_test import OpTest
import math
class TestAdagradOpV2(unittest.TestCase):
def test_v20_coverage(self):
paddle.disable_static()
inp = paddle.rand(shape=[10, 10])
linear = paddle.nn.Linear(10, 10)
out = linear(inp)
loss = paddle.mean(out)
adagrad = paddle.optimizer.Adagrad(
learning_rate=0.1, parameters=linear.parameters())
out.backward()
adagrad.step()
adagrad.clear_grad()
if __name__ == "__main__":
unittest.main()
...@@ -1369,7 +1369,7 @@ class TestLayer(LayerTest): ...@@ -1369,7 +1369,7 @@ class TestLayer(LayerTest):
dy_rlt_value = dy_ret.numpy() dy_rlt_value = dy_ret.numpy()
with self.dynamic_graph(): with self.dynamic_graph():
instanceNorm = paddle.nn.InstanceNorm(num_channels=shape[1]) instanceNorm = nn.InstanceNorm(num_channels=shape[1])
dy_ret = instanceNorm(base.to_variable(input)) dy_ret = instanceNorm(base.to_variable(input))
dy_rlt_value2 = dy_ret.numpy() dy_rlt_value2 = dy_ret.numpy()
...@@ -1380,7 +1380,7 @@ class TestLayer(LayerTest): ...@@ -1380,7 +1380,7 @@ class TestLayer(LayerTest):
with self.static_graph(): with self.static_graph():
# the input of InstanceNorm must be Variable. # the input of InstanceNorm must be Variable.
def test_Variable(): def test_Variable():
instanceNorm = paddle.nn.InstanceNorm(num_channels=shape[1]) instanceNorm = nn.InstanceNorm(num_channels=shape[1])
ret1 = instanceNorm(input) ret1 = instanceNorm(input)
self.assertRaises(TypeError, test_Variable) self.assertRaises(TypeError, test_Variable)
...@@ -1388,7 +1388,7 @@ class TestLayer(LayerTest): ...@@ -1388,7 +1388,7 @@ class TestLayer(LayerTest):
# the input dtype of InstanceNorm must be float32 or float64 # the input dtype of InstanceNorm must be float32 or float64
def test_type(): def test_type():
input = np.random.random(shape).astype('int32') input = np.random.random(shape).astype('int32')
instanceNorm = paddle.nn.InstanceNorm(num_channels=shape[1]) instanceNorm = nn.InstanceNorm(num_channels=shape[1])
ret2 = instanceNorm(input) ret2 = instanceNorm(input)
self.assertRaises(TypeError, test_type) self.assertRaises(TypeError, test_type)
......
...@@ -139,7 +139,6 @@ from .layer.norm import SyncBatchNorm #DEFINE_ALIAS ...@@ -139,7 +139,6 @@ from .layer.norm import SyncBatchNorm #DEFINE_ALIAS
from .layer.norm import GroupNorm #DEFINE_ALIAS from .layer.norm import GroupNorm #DEFINE_ALIAS
from .layer.norm import LayerNorm #DEFINE_ALIAS from .layer.norm import LayerNorm #DEFINE_ALIAS
from .layer.norm import SpectralNorm #DEFINE_ALIAS from .layer.norm import SpectralNorm #DEFINE_ALIAS
from .layer.norm import InstanceNorm #DEFINE_ALIAS
from .layer.norm import InstanceNorm1d #DEFINE_ALIAS from .layer.norm import InstanceNorm1d #DEFINE_ALIAS
from .layer.norm import InstanceNorm2d #DEFINE_ALIAS from .layer.norm import InstanceNorm2d #DEFINE_ALIAS
from .layer.norm import InstanceNorm3d #DEFINE_ALIAS from .layer.norm import InstanceNorm3d #DEFINE_ALIAS
......
...@@ -102,7 +102,7 @@ from .norm import SyncBatchNorm #DEFINE_ALIAS ...@@ -102,7 +102,7 @@ from .norm import SyncBatchNorm #DEFINE_ALIAS
from .norm import GroupNorm #DEFINE_ALIAS from .norm import GroupNorm #DEFINE_ALIAS
from .norm import LayerNorm #DEFINE_ALIAS from .norm import LayerNorm #DEFINE_ALIAS
from .norm import SpectralNorm #DEFINE_ALIAS from .norm import SpectralNorm #DEFINE_ALIAS
from .norm import InstanceNorm #DEFINE_ALIAS #from .norm import InstanceNorm #DEFINE_ALIAS
from .norm import LocalResponseNorm #DEFINE_ALIAS from .norm import LocalResponseNorm #DEFINE_ALIAS
# from .rnn import RNNCell #DEFINE_ALIAS # from .rnn import RNNCell #DEFINE_ALIAS
# from .rnn import GRUCell #DEFINE_ALIAS # from .rnn import GRUCell #DEFINE_ALIAS
......
...@@ -28,7 +28,7 @@ ...@@ -28,7 +28,7 @@
# TODO: define normalization api # TODO: define normalization api
import six import six
from ...fluid.dygraph.nn import InstanceNorm #from ...fluid.dygraph.nn import InstanceNorm
from ...fluid.dygraph import BatchNorm #DEFINE_ALIAS from ...fluid.dygraph import BatchNorm #DEFINE_ALIAS
#from ...fluid.dygraph import GroupNorm #DEFINE_ALIAS #from ...fluid.dygraph import GroupNorm #DEFINE_ALIAS
...@@ -54,19 +54,9 @@ from ...fluid.dygraph.base import no_grad ...@@ -54,19 +54,9 @@ from ...fluid.dygraph.base import no_grad
from .. import functional as F from .. import functional as F
__all__ = [ __all__ = [
'BatchNorm', 'BatchNorm', 'GroupNorm', 'LayerNorm', 'SpectralNorm', 'BatchNorm1d',
'GroupNorm', 'BatchNorm2d', 'BatchNorm3d', 'InstanceNorm1d', 'InstanceNorm2d',
'LayerNorm', 'InstanceNorm3d', 'SyncBatchNorm', 'LocalResponseNorm'
'SpectralNorm',
'InstanceNorm',
'BatchNorm1d',
'BatchNorm2d',
'BatchNorm3d',
'InstanceNorm1d',
'InstanceNorm2d',
'InstanceNorm3d',
'SyncBatchNorm',
'LocalResponseNorm',
] ]
......
...@@ -20,11 +20,12 @@ __all__ = [ ...@@ -20,11 +20,12 @@ __all__ = [
] ]
from ..fluid.optimizer import Momentum, Adagrad, Dpsgd, DecayedAdagrad, Ftrl,\ from ..fluid.optimizer import Momentum, Dpsgd, DecayedAdagrad, Ftrl,\
AdagradOptimizer, DpsgdOptimizer, DecayedAdagradOptimizer, \ AdagradOptimizer, DpsgdOptimizer, DecayedAdagradOptimizer, \
FtrlOptimizer, AdadeltaOptimizer FtrlOptimizer, AdadeltaOptimizer
from .optimizer import Optimizer from .optimizer import Optimizer
from .adagrad import Adagrad
from .adam import Adam from .adam import Adam
from .adamw import AdamW from .adamw import AdamW
from .adamax import Adamax from .adamax import Adamax
......
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from .optimizer import Optimizer
from ..fluid import core
from ..fluid import framework
from ..fluid.framework import Variable
__all__ = ["Adagrad"]
class Adagrad(Optimizer):
"""
The Adaptive Gradient optimizer (Adagrad for short) use an optimization described
in paper: `Adaptive Subgradient Methods for Online Learning and
Stochastic Optimization <http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf>`_.
The parameter ``param_out`` update rule with gradient ``grad``:
.. math::
moment\_out &= moment + grad * grad
param\_out &= param - \\frac{learning\_rate * grad}{\sqrt{moment\_out} + \epsilon}
The original paper does not have the ``epsilon`` attribute. It is added here
in our implementation as also proposed `Per-parameter adaptive learning rate
methods <http://cs231n.github.io/neural-networks-3/#ada>`_
for numerical stability to avoid the division by zero error.
Args:
learning_rate (float|Tensor): The learning rate used to update ``Parameter``.
It can be a float value or a ``Variable`` with a float type.
epsilon (float, optional): A small float value for numerical stability.
The default value is 1e-06.
parameters (list, optional): List of ``Tensor`` to update to minimize ``loss``. \
This parameter is required in dygraph mode. \
The default value is None in static mode, at this time all parameters will be updated.
weight_decay (float|WeightDecayRegularizer, optional): The strategy of regularization. \
It canbe a float value as coeff of L2 regularization or \
:ref:`api_fluid_regularizer_L1Decay`, :ref:`api_fluid_regularizer_L2Decay`.
If a parameter has set regularizer using :ref:`api_fluid_ParamAttr` already, \
the regularization setting here in optimizer will be ignored for this parameter. \
Otherwise, the regularization setting here in optimizer will take effect. \
Default None, meaning there is no regularization.
grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of
some derived class of ``GradientClipBase`` . There are three cliping strategies,
ClipGradByGlobalNorm, ClipGradByNorm and ClipGradByValue. Default None,
meaning there is no gradient clipping.
name (str, optional): Normally there is no need for user to set this property.
For more information, please refer to :ref:`api_guide_Name`.
The default value is None.
initial_accumulator_value (float, optional): Initial value for moment accumulator.
The default value is 0.0.
Examples:
.. code-block:: python
import paddle
import numpy as np
paddle.disable_static()
inp = paddle.rand(shape=[10, 10])
linear = paddle.nn.Linear(10, 10)
out = linear(inp)
loss = paddle.mean(out)
adagrad = paddle.optimizer.Adagrad(learning_rate=0.1,
parameters=linear.parameters())
out.backward()
adagrad.step()
adagrad.clear_grad()
"""
_moment_acc_str = "moment"
def __init__(self,
learning_rate,
epsilon=1.0e-6,
parameters=None,
weight_decay=None,
grad_clip=None,
name=None,
initial_accumulator_value=0.0):
assert learning_rate is not None
assert epsilon is not None
super(Adagrad, self).__init__(
learning_rate=learning_rate,
parameters=parameters,
weight_decay=weight_decay,
grad_clip=grad_clip,
name=name)
self.type = "adagrad"
self._epsilon = epsilon
self.initial_accumulator_value = initial_accumulator_value
def _create_accumulators(self, block, parameters):
assert isinstance(block, framework.Block)
for p in parameters:
self._add_accumulator(
self._moment_acc_str,
p,
fill_value=self.initial_accumulator_value)
def _append_optimize_op(self, block, param_and_grad):
assert isinstance(block, framework.Block)
moment_acc = self._get_accumulator(self._moment_acc_str,
param_and_grad[0])
# Create the adagrad optimizer op
adagrad_op = block.append_op(
type=self.type,
inputs={
"Param": param_and_grad[0],
"Grad": param_and_grad[1],
"Moment": moment_acc,
"LearningRate": self._create_param_lr(param_and_grad)
},
outputs={"ParamOut": param_and_grad[0],
"MomentOut": moment_acc},
attrs={"epsilon": self._epsilon},
stop_gradient=True)
return adagrad_op
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册
反馈
建议
客服 返回
顶部