diff --git a/paddle/fluid/API.spec b/paddle/fluid/API.spec index 5e8699d2be9efd85e8e208efd1967650773ed6c9..e90eb305f8f62a1e881404e83790fda11cc3918b 100644 --- a/paddle/fluid/API.spec +++ b/paddle/fluid/API.spec @@ -995,7 +995,7 @@ paddle.fluid.optimizer.LarsMomentumOptimizer.backward (ArgSpec(args=['self', 'lo paddle.fluid.optimizer.LarsMomentumOptimizer.get_opti_var_name_list (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) paddle.fluid.optimizer.LarsMomentumOptimizer.load (ArgSpec(args=['self', 'stat_dict'], varargs=None, keywords=None, defaults=None), ('document', '649a92cf7f1ea28666fd00c4ea01acde')) paddle.fluid.optimizer.LarsMomentumOptimizer.minimize (ArgSpec(args=['self', 'loss', 'startup_program', 'parameter_list', 'no_grad_set', 'grad_clip'], varargs=None, keywords=None, defaults=(None, None, None, None)), ('document', 'b15cffad0903fc81af77a0580ceb2a9b')) -paddle.fluid.optimizer.DGCMomentumOptimizer ('paddle.fluid.optimizer.DGCMomentumOptimizer', ('document', 'c0384e036f5c78c569f0e2b266812c0f')) +paddle.fluid.optimizer.DGCMomentumOptimizer ('paddle.fluid.optimizer.DGCMomentumOptimizer', ('document', 'facdbef1b4871d0cf74c736ff2e94720')) paddle.fluid.optimizer.DGCMomentumOptimizer.__init__ (ArgSpec(args=['self', 'learning_rate', 'momentum', 'rampup_begin_step', 'rampup_step', 'sparsity', 'use_nesterov', 'local_grad_clip_norm', 'num_trainers', 'regularization', 'name'], varargs=None, keywords=None, defaults=(1, [0.999], False, None, None, None, None)), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) paddle.fluid.optimizer.DGCMomentumOptimizer.apply_gradients (ArgSpec(args=['self', 'params_grads'], varargs=None, keywords=None, defaults=None), ('document', '80ea99c9af7ef5fac7e57fb302103610')) paddle.fluid.optimizer.DGCMomentumOptimizer.apply_optimize (ArgSpec(args=['self', 'loss', 'startup_program', 'params_grads'], varargs=None, keywords=None, defaults=None), ('document', '5c46d1926a40f1f873ffe9f37ac89dae')) diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index 5cb76e51294f4a7b1c234d1aa3b31c44dbd50cd1..3feff3a1d97d12d4799326e436cbfc083f3c483b 100644 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -811,8 +811,7 @@ class MomentumOptimizer(Optimizer): class DGCMomentumOptimizer(MomentumOptimizer): """ - - Original paper is https://arxiv.org/abs/1712.01887 + DGC (Deep Gradient Compression) Momentum Optimizer. Original paper is https://arxiv.org/abs/1712.01887 DGC reduces the communication bandwidth by sending only the important gradients (sparse update):\ only gradients larger than a threshold are transmitted. @@ -821,7 +820,7 @@ class DGCMomentumOptimizer(MomentumOptimizer): Eventually, these gradients become large enough to be transmitted. - Thus, DGC sends the large gradients immediately but eventually send all of the gradients over time. + Thus, DGC sends the large gradients immediately but eventually sends all of the gradients over time. To ensure no loss of accuracy, DGC employs momentum correction and local gradient clipping on top of the gradient sparsification to maintain model performance. @@ -832,23 +831,27 @@ class DGCMomentumOptimizer(MomentumOptimizer): 1. Compress the gradient by get TopK import value from tensor \ and use it for allreduce to reduce network bandwidth. - 2. Call momentum to optimize on the cost. + 2. Call momentum to optimize the cost. Args: - learning_rate (float|Variable): the learning rate used to update parameters. \ - Can be a float value or a Variable with one float value as data element. + learning_rate (float|Variable): The learning rate used to update parameters. \ + It can be a float value or a Variable with one float value as a data element. momentum (float): Momentum factor. rampup_begin_step (int): The beginning step from which gradient compression is implemented. - rampup_step (int): How long it use the sparsity periods. Default is 1. - for example: If the sparsity is [0.75, 0.9375, 0.984375, 0.996, 0.999], and the rampup_step is 5, \ - it will use 0.75 at 0 step, and 0.9375 at 1 step, and so on. And when reach sparsity array ends, \ - it will use 0.999 then and after. - sparsity (list[float]): Get top important element from gradient tensor, the ratio is (1 - current sparsity). - use_nesterov (bool): Enables Nesterov momentum. True means use nesterov. - local_grad_clip_norm (float): Clip norm value if needed. - num_trainers: The number of training nodes. - regularization: A Regularizer, such as fluid.regularizer.L2DecayRegularizer. - name: An optional name prefix. + rampup_step (int): Time steps used in sparsity warm-up periods. Default is 1. + For example, if the sparsity is [0.75, 0.9375, 0.984375, 0.996, 0.999], and the rampup_step is 100, \ + it will use 0.75 at 0~19 steps, and 0.9375 at 20~39 steps, and so on. \ + And when reach sparsity array ends, it will use 0.999 then and after. + sparsity (list[float]): Get top important element from gradient tensor, the ratio is (1 - current sparsity). \ + Default is [0.999]. For example, if the sparsity is [0.99, 0.999], \ + the top [1%, 0.1%] important element will be transmitted. + use_nesterov (bool): Enables Nesterov momentum. True means use Nesterov. Default is False. + local_grad_clip_norm (float, optional): Local gradient clip norm value. Optional, default is None, represent no need clip. + num_trainers (int, optional): The number of training nodes. Optional, default is None. + regularization (WeightDecayRegularizer, optional): A Regularizer, such as \ + :ref:`api_fluid_regularizer_L2DecayRegularizer`. Optional, default is None. + name (str, optional): This parameter is used by developers to print debugging information. \ + For details, please refer to :ref:`api_guide_Name`. Default is None. Examples: .. code-block:: python