optimizer.py 214.1 KB
Newer Older
1
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
14 15

from __future__ import print_function
16

17
import numpy as np
18
import six
19
import logging
20
from collections import defaultdict
21

Q
Qiao Longfei 已提交
22
from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table
23
from paddle.fluid.framework import Program, Variable, name_scope, default_main_program, default_startup_program, device_guard
24

25 26
from . import framework
from . import layers
27
from . import unique_name
28
from .backward import append_backward, _some_in_set_, _append_grad_suffix_, _get_no_grad_set_name
29
from .clip import GradientClipBase, GradientClipByNorm, error_clip_callback, append_gradient_clip_ops
30 31 32
from .framework import program_guard
from .initializer import Constant
from .layer_helper import LayerHelper
S
sneaxiy 已提交
33
from .layers import ops
34
from .regularizer import append_regularization_ops
35
from .dygraph import base as imperative_base
36
from .dygraph import no_grad
37
from .dygraph.learning_rate_scheduler import LearningRateDecay, _LearningRateEpochDecay
38 39 40
from paddle.fluid import core
from paddle.fluid.layers import tensor
from functools import reduce
41
from .wrapped_decorator import signature_safe_contextmanager
M
mapingshuo 已提交
42
from .. import compat as cpt
43

44
__all__ = [
45 46 47 48
    'SGD', 'Momentum', 'Adagrad', 'Adam', 'Adamax', 'Dpsgd', 'DecayedAdagrad',
    'Ftrl', 'SGDOptimizer', 'MomentumOptimizer', 'AdagradOptimizer',
    'AdamOptimizer', 'AdamaxOptimizer', 'DpsgdOptimizer',
    'DecayedAdagradOptimizer', 'RMSPropOptimizer', 'FtrlOptimizer', 'Adadelta',
Z
Zeng Jinle 已提交
49 50 51 52
    'AdadeltaOptimizer', 'ModelAverage', 'LarsMomentum',
    'LarsMomentumOptimizer', 'DGCMomentumOptimizer', 'LambOptimizer',
    'ExponentialMovingAverage', 'PipelineOptimizer', 'LookaheadOptimizer',
    'RecomputeOptimizer'
53
]
Q
Qiao Longfei 已提交
54 55 56 57 58 59


class Optimizer(object):
    """Optimizer Base class.

    Define the common interface of an optimizer.
60 61
    User should not use this class directly,
    but need to use one of it's implementation.
Q
Qiao Longfei 已提交
62 63
    """

64
    @imperative_base.no_grad
65 66 67 68
    def __init__(self,
                 learning_rate,
                 parameter_list=None,
                 regularization=None,
69
                 grad_clip=None,
70
                 name=None):
H
hong 已提交
71 72
        self._parameter_list = list(
            parameter_list) if parameter_list is not None else None
73
        self._name = name
L
lujun 已提交
74
        if framework.in_dygraph_mode():
M
minqiyang 已提交
75 76 77 78 79
            if not isinstance(learning_rate, float) and \
                    not isinstance(learning_rate, LearningRateDecay):
                raise TypeError(
                    "learning rate should be float or LearningRateDecay, got %s here"
                    % type(learning_rate))
80
            if self._parameter_list is None:
81 82 83
                raise AttributeError(
                    "parameter_list argument given to the Optimizer should not be None in dygraph mode."
                )
84 85 86 87 88 89 90 91
            if regularization is not None:
                for param in self._parameter_list:
                    if param.regularizer is not None:
                        logging.info(
                            "If regularizer of a Parameter has been set by 'fluid.ParamAttr' or 'fluid.WeightNormParamAttr' already. "
                            "The Regularization[%s] in Optimizer will not take effect, and it will only be applied to other Parameters!"
                            % regularization.__str__())
                        break
M
minqiyang 已提交
92 93 94 95 96 97 98
        else:
            if not isinstance(learning_rate, float) and \
                    not isinstance(learning_rate, framework.Variable):
                raise TypeError(
                    "learning rate should be float or Variable, got %s here" %
                    type(learning_rate))

99 100 101 102 103
        if grad_clip is not None:
            if not isinstance(grad_clip, GradientClipBase):
                raise TypeError(
                    "'grad_clip' should be an instance of GradientClipBase's derived class"
                )
D
dzhwinter 已提交
104
        self.regularization = regularization
105
        self._grad_clip = grad_clip
106
        self._learning_rate = learning_rate
D
dzhwinter 已提交
107 108
        # the learning rate type should be inferenced from loss
        self._dtype = None
109
        # each program should have a independent learning rate
110
        # program -> Variable(learning_rate)
Q
qiaolongfei 已提交
111
        self._learning_rate_map = dict()
112
        if isinstance(self._learning_rate, framework.Variable):
113 114
            self._learning_rate_map[framework.default_main_program(
            )] = self._learning_rate
115 116 117 118 119
        # Dictionary of accumulators. Some optimizer subclasses need to
        # allocate and manage extra variables associated with the parameters
        # to train. These variables are called accumulators.
        # {accum_name : { paramter_name : accumulator_for_parameter, ...}, ...}
        self._accumulators = defaultdict(lambda: dict())
Q
Qiao Longfei 已提交
120
        self.helper = None
121
        self._opti_name_list = []
H
hong 已提交
122
        self._accumulators_holder = {}
123
        self._param_device_map = dict()
H
hong 已提交
124 125 126 127

    @framework.dygraph_only
    def state_dict(self):
        '''
T
tianshuo78520a 已提交
128 129
        Get state dict information from optimizer. It contain all the variable used by optimizer. For Adam optimizer, contains beta1, beta2, momentum etc. If LearningRateDecay have been used, global_step will be include in state dict.
        If the optimizer never be called(minimize function), the state_dict is empty.
H
hong 已提交
130 131 132

        Args: None
        Return:
T
tianshuo78520a 已提交
133
            state_dict(dict) : dict contains all the variable used by optimizer
H
hong 已提交
134 135 136 137 138
        
        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
139 140 141 142 143 144

                with fluid.dygraph.guard():
                    emb = fluid.dygraph.Embedding([10, 10])

                    adam = fluid.optimizer.Adam(0.001, parameter_list=emb.parameters())
                    state_dict = adam.state_dict()
H
hong 已提交
145 146 147 148 149 150 151 152

        '''
        state_dict = {}
        for k, v in self._accumulators.items():
            for para_name, var_tmp in v.items():
                state_dict[var_tmp.name] = var_tmp
        # global step if use lr decay
        if isinstance(self._learning_rate, LearningRateDecay):
153 154 155 156
            state_dict["LR_Scheduler"] = self._learning_rate.state_dict()

            if not isinstance(self._learning_rate, _LearningRateEpochDecay):
                var_tmp = None
157 158 159
                var_temp = framework._varbase_creator(
                    None, name='global_step', dtype='int32')

160 161
                tensor.fill_constant(
                    [1], "int32", self._learning_rate.step_num, out=var_temp)
H
hong 已提交
162

163
                state_dict['global_step'] = var_temp
H
hong 已提交
164 165 166 167 168
        return state_dict

    @framework.dygraph_only
    def set_dict(self, state_dict):
        '''
T
tianshuo78520a 已提交
169
        Load optimizer state dict. For Adam optimizer, contains beta1, beta2, momentum etc. If LearningRateDecay have been used, global_step will be changed.
H
hong 已提交
170 171 172 173 174 175 176 177

        Args: 
            state_dict(dict) : Dict contains all the Variable needed by optimizer
        Return:
            None
        
        Examples:
            .. code-block:: python
178

H
hong 已提交
179
                with fluid.dygraph.guard():
180
                    emb = fluid.dygraph.Embedding([10, 10])
181

H
hong 已提交
182
                    state_dict = emb.state_dict()
183
                    fluid.save_dygraph(state_dict, "paddle_dy")
184

185 186
                    adam = fluid.optimizer.Adam(learning_rate=fluid.layers.noam_decay( 100, 10000), 
                                                parameter_list=emb.parameters())
H
hong 已提交
187
                    state_dict = adam.state_dict()
188
                    fluid.save_dygraph(state_dict, "paddle_dy")
189

H
hong 已提交
190
                    para_state_dict, opti_state_dict = fluid.load_dygraph( "paddle_dy")
191

192
                    adam.set_dict(opti_state_dict)
H
hong 已提交
193 194 195 196

        '''

        if isinstance(self._learning_rate, LearningRateDecay):
197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
            self._learning_rate.set_dict(state_dict["LR_Scheduler"])

            if not isinstance(self._learning_rate, _LearningRateEpochDecay):
                assert 'global_step' in state_dict, \
                        'Global step not in state dict, Dygraph use LearningRateDecay, global_step must in state_dict'
                global_step = state_dict['global_step']

                if isinstance(global_step, Variable):
                    step_np = global_step
                    step_np = np.array(step_np.value().get_tensor())
                    assert step_np.shape == (1,),  \
                            "global step shape is (1,), the shape is {}".format( step_np.shape )

                    self._learning_rate.step_num = int(step_np[0])
                elif isinstance(global_step, np.ndarray):
                    assert global_step.shape == (1,),  \
                            "global step shape is (1,), the shape is {}".format( global_step.shape )
                    self._learning_rate.step_num = global_step[0]
                else:
                    raise RuntimeError(
                        "Type not supprt, value in state dict must be [VarBase, Variable, numpy], the type is ",
                        type(global_step))
H
hong 已提交
219 220 221 222 223 224

        self._accumulators_holder = state_dict
        for k, v in self._accumulators.items():
            for para_name, var_tmp in v.items():
                assert var_tmp.name in state_dict, \
                        "optimizer variable {} not found".format( var_tmp.name )
225
                var = var_tmp.value()
H
hong 已提交
226 227 228 229 230 231 232 233
                tensor = var.get_tensor()
                model_np = np.array(tensor)

                load_para = state_dict[var_tmp.name]

                if isinstance(load_para, Variable):
                    load_para_np = load_para.numpy()
                elif isinstance(load_para, core.VarBase):
234
                    load_para_np = load_para.numpy()
H
hong 已提交
235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
                elif isinstance(load_para, np.ndarray):
                    load_para_np = load_para
                else:
                    raise RuntimeError("State dict type {} not supprt".format(
                        str(type(load_para))))

                assert model_np.shape == load_para_np.shape,  \
                                          "Parameter shape not match, Dygraph Parameter [ {} ] need tensor with shape {} but load tensor with shape {}".format(
                                                 item.name, model_np.shape, load_para_np.shape)

                assert model_np.dtype == load_para_np.dtype, \
                                          "Parameter dtype not match, Dygraph Parameter [ {} ] need tensor with dtype {}  but load tensor with dtype {}".format(
                                                item.name, model_np.dtype, load_para_np.dtype)

                tensor.set(load_para_np, framework._current_expected_place())
250

251 252
    def get_opti_var_name_list(self):
        return self._opti_name_list
Q
Qiao Longfei 已提交
253

Q
Qiao Longfei 已提交
254
    def _create_global_learning_rate(self):
255 256 257
        if imperative_base.enabled():
            # create learning rate Variable
            if isinstance(self._learning_rate, float):
M
minqiyang 已提交
258 259 260 261 262 263 264 265 266 267 268 269
                lr = self._global_learning_rate()

                if isinstance(lr, framework.Variable):
                    return
                else:
                    self._learning_rate_map[framework.default_main_program(
                    )] = layers.create_global_var(
                        name=unique_name.generate("learning_rate"),
                        shape=[1],
                        value=float(self._learning_rate),
                        dtype='float32' if self._dtype is None else self._dtype,
                        persistable=True)
270
            # get learning rate Variable from LearningRateDecay
M
minqiyang 已提交
271
            elif isinstance(self._learning_rate, LearningRateDecay):
272 273 274
                self._learning_rate_map[framework.default_main_program(
                )] = self._learning_rate()
            else:
Q
qiaolongfei 已提交
275
                raise TypeError(
276 277
                    "optimizer's learning rate must be float or LearningRateDecay"
                )
278
        else:
279 280 281 282
            lr = self._global_learning_rate()

            if isinstance(lr, framework.Variable):
                return
M
minqiyang 已提交
283 284 285 286 287 288
            else:
                if not isinstance(self._learning_rate, float):
                    raise TypeError(
                        "learning rate variable is create outside optimizer,"
                        "can not create new learning rate variable for new program"
                    )
Q
Qiao Longfei 已提交
289

290 291 292 293 294 295 296 297
            # create learning rate in the current main program
            self._learning_rate_map[framework.default_main_program(
            )] = layers.create_global_var(
                name=unique_name.generate("learning_rate"),
                shape=[1],
                value=float(self._learning_rate),
                dtype='float32' if self._dtype is None else self._dtype,
                persistable=True)
298

299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
    @framework.dygraph_only
    def set_lr(self, value):
        """
        :api_attr: imperative
        
        Set the value of the learning rate manually in the optimizer. If the optimizer use LearningRateDecay,
        this API cannot be invoked, because it will lead to conflict.

        Args:
            value (float|Variable): the value of learning rate

        Returns:
            None
          
        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                        
                with fluid.dygraph.guard():
                    linear = fluid.dygraph.nn.Linear(10, 10)

                    adam = fluid.optimizer.Adam(0.1, parameter_list=linear.parameters())

                    # set learning rate manually by python float value
                    lr_list = [0.2, 0.3, 0.4, 0.5, 0.6]
                    for i in range(5):
                        adam.set_lr(lr_list[i])
                        lr = adam.current_step_lr()
                        print("current lr is {}".format(lr))
                    # Print:
                    #    current lr is 0.2
                    #    current lr is 0.3
                    #    current lr is 0.4
                    #    current lr is 0.5
                    #    current lr is 0.6


                    # set learning rate manually by framework Variable
                    lr_var = fluid.layers.create_global_var(
                        shape=[1], value=0.7, dtype='float32')
                    adam.set_lr(lr_var)
                    lr = adam.current_step_lr()
                    print("current lr is {}".format(lr))
                    # Print:
                    #    current lr is 0.7



        """
        if not isinstance(value, (framework.Variable, float)):
            raise TypeError(
                "The type of 'value' in optimizer.set_lr must be (float, Variable), but received %s."
                % (type(value)))
        if isinstance(self._learning_rate, LearningRateDecay):
            raise RuntimeError(
                "optimizer's learning rate can't be LearningRateDecay when invoke this API, because this will lead to conflict."
            )
        if isinstance(value, float):
            self._learning_rate = value
            current_lr = self._global_learning_rate()
            if current_lr is not None:
                global_block = framework.default_main_program().global_block()
                global_block.append_op(
                    type='fill_constant',
                    outputs={'Out': [current_lr]},
                    attrs={
                        'dtype': current_lr.dtype,
                        'shape': list(current_lr.shape),
                        'value': float(value)
                    },
                    stop_gradient=True)
        else:
            assert len(value.shape) == 1 and value.shape[
                0] == 1, "optimizer's learning rate must be 1-D Tensor with shape[1]"
            self._learning_rate_map[framework.default_main_program()] = value

376 377 378
    @framework.dygraph_only
    def current_step_lr(self):
        """
379
        :api_attr: imperative
380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424
        
        Get current step learning rate. The return value is all the same When LearningRateDecay is not used,
        otherwise return the step learning rate.

        Returns:
            float: The learning rate of the current step.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

                # example1: LearningRateDecay is not used, return value is all the same
                with fluid.dygraph.guard():
                    emb = fluid.dygraph.Embedding([10, 10])
                    adam = fluid.optimizer.Adam(0.001, parameter_list = emb.parameters())
                    lr = adam.current_step_lr()
                    print(lr) # 0.001

                # example2: PiecewiseDecay is used, return the step learning rate
                with fluid.dygraph.guard():
                    inp = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32")
                    linear = fluid.dygraph.nn.Linear(10, 10)
                    inp = fluid.dygraph.to_variable(inp)
                    out = linear(inp)
                    loss = fluid.layers.reduce_mean(out)
                    
                    bd = [2, 4, 6, 8]
                    value = [0.2, 0.4, 0.6, 0.8, 1.0]
                    adam = fluid.optimizer.Adam(fluid.dygraph.PiecewiseDecay(bd, value, 0),
                                           parameter_list=linear.parameters())

                    # first step: learning rate is 0.2
                    np.allclose(adam.current_step_lr(), 0.2, rtol=1e-06, atol=0.0) # True

                    # learning rate for different steps
                    ret = [0.2, 0.2, 0.4, 0.4, 0.6, 0.6, 0.8, 0.8, 1.0, 1.0, 1.0, 1.0]
                    for i in range(12):
                        adam.minimize(loss)
                        lr = adam.current_step_lr()
                        np.allclose(lr, ret[i], rtol=1e-06, atol=0.0) # True

        """
        current_lr = self._global_learning_rate()
425
        if isinstance(current_lr, framework.Variable):
426 427 428 429
            return self._global_learning_rate().numpy()[0]

        if isinstance(self._learning_rate, float):
            return self._learning_rate
430 431 432
        elif isinstance(self._learning_rate, _LearningRateEpochDecay):
            step_lr = self._learning_rate()
            return step_lr.numpy()[0]
433 434 435 436 437 438 439
        else:
            step_lr = self._learning_rate.step()
            if isinstance(step_lr, (float, int)):
                return step_lr
            else:
                return step_lr.numpy()[0]

Y
yuyang18 已提交
440
    def _global_learning_rate(self, program=None):
Q
Qiao Longfei 已提交
441 442 443 444
        """
        get global decayed learning rate
        :return:
        """
445 446
        if program is None:
            program = framework.default_main_program()
Q
qiaolongfei 已提交
447
        return self._learning_rate_map.get(program, None)
Q
Qiao Longfei 已提交
448

Q
Qiao Longfei 已提交
449 450 451 452 453
    def _append_optimize_op(self, block, param_and_grad):
        """ append optimize operator to block and return all the added optimize_op
        """
        raise NotImplementedError()

454 455 456 457
    def _create_param_lr(self, param_and_grad):
        # create learning rate variable for every parameter
        param = param_and_grad[0]
        param_lr = param.optimize_attr['learning_rate']
W
Wu Yi 已提交
458 459
        if type(param_lr) == Variable:
            return param_lr
Q
qiaolongfei 已提交
460
        else:
W
Wu Yi 已提交
461
            if param_lr == 1.0:
Y
yuyang18 已提交
462
                return self._global_learning_rate()
W
Wu Yi 已提交
463
            else:
X
Xin Pan 已提交
464 465 466
                with default_main_program()._lr_schedule_guard(
                        is_with_opt=True), framework.name_scope(
                            'scale_with_param_lr'):
467
                    return self._global_learning_rate() * param_lr
468 469 470 471 472 473 474

    def _create_accumulators(self, block, parameters):
        """Create all accumulators needed by the parameters

        Args:
            block: the block in which the loss variable is present
            parameters: list of parameter variables for the optimizer
Q
Qiao Longfei 已提交
475
        """
476 477
        pass

478
    def _finish_update(self, block, parameters_and_grads):
479 480 481 482 483 484 485 486
        """Finish any custom updates needed
           before completing an optimization step

        Args:
            block: the block in which the loss variable is present
            parameters: list of parameter variables for the optimizer

        Returns:
Q
qiaolongfei 已提交
487
            None
488 489 490
        """
        pass

491 492 493 494 495
    def _add_accumulator(self,
                         name,
                         param,
                         dtype=None,
                         fill_value=0.0,
496
                         shape=None,
497
                         type=None,
498
                         device=None):
499 500 501 502 503 504 505 506 507
        """Utility function to add an accumulator for a parameter

        Args:
            block: the block in which the loss variable is present
            name: name of the accumulator
            param: parameter variable for which accumulator is to be added
            dtype: data type of the accumulator variable
            fill_value: value to initialize the accumulator variable
        """
W
whs 已提交
508 509
        if self._name is not None:
            name = self._name + "_" + name
510 511
        if (name in self._accumulators and
                param.name in self._accumulators[name]):
L
lujun 已提交
512
            if framework.in_dygraph_mode():
X
polish  
Xin Pan 已提交
513
                return self._accumulators[name][param.name]
514
            raise Exception("Accumulator {} already exists for parameter {}".
515
                            format(name, param.name))
516 517
        if shape == None:
            shape = param.shape
Q
Qiao Longfei 已提交
518
        assert isinstance(self.helper, LayerHelper)
519 520 521 522 523

        var_name = param.name + "_" + name
        var_name = unique_name.generate(var_name)
        self._opti_name_list.append(var_name)

Q
Qiao Longfei 已提交
524
        var = self.helper.create_global_variable(
525
            name=var_name,
Q
Qiao Longfei 已提交
526
            persistable=True,
F
fengjiayi 已提交
527
            dtype=dtype or param.dtype,
528
            type=param.type if type is None else type,
H
hong 已提交
529 530
            shape=shape,
            belong_to_optimizer=True)
531 532 533 534 535
        if device is None:
            device = self._get_device_for_param(param.name)
        with device_guard(device):
            self.helper.set_variable_initializer(
                var, initializer=Constant(value=float(fill_value)))
H
hong 已提交
536 537 538 539 540 541 542

        if framework.in_dygraph_mode():
            if len(self._accumulators_holder) > 0:
                assert var_name in self._accumulators_holder, \
                        "Optimizer set error, {} should in state dict".format( var_name )
                var.set_value(self._accumulators_holder[var_name])

Q
Qiao Longfei 已提交
543
        self._accumulators[name][param.name] = var
544
        return var
545 546 547 548 549 550 551 552 553 554 555

    def _get_accumulator(self, name, param):
        """Utility function to fetch an accumulator for a parameter

        Args:
            name: name of the accumulator
            param: parameter variable for which accumulator is to be fetched

        Returns:
            accumulator variable for the parameter
        """
W
whs 已提交
556 557
        if self._name is not None:
            name = self._name + "_" + name
558 559 560 561 562 563
        if (name not in self._accumulators or
                param.name not in self._accumulators[name]):
            raise Exception("Accumulator {} does not exist for parameter {}".
                            format(name, param.name))
        return self._accumulators[name][param.name]

564 565 566 567 568 569 570 571 572 573 574 575
    def _update_param_device_map(self, parameters_and_grads, target_block):
        for param_and_grad in parameters_and_grads:
            if param_and_grad[0].trainable is True:
                param_name = param_and_grad[0].name
                ops = target_block.ops
                device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName(
                )
                for op in ops:
                    input_arg_names = op.input_arg_names
                    if param_name in input_arg_names:
                        self._param_device_map[param_name] = op.attr(
                            device_attr_name)
576
                        break
577 578 579 580 581 582 583

    def _get_device_for_param(self, param_name):
        device = None
        if param_name in self._param_device_map:
            device = self._param_device_map[param_name]
        return device

584
    def _create_optimization_pass(self, parameters_and_grads):
Q
Qiao Longfei 已提交
585 586 587
        """Add optimization operators to update gradients to variables.

        Args:
Q
qiaolongfei 已提交
588
          parameters_and_grads(list(tuple(Variable, Variable))):
589
            a list of (variable, gradient) pair to update.
Q
Qiao Longfei 已提交
590 591

        Returns:
592
          return_op_list: a list of operators that will complete one step of
593 594 595
            optimization. This will include parameter update ops, global step
            update ops and any other custom ops required by subclasses to manage
            their internal state.
Q
Qiao Longfei 已提交
596
        """
597 598 599 600 601
        # This is a default implementation of create_optimization_pass that
        # can be shared by most optimizers. This implementation assumes that
        # the subclass will implement the _append_optimize_op method and the
        #  _initialize_tensors method. The subclass can extend the
        # _create_accumulators method if it needs to create accumulators
602
        # for parameters and extend _finish_update method to add custom ops.
603

604
        # Allways called under program_guard use global block as loss block
605 606 607
        # But if current block is in control flow, append optimize op in the
        # grad block of current block

608
        global_block = framework.default_main_program().global_block()
609 610 611 612 613 614 615 616 617
        target_block = global_block
        current_block = framework.default_main_program().current_block()
        if current_block.idx != global_block.idx:
            assert current_block.backward_block_idx != -1, \
                "current block is not global_block, but it doesn't have backward block."
            target_block = framework.default_main_program().blocks[
                current_block.backward_block_idx]

        start = len(target_block.ops)
618
        self.helper = LayerHelper(self.__class__.__name__)
619
        self._update_param_device_map(parameters_and_grads, target_block)
C
chengduo 已提交
620
        self._create_accumulators(
621
            target_block,
C
chengduo 已提交
622
            [p[0] for p in parameters_and_grads if p[0].trainable])
623 624
        self._create_global_learning_rate()

M
minqiyang 已提交
625
        if framework.in_dygraph_mode():
626 627 628
            for param_and_grad in parameters_and_grads:
                if param_and_grad[1] is None:
                    continue
629 630
                if param_and_grad[0].trainable is True:
                    self._append_optimize_op(target_block, param_and_grad)
631 632 633 634 635 636 637
        else:
            for param_and_grad in parameters_and_grads:
                if param_and_grad[1] is None:
                    continue
                with param_and_grad[0].block.program._optimized_guard(
                        param_and_grad), name_scope("optimizer"):
                    if param_and_grad[0].trainable is True:
638 639 640 641 642
                        device = self._get_device_for_param(param_and_grad[0]
                                                            .name)
                        with device_guard(device):
                            optimize_op = self._append_optimize_op(
                                target_block, param_and_grad)
643 644 645

        # Get custom finish ops for subclasses
        # FIXME: Need to fix this once we figure out how to handle dependencies
646
        self._finish_update(target_block, parameters_and_grads)
647

648 649
        end = len(target_block.ops)
        return target_block._slice_ops(start, end)
650 651

    def _process_distribute_lookuptable(self, param_grads):
Q
Qiao Longfei 已提交
652 653 654 655 656 657 658 659 660
        """
        Because distribute lookup table only support SGD optimizer for now, not support
        other optimizer and regularization, so we should find the table parameter out,
        and avoid to add regularization and other op for it, and add sgd optimize op
        for it independently.
        :param param_grads(list((Var, Var))): list of (param, grad) pair.
        :param loss: the loss variable.
        :param startup_program: the startup program
        """
661 662
        program = framework.default_main_program()
        global_block = framework.default_main_program().global_block()
Q
Qiao Longfei 已提交
663 664 665 666 667 668 669 670 671 672 673 674 675 676 677
        table_name = find_distributed_lookup_table(program)
        table_param = None
        table_grad = None
        new_param_grads = []
        for p, g in param_grads:
            if p.name == table_name:
                if table_param is not None:
                    raise RuntimeError(
                        "multi dist table var found, only support one now!")
                table_param = p
                table_grad = g
            else:
                new_param_grads.append((p, g))
        sgd_op = None
        if table_param is not None:
678 679 680 681 682 683 684 685 686 687 688 689 690
            param_and_grad = [table_param, table_grad]
            with table_param.block.program._optimized_guard(param_and_grad), \
                    framework.name_scope("optimizer"):
                self._create_global_learning_rate()
                # create the optimize op
                sgd_op = global_block.append_op(
                    type='sgd',
                    inputs={
                        "Param": table_param,
                        "Grad": table_grad,
                        "LearningRate": self._create_param_lr(param_and_grad)
                    },
                    outputs={"ParamOut": param_and_grad[0]})
Q
Qiao Longfei 已提交
691 692
        return new_param_grads, (table_param, table_grad), sgd_op

693 694 695
    def _append_dgc_ops(self, param_and_grad):
        pass

696 697 698 699 700 701 702
    def backward(self,
                 loss,
                 startup_program=None,
                 parameter_list=None,
                 no_grad_set=None,
                 callbacks=None):
        """
703
        The first part of ``minimize``, do auto-diff to append backward operations for
704 705 706
        the current program.

        Args:
707 708 709 710
            loss (Variable): ``loss`` variable to run optimizations.
            startup_program (Program, optional): :ref:`api_fluid_Program` for
                initializing parameters in ``parameter_list``. The default value
                is None, at this time :ref:`api_fluid_default_startup_program` will be used.
H
hong 已提交
711
            parameter_list (Iterable, optional): Iterable of ``Variable`` or ``Variable.name`` to update
712 713
                to minimize ``loss``. The default value is None, at this time all parameters
                will be updated.
714
            no_grad_set (set, optional): Set of ``Variable``  or ``Variable.name`` that don't need
715 716 717
                to be updated. The default value is None.
            callbacks (list, optional): list of callable objects to run when appending backward
                operator for one parameter. The default value is None.
M
minqiyang 已提交
718

719
        Return:
720 721
            list: list of (param, grad) variable pairs, param is ``Parameter``,
                grad is the gradient value corresponding to the parameter.
M
minqiyang 已提交
722

723
        Examples:
724
            See examples in ``apply_gradients``.
725
        """
726
        act_no_grad_set = None
L
Leo Chen 已提交
727
        if framework.in_dygraph_mode():
728
            pass
L
Leo Chen 已提交
729 730
        else:
            act_no_grad_set = self._get_no_grad_set(loss, no_grad_set)
G
gongweibao 已提交
731

C
chengduo 已提交
732
        self._dtype = loss.dtype
L
lujun 已提交
733
        if framework.in_dygraph_mode():
C
chengduo 已提交
734
            params_grads = []
735
            for param in self._parameter_list:
C
chengduo 已提交
736 737
                if not param.trainable:
                    continue
738
                if param._grad_ivar() is not None:
C
chengduo 已提交
739
                    # create gradient variable
740
                    grad_var = param._grad_ivar()
C
chengduo 已提交
741
                    params_grads.append((param, grad_var))
742
        else:
C
chengduo 已提交
743 744 745 746 747
            if callbacks is None:
                callbacks = [error_clip_callback]
            else:
                assert (isinstance(callbacks, list))
            program = loss.block.program
C
chengduo 已提交
748 749 750 751
            assert len(loss.shape) == 1 and loss.shape[0] == 1, \
                "The loss.shape should be (1L,), but the current loss.shape is {}. " \
                "Maybe that you should call fluid.layers.mean to process the current loss.".format(
                    loss.shape)
752 753
            parameter_list = parameter_list if parameter_list \
                else self._parameter_list
C
chengduo 已提交
754 755
            with program_guard(program, startup_program):
                params_grads = append_backward(loss, parameter_list,
756
                                               act_no_grad_set, callbacks)
C
chengduo 已提交
757 758 759 760
                # Note: since we can't use all_reduce_op now,
                #  dgc_op should be the last op of one grad.
                self._append_dgc_ops(params_grads)
        return params_grads
761 762 763 764 765 766 767 768

    def apply_gradients(self, params_grads):
        """
        Second part of `minimize`, appending optimization operators for
        given `params_grads` pairs.

        Args:
            params_grads (list): list of (param, grad) pair to do optimization.
M
minqiyang 已提交
769

770 771
        Returns:
            list: A list of operators appended to the current program.
M
minqiyang 已提交
772

773 774 775
        Examples:
            .. code-block:: python

776
                import paddle.fluid as fluid
777 778 779 780 781 782 783
                loss = network()
                optimizer = fluid.optimizer.SGD(learning_rate=0.1)
                params_grads = optimizer.backward(loss)
                # you may append operations for params_grads here
                # ...
                optimizer.apply_gradients(params_grads)
        """
784

785 786
        params_grads = sorted(params_grads, key=lambda x: x[0].name)

787
        # 'optimizer(grad_clip)' or 'set_gradient_clip'
788 789 790 791
        if self._grad_clip is not None:
            params_grads = self._grad_clip(params_grads)
        else:
            params_grads = append_gradient_clip_ops(params_grads)
792 793

        # Add regularization if any
794 795
        params_grads = append_regularization_ops(
            params_grads, self.regularization, self._param_device_map)
796 797 798 799

        optimize_ops = self._create_optimization_pass(params_grads)
        return optimize_ops

C
chengduo 已提交
800 801 802 803 804 805 806 807 808 809 810 811
    def apply_optimize(self, loss, startup_program, params_grads):
        """
        Second part of `minimize`, appending optimization operators for
        given `params_grads` pairs.
        Args:
            loss (Variable): loss variable to run optimizations.
            startup_program (Program): startup_program for initializing parameters
                in `parameter_list`.
            params_grads (list): list of (param, grad) pair to do optimization.
        Returns:
            list: A list of operators appended to the current program.
        """
L
lujun 已提交
812
        if framework.in_dygraph_mode():
C
chengduo 已提交
813 814
            with program_guard(framework.default_main_program(),
                               framework.default_startup_program()):
815 816
                if self._grad_clip is not None:
                    params_grads = self._grad_clip(params_grads)
817 818
                params_grads = append_regularization_ops(params_grads,
                                                         self.regularization)
C
chengduo 已提交
819 820 821 822 823 824 825
                optimize_ops = self._create_optimization_pass(params_grads)
        else:
            program = loss.block.program
            with program_guard(program, startup_program):
                optimize_ops = self.apply_gradients(params_grads)
        return optimize_ops

G
gongweibao 已提交
826
    def _get_no_grad_set(self, loss, no_grad_set=None):
827
        no_grad_set = _get_no_grad_set_name(no_grad_set)
G
gongweibao 已提交
828 829 830 831 832 833 834 835
        parameters = loss.block.program.global_block().all_parameters()
        param_no_trainable = set(
            [param.name for param in parameters if param.trainable is False])
        # If the parameter is no trainable, it should not have a gradient.
        no_grad_set.update(param_no_trainable)

        return no_grad_set

836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866
    @framework.dygraph_only
    def clear_gradients(self):
        """
        Clear the gradients of all optimized parameters for model.
        
        Returns:
            None
        
        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

                with fluid.dygraph.guard():
                    value = np.arange(26).reshape(2, 13).astype("float32")
                    a = fluid.dygraph.to_variable(value)
                    linear = fluid.Linear(13, 5, dtype="float32")
                    # This can be any optimizer supported by dygraph.
                    adam = fluid.optimizer.Adam(learning_rate = 0.01, 
                                                parameter_list = linear.parameters())
                    out = linear(a)
                    out.backward()
                    adam.minimize(out)
                    adam.clear_gradients()

        """
        for p in self._parameter_list:
            if p.trainable:
                p.clear_gradient()

867
    @imperative_base.no_grad
Q
Qiao Longfei 已提交
868 869
    def minimize(self,
                 loss,
870
                 startup_program=None,
Q
Qiao Longfei 已提交
871
                 parameter_list=None,
872
                 no_grad_set=None):
873
        """
874
        Add operations to minimize ``loss`` by updating ``parameter_list``.
M
minqiyang 已提交
875

876
        Args:
877 878 879 880
            loss (Variable): A ``Variable`` containing the value to minimize.
            startup_program (Program, optional): :ref:`api_fluid_Program` for
                initializing parameters in ``parameter_list``. The default value
                is None, at this time :ref:`api_fluid_default_startup_program` will be used.
H
hong 已提交
881
            parameter_list (Iterable, optional): Iterable of ``Variable`` or ``Variable.name`` to update
882 883
                to minimize ``loss``. The default value is None, at this time all parameters
                will be updated.
884
            no_grad_set (set, optional): Set of ``Variable``  or ``Variable.name`` that don't need
885
                to be updated. The default value is None.
Q
Qiao Longfei 已提交
886

887
        Returns:
888 889 890
            tuple: tuple (optimize_ops, params_grads), A list of operators appended
            by minimize and a list of (param, grad) variable pairs, param is
            ``Parameter``, grad is the gradient value corresponding to the parameter.
891 892 893
            The returned tuple can be passed to ``fetch_list`` in ``Executor.run()`` to 
            indicate program pruning. If so, the program will be pruned by ``feed`` and 
            ``fetch_list`` before run, see details in ``Executor``.
894 895 896

        Examples:
            Please refer to the example of current Optimizer.
Q
Qiao Longfei 已提交
897
        """
C
chengduo 已提交
898
        assert isinstance(loss, Variable), "The loss should be an Variable."
899

900 901
        parameter_list = parameter_list if parameter_list \
            else self._parameter_list
C
chengduo 已提交
902 903 904 905 906
        params_grads = self.backward(
            loss,
            startup_program=startup_program,
            parameter_list=parameter_list,
            no_grad_set=no_grad_set)
907

C
chengduo 已提交
908 909
        optimize_ops = self.apply_optimize(
            loss, startup_program=startup_program, params_grads=params_grads)
M
minqiyang 已提交
910

Q
Qiao Longfei 已提交
911
        return optimize_ops, params_grads
Q
Qiao Longfei 已提交
912 913 914


class SGDOptimizer(Optimizer):
Q
qiaolongfei 已提交
915 916 917 918 919 920 921
    """
    Optimizer of the stochastic gradient descent algorithm.

    .. math::

        param\_out = param - learning\_rate * grad

922 923 924
    Parameters:
        learning_rate (float|Variable): The learning rate used to update parameters. \
            Can be a float value or a Variable with one float value as data element.
H
hong 已提交
925
        parameter_list (Iterable, optional):  Iterable of ``Variable`` names to update to minimize ``loss``. \
926 927
            This parameter is required in dygraph mode. \
            The default value is None in static mode, at this time all parameters will be updated.
928 929 930 931 932
        regularization (WeightDecayRegularizer, optional): The strategy of regularization. There are two method: \
             :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If a parameter has set \
            regularizer using :ref:`api_fluid_ParamAttr` already, the regularization setting here in optimizer will be \
            ignored for this parameter. Otherwise, the regularization setting here in optimizer will take effect.  \
            Default None, meaning there is no regularization.
933 934 935 936
        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of 
            some derived class of ``GradientClipBase`` . There are three cliping strategies 
            ( :ref:`api_fluid_clip_GradientClipByGlobalNorm` , :ref:`api_fluid_clip_GradientClipByNorm` , 
            :ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping.
937 938
        name (str, optional): This parameter is used by developers to print debugging information. \
            For details, please refer to :ref:`api_guide_Name`. Default is None.
Q
qiaolongfei 已提交
939 940 941 942

    Examples:
        .. code-block:: python

943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967
            import paddle
            import paddle.fluid as fluid
            import numpy as np

            place = fluid.CPUPlace()
            main = fluid.Program()
            with fluid.program_guard(main):
                x = fluid.layers.data(name='x', shape=[13], dtype='float32')
                y = fluid.layers.data(name='y', shape=[1], dtype='float32')
                y_predict = fluid.layers.fc(input=x, size=1, act=None)
                cost = fluid.layers.square_error_cost(input=y_predict, label=y)
                avg_cost = fluid.layers.mean(cost)

                sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001)
                sgd_optimizer.minimize(avg_cost)

                fetch_list = [avg_cost]
                train_reader = paddle.batch(
                    paddle.dataset.uci_housing.train(), batch_size=1)
                feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
                exe = fluid.Executor(place)
                exe.run(fluid.default_startup_program())
                for data in train_reader():
                    exe.run(main, feed=feeder.feed(data), fetch_list=fetch_list)

Q
Qiao Longfei 已提交
968 969
    """

970 971 972 973
    def __init__(self,
                 learning_rate,
                 parameter_list=None,
                 regularization=None,
974
                 grad_clip=None,
975
                 name=None):
Q
Qiao Longfei 已提交
976
        assert learning_rate is not None
Q
Qiao Longfei 已提交
977
        super(SGDOptimizer, self).__init__(
X
Xin Pan 已提交
978
            learning_rate=learning_rate,
979
            parameter_list=parameter_list,
X
Xin Pan 已提交
980
            regularization=regularization,
981
            grad_clip=grad_clip,
X
Xin Pan 已提交
982
            name=name)
Q
Qiao Longfei 已提交
983 984
        self.type = "sgd"

985
    @no_grad
986
    def _append_optimize_op(self, block, param_and_grad):
987
        lr = self._create_param_lr(param_and_grad)
988
        if framework.in_dygraph_mode():
989 990 991
            core.ops.sgd(param_and_grad[0], lr, param_and_grad[1],
                         param_and_grad[0])
            return None
992

993
        assert isinstance(block, framework.Block)
Q
Qiao Longfei 已提交
994 995 996 997 998 999
        # create the optimize op
        sgd_op = block.append_op(
            type=self.type,
            inputs={
                "Param": param_and_grad[0],
                "Grad": param_and_grad[1],
1000
                "LearningRate": lr
Q
Qiao Longfei 已提交
1001
            },
M
minqiyang 已提交
1002 1003
            outputs={"ParamOut": param_and_grad[0]},
            stop_gradient=True)
Q
Qiao Longfei 已提交
1004 1005

        return sgd_op
1006 1007 1008


class MomentumOptimizer(Optimizer):
Q
qiaolongfei 已提交
1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
    """

    Simple Momentum optimizer with velocity state

    This optimizer has a flag for Nestrov Momentum.

    The update equations are as follows:

    .. math::

        & velocity = mu * velocity + gradient

        & if (use\_nesterov):

1023
        &\quad   param = param - (gradient + mu * velocity) * learning\_rate
Q
qiaolongfei 已提交
1024 1025 1026

        & else:

Q
qiaolongfei 已提交
1027
        &\quad   param = param - learning\_rate * velocity
Q
qiaolongfei 已提交
1028

1029 1030 1031 1032
    Parameters:
        learning_rate (float|Variable): The learning rate used to update parameters. \
            Can be a float value or a Variable with one float value as data element.
        momentum (float): Momentum factor
H
hong 已提交
1033
        parameter_list (Iterable, optional):  Iterable of ``Variable`` names to update to minimize ``loss``. \
1034 1035
            This parameter is required in dygraph mode. \
            The default value is None in static mode, at this time all parameters will be updated.
1036
        use_nesterov (bool, optional): Enables Nesterov momentum, default is false.
1037 1038 1039 1040 1041
        regularization (WeightDecayRegularizer, optional): The strategy of regularization. There are two method: \
             :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If a parameter has set \
            regularizer using :ref:`api_fluid_ParamAttr` already, the regularization setting here in optimizer will be \
            ignored for this parameter. Otherwise, the regularization setting here in optimizer will take effect.  \
            Default None, meaning there is no regularization.
1042 1043 1044 1045
        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of 
            some derived class of ``GradientClipBase`` . There are three cliping strategies 
            ( :ref:`api_fluid_clip_GradientClipByGlobalNorm` , :ref:`api_fluid_clip_GradientClipByNorm` , 
            :ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping.
1046 1047
        name (str, optional): This parameter is used by developers to print debugging information. \
            For details, please refer to :ref:`api_guide_Name`. Default is None.
Q
qiaolongfei 已提交
1048 1049 1050 1051

    Examples:
        .. code-block:: python

1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
            import paddle
            import paddle.fluid as fluid
            import numpy as np

            place = fluid.CPUPlace()
            main = fluid.Program()
            with fluid.program_guard(main):
                x = fluid.layers.data(name='x', shape=[13], dtype='float32')
                y = fluid.layers.data(name='y', shape=[1], dtype='float32')
                y_predict = fluid.layers.fc(input=x, size=1, act=None)
                cost = fluid.layers.square_error_cost(input=y_predict, label=y)
                avg_cost = fluid.layers.mean(cost)

                moment_optimizer = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, momentum=0.9)
                moment_optimizer.minimize(avg_cost)

                fetch_list = [avg_cost]
                train_reader = paddle.batch(
                    paddle.dataset.uci_housing.train(), batch_size=1)
                feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
                exe = fluid.Executor(place)
                exe.run(fluid.default_startup_program())
                for data in train_reader():
                    exe.run(main, feed=feeder.feed(data), fetch_list=fetch_list)

1077 1078 1079
    """
    _velocity_acc_str = "velocity"

X
Xin Pan 已提交
1080 1081 1082
    def __init__(self,
                 learning_rate,
                 momentum,
1083
                 parameter_list=None,
X
Xin Pan 已提交
1084 1085
                 use_nesterov=False,
                 regularization=None,
1086
                 grad_clip=None,
X
Xin Pan 已提交
1087
                 name=None):
1088 1089
        assert learning_rate is not None
        assert momentum is not None
Q
Qiao Longfei 已提交
1090
        super(MomentumOptimizer, self).__init__(
X
Xin Pan 已提交
1091
            learning_rate=learning_rate,
1092
            parameter_list=parameter_list,
X
Xin Pan 已提交
1093
            regularization=regularization,
1094
            grad_clip=grad_clip,
X
Xin Pan 已提交
1095
            name=name)
1096 1097
        self.type = "momentum"
        self._momentum = momentum
1098
        self._use_nesterov = bool(use_nesterov)
1099 1100 1101 1102 1103

    def _create_accumulators(self, block, parameters):
        assert isinstance(block, framework.Block)

        for p in parameters:
Q
Qiao Longfei 已提交
1104
            self._add_accumulator(self._velocity_acc_str, p)
1105 1106 1107 1108 1109 1110

    def _append_optimize_op(self, block, param_and_grad):
        assert isinstance(block, framework.Block)

        velocity_acc = self._get_accumulator(self._velocity_acc_str,
                                             param_and_grad[0])
1111 1112 1113 1114 1115 1116 1117 1118
        lr = self._create_param_lr(param_and_grad)

        if framework.in_dygraph_mode():
            _, _ = core.ops.momentum(param_and_grad[0], param_and_grad[1],
                                     velocity_acc, lr, param_and_grad[0],
                                     velocity_acc, 'mu', self._momentum,
                                     'use_nesterov', self._use_nesterov)
            return None
1119

1120
        attrs = {"mu": self._momentum, "use_nesterov": self._use_nesterov}
1121 1122 1123 1124
        inputs = {
            "Param": [param_and_grad[0]],
            "Grad": [param_and_grad[1]],
            "Velocity": [velocity_acc],
1125
            "LearningRate": [lr]
1126 1127 1128 1129 1130 1131
        }

        outputs = {
            "ParamOut": [param_and_grad[0]],
            "VelocityOut": [velocity_acc]
        }
1132 1133 1134
        # create the momentum optimize op
        momentum_op = block.append_op(
            type=self.type,
1135 1136 1137
            inputs=inputs,
            outputs=outputs,
            attrs=attrs,
M
minqiyang 已提交
1138
            stop_gradient=True)
1139 1140

        return momentum_op
1141 1142


1143
class DGCMomentumOptimizer(Optimizer):
1144
    """
1145
	:api_attr: Static Graph
S
swtkiwi 已提交
1146

1147
    DGC (Deep Gradient Compression) Momentum Optimizer. Original paper is https://arxiv.org/abs/1712.01887
1148

G
gongweibao 已提交
1149
    DGC reduces the communication bandwidth by sending only the important gradients (sparse update):\
1150 1151
        only gradients larger than a threshold are transmitted.

G
gongweibao 已提交
1152
    To avoid losing information, DGC accumulates the rest of the gradients locally.
1153 1154 1155

    Eventually, these gradients become large enough to be transmitted.

1156
    Thus, DGC sends the large gradients immediately but eventually sends all of the gradients over time.
1157

G
gongweibao 已提交
1158
    To ensure no loss of accuracy, DGC employs momentum correction and local gradient clipping on top of the gradient sparsification to maintain model performance.
1159 1160 1161 1162

    DGC also uses momentum factor masking and warmup training to overcome the staleness problem caused by reduced communication.

    This optimizer will do two things:
1163

1164 1165
        1. Compress the gradient by get TopK import value from tensor \
            and use it for allreduce to reduce network bandwidth.
1166

1167
        2. Call momentum to optimize the cost.
1168 1169

    Args:
1170 1171
        learning_rate (float|Variable): The learning rate used to update parameters. \
            It can be a float value or a Variable with one float value as a data element.
1172
        momentum (float): Momentum factor.
G
gongweibao 已提交
1173
        rampup_begin_step (int): The beginning step from which gradient compression is implemented.
1174 1175 1176 1177 1178 1179 1180
        rampup_step (int): Time steps used in sparsity warm-up periods. Default is 1.
            For example, if the sparsity is [0.75, 0.9375, 0.984375, 0.996, 0.999], and the rampup_step is 100, \
                it will use 0.75 at 0~19 steps, and 0.9375 at 20~39 steps, and so on. \
                And when reach sparsity array ends, it will use 0.999 then and after.
        sparsity (list[float]): Get top important element from gradient tensor, the ratio is (1 - current sparsity). \
            Default is [0.999]. For example, if the sparsity is [0.99, 0.999], \
                the top [1%, 0.1%] important element will be transmitted.
H
hong 已提交
1181
        parameter_list (Iterable, optional):  Iterable of ``Variable`` names to update to minimize ``loss``. \
1182 1183
            This parameter is required in dygraph mode. \
            The default value is None in static mode, at this time all parameters will be updated.
1184
        use_nesterov (bool): Enables Nesterov momentum. True means use Nesterov. Default is False.
1185 1186 1187 1188 1189
        regularization (WeightDecayRegularizer, optional): The strategy of regularization. There are two method: \
             :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If a parameter has set \
            regularizer using :ref:`api_fluid_ParamAttr` already, the regularization setting here in optimizer will be \
            ignored for this parameter. Otherwise, the regularization setting here in optimizer will take effect.  \
            Default None, meaning there is no regularization.
1190 1191 1192
        grad_clip (GradientClipByNorm, optional): Gradient cliping strategy. ``DGCMomentumOptimizer`` only support 
            :ref:`api_fluid_clip_GradientClipByNorm` , and if not, it will raise TypeError. Default None, 
            meaning there is no gradient clipping.
1193 1194
        name (str, optional): This parameter is used by developers to print debugging information. \
            For details, please refer to :ref:`api_guide_Name`. Default is None.
1195 1196 1197 1198

    Examples:
        .. code-block:: python

1199
            import paddle.fluid as fluid
1200
            optimizer = fluid.optimizer.DGCMomentumOptimizer(
G
gongweibao 已提交
1201 1202 1203 1204 1205
                        learning_rate=0.0001,
                        momentum=0.9,
                        rampup_step=1000,
                        rampup_begin_step=1252,
                        sparsity=[0.999, 0.999])
1206 1207

    """
1208 1209
    _u_velocity_acc_str = "_dgc_u_"
    _v_velocity_acc_str = "_dgc_v_"
1210 1211 1212 1213 1214 1215 1216

    def __init__(self,
                 learning_rate,
                 momentum,
                 rampup_begin_step,
                 rampup_step=1,
                 sparsity=[0.999],
1217
                 parameter_list=None,
1218 1219 1220
                 use_nesterov=False,
                 num_trainers=None,
                 regularization=None,
1221
                 grad_clip=None,
1222
                 name=None):
Z
zhongpu 已提交
1223 1224
        if framework.in_dygraph_mode():
            raise Exception("In dygraph, don't support DGCMomentumOptimizer.")
1225 1226 1227 1228

        assert core.is_compiled_with_cuda(), \
            "Paddle is not compiled with CUDA. DGC is only support GPU for now."

1229 1230 1231 1232
        assert learning_rate is not None
        assert momentum is not None
        super(DGCMomentumOptimizer, self).__init__(
            learning_rate=learning_rate,
1233
            parameter_list=parameter_list,
1234
            regularization=regularization,
1235
            grad_clip=grad_clip,
1236 1237 1238 1239
            name=name)
        self.type = "dgc_momentum"
        self._momentum = momentum
        self._use_nesterov = bool(use_nesterov)
1240

1241
        assert rampup_begin_step >= 0, "rampup_begin_step must >= 0"
1242
        self._rampup_begin_step = rampup_begin_step
1243 1244
        self._rampup_step = rampup_step
        self._sparsity = sparsity
1245

1246
        self._rampup_begin_step_var = None
1247
        self._global_step_var = None
1248

1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259
        self._dgc_clip_norm = None
        if grad_clip is not None:
            if not isinstance(grad_clip, GradientClipByNorm):
                raise TypeError(
                    "The type of grad_clip should be 'GradientClipByNorm', because DGCMomentumOptimizer only support GradientClipByNorm"
                )
            assert isinstance(
                num_trainers, int
            ), "The type of num_trainers should be 'int', but received %s" % type(
                value)
            assert num_trainers > 0, "The value of num_trainers should be greater than 0!"
1260 1261

            self._num_trainers = num_trainers
1262
            self._dgc_clip_norm = grad_clip.clip_norm * (num_trainers**-0.5)
1263

1264 1265
        self.regular_type, self.regular_coeff = self._get_regularization_param(
            self.regularization)
1266

1267 1268 1269
    def _get_regularization_param(self, regularization):
        regular_type = 0
        regular_coeff = 0.0
1270

1271 1272
        if regularization is not None:
            regular_coeff = regularization._regularization_coeff
1273
            from .regularizer import L1Decay, L2Decay
1274 1275 1276 1277
            if isinstance(regularization, L1Decay):
                regular_type = 1
            elif isinstance(regularization, L2Decay):
                regular_type = 2
1278 1279
            else:
                assert False, 'regularization must be None|L1Decay|L2Deacy'
1280
        return regular_type, regular_coeff
1281

1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
    def _is_use_dgc(self, param_var, grad_var):
        var_numel = abs(reduce(lambda x, y: x * y, param_var.shape))
        if var_numel < 16384 or \
           param_var.type == core.VarDesc.VarType.SELECTED_ROWS  or \
           grad_var.type == core.VarDesc.VarType.SELECTED_ROWS  or  \
               param_var.dtype != core.VarDesc.VarType.FP32 :
            return False
        return True

    def _append_optimize_op(self, block, param_and_grad):
        assert isinstance(block, framework.Block)
1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307
        velocity_acc = self._get_accumulator(self._u_velocity_acc_str,
                                             param_and_grad[0])
        assert velocity_acc is not None

        inputs = {
            "Param": param_and_grad[0],
            "Grad": param_and_grad[1],
            "Velocity": velocity_acc,
            "LearningRate": self._create_param_lr(param_and_grad),
        }
        outputs = {
            "ParamOut": param_and_grad[0],
            "VelocityOut": velocity_acc,
        }
        attrs = {"mu": self._momentum, "use_nesterov": self._use_nesterov}
1308 1309

        if not self._is_use_dgc(param_and_grad[0], param_and_grad[1]):
1310 1311 1312
            type = "momentum"
        else:
            type = "dgc_momentum"
1313 1314 1315 1316 1317
            inputs.update({
                "current_step": self._global_step_var,
                "nranks": self._nranks_var
            })
            outputs.update({'Grad_out': param_and_grad[1]})
1318
            attrs.update({"rampup_begin_step": float(self._rampup_begin_step)})
1319 1320 1321

        # create the dgc momentum optimize op
        dgc_momentum_op = block.append_op(
1322 1323 1324 1325
            type=type,
            inputs=inputs,
            outputs=outputs,
            attrs=attrs,
1326 1327 1328
            stop_gradient=True)
        return dgc_momentum_op

1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347
    def _add_auto_increment_var(self, counter_name, begin, step=1):
        helper = LayerHelper('global_step_counter')
        counter, is_new_var = helper.create_or_get_global_variable(
            name=counter_name, dtype='float32', shape=[1], persistable=True)
        if is_new_var:
            helper.set_variable_initializer(
                counter,
                initializer=Constant(
                    value=float(begin - 1), force_cpu=True))
            helper.main_program.global_block()._prepend_op(
                type='increment',
                inputs={'X': [counter]},
                outputs={'Out': [counter]},
                attrs={'step': float(step)},
                stop_gradient=True)
            counter.stop_gradient = True

        return counter

1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360
    def _add_nranks_var(self, name, value=-1):
        helper = LayerHelper('global_step_counter')
        counter, is_new_var = helper.create_or_get_global_variable(
            name=name, dtype='float32', shape=[1], persistable=True)
        if is_new_var:
            helper.set_variable_initializer(
                counter,
                initializer=Constant(
                    value=float(value), force_cpu=True))
            counter.stop_gradient = True

        return counter

1361 1362 1363 1364 1365 1366
    def _append_dgc_ops(self, param_and_grads):
        main_program = default_main_program()
        main_program._enable_dgc = True

        # step counter
        self._global_step_var = self._add_auto_increment_var(
G
gongweibao 已提交
1367
            counter_name=core.dgc.kDGCCounterName(), begin=0)
1368

1369 1370 1371
        self._nranks_var = self._add_nranks_var(
            name=core.dgc.kDGCNRanksName(), value=-1)

1372 1373 1374 1375 1376
        # rampup begin step var for all_reduce_op_handle
        self._rampup_begin_step_var = tensor.create_global_var(
            shape=[1],
            dtype=core.VarDesc.VarType.FP32,
            persistable=True,
G
gongweibao 已提交
1377
            name=core.dgc.kDGCRampUpBeginStepName(),
1378 1379 1380
            value=self._rampup_begin_step * 1.0,
            force_cpu=True)

1381 1382
        self.helper = LayerHelper(self.__class__.__name__)

1383
        for param_var, grad_var in param_and_grads:
1384 1385 1386
            # reuse velocity in dgc_op and dgc_momentum_op
            u_var = self._add_accumulator(self._u_velocity_acc_str, param_var)

1387
            if not self._is_use_dgc(param_var, grad_var):
1388 1389
                continue

1390
            v_var = self._add_accumulator(self._v_velocity_acc_str, param_var)
1391 1392 1393 1394 1395

            k_var = tensor.create_global_var(
                shape=[1],
                dtype=param_var.dtype,
                persistable=True,
G
gongweibao 已提交
1396
                name=param_var.name + core.dgc.kDGCKName(),
1397 1398 1399 1400 1401 1402 1403
                value=0.0,
                force_cpu=True)

            encoded_var = tensor.create_global_var(
                shape=[1],
                dtype=param_var.dtype,
                persistable=True,
G
gongweibao 已提交
1404
                name=param_var.name + core.dgc.kDGCEncodedName(),
1405 1406 1407
                value=0.0,
                force_cpu=False)

1408 1409 1410 1411 1412 1413 1414 1415
            gather_var = tensor.create_global_var(
                shape=[1],
                dtype=param_var.dtype,
                persistable=True,
                name=param_var.name + core.dgc.kDGCGatherName(),
                value=0.0,
                force_cpu=False)

1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434
            # del back oprolevarname
            op_maker = core.op_proto_and_checker_maker
            backward = core.op_proto_and_checker_maker.OpRole.Backward
            for op in main_program.global_block().ops:
                if not self._is_the_backward_op(op):
                    continue

                var_attr = op.all_attrs()[op_maker.kOpRoleVarAttrName()]
                if param_var.name not in var_attr:
                    continue

                var_attr.remove(param_var.name)
                var_attr.remove(grad_var.name)
                if len(var_attr) > 1:
                    op._set_attr(op_maker.kOpRoleVarAttrName(), var_attr)
                else:
                    op._remove_attr(op_maker.kOpRoleVarAttrName())

            clip_var = grad_var
1435 1436
            if self._dgc_clip_norm is not None:
                clip_var = self._append_clip_norm(grad_var, self._dgc_clip_norm)
1437
            self._dgc_op(param_var, clip_var, grad_var, u_var, v_var, k_var,
1438
                         encoded_var, gather_var)
1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453

    def _is_the_backward_op(self, op):
        op_maker = core.op_proto_and_checker_maker
        backward = core.op_proto_and_checker_maker.OpRole.Backward
        if op_maker.kOpRoleVarAttrName() in op.attr_names and \
                int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(backward):
            return True
        return False

    def _clip_by_norm(self, x, max_norm, name=None):
        args = {'x': x, 'max_norm': max_norm, 'name': name}

        helper = LayerHelper("dgc_clip_by_norm_op", **args)

        if name is None:
1454 1455
            name = unique_name.generate_with_ignorable_key(".".join(
                [helper.name, 'tmp']))
1456 1457 1458 1459 1460

        out = helper.create_variable(
            type=x.type, name=name, dtype=x.dtype, persistable=False)

        helper.append_op(
G
gongweibao 已提交
1461
            type="dgc_clip_by_norm",
1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473
            inputs={"X": x,
                    "current_step": self._global_step_var},
            attrs={
                "max_norm": max_norm,
                "rampup_begin_step": float(self._rampup_begin_step)
            },
            outputs={"Out": out})
        return out

    def _append_clip_norm(self, grad_var, clip_norm):
        with grad_var.block.program._backward_role_guard():
            return self._clip_by_norm(
G
gongweibao 已提交
1474
                x=grad_var, max_norm=clip_norm, name=grad_var.name)
1475 1476

    def _dgc_op(self, param_var, clip_var, grad_var, u_var, v_var, k_var,
1477
                encoded_var, gather_var):
1478 1479
        block = framework.default_main_program().global_block()
        op_maker = core.op_proto_and_checker_maker
1480

1481 1482 1483 1484 1485 1486 1487
        regular_type = self.regular_type
        regular_coeff = self.regular_coeff
        # The regularizer of the Parameters have higher priority
        if param_var.regularizer is not None:
            regular_type, regular_coeff = self._get_regularization_param(
                param_var.regularizer)

1488 1489 1490 1491 1492 1493
        dgc_op = block.append_op(
            type="dgc",
            inputs={
                "U": u_var,
                "V": v_var,
                "Grad": clip_var,
1494
                "Param": param_var,
1495 1496
                "current_step": self._global_step_var,
                "nranks": self._nranks_var,
1497 1498 1499 1500 1501 1502
            },
            outputs={
                "U_out": u_var,
                "V_out": v_var,
                "EncodeGrad": encoded_var,
                "k": k_var,
1503 1504
                "Grad_out": grad_var,
                "GatherBuff": gather_var,
1505 1506 1507 1508 1509 1510
            },
            attrs={
                "m": self._momentum,
                "sparsity": self._sparsity,
                "use_nesterov": self._use_nesterov,
                "rampup_begin_step": float(self._rampup_begin_step),
1511
                "rampup_step": float(self._rampup_step),
1512 1513
                "regular_coeff": float(regular_coeff),
                "regular_type": int(regular_type),
1514 1515 1516 1517 1518 1519 1520 1521
            },
            stop_gradient=True)

        backward = op_maker.OpRole.Backward
        dgc_op._set_attr(op_maker.kOpRoleAttrName(), backward)
        dgc_op._set_attr(op_maker.kOpRoleVarAttrName(),
                         [param_var.name, grad_var.name])

1522
    @imperative_base.no_grad
1523 1524 1525 1526 1527 1528 1529
    def apply_gradients(self, params_grads):
        params_grads = sorted(params_grads, key=lambda x: x[0].name)
        params_grads, table_param_and_grad, table_optimize_op = \
            self._process_distribute_lookuptable(params_grads)

        not_dgc_params_grads = []
        dgc_params_grads = []
1530
        # DGC clip and regularization in optimizer.backward
1531 1532 1533 1534 1535 1536
        for param, grad in params_grads:
            if not self._is_use_dgc(param, grad):
                not_dgc_params_grads.append((param, grad))
            else:
                dgc_params_grads.append((param, grad))

1537
        # 'optimizer(grad_clip)' or 'set_gradient_clip'
1538 1539 1540 1541 1542
        if self._grad_clip is not None:
            not_dgc_params_grads = self._grad_clip(not_dgc_params_grads)
        else:
            not_dgc_params_grads = append_gradient_clip_ops(
                not_dgc_params_grads)
1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556

        not_dgc_params_grads = append_regularization_ops(not_dgc_params_grads,
                                                         self.regularization)

        params_grads = not_dgc_params_grads + dgc_params_grads
        params_grads = sorted(params_grads, key=lambda x: x[0].name)

        optimize_ops = self._create_optimization_pass(params_grads)
        if table_optimize_op is not None:
            optimize_ops.append(table_optimize_op)
            params_grads.append(table_param_and_grad)

        return optimize_ops

1557

1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572
class LarsMomentumOptimizer(Optimizer):
    """
    Momentum optimizer with LARS support

    The update equations are as follows:

    .. math::

        & local\_learning\_rate = learning\_rate * lars\_coeff * \\
          \\frac{||param||}{||gradient|| + lars\_weight\_decay * ||param||}

        & velocity = mu * velocity + local\_learning\_rate * (gradient + lars\_weight\_decay * param)

        & param = param - velocity

1573 1574 1575 1576 1577 1578
    Parameters:
        learning_rate (float|Variable): The learning rate used to update parameters. \
            Can be a float value or a Variable with one float value as data element. \
            momentum (float): momentum factor
        lars_coeff (float): Defines how much we trust the layer to change its weights.
        lars_weight_decay (float): Weight decay coefficient for decaying using LARS.
H
hong 已提交
1579
        parameter_list (Iterable, optional):  Iterable of ``Variable`` names to update to minimize ``loss``. \
1580 1581
            This parameter is required in dygraph mode. \
            The default value is None in static mode, at this time all parameters will be updated.
1582 1583 1584 1585 1586
        regularization (WeightDecayRegularizer, optional): The strategy of regularization. There are two method: \
             :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If a parameter has set \
            regularizer using :ref:`api_fluid_ParamAttr` already, the regularization setting here in optimizer will be \
            ignored for this parameter. Otherwise, the regularization setting here in optimizer will take effect.  \
            Default None, meaning there is no regularization.
1587 1588 1589 1590
        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of 
            some derived class of ``GradientClipBase`` . There are three cliping strategies 
            ( :ref:`api_fluid_clip_GradientClipByGlobalNorm` , :ref:`api_fluid_clip_GradientClipByNorm` , 
            :ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping.
1591 1592
        name (str, optional): This parameter is used by developers to print debugging information. \
            For details, please refer to :ref:`api_guide_Name`. Default is None.
1593 1594 1595 1596

    Examples:
        .. code-block:: python

1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612
            import paddle.fluid as fluid
            import numpy as np

            np_inp = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)
            inp = fluid.layers.data(
                name="inp", shape=[2, 2], append_batch_size=False)
            out = fluid.layers.fc(inp, size=3)
            out = fluid.layers.reduce_sum(out)
            optimizer = fluid.optimizer.LarsMomentumOptimizer(learning_rate=0.001, momentum=0.9)
            optimizer.minimize(out)

            exe = fluid.Executor(fluid.CPUPlace())
            exe.run(fluid.default_startup_program())
            exe.run(
                feed={"inp": np_inp},
                fetch_list=[out.name])
1613 1614 1615 1616 1617 1618 1619 1620
    """
    _velocity_acc_str = "velocity"

    def __init__(self,
                 learning_rate,
                 momentum,
                 lars_coeff=0.001,
                 lars_weight_decay=0.0005,
1621
                 parameter_list=None,
1622
                 regularization=None,
1623
                 grad_clip=None,
1624 1625 1626 1627 1628
                 name=None):
        assert learning_rate is not None
        assert momentum is not None
        super(LarsMomentumOptimizer, self).__init__(
            learning_rate=learning_rate,
1629
            parameter_list=parameter_list,
1630
            regularization=regularization,
1631
            grad_clip=grad_clip,
1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665
            name=name)
        self.type = "lars_momentum"
        self._momentum = momentum
        self._lars_coeff = float(lars_coeff)
        self._lars_weight_decay = float(lars_weight_decay)

    def _create_accumulators(self, block, parameters):
        assert isinstance(block, framework.Block)

        for p in parameters:
            self._add_accumulator(self._velocity_acc_str, p)

    def _append_optimize_op(self, block, param_and_grad):
        assert isinstance(block, framework.Block)

        velocity_acc = self._get_accumulator(self._velocity_acc_str,
                                             param_and_grad[0])
        # create the momentum optimize op
        momentum_op = block.append_op(
            type=self.type,
            inputs={
                "Param": param_and_grad[0],
                "Grad": param_and_grad[1],
                "Velocity": velocity_acc,
                "LearningRate": self._create_param_lr(param_and_grad)
            },
            outputs={
                "ParamOut": param_and_grad[0],
                "VelocityOut": velocity_acc
            },
            attrs={
                "mu": self._momentum,
                "lars_coeff": self._lars_coeff,
                "lars_weight_decay": self._lars_weight_decay
M
minqiyang 已提交
1666 1667
            },
            stop_gradient=True)
1668 1669 1670 1671

        return momentum_op


1672
class AdagradOptimizer(Optimizer):
Q
qiaolongfei 已提交
1673
    """
1674 1675
    The Adaptive Gradient optimizer (Adagrad for short) can adaptively assign
    different learning rates to individual parameters.
Q
qiaolongfei 已提交
1676

1677
    The parameter ``param_out`` update rule with gradient ``grad``:
Q
qiaolongfei 已提交
1678 1679 1680 1681 1682 1683 1684

    .. math::

        moment\_out &= moment + grad * grad

        param\_out &= param - \\frac{learning\_rate * grad}{\sqrt{moment\_out} + \epsilon}

1685 1686 1687 1688 1689 1690
    Related paper: `Adaptive Subgradient Methods for Online Learning and
    Stochastic Optimization <http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf>`_.

    The original paper does not have the ``epsilon`` attribute. It is added here
    in our implementation as also proposed `Per-parameter adaptive learning rate
    methods <http://cs231n.github.io/neural-networks-3/#ada>`_
Q
qiaolongfei 已提交
1691 1692 1693
    for numerical stability to avoid the division by zero error.

    Args:
1694 1695 1696 1697
        learning_rate (float|Variable): The learning rate used to update ``Parameter``.
            It can be a float value or a ``Variable`` with a float type.
        epsilon (float, optional): A small float value for numerical stability.
            The default value is 1e-06.
H
hong 已提交
1698
        parameter_list (Iterable, optional):  Iterable of ``Variable`` names to update to minimize ``loss``. \
1699 1700
            This parameter is required in dygraph mode. \
            The default value is None in static mode, at this time all parameters will be updated.
1701 1702 1703 1704 1705
        regularization (WeightDecayRegularizer, optional): The strategy of regularization. There are two method: \
             :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If a parameter has set \
            regularizer using :ref:`api_fluid_ParamAttr` already, the regularization setting here in optimizer will be \
            ignored for this parameter. Otherwise, the regularization setting here in optimizer will take effect.  \
            Default None, meaning there is no regularization.
1706 1707 1708 1709
        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of 
            some derived class of ``GradientClipBase`` . There are three cliping strategies 
            ( :ref:`api_fluid_clip_GradientClipByGlobalNorm` , :ref:`api_fluid_clip_GradientClipByNorm` , 
            :ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping.
1710 1711 1712 1713 1714
        name (str, optional): Normally there is no need for user to set this property.
            For more information, please refer to :ref:`api_guide_Name`.
            The default value is None.
        initial_accumulator_value (float, optional): Initial value for moment accumulator.
            The default value is 0.0.
Q
qiaolongfei 已提交
1715 1716 1717 1718

    Examples:
        .. code-block:: python

1719
            import numpy as np
1720
            import paddle.fluid as fluid
1721 1722

            np_inp = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)
1723
            inp = fluid.data(name="inp", shape=[2, 2])
1724 1725
            out = fluid.layers.fc(inp, size=3)
            out = fluid.layers.reduce_sum(out)
1726
            optimizer = fluid.optimizer.AdagradOptimizer(learning_rate=0.2)
1727 1728 1729 1730 1731 1732 1733
            optimizer.minimize(out)

            exe = fluid.Executor(fluid.CPUPlace())
            exe.run(fluid.default_startup_program())
            exe.run(
                feed={"inp": np_inp},
                fetch_list=[out.name])
1734 1735 1736
    """
    _moment_acc_str = "moment"

X
Xin Pan 已提交
1737 1738 1739
    def __init__(self,
                 learning_rate,
                 epsilon=1.0e-6,
1740
                 parameter_list=None,
X
Xin Pan 已提交
1741
                 regularization=None,
1742
                 grad_clip=None,
1743
                 name=None,
X
xuezhong 已提交
1744
                 initial_accumulator_value=0.0):
1745 1746
        assert learning_rate is not None
        assert epsilon is not None
Q
Qiao Longfei 已提交
1747
        super(AdagradOptimizer, self).__init__(
X
Xin Pan 已提交
1748
            learning_rate=learning_rate,
1749
            parameter_list=parameter_list,
X
Xin Pan 已提交
1750
            regularization=regularization,
1751
            grad_clip=grad_clip,
X
Xin Pan 已提交
1752
            name=name)
1753 1754
        self.type = "adagrad"
        self._epsilon = epsilon
1755
        self.initial_accumulator_value = initial_accumulator_value
1756 1757 1758 1759 1760

    def _create_accumulators(self, block, parameters):
        assert isinstance(block, framework.Block)

        for p in parameters:
Z
zhongpu 已提交
1761 1762 1763 1764
            self._add_accumulator(
                self._moment_acc_str,
                p,
                fill_value=self.initial_accumulator_value)
1765 1766 1767 1768 1769 1770

    def _append_optimize_op(self, block, param_and_grad):
        assert isinstance(block, framework.Block)

        moment_acc = self._get_accumulator(self._moment_acc_str,
                                           param_and_grad[0])
1771
        # Create the adagrad optimizer op
1772 1773 1774 1775 1776 1777
        adagrad_op = block.append_op(
            type=self.type,
            inputs={
                "Param": param_and_grad[0],
                "Grad": param_and_grad[1],
                "Moment": moment_acc,
1778
                "LearningRate": self._create_param_lr(param_and_grad)
1779 1780 1781
            },
            outputs={"ParamOut": param_and_grad[0],
                     "MomentOut": moment_acc},
M
minqiyang 已提交
1782 1783
            attrs={"epsilon": self._epsilon},
            stop_gradient=True)
1784 1785

        return adagrad_op
1786 1787 1788


class AdamOptimizer(Optimizer):
Q
qiaolongfei 已提交
1789
    """
T
tianshuo78520a 已提交
1790
    The Adam optimizer uses an optimization described at the end
1791 1792 1793 1794 1795
    of section 2 of `Adam paper <https://arxiv.org/abs/1412.6980>`_ ,
    it can dynamically adjusts the learning rate of each parameter using
    the 1st moment estimates and the 2nd moment estimates of the gradient.
    
    The parameter ``param_out`` update rule with gradient ``grad``:
Q
qiaolongfei 已提交
1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809

    .. math::

        t & = t + 1

        moment\_1\_out & = {\\beta}_1 * moment\_1 + (1 - {\\beta}_1) * grad

        moment\_2\_out & = {\\beta}_2 * moment\_2 + (1 - {\\beta}_2) * grad * grad

        learning\_rate & = learning\_rate * \\
                          \\frac{\sqrt{1 - {\\beta}_2^t}}{1 - {\\beta}_1^t}

        param\_out & = param - learning\_rate * \\frac{moment\_1}{\sqrt{moment\_2} + \epsilon}

1810 1811
    Related paper: `Adam: A Method for Stochastic Optimization <https://arxiv.org/abs/1412.6980>`_

Q
qiaolongfei 已提交
1812
    Args:
1813 1814
        learning_rate (float|Variable, optional): The learning rate used to update ``Parameter``.
            It can be a float value or a ``Variable`` with a float type. The default value is 0.001.
1815 1816
        beta1 (float|Variable, optional): The exponential decay rate for the 1st moment estimates.
            It should be a float number or a Variable with shape [1] and data type as float32.
1817
            The default value is 0.9.
1818 1819
        beta2 (float|Variable, optional): The exponential decay rate for the 2nd moment estimates.
            It should be a float number or a Variable with shape [1] and data type as float32.
1820 1821 1822
            The default value is 0.999.
        epsilon (float, optional): A small float value for numerical stability.
            The default value is 1e-08.
H
hong 已提交
1823
        parameter_list (Iterable, optional):  Iterable of ``Variable`` names to update to minimize ``loss``. \
1824 1825
            This parameter is required in dygraph mode. \
            The default value is None in static mode, at this time all parameters will be updated.
1826 1827 1828 1829 1830
        regularization (WeightDecayRegularizer, optional): The strategy of regularization. There are two method: \
             :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If a parameter has set \
            regularizer using :ref:`api_fluid_ParamAttr` already, the regularization setting here in optimizer will be \
            ignored for this parameter. Otherwise, the regularization setting here in optimizer will take effect.  \
            Default None, meaning there is no regularization.
1831 1832 1833 1834
        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of 
            some derived class of ``GradientClipBase`` . There are three cliping strategies 
            ( :ref:`api_fluid_clip_GradientClipByGlobalNorm` , :ref:`api_fluid_clip_GradientClipByNorm` , 
            :ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping.
1835 1836 1837 1838 1839 1840 1841 1842 1843 1844
        name (str, optional): Normally there is no need for user to set this property.
            For more information, please refer to :ref:`api_guide_Name`.
            The default value is None.
        lazy_mode (bool, optional): The official Adam algorithm has two moving-average accumulators.
            The accumulators are updated at every step. Every element of the two moving-average
            is updated in both dense mode and sparse mode. If the size of parameter is very large,
            then the update may be very slow. The lazy mode only update the element that has
            gradient in current mini-batch, so it will be much more faster. But this mode has
            different semantics with the original Adam algorithm and may lead to different result.
            The default value is False.
Q
qiaolongfei 已提交
1845 1846 1847 1848

    Examples:
        .. code-block:: python

1849 1850 1851 1852 1853 1854
            import paddle
            import paddle.fluid as fluid

            place = fluid.CPUPlace()
            main = fluid.Program()
            with fluid.program_guard(main):
1855 1856
                x = fluid.data(name='x', shape=[None, 13], dtype='float32')
                y = fluid.data(name='y', shape=[None, 1], dtype='float32')
1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871
                y_predict = fluid.layers.fc(input=x, size=1, act=None)
                cost = fluid.layers.square_error_cost(input=y_predict, label=y)
                avg_cost = fluid.layers.mean(cost)

                adam_optimizer = fluid.optimizer.AdamOptimizer(0.01)
                adam_optimizer.minimize(avg_cost)

                fetch_list = [avg_cost]
                train_reader = paddle.batch(
                    paddle.dataset.uci_housing.train(), batch_size=1)
                feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
                exe = fluid.Executor(place)
                exe.run(fluid.default_startup_program())
                for data in train_reader():
                    exe.run(main, feed=feeder.feed(data), fetch_list=fetch_list)
Q
qiaolongfei 已提交
1872

1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889
        .. code-block:: python

            # Adam with beta1/beta2 as Variable
            import paddle
            import paddle.fluid as fluid
            import paddle.fluid.layers.learning_rate_scheduler as lr_scheduler

            place = fluid.CPUPlace()
            main = fluid.Program()
            with fluid.program_guard(main):
                x = fluid.data(name='x', shape=[None, 13], dtype='float32')
                y = fluid.data(name='y', shape=[None, 1], dtype='float32')
                y_predict = fluid.layers.fc(input=x, size=1, act=None)
                cost = fluid.layers.square_error_cost(input=y_predict, label=y)
                avg_cost = fluid.layers.mean(cost)

                # define beta decay variable
1890
                def get_decayed_betas(beta1_init, beta2_init, decay_steps, decay_rate):
1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918
                    global_step = lr_scheduler._decay_step_counter()

                    beta1 = fluid.layers.create_global_var(
                        shape=[1],
                        value=float(beta1_init),
                        dtype='float32',
                        # set persistable for save checkpoints and resume
                        persistable=True,
                        name="beta1")
                    beta2 = fluid.layers.create_global_var(
                        shape=[1],
                        value=float(beta2_init),
                        dtype='float32',
                        # set persistable for save checkpoints and resume
                        persistable=True,
                        name="beta2")

                    div_res = global_step / decay_steps
                    decayed_beta1 = beta1_init * (decay_rate**div_res)
                    decayed_beta2 = beta2_init * (decay_rate**div_res)
                    fluid.layers.assign(decayed_beta1, beta1)
                    fluid.layers.assign(decayed_beta2, beta2)

                    return beta1, beta2

                beta1, beta2 = get_decayed_betas(0.9, 0.99, 1e5, 0.9)
                adam_optimizer = fluid.optimizer.AdamOptimizer(
                                                    learning_rate=0.01,
1919
                                                    beta1=beta1,
1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930
                                                    beta2=beta2)
                adam_optimizer.minimize(avg_cost)

                fetch_list = [avg_cost]
                train_reader = paddle.batch(
                    paddle.dataset.uci_housing.train(), batch_size=1)
                feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
                exe = fluid.Executor(place)
                exe.run(fluid.default_startup_program())
                for data in train_reader():
                    exe.run(main, feed=feeder.feed(data), fetch_list=fetch_list)
1931 1932 1933
    """
    _moment1_acc_str = "moment1"
    _moment2_acc_str = "moment2"
Q
qiaolongfei 已提交
1934 1935
    _beta1_pow_acc_str = "beta1_pow_acc"
    _beta2_pow_acc_str = "beta2_pow_acc"
1936 1937 1938 1939 1940

    def __init__(self,
                 learning_rate=0.001,
                 beta1=0.9,
                 beta2=0.999,
1941
                 epsilon=1e-8,
1942
                 parameter_list=None,
X
Xin Pan 已提交
1943
                 regularization=None,
1944
                 grad_clip=None,
Q
Qiao Longfei 已提交
1945
                 name=None,
Q
Qiao Longfei 已提交
1946
                 lazy_mode=False):
1947 1948 1949 1950
        assert learning_rate is not None
        assert beta1 is not None
        assert beta2 is not None
        assert epsilon is not None
Q
Qiao Longfei 已提交
1951
        super(AdamOptimizer, self).__init__(
X
Xin Pan 已提交
1952
            learning_rate=learning_rate,
1953
            parameter_list=parameter_list,
X
Xin Pan 已提交
1954
            regularization=regularization,
1955
            grad_clip=grad_clip,
X
Xin Pan 已提交
1956
            name=name)
1957 1958 1959 1960
        self.type = "adam"
        self._beta1 = beta1
        self._beta2 = beta2
        self._epsilon = epsilon
Q
Qiao Longfei 已提交
1961
        self._lazy_mode = lazy_mode
1962 1963 1964 1965 1966 1967

    def _create_accumulators(self, block, parameters):
        assert isinstance(block, framework.Block)

        # Create accumulator tensors for first and second moments
        for p in parameters:
Q
Qiao Longfei 已提交
1968 1969
            self._add_accumulator(self._moment1_acc_str, p)
            self._add_accumulator(self._moment2_acc_str, p)
Q
qiaolongfei 已提交
1970 1971 1972
            self._add_accumulator(
                name=self._beta1_pow_acc_str,
                param=p,
1973 1974
                fill_value=0.9 if isinstance(self._beta1, Variable) \
                        else self._beta1,
1975
                shape=[1],
1976
                type=core.VarDesc.VarType.LOD_TENSOR, device='cpu')
Q
qiaolongfei 已提交
1977 1978 1979
            self._add_accumulator(
                name=self._beta2_pow_acc_str,
                param=p,
1980 1981
                fill_value=0.999 if isinstance(self._beta2, Variable) \
                        else self._beta2,
1982
                shape=[1],
1983
                type=core.VarDesc.VarType.LOD_TENSOR, device='cpu')
1984 1985 1986 1987 1988 1989 1990 1991

    def _append_optimize_op(self, block, param_and_grad):
        assert isinstance(block, framework.Block)

        moment1 = self._get_accumulator(self._moment1_acc_str,
                                        param_and_grad[0])
        moment2 = self._get_accumulator(self._moment2_acc_str,
                                        param_and_grad[0])
Q
qiaolongfei 已提交
1992 1993 1994 1995
        beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str,
                                              param_and_grad[0])
        beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str,
                                              param_and_grad[0])
1996
        lr = self._create_param_lr(param_and_grad)
1997
        # create the adam optimize op
1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012

        if framework.in_dygraph_mode():
            _beta1 = self._beta1 if not isinstance(
                self._beta1, Variable) else self._beta1.numpy().item(0)
            _beta2 = self._beta2 if not isinstance(
                self._beta2, Variable) else self._beta2.numpy().item(0)
            _, _, _, _, _ = core.ops.adam(
                param_and_grad[0], param_and_grad[1], lr, moment1, moment2,
                beta1_pow_acc, beta2_pow_acc, param_and_grad[0], moment1,
                moment2, beta1_pow_acc, beta2_pow_acc, 'epsilon', self._epsilon,
                'lazy_mode', self._lazy_mode, 'min_row_size_to_use_multithread',
                1000, 'beta1', _beta1, 'beta2', _beta2)

            return None

2013
        inputs = {
2014 2015
            "Param": [param_and_grad[0]],
            "Grad": [param_and_grad[1]],
2016
            "LearningRate": [lr],
2017 2018 2019 2020
            "Moment1": [moment1],
            "Moment2": [moment2],
            "Beta1Pow": [beta1_pow_acc],
            "Beta2Pow": [beta2_pow_acc]
2021 2022
        }
        outputs = {
2023 2024 2025 2026 2027
            "ParamOut": [param_and_grad[0]],
            "Moment1Out": [moment1],
            "Moment2Out": [moment2],
            "Beta1PowOut": [beta1_pow_acc],
            "Beta2PowOut": [beta2_pow_acc],
2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043
        }
        attrs = {
            "epsilon": self._epsilon,
            "lazy_mode": self._lazy_mode,
            "min_row_size_to_use_multithread": 1000
        }

        if isinstance(self._beta1, Variable):
            inputs['Beta1Tensor'] = self._beta1
        else:
            attrs['beta1'] = self._beta1
        if isinstance(self._beta2, Variable):
            inputs['Beta2Tensor'] = self._beta2
        else:
            attrs['beta2'] = self._beta2

2044 2045
        adam_op = block.append_op(
            type=self.type,
2046 2047 2048
            inputs=inputs,
            outputs=outputs,
            attrs=attrs,
M
minqiyang 已提交
2049
            stop_gradient=True)
2050 2051 2052

        return adam_op

2053 2054

class AdamaxOptimizer(Optimizer):
Q
qiaolongfei 已提交
2055
    """
2056 2057 2058 2059
    The Adamax optimizer is implemented based on the Adamax Optimization 
    in Section 7 of `Adam paper <https://arxiv.org/abs/1412.6980>`_.
    The Adamax algorithm is a variant of the Adam algorithm based on the infinite norm,
    which makes the learning rate update algorithm more stable and simple.
Q
qiaolongfei 已提交
2060

2061
    The parameter ``param_out`` update rule with gradient ``grad``:
Q
qiaolongfei 已提交
2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074

    .. math::

        t & = t + 1

        moment\_out & = {\\beta}_1 * moment + (1 - {\\beta}_1) * grad

        inf\_norm\_out & = max({\\beta}_2 * inf\_norm + \epsilon, |grad|)

        learning\_rate & = \\frac{learning\_rate}{1 - {\\beta}_1^t}

        param\_out & = param - learning\_rate * \\frac{moment\_out}{inf\_norm\_out}

2075
    Related paper: `Adam: A Method for Stochastic Optimization <https://arxiv.org/abs/1412.6980>`_
Q
qiaolongfei 已提交
2076

2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088
    The original paper does not have an ``epsilon`` attribute,
    it is added here for numerical stability to prevent the division by 0 error.

    Args:
        learning_rate (float|Variable, optional): The learning rate used to update ``Parameter``.
            It can be a float value or a ``Variable`` with a float type. The default value is 0.001.
        beta1 (float, optional): The exponential decay rate for the 1st moment estimates.
            The default value is 0.9.
        beta2 (float, optional): The exponential decay rate for the 2nd moment estimates.
            The default value is 0.999.
        epsilon (float, optional): A small float value for numerical stability.
            The default value is 1e-08.
H
hong 已提交
2089
        parameter_list (Iterable, optional):  Iterable of ``Variable`` names to update to minimize ``loss``. \
2090 2091
            This parameter is required in dygraph mode. \
            The default value is None in static mode, at this time all parameters will be updated.
2092 2093 2094 2095 2096
        regularization (WeightDecayRegularizer, optional): The strategy of regularization. There are two method: \
             :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If a parameter has set \
            regularizer using :ref:`api_fluid_ParamAttr` already, the regularization setting here in optimizer will be \
            ignored for this parameter. Otherwise, the regularization setting here in optimizer will take effect.  \
            Default None, meaning there is no regularization.
2097 2098 2099 2100
        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of 
            some derived class of ``GradientClipBase`` . There are three cliping strategies 
            ( :ref:`api_fluid_clip_GradientClipByGlobalNorm` , :ref:`api_fluid_clip_GradientClipByNorm` , 
            :ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping.
2101 2102 2103 2104 2105 2106
        name (str, optional): Normally there is no need for user to set this property.
            For more information, please refer to :ref:`api_guide_Name`.
            The default value is None.

    **Notes**:
        **Currently, AdamaxOptimizer doesn't support sparse parameter optimization.**
Q
qiaolongfei 已提交
2107

2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120
    Examples:
        .. code-block:: python

          import paddle.fluid as fluid
          import numpy

          # First create the Executor.
          place = fluid.CPUPlace() # fluid.CUDAPlace(0)
          exe = fluid.Executor(place)

          train_program = fluid.Program()
          startup_program = fluid.Program()
          with fluid.program_guard(train_program, startup_program):
2121
              data = fluid.data(name='X', shape=[None, 1], dtype='float32')
2122 2123
              hidden = fluid.layers.fc(input=data, size=10)
              loss = fluid.layers.mean(hidden)
2124
              adam = fluid.optimizer.AdamaxOptimizer(learning_rate=0.2)
2125 2126 2127 2128 2129 2130 2131 2132 2133
              adam.minimize(loss)

          # Run the startup program once and only once.
          exe.run(startup_program)

          x = numpy.random.random(size=(10, 1)).astype('float32')
          outs = exe.run(program=train_program,
                        feed={'X': x},
                         fetch_list=[loss.name])
2134 2135 2136
    """
    _moment_acc_str = "moment"
    _inf_norm_acc_str = "inf_norm"
Q
qiaolongfei 已提交
2137
    _beta1_pow_acc_str = "beta1_pow_acc"
2138 2139 2140 2141 2142

    def __init__(self,
                 learning_rate=0.001,
                 beta1=0.9,
                 beta2=0.999,
2143
                 epsilon=1e-8,
2144
                 parameter_list=None,
X
Xin Pan 已提交
2145
                 regularization=None,
2146
                 grad_clip=None,
X
Xin Pan 已提交
2147
                 name=None):
2148 2149 2150 2151
        assert learning_rate is not None
        assert beta1 is not None
        assert beta2 is not None
        assert epsilon is not None
Q
Qiao Longfei 已提交
2152
        super(AdamaxOptimizer, self).__init__(
X
Xin Pan 已提交
2153
            learning_rate=learning_rate,
2154
            parameter_list=parameter_list,
X
Xin Pan 已提交
2155
            regularization=regularization,
2156
            grad_clip=grad_clip,
X
Xin Pan 已提交
2157
            name=name)
2158 2159 2160 2161 2162 2163 2164 2165
        self.type = "adamax"
        self._beta1 = beta1
        self._beta2 = beta2
        self._epsilon = epsilon

    def _create_accumulators(self, block, parameters):
        # Create accumulator tensors for first moment and infinity norm
        for p in parameters:
Q
Qiao Longfei 已提交
2166 2167
            self._add_accumulator(self._moment_acc_str, p)
            self._add_accumulator(self._inf_norm_acc_str, p)
Q
qiaolongfei 已提交
2168 2169 2170 2171 2172
            self._add_accumulator(
                name=self._beta1_pow_acc_str,
                param=p,
                fill_value=self._beta1,
                shape=[1])
2173 2174 2175 2176 2177 2178 2179

    def _append_optimize_op(self, block, param_and_grad):
        assert isinstance(block, framework.Block)

        moment = self._get_accumulator(self._moment_acc_str, param_and_grad[0])
        inf_norm = self._get_accumulator(self._inf_norm_acc_str,
                                         param_and_grad[0])
Q
qiaolongfei 已提交
2180 2181
        beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str,
                                              param_and_grad[0])
2182 2183 2184 2185 2186 2187
        # create the adamax optimize op
        adamax_op = block.append_op(
            type=self.type,
            inputs={
                "Param": param_and_grad[0],
                "Grad": param_and_grad[1],
2188
                "LearningRate": self._create_param_lr(param_and_grad),
2189 2190
                "Moment": moment,
                "InfNorm": inf_norm,
Q
qiaolongfei 已提交
2191
                "Beta1Pow": beta1_pow_acc
2192 2193 2194 2195 2196 2197 2198 2199 2200 2201
            },
            outputs={
                "ParamOut": param_and_grad[0],
                "MomentOut": moment,
                "InfNormOut": inf_norm
            },
            attrs={
                "beta1": self._beta1,
                "beta2": self._beta2,
                "epsilon": self._epsilon
M
minqiyang 已提交
2202 2203
            },
            stop_gradient=True)
2204 2205 2206

        return adamax_op

2207
    def _finish_update(self, block, parameters_and_grads):
2208 2209 2210
        """Update Beta1 Power accumulator
        """
        assert isinstance(block, framework.Block)
2211
        for param, grad in parameters_and_grads:
C
chengduo 已提交
2212
            if grad is None or param.trainable is False:
2213
                continue
X
Xin Pan 已提交
2214 2215
            with param.block.program._optimized_guard(
                [param, grad]), name_scope('adamx'):
2216 2217
                beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str,
                                                      param)
2218
                block.append_op(
2219 2220 2221
                    type="scale",
                    inputs={"X": beta1_pow_acc},
                    outputs={"Out": beta1_pow_acc},
M
minqiyang 已提交
2222 2223
                    attrs={"scale": self._beta1},
                    stop_gradient=True)
2224 2225


2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263
class DpsgdOptimizer(Optimizer):
    """
    We implement the Dpsgd optimizer according to CCS16 paper -
    Deep Learning with Differential Privacy.

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid
          import numpy

          # First create the Executor.
          place = fluid.CPUPlace() # fluid.CUDAPlace(0)
          exe = fluid.Executor(place)

          train_program = fluid.Program()
          startup_program = fluid.Program()
          with fluid.program_guard(train_program, startup_program):
              data = fluid.layers.data(name='X', shape=[1], dtype='float32')
              hidden = fluid.layers.fc(input=data, size=10)
              loss = fluid.layers.mean(hidden)
              optimizer = fluid.optimizer.Dpsgd(learning_rate=0.01, clip=10.0, batch_size=16.0, sigma=1.0)
              optimizer.minimize(loss)

          # Run the startup program once and only once.
          exe.run(startup_program)

          x = numpy.random.random(size=(10, 1)).astype('float32')
          outs = exe.run(program=train_program,
                        feed={'X': x},
                         fetch_list=[loss.name])

    Args:
        learning_rate (float|Variable): the learning rate used to update parameters. \
        Can be a float value or a Variable with one float value as data element.
        clip (float): clipping threshold
        batch_size (float): batch size.
        sigma (float): for gaussian noise.
H
hong 已提交
2264
        parameter_list (Iterable, optional):  Iterable of ``Variable`` names to update to minimize ``loss``. \
2265 2266
            This parameter is required in dygraph mode. \
            The default value is None in static mode, at this time all parameters will be updated.
2267 2268 2269 2270 2271 2272 2273 2274
    Notes:
       Currently, DpsgdOptimizer doesn't support sparse parameter optimization.
    """

    def __init__(self,
                 learning_rate=0.001,
                 clip=0.9,
                 batch_size=0.999,
2275 2276
                 sigma=1e-8,
                 parameter_list=None):
2277 2278 2279 2280
        assert learning_rate is not None
        assert clip is not None
        assert batch_size is not None
        assert sigma is not None
2281 2282
        super(DpsgdOptimizer, self).__init__(
            learning_rate=learning_rate, parameter_list=parameter_list)
2283 2284 2285 2286
        self.type = "dpsgd"
        self._clip = clip
        self._batch_size = batch_size
        self._sigma = sigma
Z
zhongpu 已提交
2287 2288 2289 2290 2291 2292 2293
        '''
        Note(wangzhongpu):
        This property is only used for debugging, do not need to set it!
        Dpsgd operator use time(NULL) as random seed to generate random number.
        However, during debugging, we need determinated result, so we will set self._seed to a fixed number.
        '''
        self._seed = None
2294 2295 2296 2297 2298

    def _append_optimize_op(self, block, param_and_grad):
        assert isinstance(block, framework.Block)

        # create the dpsgd optimize op
Z
zhongpu 已提交
2299 2300 2301
        if self._seed == None:
            self._seed = 0

2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312
        dpsgd_op = block.append_op(
            type=self.type,
            inputs={
                "Param": param_and_grad[0],
                "Grad": param_and_grad[1],
                "LearningRate": self._create_param_lr(param_and_grad)
            },
            outputs={"ParamOut": param_and_grad[0]},
            attrs={
                "clip": self._clip,
                "batch_size": self._batch_size,
Z
zhongpu 已提交
2313 2314
                "sigma": self._sigma,
                "seed": self._seed
2315 2316 2317 2318 2319 2320
            },
            stop_gradient=True)

        return dpsgd_op


2321
class DecayedAdagradOptimizer(Optimizer):
2322
    """
2323 2324 2325
    The Decayed Adagrad optimizer can be seen as an Adagrad algorithm that introduces
    the decay rate to solve the problem of a sharp drop in the learning rate
    during model training when using the AdagradOptimizer.
2326

2327
    The parameter ``param_out`` update rule with gradient ``grad``:
2328 2329 2330 2331 2332 2333 2334

    .. math::

        moment\_out & = decay * moment + (1 - decay) * grad * grad

        param\_out & = param - \\frac{learning\_rate * grad}{\sqrt{moment\_out} + \epsilon}

2335 2336 2337 2338
    Related paper: `Adaptive Subgradient Methods for Online Learning and Stochastic
    Optimization <http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf>`_.

    The original paper does not have an ``epsilon`` attribute. It is added here for numerical
2339 2340 2341
    stability to avoid the division by zero error.

    Args:
2342 2343 2344 2345 2346
        learning_rate (float|Variable): The learning rate used to update ``Parameter``.
            It can be a float value or a ``Variable`` with a float type.
        decay (float, optional): The decay rate. The default value is 0.95.
        epsilon (float, optional): A small float value for numerical stability.
            The default value is 1e-06.
H
hong 已提交
2347
        parameter_list (Iterable, optional):  Iterable of ``Variable`` names to update to minimize ``loss``. \
2348 2349
            This parameter is required in dygraph mode. \
            The default value is None in static mode, at this time all parameters will be updated.
2350 2351 2352 2353 2354
        regularization (WeightDecayRegularizer, optional): The strategy of regularization. There are two method: \
             :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If a parameter has set \
            regularizer using :ref:`api_fluid_ParamAttr` already, the regularization setting here in optimizer will be \
            ignored for this parameter. Otherwise, the regularization setting here in optimizer will take effect.  \
            Default None, meaning there is no regularization.
2355 2356 2357 2358
        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of 
            some derived class of ``GradientClipBase`` . There are three cliping strategies 
            ( :ref:`api_fluid_clip_GradientClipByGlobalNorm` , :ref:`api_fluid_clip_GradientClipByNorm` , 
            :ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping.
2359 2360 2361 2362 2363 2364
        name (str, optional): Normally there is no need for user to set this property.
            For more information, please refer to :ref:`api_guide_Name`.
            The default value is None.

    **Notes**:
        **Currently, DecayedAdagradOptimizer doesn't support sparse parameter optimization.**
2365 2366 2367 2368

    Examples:
        .. code-block:: python

2369 2370
            import paddle.fluid as fluid

2371 2372 2373 2374
            x = fluid.data( name='x', shape=[None, 10], dtype='float32' )
            trans = fluid.layers.fc( x, 100 )
            cost = fluid.layers.reduce_mean( trans )
            optimizer = fluid.optimizer.DecayedAdagradOptimizer(learning_rate=0.2)
2375
            optimizer.minimize(cost)
2376 2377 2378
    """
    _moment_acc_str = "moment"

X
Xin Pan 已提交
2379 2380 2381 2382
    def __init__(self,
                 learning_rate,
                 decay=0.95,
                 epsilon=1.0e-6,
2383
                 parameter_list=None,
X
Xin Pan 已提交
2384
                 regularization=None,
2385
                 grad_clip=None,
X
Xin Pan 已提交
2386
                 name=None):
2387 2388 2389 2390
        assert learning_rate is not None
        assert decay is not None
        assert epsilon is not None

Q
Qiao Longfei 已提交
2391
        super(DecayedAdagradOptimizer, self).__init__(
X
Xin Pan 已提交
2392
            learning_rate=learning_rate,
2393
            parameter_list=parameter_list,
X
Xin Pan 已提交
2394
            regularization=regularization,
2395
            grad_clip=grad_clip,
X
Xin Pan 已提交
2396
            name=name)
2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423
        self.type = "decayed_adagrad"
        self._decay = decay
        self._epsilon = epsilon

    def _create_accumulators(self, block, parameters):
        assert isinstance(block, framework.Block)

        for p in parameters:
            self._add_accumulator(self._moment_acc_str, p)

    def _append_optimize_op(self, block, param_and_grad):
        assert isinstance(block, framework.Block)

        moment_acc = self._get_accumulator(self._moment_acc_str,
                                           param_and_grad[0])

        # Create the decayed adagrad optimizer op
        decayed_adagrad_op = block.append_op(
            type=self.type,
            inputs={
                "Param": param_and_grad[0],
                "Grad": param_and_grad[1],
                "Moment": moment_acc,
                "LearningRate": self._create_param_lr(param_and_grad)
            },
            outputs={"ParamOut": param_and_grad[0],
                     "MomentOut": moment_acc},
2424 2425
            attrs={"epsilon": self._epsilon,
                   "decay": self._decay},
M
minqiyang 已提交
2426
            stop_gradient=True)
2427 2428

        return decayed_adagrad_op
2429 2430


2431
class AdadeltaOptimizer(Optimizer):
2432
    """
Z
Zeng Jinle 已提交
2433
    **Notes: This API does not support sparse parameter optimization.**
Q
qiaolongfei 已提交
2434

Z
Zeng Jinle 已提交
2435
    Adadelta Optimizer. Please refer to this for details:
Z
Zeng Jinle 已提交
2436 2437 2438
    `ADADELTA: AN ADAPTIVE LEARNING RATE METHOD <https://arxiv.org/abs/1212.5701>`_.

    The update is done as follows:
2439

Z
Zeng Jinle 已提交
2440 2441
    .. math::

Z
Zeng Jinle 已提交
2442
        E(g_t^2) &= \\rho * E(g_{t-1}^2) + (1-\\rho) * g^2
2443

Z
Zeng Jinle 已提交
2444
        learning\_rate &= \sqrt{ ( E(dx_{t-1}^2) + \\epsilon ) / ( E(g_t^2) + \\epsilon ) }
Z
Zeng Jinle 已提交
2445

Z
Zeng Jinle 已提交
2446
        E(dx_t^2) &= \\rho * E(dx_{t-1}^2) + (1-\\rho) * (-g*learning\_rate)^2
2447 2448

    Args:
Z
Zeng Jinle 已提交
2449 2450 2451
        learning_rate (float|Variable): global learning rate.
        epsilon (float): a small float number for numeric stability. Default 1.0e-6.
        rho (float): a floating point value indicating the decay rate. Default 0.95.
H
hong 已提交
2452
        parameter_list (Iterable, optional):  Iterable of ``Variable`` names to update to minimize ``loss``. \
2453 2454
            This parameter is required in dygraph mode. \
            The default value is None in static mode, at this time all parameters will be updated.
2455 2456 2457 2458 2459
        regularization (WeightDecayRegularizer, optional): The strategy of regularization. There are two method: \
             :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If a parameter has set \
            regularizer using :ref:`api_fluid_ParamAttr` already, the regularization setting here in optimizer will be \
            ignored for this parameter. Otherwise, the regularization setting here in optimizer will take effect.  \
            Default None, meaning there is no regularization.
2460 2461 2462 2463
        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of 
            some derived class of ``GradientClipBase`` . There are three cliping strategies 
            ( :ref:`api_fluid_clip_GradientClipByGlobalNorm` , :ref:`api_fluid_clip_GradientClipByNorm` , 
            :ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping.
2464 2465 2466
        name (str, optional): The default value is None. Normally there is no need for user
                to set this property. For more information, please refer to
                :ref:`api_guide_Name` .
2467 2468 2469 2470

    Examples:
        .. code-block:: python

2471
            import paddle.fluid as fluid
Z
Zeng Jinle 已提交
2472

2473
            image = fluid.data(name='image', shape=[None, 28], dtype='float32')
Z
Zeng Jinle 已提交
2474 2475
            fc = fluid.layers.fc(image, size=10)
            cost = fluid.layers.reduce_mean(fc)
2476 2477
            optimizer = fluid.optimizer.Adadelta(
                learning_rate=0.0003, epsilon=1.0e-6, rho=0.95)
C
chengduo 已提交
2478

Z
Zeng Jinle 已提交
2479 2480 2481 2482
            # optimizer_ops is a list of optimizer operators to update parameters
            # params_grads is a list of (param, param_grad), where param is each
            # parameter and param_grad is the gradient variable of param.
            optimizer_ops, params_grads = optimizer.minimize(cost)
2483
    """
2484

2485 2486 2487
    _avg_squared_grad_acc_str = "_avg_squared_grad"
    _avg_squared_update_acc_str = "_avg_squared_update"

X
Xin Pan 已提交
2488 2489 2490 2491
    def __init__(self,
                 learning_rate,
                 epsilon=1.0e-6,
                 rho=0.95,
2492
                 parameter_list=None,
X
Xin Pan 已提交
2493
                 regularization=None,
2494
                 grad_clip=None,
X
Xin Pan 已提交
2495
                 name=None):
2496 2497 2498 2499 2500 2501
        if learning_rate is None:
            raise ValueError("learning_rate is not set.")
        if epsilon is None:
            raise ValueError("epsilon is not set.")
        if rho is None:
            raise ValueError("rho is not set.")
2502
        super(AdadeltaOptimizer, self).__init__(
X
Xin Pan 已提交
2503
            learning_rate=learning_rate,
2504
            parameter_list=parameter_list,
X
Xin Pan 已提交
2505
            regularization=regularization,
2506
            grad_clip=grad_clip,
X
Xin Pan 已提交
2507
            name=name)
2508 2509 2510 2511 2512
        self.type = "adadelta"
        self._epsilon = epsilon
        self._rho = rho

    def _create_accumulators(self, block, parameters):
2513 2514
        if not isinstance(block, framework.Block):
            raise TypeError("block is not instance of framework.Block.")
2515 2516 2517 2518 2519 2520

        for p in parameters:
            self._add_accumulator(self._avg_squared_grad_acc_str, p)
            self._add_accumulator(self._avg_squared_update_acc_str, p)

    def _append_optimize_op(self, block, param_and_grad):
2521 2522
        if not isinstance(block, framework.Block):
            raise TypeError("block is not instance of framework.Block.")
2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543

        avg_squared_grad_acc = self._get_accumulator(
            self._avg_squared_grad_acc_str, param_and_grad[0])
        avg_squared_update_acc = self._get_accumulator(
            self._avg_squared_update_acc_str, param_and_grad[0])

        # Create the adadelta optimizer op
        adadelta_op = block.append_op(
            type=self.type,
            inputs={
                "Param": param_and_grad[0],
                "Grad": param_and_grad[1],
                "AvgSquaredGrad": avg_squared_grad_acc,
                "AvgSquaredUpdate": avg_squared_update_acc
            },
            outputs={
                "ParamOut": param_and_grad[0],
                "AvgSquaredGradOut": avg_squared_grad_acc,
                "AvgSquaredUpdateOut": avg_squared_update_acc
            },
            attrs={"epsilon": self._epsilon,
M
minqiyang 已提交
2544 2545
                   "rho": self._rho},
            stop_gradient=True)
2546 2547 2548 2549

        return adadelta_op


Q
qingqing01 已提交
2550 2551 2552 2553 2554 2555 2556 2557 2558 2559
class RMSPropOptimizer(Optimizer):
    """
    Root Mean Squared Propagation (RMSProp) is an unpublished, adaptive learning
    rate method. The original slides proposed RMSProp: Slide 29 of
    http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf .

    The original equation is as follows:

    ..  math::

Q
qiaolongfei 已提交
2560
        r(w, t) & = \\rho r(w, t-1) + (1 - \\rho)(\\nabla Q_{i}(w))^2
Q
qingqing01 已提交
2561 2562 2563 2564

        w & = w - \\frac{\\eta} {\\sqrt{r(w,t) + \\epsilon}} \\nabla Q_{i}(w)

    The first equation calculates moving average of the squared gradient for
Q
qiaolongfei 已提交
2565
    each weight. Then dividing the gradient by :math:`sqrt{v(w,t)}`.
Q
qingqing01 已提交
2566 2567 2568 2569 2570 2571

    In some cases, adding a momentum term :math: `\\beta` is beneficial.
    In our implementation, Nesterov momentum is used:

    ..  math::

Q
qiaolongfei 已提交
2572
        r(w, t) & = \\rho r(w, t-1) + (1 - \\rho)(\\nabla Q_{i}(w))^2
Q
qingqing01 已提交
2573

2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587
        v(w, t) & = \\beta v(w, t-1) + \\frac{\\eta} {\\sqrt{r(w,t) +
            \\epsilon}} \\nabla Q_{i}(w)

        w & = w - v(w, t)

    if centered is True:

    ..  math::

        r(w, t) & = \\rho r(w, t-1) + (1 - \\rho)(\\nabla Q_{i}(w))^2

        g(w, t) & = \\rho g(w, t-1) + (1 - \\rho)\\nabla Q_{i}(w)

        v(w, t) & = \\beta v(w, t-1) + \\frac{\\eta} {\\sqrt{r(w,t) - (g(w, t))^2 +
Q
qingqing01 已提交
2588 2589 2590 2591
            \\epsilon}} \\nabla Q_{i}(w)

        w & = w - v(w, t)

Q
qiaolongfei 已提交
2592
    where, :math:`\\rho` is a hyperparameter and typical values are 0.9, 0.95
Q
qingqing01 已提交
2593 2594 2595 2596 2597
    and so on. :math: `beta` is the momentum term. :math: `\\epsilon` is a
    smoothing term to avoid division by zero, usually set somewhere in range
    from 1e-4 to 1e-8.


2598 2599 2600
    Parameters:
        learning_rate(float): Global learning rate.
        rho(float): rho is :math: `\\rho` in equation, default is 0.95.
Q
qingqing01 已提交
2601
        epsilon(float): :math: `\\epsilon` in equation is smoothing term to
2602
            avoid division by zero, default is 1e-6.
Q
qiaolongfei 已提交
2603
        momentum(float): :math:`\\beta` in equation is the momentum term,
2604
            default is 0.0.
2605 2606 2607 2608
        centered(bool): If True, gradients are normalized by the estimated variance of
            the gradient; if False, by the uncentered second moment. Setting this to
            True may help with training, but is slightly more expensive in terms of
            computation and memory. Defaults to False.
H
hong 已提交
2609
        parameter_list (Iterable, optional):  Iterable of ``Variable`` names to update to minimize ``loss``. \
2610 2611
            This parameter is required in dygraph mode. \
            The default value is None in static mode, at this time all parameters will be updated.
2612 2613 2614 2615 2616
        regularization (WeightDecayRegularizer, optional): The strategy of regularization. There are two method: \
             :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If a parameter has set \
            regularizer using :ref:`api_fluid_ParamAttr` already, the regularization setting here in optimizer will be \
            ignored for this parameter. Otherwise, the regularization setting here in optimizer will take effect.  \
            Default None, meaning there is no regularization.
2617 2618 2619 2620
        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of 
            some derived class of ``GradientClipBase`` . There are three cliping strategies 
            ( :ref:`api_fluid_clip_GradientClipByGlobalNorm` , :ref:`api_fluid_clip_GradientClipByNorm` , 
            :ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping.
2621 2622
        name (str, optional): This parameter is used by developers to print debugging information. \
            For details, please refer to :ref:`api_guide_Name`. Default is None.
Q
qingqing01 已提交
2623 2624 2625 2626 2627 2628 2629

    Raises:
        ValueError: If learning_rate, rho, epsilon, momentum are None.

    Examples:
          .. code-block:: python

2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654
            import paddle
            import paddle.fluid as fluid
            import numpy as np

            place = fluid.CPUPlace()
            main = fluid.Program()
            with fluid.program_guard(main):
                x = fluid.layers.data(name='x', shape=[13], dtype='float32')
                y = fluid.layers.data(name='y', shape=[1], dtype='float32')
                y_predict = fluid.layers.fc(input=x, size=1, act=None)
                cost = fluid.layers.square_error_cost(input=y_predict, label=y)
                avg_cost = fluid.layers.mean(cost)

                rms_optimizer = fluid.optimizer.RMSProp(learning_rate=0.1)
                rms_optimizer.minimize(avg_cost)

                fetch_list = [avg_cost]
                train_reader = paddle.batch(
                    paddle.dataset.uci_housing.train(), batch_size=1)
                feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
                exe = fluid.Executor(place)
                exe.run(fluid.default_startup_program())
                for data in train_reader():
                    exe.run(main, feed=feeder.feed(data), fetch_list=fetch_list)

Q
qingqing01 已提交
2655 2656 2657 2658
    """

    _momentum_acc_str = "momentum"
    _mean_square_acc_str = "mean_square"
2659
    _mean_grad_acc_str = "mean_grad"
Q
qingqing01 已提交
2660 2661 2662 2663 2664 2665

    def __init__(self,
                 learning_rate,
                 rho=0.95,
                 epsilon=1.0e-6,
                 momentum=0.0,
2666
                 centered=False,
2667
                 parameter_list=None,
X
Xin Pan 已提交
2668
                 regularization=None,
2669
                 grad_clip=None,
X
Xin Pan 已提交
2670
                 name=None):
Q
qingqing01 已提交
2671
        super(RMSPropOptimizer, self).__init__(
X
Xin Pan 已提交
2672
            learning_rate=learning_rate,
2673
            parameter_list=parameter_list,
X
Xin Pan 已提交
2674
            regularization=regularization,
2675
            grad_clip=grad_clip,
X
Xin Pan 已提交
2676
            name=name)
Q
qingqing01 已提交
2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689
        if learning_rate is None:
            raise ValueError("learning_rate is not set.")
        if rho is None:
            raise ValueError("rho is not set.")
        if epsilon is None:
            raise ValueError("epsilon is not set.")
        if momentum is None:
            raise ValueError("momentum is not set.")

        self.type = "rmsprop"
        self._rho = rho
        self._epsilon = epsilon
        self._momentum = momentum
2690
        self._centered = centered
Q
qingqing01 已提交
2691 2692 2693 2694 2695 2696 2697 2698

    def _create_accumulators(self, block, parameters):
        if not isinstance(block, framework.Block):
            raise TypeError("block is not instance of framework.Block.")

        for p in parameters:
            self._add_accumulator(self._momentum_acc_str, p)
            self._add_accumulator(self._mean_square_acc_str, p)
2699
            self._add_accumulator(self._mean_grad_acc_str, p)
Q
qingqing01 已提交
2700 2701 2702 2703 2704 2705 2706 2707 2708

    def _append_optimize_op(self, block, param_and_grad):
        if not isinstance(block, framework.Block):
            raise TypeError("block is not instance of framework.Block.")

        momentum_acc = self._get_accumulator(self._momentum_acc_str,
                                             param_and_grad[0])
        mean_square_acc = self._get_accumulator(self._mean_square_acc_str,
                                                param_and_grad[0])
2709 2710
        mean_grad_acc = self._get_accumulator(self._mean_grad_acc_str,
                                              param_and_grad[0])
Q
qingqing01 已提交
2711 2712 2713 2714 2715 2716 2717
        rmsprop_op = block.append_op(
            type=self.type,
            inputs={
                "Param": param_and_grad[0],
                "Grad": param_and_grad[1],
                "Moment": momentum_acc,
                "MeanSquare": mean_square_acc,
2718
                "MeanGrad": mean_grad_acc,
Q
qingqing01 已提交
2719 2720 2721 2722 2723
                "LearningRate": self._create_param_lr(param_and_grad),
            },
            outputs={
                "ParamOut": param_and_grad[0],
                "MomentOut": momentum_acc,
2724 2725
                "MeanSquareOut": mean_square_acc,
                "MeanGradOut": mean_grad_acc
Q
qingqing01 已提交
2726 2727 2728 2729
            },
            attrs={
                "epsilon": self._epsilon,
                "decay": self._rho,
2730 2731
                "momentum": self._momentum,
                "centered": self._centered
M
minqiyang 已提交
2732 2733
            },
            stop_gradient=True)
Q
qingqing01 已提交
2734 2735 2736 2737

        return rmsprop_op


Q
qiaolongfei 已提交
2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777
class FtrlOptimizer(Optimizer):
    """
    FTRL (Follow The Regularized Leader) Optimizer.

    The paper that proposed Follow The Regularized Leader (FTRL):
    (https://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf)

    ..  math::

        &new\_accum = squared\_accum + grad^2

        &if (lr\_power == -0.5):

        &\quad  linear\_accum += grad - \\frac{\\sqrt{new\_accum} - \\sqrt{squared\_accum}}{learning\_rate * param}

        &else:

        &\quad   linear\_accum += grad - \\frac{new\_accum^{-lr\_power} - accum^{-lr\_power}}{learning\_rate * param}


        &x = l1 * sign(linear\_accum) - linear\_accum

        &if (lr\_power == -0.5):

        &\quad   y = \\frac{\\sqrt{new\_accum}}{learning\_rate} + (2 * l2)

        &\quad   pre\_shrink = \\frac{x}{y}

        &\quad   param = (abs(linear\_accum) > l1).select(pre\_shrink, 0.0)

        &else:

        &\quad   y = \\frac{new\_accum^{-lr\_power}}{learning\_rate} + (2 * l2)

        &\quad   pre\_shrink = \\frac{x}{y}

        &\quad   param = (abs(linear\_accum) > l1).select(pre\_shrink, 0.0)

        &squared\_accum += grad^2

2778 2779 2780 2781 2782
    Parameters:
        learning_rate (float|Variable): Global learning rate.
        l1 (float): L1 regularization strength, default is 0.0.
        l2 (float): L2 regularization strength, default is 0.0.
        lr_power (float): Learning Rate Power, default is -0.5.
H
hong 已提交
2783
        parameter_list (Iterable, optional):  Iterable of ``Variable`` names to update to minimize ``loss``. \
2784 2785
            This parameter is required in dygraph mode. \
            The default value is None in static mode, at this time all parameters will be updated.
2786 2787 2788 2789 2790
        regularization (WeightDecayRegularizer, optional): The strategy of regularization. There are two method: \
             :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If a parameter has set \
            regularizer using :ref:`api_fluid_ParamAttr` already, the regularization setting here in optimizer will be \
            ignored for this parameter. Otherwise, the regularization setting here in optimizer will take effect.  \
            Default None, meaning there is no regularization.
2791 2792 2793 2794
        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of 
            some derived class of ``GradientClipBase`` . There are three cliping strategies 
            ( :ref:`api_fluid_clip_GradientClipByGlobalNorm` , :ref:`api_fluid_clip_GradientClipByNorm` , 
            :ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping.
2795 2796
        name (str, optional): This parameter is used by developers to print debugging information. \
            For details, please refer to :ref:`api_guide_Name`. Default is None.
Q
qiaolongfei 已提交
2797 2798 2799 2800 2801 2802 2803

    Raises:
        ValueError: If learning_rate, rho, epsilon, momentum are None.

    Examples:
          .. code-block:: python

2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827
            import paddle
            import paddle.fluid as fluid
            import numpy as np

            place = fluid.CPUPlace()
            main = fluid.Program()
            with fluid.program_guard(main):
                x = fluid.layers.data(name='x', shape=[13], dtype='float32')
                y = fluid.layers.data(name='y', shape=[1], dtype='float32')
                y_predict = fluid.layers.fc(input=x, size=1, act=None)
                cost = fluid.layers.square_error_cost(input=y_predict, label=y)
                avg_cost = fluid.layers.mean(cost)

                ftrl_optimizer = fluid.optimizer.Ftrl(learning_rate=0.1)
                ftrl_optimizer.minimize(avg_cost)

                fetch_list = [avg_cost]
                train_reader = paddle.batch(
                    paddle.dataset.uci_housing.train(), batch_size=1)
                feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
                exe = fluid.Executor(place)
                exe.run(fluid.default_startup_program())
                for data in train_reader():
                    exe.run(main, feed=feeder.feed(data), fetch_list=fetch_list)
C
chengduo 已提交
2828

2829
    NOTE:
C
chengduo 已提交
2830
       Currently, FtrlOptimizer doesn't support sparse parameter optimization.
Q
qiaolongfei 已提交
2831 2832 2833 2834 2835
    """

    _squared_acc_str = "squared"
    _linear_acc_str = "linear"

X
Xin Pan 已提交
2836 2837 2838 2839 2840
    def __init__(self,
                 learning_rate,
                 l1=0.0,
                 l2=0.0,
                 lr_power=-0.5,
2841
                 parameter_list=None,
X
Xin Pan 已提交
2842
                 regularization=None,
2843
                 grad_clip=None,
X
Xin Pan 已提交
2844
                 name=None):
Q
qiaolongfei 已提交
2845
        super(FtrlOptimizer, self).__init__(
X
Xin Pan 已提交
2846
            learning_rate=learning_rate,
2847
            parameter_list=parameter_list,
X
Xin Pan 已提交
2848
            regularization=regularization,
2849
            grad_clip=grad_clip,
X
Xin Pan 已提交
2850
            name=name)
Q
qiaolongfei 已提交
2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889
        if learning_rate is None:
            raise ValueError("learning_rate is not set.")

        self.type = "ftrl"
        self._l1 = l1
        self._l2 = l2
        self._lr_power = lr_power

    def _create_accumulators(self, block, parameters):
        if not isinstance(block, framework.Block):
            raise TypeError("block is not instance of framework.Block.")

        for p in parameters:
            self._add_accumulator(self._squared_acc_str, p)
            self._add_accumulator(self._linear_acc_str, p)

    def _append_optimize_op(self, block, param_and_grad):
        if not isinstance(block, framework.Block):
            raise TypeError("block is not instance of framework.Block.")

        squared_acc = self._get_accumulator(self._squared_acc_str,
                                            param_and_grad[0])
        linear_acc = self._get_accumulator(self._linear_acc_str,
                                           param_and_grad[0])
        ftrl_op = block.append_op(
            type=self.type,
            inputs={
                "Param": param_and_grad[0],
                "Grad": param_and_grad[1],
                "SquaredAccumulator": squared_acc,
                "LinearAccumulator": linear_acc,
                "LearningRate": self._create_param_lr(param_and_grad),
            },
            outputs={
                "ParamOut": param_and_grad[0],
                "SquaredAccumOut": squared_acc,
                "LinearAccumOut": linear_acc
            },
            attrs={"l1": self._l1,
2890
                   "l2": self._l2,
M
minqiyang 已提交
2891 2892
                   "lr_power": self._lr_power},
            stop_gradient=True)
Q
qiaolongfei 已提交
2893 2894 2895 2896

        return ftrl_op


Y
Yibing Liu 已提交
2897 2898 2899 2900 2901 2902
class LambOptimizer(AdamOptimizer):
    """
    LAMB (Layer-wise Adaptive Moments optimizer for Batching training) Optimizer.

    LAMB Optimizer is designed to scale up the batch size of training without losing 
    accuracy, which supports adaptive element-wise updating and accurate layer-wise 
Y
Yibing Liu 已提交
2903 2904
    correction. For more information, please refer to `Large Batch Optimization for 
    Deep Learning: Training BERT in 76 minutes <https://arxiv.org/abs/1904.00962>`_ .
Y
Yibing Liu 已提交
2905 2906 2907 2908 2909

    The updating of parameters follows:

    ..  math::

Y
Yibing Liu 已提交
2910
        m_t &= \\beta_1 m_{t - 1}+ (1 - \\beta_1)g_t 
Y
Yibing Liu 已提交
2911

Y
Yibing Liu 已提交
2912
        v_t &= \\beta_2 v_{t - 1}  + (1 - \\beta_2)g_t^2
Y
Yibing Liu 已提交
2913

Y
Yibing Liu 已提交
2914
        r_t &= \\frac{m_t}{\\sqrt{v_t}+\\epsilon}
Y
Yibing Liu 已提交
2915

Y
Yibing Liu 已提交
2916
        w_t &= w_{t-1} -\\eta_t \\frac{\\left \| w_{t-1}\\right \|}{\\left \| r_t + \\lambda w_{t-1}\\right \|} (r_t + \\lambda w_{t-1})
Y
Yibing Liu 已提交
2917 2918 2919 2920 2921 2922


    where :math:`m` is the 1st moment, and :math:`v` the 2nd moment, :math:`\\eta` the 
    learning rate, :math:`\\lambda` the LAMB weight decay rate.

    Args:
Y
Yibing Liu 已提交
2923 2924 2925 2926 2927 2928 2929 2930
        learning_rate (float|Variable, optional): the learning rate used to update parameters. \
            Can be a float value or a Variable with data type float32. Default 0.001.
        lamb_weight_decay (float, optional): The LAMB weight decay rate. Default 0.01.
        beta1 (float, optional): The exponential decay rate for the 1st moment estimates.
            Default 0.9.
        beta2 (float, optional): The exponential decay rate for the 2nd moment estimates.
            Default 0.999.
        epsilon (float, optional): A small float value for numerical stability. Default 1e-6.
H
hong 已提交
2931
        parameter_list (Iterable, optional):  Iterable of ``Variable`` names to update to minimize ``loss``. \
2932 2933
            This parameter is required in dygraph mode. \
            The default value is None in static mode, at this time all parameters will be updated.
2934 2935 2936 2937 2938
        regularization (WeightDecayRegularizer, optional): The strategy of regularization. There are two method: \
             :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If a parameter has set \
            regularizer using :ref:`api_fluid_ParamAttr` already, the regularization setting here in optimizer will be \
            ignored for this parameter. Otherwise, the regularization setting here in optimizer will take effect.  \
            Default None, meaning there is no regularization.
2939 2940 2941 2942
        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of 
            some derived class of ``GradientClipBase`` . There are three cliping strategies 
            ( :ref:`api_fluid_clip_GradientClipByGlobalNorm` , :ref:`api_fluid_clip_GradientClipByNorm` , 
            :ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping.
Y
Yibing Liu 已提交
2943 2944 2945 2946 2947
        exclude_from_weight_decay_fn (function|None): Exclude a parameter from weight 
            decay when **exclude_from_weight_decay_fn(parameter)** returns true. 
            Default None.
        name(str|None): For detailed information, please refer to 
            :ref:`api_guide_Name` . Usually name is no need to set and None by default.
Y
Yibing Liu 已提交
2948 2949 2950 2951 2952 2953

    Examples:
        .. code-block:: python
            
            import paddle.fluid as fluid 

Y
Yibing Liu 已提交
2954
            data = fluid.data(name='x', shape=[-1, 5], dtype='float32')
Y
Yibing Liu 已提交
2955 2956 2957
            hidden = fluid.layers.fc(input=data, size=10)
            cost = fluid.layers.mean(hidden)

Y
Yibing Liu 已提交
2958 2959 2960 2961 2962
            def exclude_fn(param):
                return param.name.endswith('.b_0')

            optimizer = fluid.optimizer.Lamb(learning_rate=0.002,
                                             exclude_from_weight_decay_fn=exclude_fn)
Y
Yibing Liu 已提交
2963 2964 2965 2966
            optimizer.minimize(cost)
    """
    _moment1_acc_str = "moment1"
    _moment2_acc_str = "moment2"
Y
Yibing Liu 已提交
2967
    # these two not used in op temporarily
Y
Yibing Liu 已提交
2968 2969 2970 2971 2972 2973 2974 2975 2976
    _beta1_pow_acc_str = "beta1_pow_acc"
    _beta2_pow_acc_str = "beta2_pow_acc"

    def __init__(self,
                 learning_rate=0.001,
                 lamb_weight_decay=0.01,
                 beta1=0.9,
                 beta2=0.999,
                 epsilon=1e-6,
2977
                 parameter_list=None,
Y
Yibing Liu 已提交
2978
                 regularization=None,
2979
                 grad_clip=None,
Y
Yibing Liu 已提交
2980
                 exclude_from_weight_decay_fn=None,
Y
Yibing Liu 已提交
2981 2982 2983 2984 2985 2986 2987 2988
                 name=None):
        assert learning_rate is not None
        assert lamb_weight_decay is not None
        assert beta1 is not None
        assert beta2 is not None
        assert epsilon is not None
        super(LambOptimizer, self).__init__(
            learning_rate=learning_rate,
2989
            parameter_list=parameter_list,
Y
Yibing Liu 已提交
2990
            regularization=regularization,
2991
            grad_clip=grad_clip,
Y
Yibing Liu 已提交
2992 2993 2994 2995 2996 2997
            beta1=beta1,
            beta2=beta2,
            epsilon=epsilon,
            name=name)
        self.type = "lamb"
        self._weight_decay = lamb_weight_decay
Y
Yibing Liu 已提交
2998
        self._exclude_from_weight_decay_fn = exclude_from_weight_decay_fn
Y
Yibing Liu 已提交
2999 3000 3001

    def _append_optimize_op(self, block, param_and_grad):
        assert isinstance(block, framework.Block)
3002
        block.program._use_lamb = True
Y
Yibing Liu 已提交
3003 3004 3005 3006 3007 3008 3009 3010 3011 3012

        moment1 = self._get_accumulator(self._moment1_acc_str,
                                        param_and_grad[0])
        moment2 = self._get_accumulator(self._moment2_acc_str,
                                        param_and_grad[0])
        beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str,
                                              param_and_grad[0])
        beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str,
                                              param_and_grad[0])

Y
Yibing Liu 已提交
3013 3014 3015 3016 3017 3018
        if self._exclude_from_weight_decay_fn is not None \
            and self._exclude_from_weight_decay_fn(param_and_grad[0]):
            weight_decay = 0.0
        else:
            weight_decay = self._weight_decay

Y
Yibing Liu 已提交
3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039
        # create the lamb optimize op
        lamb_op = block.append_op(
            type=self.type,
            inputs={
                "Param": param_and_grad[0],
                "Grad": param_and_grad[1],
                "LearningRate": self._create_param_lr(param_and_grad),
                "Moment1": moment1,
                "Moment2": moment2,
                "Beta1Pow": beta1_pow_acc,
                "Beta2Pow": beta2_pow_acc
            },
            outputs={
                "ParamOut": param_and_grad[0],
                "Moment1Out": moment1,
                "Moment2Out": moment2
            },
            attrs={
                "beta1": self._beta1,
                "beta2": self._beta2,
                "epsilon": self._epsilon,
Y
Yibing Liu 已提交
3040
                "weight_decay": weight_decay
Y
Yibing Liu 已提交
3041 3042 3043 3044 3045 3046
            },
            stop_gradient=True)

        return lamb_op


3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059
# We short the class name, since users will use the optimizer with the package
# name. The sample code:
#
# import paddle.fluid as fluid
#
# sgd = fluid.optimizer.SGD(...)
#
# It is no need to add an `Optimizer` as the class suffix
SGD = SGDOptimizer
Momentum = MomentumOptimizer
Adagrad = AdagradOptimizer
Adam = AdamOptimizer
Adamax = AdamaxOptimizer
3060
Dpsgd = DpsgdOptimizer
3061
DecayedAdagrad = DecayedAdagradOptimizer
3062
Adadelta = AdadeltaOptimizer
Q
qingqing01 已提交
3063
RMSProp = RMSPropOptimizer
Q
qiaolongfei 已提交
3064
Ftrl = FtrlOptimizer
3065
LarsMomentum = LarsMomentumOptimizer
Y
Yibing Liu 已提交
3066
Lamb = LambOptimizer
3067 3068 3069


class ModelAverage(Optimizer):
3070
    """
3071
	:api_attr: Static Graph
S
swtkiwi 已提交
3072

3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090
    The ModelAverage optimizer accumulates specific continuous historical parameters
    during training. The accumulated historical range can be controlled by the passed
    ``average_window_rate`` argument. The averaged ``Parameter`` are used in the prediction,
    which usually can improve the accuracy of the prediction.

    Accumulate the average of the ``Parameter`` in the sliding window, the result will be saved
    in a temporary variable, can be applied to the current model's ``Parameter`` by calling
    the ``apply()`` method, and the current model ``Parameter`` can be restored by calling
    the ``restore()`` method.

    The window size for calculating the average is determined by ``average_window_rate``,
    ``min_average_window``, ``max_average_window`` and the current ``Parameter`` update times (num_updates).

    When the cumulative times (num_accumulates) is greater than the specific window
    threshold (average_window), the accumulated ``Parameter`` temporary variable is set to 0.0.
    The following example will help to understand the role of these arguments:

    ::
3091

3092 3093 3094 3095 3096 3097 3098 3099 3100
        if num_accumulates >= min_average_window and num_accumulates >= min(max_average_window, num_updates * average_window_rate):
            num_accumulates = 0

    In the above conditional judgment statement, ``num_accumulates`` indicates the current
    accumulated number, which can be abstractly understood as the length of the cumulative window.
    The length of the window must be at least the length set by the ``min_average_window`` argument,
    and cannot exceed the length specified by the ``max_average_window`` argument or
    ``num_updates * average_window_rate``, where ``num_updates`` indicates the current ``Parameter``
    update times, ``average_window_rate`` is a coefficient that calculates the length of the window.
3101 3102

    Args:
3103 3104 3105
        average_window_rate (float): The calculate ratio of the window length relative to ``Parameter`` update times.
        min_average_window (int, optional): the minimum size of average window length. The default value is 10000.
        max_average_window (int, optional): The maximum size of average window length. The default value is 10000.
3106 3107 3108 3109 3110
        regularization (WeightDecayRegularizer, optional): The strategy of regularization. There are two method: \
             :ref:`api_fluid_regularizer_L1Decay` , :ref:`api_fluid_regularizer_L2Decay` . If a parameter has set \
            regularizer using :ref:`api_fluid_ParamAttr` already, the regularization setting here in optimizer will be \
            ignored for this parameter. Otherwise, the regularization setting here in optimizer will take effect.  \
            Default None, meaning there is no regularization.
3111 3112 3113
        name (str, optional): Normally there is no need for user to set this property.
            For more information, please refer to :ref:`api_guide_Name`.
            The default value is None.
3114

3115
    Examples:
Q
qiaolongfei 已提交
3116 3117 3118

      .. code-block:: python

3119 3120 3121 3122 3123 3124
        import paddle.fluid as fluid
        import numpy

        # First create the Executor.
        place = fluid.CPUPlace()  # fluid.CUDAPlace(0)
        exe = fluid.Executor(place)
3125

3126 3127 3128 3129
        train_program = fluid.Program()
        startup_program = fluid.Program()
        with fluid.program_guard(train_program, startup_program):
            # build net
3130
            data = fluid.data(name='X', shape=[None, 1], dtype='float32')
3131 3132 3133 3134 3135 3136 3137 3138
            hidden = fluid.layers.fc(input=data, size=10)
            loss = fluid.layers.mean(hidden)
            optimizer = fluid.optimizer.Momentum(learning_rate=0.2, momentum=0.1)
            optimizer.minimize(loss)

            # build ModelAverage optimizer
            model_average = fluid.optimizer.ModelAverage(0.15,
                                                         min_average_window=10000,
3139
                                                         max_average_window=12500)
3140 3141

            exe.run(startup_program)
3142 3143 3144 3145 3146
            for i in range(12500):
                x = numpy.random.random(size=(10, 1)).astype('float32')
                outs = exe.run(program=train_program,
                               feed={'X': x},
                               fetch_list=[loss.name])
3147 3148

            # apply ModelAverage
3149
            with model_average.apply(exe):
3150 3151 3152 3153
                x = numpy.random.random(size=(10, 1)).astype('float32')
                exe.run(program=train_program,
                        feed={'X': x},
                        fetch_list=[loss.name])
3154 3155 3156
    """

    def __init__(self,
W
wanghaoshuang 已提交
3157
                 average_window_rate,
3158 3159
                 min_average_window=10000,
                 max_average_window=10000,
X
Xin Pan 已提交
3160 3161
                 regularization=None,
                 name=None):
Z
zhongpu 已提交
3162 3163
        if framework.in_dygraph_mode():
            raise Exception("In dygraph, don't support ModelAverage.")
X
Xin Pan 已提交
3164 3165
        super(ModelAverage, self).__init__(
            0.0, regularization=regularization, name=name)
3166 3167 3168
        self.average_window = average_window_rate
        self.min_average_window = min_average_window
        self.max_average_window = max_average_window
3169

3170
        self.params_grads = []
3171 3172
        for param in framework.default_main_program().global_block(
        ).all_parameters():
3173
            if param.do_model_average != False:
3174
                grad = param.block.create_var(
3175 3176
                    name=unique_name.generate_with_ignorable_key(".".join(
                        [param.name, 'tmp'])),
3177 3178
                    dtype=param.dtype,
                    persistable=False,
W
wanghaoshuang 已提交
3179
                    stop_gradient=True)
3180
                self.params_grads.append((param, grad))
3181

3182
        for param, grad in self.params_grads:
3183 3184
            if grad is None:
                continue
X
Xin Pan 已提交
3185 3186
            with param.block.program._optimized_guard(
                [param, grad]), name_scope('move_average'):
3187
                self._append_average_accumulate_op(param)
3188

3189 3190 3191 3192
        self.apply_program = Program()
        block = self.apply_program.global_block()
        with program_guard(main_program=self.apply_program):
            for param_grad in self.params_grads:
3193
                self._add_average_apply_op(block, param_grad)
3194 3195 3196 3197 3198

        self.restore_program = Program()
        block = self.restore_program.global_block()
        with program_guard(main_program=self.restore_program):
            for param_grad in self.params_grads:
3199
                self._add_average_restore_op(block, param_grad)
3200

3201
    def _add_average_apply_op(self, block, param_grad):
L
Luo Tao 已提交
3202 3203 3204 3205 3206 3207
        param = block._clone_variable(param_grad[0])
        grad = block._clone_variable(param_grad[1])
        sum_1 = block._clone_variable(self._get_accumulator('sum_1', param))
        sum_2 = block._clone_variable(self._get_accumulator('sum_2', param))
        sum_3 = block._clone_variable(self._get_accumulator('sum_3', param))
        num_accumulates = block._clone_variable(
3208
            self._get_accumulator('num_accumulates', param))
L
Luo Tao 已提交
3209
        old_num_accumulates = block._clone_variable(
3210
            self._get_accumulator('old_num_accumulates', param))
L
Luo Tao 已提交
3211
        num_updates = block._clone_variable(
3212 3213 3214 3215 3216 3217
            self._get_accumulator('num_updates', param))
        # backup param value to grad
        layers.assign(input=param, output=grad)
        # param = (sum_1 + sum_2 + sum_3) / (num_accumulates + old_num_accumulates)
        tmp = layers.sum(x=[num_accumulates, old_num_accumulates])
        sum = layers.sum(x=[sum_1, sum_2, sum_3])
D
dzhwinter 已提交
3218 3219 3220 3221
        tmp = layers.cast(
            x=tmp, dtype='float32' if self._dtype == None else self._dtype)
        sum = layers.cast(
            x=sum, dtype='float32' if self._dtype == None else self._dtype)
S
sneaxiy 已提交
3222
        ops._elementwise_div(x=sum, y=tmp, out=param)
3223 3224

    def _add_average_restore_op(self, block, param_grad):
L
Luo Tao 已提交
3225 3226
        param = block._clone_variable(param_grad[0])
        grad = block._clone_variable(param_grad[1])
3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263
        layers.assign(input=grad, output=param)

    def _append_average_accumulate_op(self, param):
        self.helper = LayerHelper("average_accumulate")
        sum_1 = self._add_accumulator('sum_1', param)
        sum_2 = self._add_accumulator('sum_2', param)
        sum_3 = self._add_accumulator('sum_3', param)
        num_accumulates = self._add_accumulator(
            'num_accumulates', param, dtype='int64', shape=[1])
        old_num_accumulates = self._add_accumulator(
            'old_num_accumulates', param, dtype='int64', shape=[1])
        num_updates = self._add_accumulator(
            'num_updates', param, dtype='int64', shape=[1])

        self.helper.append_op(
            type='average_accumulates',
            inputs={
                "param": param,
                "in_sum_1": sum_1,
                "in_sum_2": sum_2,
                "in_sum_3": sum_3,
                "in_num_accumulates": num_accumulates,
                "in_old_num_accumulates": old_num_accumulates,
                "in_num_updates": num_updates
            },
            outputs={
                "out_sum_1": sum_1,
                "out_sum_2": sum_2,
                "out_sum_3": sum_3,
                "out_num_accumulates": num_accumulates,
                "out_old_num_accumulates": old_num_accumulates,
                "out_num_updates": num_updates,
            },
            attrs={
                "average_window": self.average_window,
                "min_average_window": self.min_average_window,
                "max_average_window": self.max_average_window,
M
minqiyang 已提交
3264 3265
            },
            stop_gradient=True)
3266

S
rename  
sneaxiy 已提交
3267
    @signature_safe_contextmanager
3268
    def apply(self, executor, need_restore=True):
3269 3270
        """
        Apply the average of the cumulative ``Parameter`` to the parameters of the current model.
3271 3272

        Args:
3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316
            executor(fluid.Executor): The current network executor.
            need_restore(bool): Restore flag variable, if set to True, the network will restore
                the parameters of the network to the default value, if set to False,
                it will not be restored. The default value is True.

        Examples:

          .. code-block:: python

            import paddle.fluid as fluid
            import numpy

            # First create the Executor.
            place = fluid.CPUPlace()  # fluid.CUDAPlace(0)
            exe = fluid.Executor(place)

            train_program = fluid.Program()
            startup_program = fluid.Program()
            with fluid.program_guard(train_program, startup_program):
                # build net
                data = fluid.data(name='X', shape=[None, 1], dtype='float32')
                hidden = fluid.layers.fc(input=data, size=10)
                loss = fluid.layers.mean(hidden)
                optimizer = fluid.optimizer.Momentum(learning_rate=0.2, momentum=0.1)
                optimizer.minimize(loss)

                # build ModelAverage optimizer
                model_average = fluid.optimizer.ModelAverage(0.15,
                                                            min_average_window=10000,
                                                            max_average_window=12500)

                exe.run(startup_program)
                for i in range(12500):
                    x = numpy.random.random(size=(10, 1)).astype('float32')
                    outs = exe.run(program=train_program,
                                feed={'X': x},
                                fetch_list=[loss.name])

                # apply ModelAverage
                with model_average.apply(exe):
                    x = numpy.random.random(size=(10, 1)).astype('float32')
                    exe.run(program=train_program,
                            feed={'X': x},
                            fetch_list=[loss.name])
3317
        """
3318 3319 3320 3321 3322 3323
        executor.run(self.apply_program)
        try:
            yield
        finally:
            if need_restore:
                self.restore(executor)
3324 3325

    def restore(self, executor):
3326 3327
        """
        Restore ``Parameter`` values of current model.
3328 3329
        
        Args:
3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373
            executor(fluid.Executor): The current network executor.

        Examples:

          .. code-block:: python

            import paddle.fluid as fluid
            import numpy

            # First create the Executor.
            place = fluid.CPUPlace()  # fluid.CUDAPlace(0)
            exe = fluid.Executor(place)

            train_program = fluid.Program()
            startup_program = fluid.Program()
            with fluid.program_guard(train_program, startup_program):
                # build net
                data = fluid.data(name='X', shape=[None, 1], dtype='float32')
                hidden = fluid.layers.fc(input=data, size=10)
                loss = fluid.layers.mean(hidden)
                optimizer = fluid.optimizer.Momentum(learning_rate=0.2, momentum=0.1)
                optimizer.minimize(loss)

                # build ModelAverage optimizer
                model_average = fluid.optimizer.ModelAverage(0.15,
                                                            min_average_window=10000,
                                                            max_average_window=12500)

                exe.run(startup_program)
                for i in range(12500):
                    x = numpy.random.random(size=(10, 1)).astype('float32')
                    outs = exe.run(program=train_program,
                                feed={'X': x},
                                fetch_list=[loss.name])

                # apply ModelAverage
                with model_average.apply(exe, False):
                    x = numpy.random.random(size=(10, 1)).astype('float32')
                    exe.run(program=train_program,
                            feed={'X': x},
                            fetch_list=[loss.name])

                # restore Parameters
                model_average.restore(exe)
3374
        """
3375
        executor.run(self.restore_program)
3376 3377 3378 3379


class ExponentialMovingAverage(object):
    """
3380
	:api_attr: Static Graph
S
swtkiwi 已提交
3381

3382 3383 3384 3385 3386 3387
    Compute the moving average of parameters with exponential decay.
    Given a parameter :math:`\\theta`, its exponential moving average (EMA)
    will be

    ..  math::

3388
        \\text{EMA}_0 & = 0
3389

3390 3391
	\\text{EMA}_t & = \\text{decay} * \\text{EMA}_{t-1} + (1 - \\text{decay}) * \\theta_t

Y
Yibing Liu 已提交
3392 3393 3394 3395
    The average results calculated by **update()** method will be saved in 
    temporary variables which are created and maintained by the object, and can 
    be applied to parameters of current model by calling **apply()** method. And 
    the **restore()** method is used to restore the parameters.
3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416

    **Bias correction**. All EMAs are initialized to :math:`0` and hence they will be 
    zero biased, which can be corrected by divided by a factor 
    :math:`(1 - \\text{decay}^t)` , i.e., the actual EMAs applied to parameters 
    when calling **apply()** method would be 

    ..  math::
    
        \\widehat{\\text{EMA}}_t = \\frac{\\text{EMA}_t}{1 - \\text{decay}^t}

    **Decay rate scheduling**. A large decay rate very close to 1 would result 
    in that the averages move very slowly. And a better strategy is to set a 
    relative smaller decay rate in the very beginning. The argument **thres_steps**
    allows users to pass a Variable to schedule the decay rate, in this case, 
    the actual decay rate becomes
     
    ..  math::
    
        \\min(\\text{decay}, \\frac{1 + \\text{thres_steps}}{10 + \\text{thres_steps}})

    Usually **thres_steps** can be the global training steps.
3417 3418 3419


    Args:
Y
Yibing Liu 已提交
3420 3421 3422 3423 3424 3425 3426
	decay (float, optional): The exponential decay rate, usually close to 1, such as 
            0.999, 0.9999, ... . Default 0.999.
        thres_steps (Variable|None): If not `None`, schedule the decay rate. 
            Default None.
        name (str|None): For detailed information, please refer to 
            :ref:`api_guide_Name`. Usually name is no need to set and None by 
            default.
3427 3428 3429 3430 3431


    Examples:

	.. code-block:: python
3432 3433 3434 3435 3436

	    import numpy
	    import paddle
	    import paddle.fluid as fluid

Y
Yibing Liu 已提交
3437
	    data = fluid.data(name='x', shape=[-1, 5], dtype='float32')
3438 3439 3440 3441 3442 3443 3444 3445
	    hidden = fluid.layers.fc(input=data, size=10)
	    cost = fluid.layers.mean(hidden)

	    test_program = fluid.default_main_program().clone(for_test=True)

	    optimizer = fluid.optimizer.Adam(learning_rate=0.001)
	    optimizer.minimize(cost)

3446
	    global_steps = fluid.layers.autoincreased_step_counter()
3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475
	    ema = fluid.optimizer.ExponentialMovingAverage(0.999, thres_steps=global_steps)
	    ema.update()

	    place = fluid.CPUPlace()
	    exe = fluid.Executor(place)
	    exe.run(fluid.default_startup_program())

	    for pass_id in range(3):
		for batch_id in range(6):
		    data = numpy.random.random(size=(10, 5)).astype('float32')
		    exe.run(program=fluid.default_main_program(),
			feed={'x': data}, 
			fetch_list=[cost.name])

		# usage 1
		with ema.apply(exe):
		    data = numpy.random.random(size=(10, 5)).astype('float32')
		    exe.run(program=test_program,
			    feed={'x': data}, 
			    fetch_list=[hidden.name])
			    

		 # usage 2
		with ema.apply(exe, need_restore=False):
		    data = numpy.random.random(size=(10, 5)).astype('float32')
		    exe.run(program=test_program,
			    feed={'x': data}, 
			    fetch_list=[hidden.name])
		ema.restore(exe)
3476 3477
    """

3478
    def __init__(self, decay=0.999, thres_steps=None, name=None):
Z
zhongpu 已提交
3479 3480 3481
        if framework.in_dygraph_mode():
            raise Exception(
                "In dygraph, don't support ExponentialMovingAverage.")
3482
        self._decay = decay
3483
        self._thres_steps = thres_steps
3484
        self._name = name if name is not None else ''
3485 3486
        self._decay_var = self._get_ema_decay()

3487
        self._step_counter_name = "@EMA_STEP_COUNTER@"
Y
Yibing Liu 已提交
3488
        self._params_tmps = []
3489
        for param in default_main_program().global_block().all_parameters():
3490 3491 3492 3493 3494 3495 3496
            if param.do_model_average != False:
                tmp = param.block.create_var(
                    name=unique_name.generate(".".join(
                        [self._name + param.name, 'ema_tmp'])),
                    dtype=param.dtype,
                    persistable=False,
                    stop_gradient=True)
Y
Yibing Liu 已提交
3497
                self._params_tmps.append((param, tmp))
3498

Y
Yibing Liu 已提交
3499 3500
        self._ema_vars = {}
        for param, tmp in self._params_tmps:
3501 3502
            with param.block.program._optimized_guard(
                [param, tmp]), name_scope('moving_average'):
Y
Yibing Liu 已提交
3503
                self._ema_vars[param.name] = self._create_ema_vars(param)
3504 3505 3506 3507

        self.apply_program = Program()
        block = self.apply_program.global_block()
        with program_guard(main_program=self.apply_program):
3508
            decay_pow, global_step = self._get_decay_pow(block)
Y
Yibing Liu 已提交
3509
            for param, tmp in self._params_tmps:
3510 3511
                param = block._clone_variable(param)
                tmp = block._clone_variable(tmp)
Y
Yibing Liu 已提交
3512
                ema = block._clone_variable(self._ema_vars[param.name])
3513
                layers.assign(input=param, output=tmp)
3514
                # bias correction
3515 3516 3517
                with layers.control_flow.Switch() as switch:
                    with switch.case(global_step > 0):
                        layers.assign(output=ema, input=ema / (1.0 - decay_pow))
3518 3519 3520 3521 3522
                layers.assign(input=ema, output=param)

        self.restore_program = Program()
        block = self.restore_program.global_block()
        with program_guard(main_program=self.restore_program):
Y
Yibing Liu 已提交
3523
            for param, tmp in self._params_tmps:
3524 3525 3526 3527
                tmp = block._clone_variable(tmp)
                param = block._clone_variable(param)
                layers.assign(input=tmp, output=param)

3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549
    def _get_ema_decay(self):
        with default_main_program()._lr_schedule_guard():
            decay_var = layers.tensor.create_global_var(
                shape=[1],
                value=self._decay,
                dtype='float32',
                persistable=True,
                name="scheduled_ema_decay_rate")

            if self._thres_steps is not None:
                decay_t = (self._thres_steps + 1.0) / (self._thres_steps + 10.0)
                with layers.control_flow.Switch() as switch:
                    with switch.case(decay_t < self._decay):
                        layers.tensor.assign(decay_t, decay_var)
                    with switch.default():
                        layers.tensor.assign(
                            np.array(
                                [self._decay], dtype=np.float32),
                            decay_var)
        return decay_var

    def _get_decay_pow(self, block):
3550 3551 3552 3553 3554 3555 3556
        global_step = layers.create_global_var(
            name=self._step_counter_name,
            shape=[1],
            value=0,
            dtype='int64',
            persistable=True)
        global_step = layers.cast(global_step, "float32")
3557
        decay_var = block._clone_variable(self._decay_var)
3558 3559
        decay_pow_acc = layers.elementwise_pow(decay_var, global_step)
        return decay_pow_acc, global_step
3560

Y
Yibing Liu 已提交
3561
    def _create_ema_vars(self, param):
3562 3563 3564 3565 3566 3567 3568 3569 3570
        param_ema = layers.create_global_var(
            name=unique_name.generate(self._name + param.name + '_ema'),
            shape=param.shape,
            value=0.0,
            dtype=param.dtype,
            persistable=True)

        return param_ema

Y
Yibing Liu 已提交
3571 3572 3573 3574 3575
    def update(self):
        """ 
        Update Exponential Moving Average. Should only call this method in 
        train program.
        """
3576 3577
        global_step = layers.autoincreased_step_counter(
            counter_name=self._step_counter_name)
3578
        param_master_emas = []
Y
Yibing Liu 已提交
3579 3580 3581 3582
        for param, tmp in self._params_tmps:
            with param.block.program._optimized_guard(
                [param, tmp]), name_scope('moving_average'):
                param_ema = self._ema_vars[param.name]
3583
                if param.name + '.master' in self._ema_vars:
3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600
                    master_ema = self._ema_vars[param.name + '.master']
                    param_master_emas.append([param_ema, master_ema])
                else:
                    ema_t = param_ema * self._decay_var + param * (
                        1 - self._decay_var)
                    layers.assign(input=ema_t, output=param_ema)

        # for fp16 params
        for param_ema, master_ema in param_master_emas:
            default_main_program().global_block().append_op(
                type="cast",
                inputs={"X": master_ema},
                outputs={"Out": param_ema},
                attrs={
                    "in_dtype": master_ema.dtype,
                    "out_dtype": param_ema.dtype
                })
Y
Yibing Liu 已提交
3601

3602 3603 3604 3605 3606 3607 3608
    @signature_safe_contextmanager
    def apply(self, executor, need_restore=True):
        """
        Apply moving average to parameters for evaluation.
        
        Args:
            executor (Executor): The Executor to execute applying.
Y
Yibing Liu 已提交
3609 3610
            need_restore (bool, optional): Whether to restore parameters after 
                applying. Default True.
3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625
        """
        executor.run(self.apply_program)
        try:
            yield
        finally:
            if need_restore:
                self.restore(executor)

    def restore(self, executor):
        """Restore parameters.
        
        Args:
            executor (Executor): The Executor to execute restoring.
        """
        executor.run(self.restore_program)
H
hutuxian 已提交
3626 3627 3628


class PipelineOptimizer(object):
3629
    """
3630
	:api_attr: Static Graph
S
swtkiwi 已提交
3631

3632 3633 3634 3635
    Pipeline Optimizer: Make a program to run as pipeline, that is splitting a
    program into multiple sections (sub-programs) and each section run on a
    device to enable the training of large scale models and the use of
    heterogeneous devices. Meanwhile, all sections run in the stype of pipeline.
H
hutuxian 已提交
3636

3637
    Args:
3638 3639 3640 3641
        optimizer (Optimizer): The optimizer to use, such as SGD.
        num_microbatches (int): Number of microbatches. [Optional. Default:1].
        start_cpu_core_id (int): The first cpu core id to use. [Optional. Default:0].
    
3642 3643
    Examples:
        .. code-block:: python
H
hutuxian 已提交
3644

3645
            import paddle.fluid as fluid
H
hutuxian 已提交
3646 3647
            import paddle.fluid.layers as layers

3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663
            with fluid.device_guard("gpu:0"):
                x = fluid.layers.data(name='x', shape=[1], dtype='int64', lod_level=0)
                y = fluid.layers.data(name='y', shape=[1], dtype='int64', lod_level=0)
                data_loader = fluid.io.DataLoader.from_generator(
                    feed_list=[x, y],
                    capacity=64,
                    use_double_buffer=True,
                    iterable=False)

                emb_x = layers.embedding(input=x, param_attr=fluid.ParamAttr(name="embx"), size=[10,2], is_sparse=False)
                emb_y = layers.embedding(input=y, param_attr=fluid.ParamAttr(name="emby",learning_rate=0.9), size=[10,2], is_sparse=False)

            with fluid.device_guard("gpu:1"):
                concat = layers.concat([emb_x, emb_y], axis=1)
                fc = layers.fc(input=concat, name="fc", size=1, num_flatten_dims=1, bias_attr=False)
                loss = layers.reduce_mean(fc)
H
hutuxian 已提交
3664
            optimizer = fluid.optimizer.SGD(learning_rate=0.5)
3665
            optimizer = fluid.optimizer.PipelineOptimizer(optimizer)
H
hutuxian 已提交
3666
            optimizer.minimize(loss)
3667 3668 3669 3670 3671 3672 3673 3674 3675

            def train_reader():
                for _ in range(4):
                    x = np.random.random(size=[1]).astype('int64')
                    y = np.random.random(size=[1]).astype('int64')
                    yield x, y
            data_loader.set_sample_generator(train_reader, batch_size=1)

            place = fluid.CUDAPlace(0)
H
hutuxian 已提交
3676 3677
            exe = fluid.Executor(place)
            exe.run(fluid.default_startup_program())
3678
            batch_size = 1
H
hutuxian 已提交
3679 3680 3681 3682 3683
            filelist = [] # you should set your own filelist, e.g. filelist = ["dataA.txt"]
            dataset = fluid.DatasetFactory().create_dataset("FileInstantDataset")
            dataset.set_use_var([x,y])
            dataset.set_batch_size(batch_size)
            dataset.set_filelist(filelist)
3684
            data_loader.start()
H
hutuxian 已提交
3685
            exe.train_from_dataset(
3686 3687 3688
                    fluid.default_main_program(),
                    dataset)
            data_loader.reset()
3689 3690
    """

3691
    def __init__(self, optimizer, num_microbatches=1, start_cpu_core_id=0):
Z
zhongpu 已提交
3692 3693
        if framework.in_dygraph_mode():
            raise Exception("In dygraph, don't support PipelineOptimizer.")
3694 3695 3696 3697 3698
        if not isinstance(optimizer, Optimizer):
            raise ValueError("The 'optimizer' parameter for "
                             "PipelineOptimizer must be an instance of "
                             "Optimizer, but the given type is {}.".format(
                                 type(optimizer)))
H
hutuxian 已提交
3699
        self._optimizer = optimizer
3700 3701 3702 3703 3704
        assert num_microbatches >= 1, (
            "num_microbatches must be a positive value.")
        self._num_microbatches = num_microbatches
        assert start_cpu_core_id >= 0, (
            "start_cpu_core_id must be greater than or equal to 0.")
H
hutuxian 已提交
3705
        self._start_cpu_core_id = start_cpu_core_id
3706 3707 3708 3709 3710 3711 3712
        self._place_list = None
        op_maker = core.op_proto_and_checker_maker
        self._op_role = op_maker.OpRole
        self._op_role_key = op_maker.kOpRoleAttrName()
        self._op_role_var_key = op_maker.kOpRoleVarAttrName()
        self._op_device_key = op_maker.kOpDeviceAttrName()
        self._param_device_map = dict()
H
hutuxian 已提交
3713

H
hutuxian 已提交
3714
    def _create_vars(self, block, main_program):
3715
        # Create vars for block, copied from main_program's global block
H
hutuxian 已提交
3716 3717 3718 3719 3720
        used_var_set = set()
        for op_idx in range(block.desc.op_size()):
            op_desc = block.desc.op(op_idx)
            vars = op_desc.input_arg_names() + op_desc.output_arg_names()
            for var in vars:
3721 3722 3723
                # a var whose name contains "blocking_queue" 
                # only exists in startup program 
                if var in used_var_set or "_blocking_queue" in var:
H
hutuxian 已提交
3724 3725 3726
                    continue
                used_var_set.add(var)
                source_var = main_program.block(0).var(str(var))
3727 3728 3729 3730
                if source_var.type == core.VarDesc.VarType.READER:
                    block.create_var(name=var, type=core.VarDesc.VarType.READER)
                else:
                    block._clone_variable(source_var, False)
H
hutuxian 已提交
3731

3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751
    def _is_loss_grad_op(self, op):
        if self._op_role_key not in op.attr_names:
            return False
        op_role = int(op.all_attrs()[self._op_role_key])
        return op_role & int(self._op_role.Backward) and op_role & int(
            self._op_role.Loss)

    def _is_backward_op(self, op):
        return self._op_role_key in op.attr_names and int(op.all_attrs()[
            self._op_role_key]) & int(self._op_role.Backward)

    def _is_optimize_op(self, op):
        return self._op_role_key in op.attr_names and int(op.all_attrs()[
            self._op_role_key]) & int(self._op_role.Optimize)

    def _is_update_op(self, op):
        return 'Param' in op.input_names and 'Grad' in op.input_names and (
            "LearningRate" in op.input_names)

    def _split_program(self, main_program):
H
hutuxian 已提交
3752
        """
3753 3754 3755 3756
        Split a program into sections according to devices that ops run on.

        Args:
            main_program (Program): the main program
H
hutuxian 已提交
3757
        """
3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777
        programs = []
        # Map from device to its corresponding section program info
        device_program_map = dict()
        block = main_program.block(0)

        for op in block.ops:
            device = op.attr(self._op_device_key)

            if device not in device_program_map:
                program = {"program": Program()}
                device_program_map[device] = program
            program = device_program_map[device]
            op_desc = op.desc
            ap_op = program["program"].block(0).desc.append_op()
            ap_op.copy_from(op_desc)

        for key in sorted(device_program_map.keys()):
            program = device_program_map[key]
            program['program']._sync_with_cpp()
            programs.append(program)
H
hutuxian 已提交
3778

3779
        return programs
H
hutuxian 已提交
3780

3781
    def _find_post_op(self, ops, cur_op, var_name):
H
hutuxian 已提交
3782
        """
3783 3784 3785 3786 3787 3788 3789
        Find the real post op that has variable named var_name as input.

        Args:
            ops (list): A list of ops.
            cur_op (Operator): Current operator which has variable named
                               var_name as output.
            var_name (string): Variable name.
H
hutuxian 已提交
3790
        """
3791 3792
        post_op = []
        before = True
H
hutuxian 已提交
3793
        for op in ops:
3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808
            if op == cur_op:
                before = False
                continue
            if before:
                continue
            for in_var_name in op.input_arg_names:
                if in_var_name == var_name:
                    post_op.append(op)
        if post_op:
            if not len(post_op) == 1:
                raise ValueError("Each op can only have one post op.")
            return post_op[0]
        return None

    def _find_real_prev_op(self, ops, cur_op, var_name):
H
hutuxian 已提交
3809
        """
3810 3811 3812 3813 3814 3815 3816
        Find the real previous op that outputs variable named var_name.

        Args:
            ops (list): A list of ops.
            cur_op (Operator): Current operator which has variable named
                               var_name as input.
            var_name (string): Variable name.
H
hutuxian 已提交
3817
        """
3818
        prev_op = []
H
hutuxian 已提交
3819
        for op in ops:
3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858
            if op == cur_op:
                break
            for out_var_name in op.output_arg_names:
                if out_var_name == var_name:
                    prev_op.append(op)
        if prev_op:
            # A op may have more than one prev op,
            # e.g., for 'learning_rate', there may be multiple ops have it as
            # output.
            return prev_op[-1]
        return None

    def _rename_arg(self, op, old_name, new_name):
        op_desc = op.desc
        if isinstance(op_desc, tuple):
            op_desc = op_desc[0]
        op_desc._rename_input(old_name, new_name)
        op_desc._rename_output(old_name, new_name)

    def _create_var(self, block, ref_var, name):
        """
        Create a new var for block, which has the same type,
        shape and dtype as ref_var, then rename it with the
        name `name`.
        """
        new_var = block.create_var(
            name=name,
            shape=ref_var.shape,
            dtype=ref_var.dtype,
            type=ref_var.type,
            lod_level=ref_var.lod_level,
            persistable=False,
            is_data=False,
            need_check_feed=ref_var.desc.need_check_feed())
        return new_var

    def _get_data_var_info(self, block):
        """
        Get all vars whose is_data attribute are true and then rename them.
H
hutuxian 已提交
3859

3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907
        For PipelineTrainer, all data vars are binded to
        minibatch scope, so we have to feed them to the microbatch
        to avoid conflicts. The vars feeded to microbatch have to
        be renamed.
        """
        # A map from var name to the renamed name.
        raw_name_new_name_map = dict()
        # Because we will create vars in block, it is more safe
        # to get all var_names before iteration.
        var_names = list(block.vars.keys())
        for var_name in var_names:
            var = block.var(var_name)
            if not var.is_data:
                continue
            assert var_name not in raw_name_new_name_map, (
                "{} has already been processed.".format(var_name))
            new_name = unique_name.generate(var_name)
            raw_name_new_name_map[var_name] = new_name
            new_var = self._create_var(block, var, new_name)
            new_var.is_data = False

        # map of data to devices that that data on
        data_devices_map = dict()
        for op in block.ops:
            dev_spec = op.attr(self._op_device_key)
            for var_name in op.input_arg_names:
                if var_name not in raw_name_new_name_map:
                    continue
                if not var_name in data_devices_map:
                    data_devices_map[var_name] = []
                if not dev_spec in data_devices_map[var_name]:
                    data_devices_map[var_name].append(dev_spec)
                new_name = raw_name_new_name_map[var_name]
                #self._rename_arg(op, var_name, new_name)
        return data_devices_map, raw_name_new_name_map

    def _rename_var_in_block(self, block, raw_name_new_name_map):
        """
        Rename vars whose names in raw_name_new_name_map to the corresponding
        new names.
        """
        for op in block.ops:
            if op.type == "enqueue" or op.type == "dequeue":
                continue
            for var_name in op.input_arg_names:
                if var_name in raw_name_new_name_map:
                    new_name = raw_name_new_name_map[var_name]
                    self._rename_arg(op, var_name, new_name)
H
hutuxian 已提交
3908

3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991
    def _insert_enq_deq_for_data_var(self, main_block, programs, startup,
                                     devices):
        """
        Insert enqueue and dequeue ops for data var

        Args:
            main_block (Block): Global block for main program
            programs (dict): Dictionary for section params
            startup (Program): Startup program
            devices (list): List of devices in the format (dev:dev_index)
        """
        main_program = main_block.program
        data_devices_map, raw_name_new_name_map = self._get_data_var_info(
            main_block)

        first_prog = programs[0]['program']
        first_block = first_prog.block(0)
        enqueue_index = 0
        if first_block.ops[0].type == "create_py_reader" or (
                first_block.ops[1].type == "create_py_reader"):
            for op in first_block.ops:
                if op.type == "read":
                    enqueue_index += 1
                    break
                enqueue_index += 1
        first_dev_spec = devices[0]
        for var_name in data_devices_map.keys():
            for device in data_devices_map[var_name]:
                # step1: generate queue for each pair of data var and device
                # that that data on
                queue_name = var_name + "_blocking_queue"
                queue_name = unique_name.generate(queue_name)
                queue_var = startup.block(0).create_var(
                    name=queue_name,
                    persistable=True,
                    type=core.VarDesc.VarType.RAW)
                startup.block(0).append_op(
                    type='queue_generator',
                    attrs={
                        'names': [queue_name],
                        'capacity': self._num_microbatches
                    })
                main_var = main_block.var(var_name)
                assert main_var.is_data
                if not var_name in first_block.vars:
                    self._create_var(first_block, main_var, var_name)
                first_block._insert_op(
                    index=enqueue_index,
                    type='enqueue',
                    inputs={'X': first_block.var(var_name)},
                    attrs={
                        'queue_name': queue_name,
                        self._op_device_key: first_dev_spec,
                        self._op_role_key: self._op_role.Forward
                    })
                # Get the device that that data on
                assert device in devices
                prog_index = devices.index(device)
                prog = programs[prog_index]['program']
                block = prog.block(0)
                index = 0
                if device == first_dev_spec:
                    index = enqueue_index + 1
                new_name = raw_name_new_name_map[var_name]
                source_var = main_program.block(0).var(var_name)
                new_var = self._create_var(block, source_var, new_name)
                block._insert_op(
                    index=index,
                    type='dequeue',
                    outputs={'Out': [new_var]},
                    attrs={
                        self._op_device_key: device,
                        self._op_role_key: self._op_role.Forward,
                        'queue_name': queue_name,
                    })
                self._rename_var_in_block(block, raw_name_new_name_map)

    def _strip_grad_suffix(self, name):
        """
        Strip the grad suffix from the given variable name
        """
        pos = name.find(core.grad_var_suffix())
        return name[:pos] if pos != -1 else name
H
hutuxian 已提交
3992

3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011
    def _append_grad_suffix(self, name):
        """
        Append grad suffix to the given variable name
        """
        return name + core.grad_var_suffix()

    def _update_param_device_map(self, params_grads, block):
        for param_grad in params_grads:
            if not param_grad[0].trainable: continue
            param_name = param_grad[0].name
            ops = block.ops
            for op in ops:
                input_arg_names = op.input_arg_names
                if param_name in input_arg_names:
                    self._param_device_map[param_name] = op.attr(
                        self._op_device_key)
                    break

    def _add_opdevice_attr_for_regularization_clip(self, block):
H
hutuxian 已提交
4012
        """
4013
        Add op_device attribute for regulization and clip ops.
H
hutuxian 已提交
4014
        """
4015 4016 4017
        for op in block.ops:
            # role for regularization and clip ops is optimize
            if int(op.attr(self._op_role_key)) != int(self._op_role.Optimize):
H
hutuxian 已提交
4018
                continue
4019 4020 4021 4022 4023 4024 4025 4026 4027
            if op.has_attr(self._op_device_key) and (
                    op.attr(self._op_device_key) != ""):
                continue
            assert self._op_role_var_key in op.attr_names
            op_role_var = op.all_attrs()[self._op_role_var_key]
            assert len(op_role_var) == 2
            param_name = block.vars[op_role_var[0]].name
            device = self._param_device_map[param_name]
            op._set_attr(self._op_device_key, device)
H
hutuxian 已提交
4028

4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065
    def _add_default_opdevice_attr(self, block):
        """
        1. Add default op_device attribute for lr-related ops.
           The default value is the one that of the first place.
        2. Add default op_device attribute for sum ops added during
           backward. For these ops, we set the op_device attribute
           as the one of its post op, i.e, which op has the output of the
           sum op as an input.
        """
        first_devcie = ""

        # Get the device spec of the first place.
        # device_spec: 'cpu' for cpu device and 'gpu:id' for gpu device,
        # e.g. 'gpu:0', 'gpu:1', etc.
        for op in block.ops:
            if op.has_attr(self._op_device_key) and (
                    op.attr(self._op_device_key) != ""):
                first_device = op.attr(self._op_device_key)
                break
        assert first_device

        # set op_device attr for lr-related ops
        lrsched_role = int(self._op_role.LRSched)
        for op in block.ops:
            if not op.has_attr(self._op_device_key) or (
                    op.attr(self._op_device_key) == ""):
                if op.type == "sum":
                    # For sum ops that compute the sum of @RENAMED@ vars
                    for name in op.desc.input_arg_names():
                        assert '@RENAME@' in name
                    assert len(op.desc.output_arg_names()) == 1
                    out_name = op.desc.output_arg_names()[0]
                    post_op = self._find_post_op(block.ops, op, out_name)
                    device = post_op.attr(self._op_device_key)
                    assert device
                    op._set_attr(self._op_device_key, device)
                    continue
H
hutuxian 已提交
4066

4067 4068 4069 4070
                assert op.attr(self._op_role_key) == lrsched_role, (
                    "Op whose op_device attr has not been set for pipeline"
                    " must be of the role LRSched.")
                op._set_attr(self._op_device_key, first_device)
H
hutuxian 已提交
4071

4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395
    def _check_validation(self, block):
        """
        Check whether ops in a block are all validate (i.e., the 
        op_device attribute has been set).
        Then, return all device specifications in order.
        """
        device_specs = []
        for op in block.ops:
            type = op.type
            if not op._has_kernel(type):
                assert op.type == "conditional_block" and (
                    op.attr(self._op_role_key) == int(self._op_role.LRSched)), (
                        "Now, the only supported op without kernel is "
                        "conditional_block, and its op role must be LRSched.")
            assert op.has_attr(self._op_device_key), (
                "op ({}) has no {} attribute.".format(op.type,
                                                      self._op_device_key))
            dev_spec = op.attr(self._op_device_key)
            assert dev_spec, ("op_device attribute for op "
                              "{} has not been set.".format(op.type))
            if not dev_spec in device_specs:
                device_specs.append(dev_spec)
        return device_specs

    def _insert_enq_deq_ops_for_boundaries(self, block, origin_block,
                                           startup_program):
        """
        Insert a pair of enqueue and dequeue ops for every two
        consecutive ops on different devices.
        """
        startup_block = startup_program.global_block()
        extra_index = 0

        # A map from var to device spec where op takes it as input,
        # avoiding multiple enqueue and dequeue ops.
        var_devspec = dict()

        for index, op in list(enumerate(origin_block.ops)):
            cur_device_spec = op.attr(self._op_device_key)
            for var_name in op.input_arg_names:
                # i.e., lod_tensor_blocking_queue created by DataLoader,
                # which only exists in startup program.
                if not var_name in origin_block.vars: continue
                var = block.var(var_name)
                # skip data, because we will process it later
                if var.is_data: continue
                prev_op = self._find_real_prev_op(origin_block.ops, op,
                                                  var_name)
                if prev_op is None:
                    continue
                prev_device_spec = prev_op.attr(self._op_device_key)

                if prev_device_spec != cur_device_spec:
                    if var_name not in var_devspec:
                        var_devspec[var_name] = []
                    if cur_device_spec in var_devspec[var_name]: continue
                    var_devspec[var_name].append(cur_device_spec)

                    queue_name = var_name + "_blocking_queue"
                    queue_name = unique_name.generate(queue_name)
                    queue_var = startup_block.create_var(
                        name=queue_name,
                        persistable=True,
                        type=core.VarDesc.VarType.RAW)
                    startup_block.append_op(
                        type='queue_generator',
                        attrs={
                            'names': [queue_name],
                            'capacity': self._num_microbatches
                        })
                    op_role = op.all_attrs()[self._op_role_key]
                    var = block.vars[var_name]
                    block._insert_op(
                        index=index + extra_index,
                        type='enqueue',
                        inputs={'X': var},
                        attrs={
                            'queue_name': queue_name,
                            self._op_device_key: prev_device_spec,
                            self._op_role_key: op_role
                        })
                    extra_index += 1
                    block._insert_op(
                        index=index + extra_index,
                        type='dequeue',
                        outputs={'Out': [var]},
                        attrs={
                            self._op_device_key: cur_device_spec,
                            'queue_name': queue_name,
                            self._op_role_key: op_role
                        })
                    extra_index += 1

    def _add_dequeue_ops_for_optimize(self, block, startup_program):
        startup_block = startup_program.global_block()
        grad_queue_map = dict()
        grad_device_map = dict()
        optimize_index = None
        grad_names_to_dequeue = []

        for index, op in reversed(list(enumerate(block.ops))):
            device = op.attr(self._op_device_key)
            # Optimizer pass
            if not self._is_optimize_op(op):
                optimize_index = index + 1
                break
            if not self._is_update_op(op): continue
            assert self._op_role_var_key in op.attr_names
            op_role_var = op.all_attrs()[self._op_role_var_key]
            assert len(op_role_var) == 2
            grad_name = op_role_var[1]
            assert grad_name not in grad_device_map
            assert grad_name not in grad_names_to_dequeue
            grad_device_map[grad_name] = device
            grad_names_to_dequeue.append(grad_name)

        for grad_name in grad_names_to_dequeue:
            device = grad_device_map[grad_name]
            grad_names = []
            grads = []
            queue_name = grad_name + "_blocking_queue"
            queue_name = unique_name.generate(queue_name)
            grad_queue_map[grad_name] = queue_name
            ref_var = block.vars[grad_name]
            queue_var = startup_block.create_var(
                name=queue_name,
                persistable=True,
                type=core.VarDesc.VarType.RAW)
            startup_block.append_op(
                type='queue_generator',
                attrs={
                    'names': [queue_name],
                    'capacity': self._num_microbatches
                })
            orig_var_name = self._strip_grad_suffix(grad_name)
            for _ in range(self._num_microbatches):
                u_name = unique_name.generate(orig_var_name)
                u_grad_name = self._append_grad_suffix(u_name)
                grad_var = self._create_var(block, ref_var, u_grad_name)
                grad_names.append(u_grad_name)
                grads.append(grad_var)
            block._insert_op(
                index=optimize_index,
                type='dequeue',
                outputs={'Out': grads},
                attrs={
                    self._op_device_key: device,
                    'queue_name': queue_name,
                    self._op_role_key: self._op_role.Optimize
                })
            block._insert_op(
                index=optimize_index + 1,
                type='sum',
                inputs={'X': grad_names},
                outputs={'Out': ref_var},
                attrs={
                    self._op_device_key: device,
                    self._op_role_key: self._op_role.Optimize
                })
        return grad_queue_map

    def _insert_enq_deq_ops_for_update(self, block, startup_program):
        """
        Insert enqueue and dequeue ops for gradients of parameters.
        """
        startup_block = startup_program.global_block()
        grad_queue_map = self._add_dequeue_ops_for_optimize(block,
                                                            startup_program)

        for index, op in reversed(list(enumerate(block.ops))):
            offset = index
            device = op.attr(self._op_device_key)

            # Backward pass
            if self._is_loss_grad_op(op):
                loss_grad_var = block.vars[op.output_arg_names[0]]
                scale_factor = self._num_microbatches
                block._insert_op(
                    index=index + 1,
                    type='scale',
                    inputs={'X': loss_grad_var},
                    outputs={'Out': loss_grad_var},
                    attrs={
                        'scale': 1.0 / scale_factor,
                        self._op_device_key: device,
                        self._op_role_key: self._op_role.Backward
                    })
                break
            if self._is_backward_op(op) and (
                    self._op_role_var_key in op.attr_names):
                op_role_var = op.all_attrs()[self._op_role_var_key]

                if len(op_role_var) == 0:
                    continue
                assert len(op_role_var) % 2 == 0
                for i in range(0, len(op_role_var), 2):
                    grad_name = op_role_var[i + 1]
                    grad_var = block.vars[grad_name]
                    assert grad_name in grad_queue_map
                    queue_name = grad_queue_map[grad_name]
                    block._insert_op(
                        index=offset + 1,
                        type='enqueue',
                        inputs={'X': block.vars[grad_name]},
                        attrs={
                            'queue_name': queue_name,
                            self._op_device_key: device,
                            self._op_role_key: self._op_role.Backward
                        })
                    offset += 1

    def _add_sub_blocks(self, main_block, program_list):
        main_program = main_block.program
        for prog_info in program_list:
            prog = prog_info['program']
            for op in prog.block(0).ops:
                if not op.has_attr('sub_block'):
                    continue
                origin_sub_block_id = op.attr('sub_block').id
                origin_sub_block = main_program.block(origin_sub_block_id)
                new_sub_block = prog._create_block(parent_idx=0)
                for op in origin_sub_block.ops:
                    op_desc = op.desc
                    ap_op = new_sub_block.desc.append_op()
                    ap_op.copy_from(op_desc)
                new_sub_block._sync_with_cpp()
                op._set_attr('sub_block:', new_sub_block)

    def _get_device_info(self, block):
        for op in block.ops:
            if not op._has_kernel(op.type): continue
            op_device = op.attr(self._op_device_key)
            return op_device

    def _process_persistable_vars_in_multi_sections(self, main_program,
                                                    startup_prog, program_list):
        """
        Special Case: process persistable vars that exist in
        multiple sections, e.g., shared weight
        """
        # var_info = {var_name: [program1, program2...]},
        # persistable var only
        var_info = dict()
        for prog_info in program_list:
            prog = prog_info['program']
            block = prog.block(0)
            for var_name in block.vars:
                var = block.var(var_name)
                if not var.persistable: continue
                if not var_name in var_info:
                    var_info[var_name] = []
                if not prog in var_info[var_name]:
                    var_info[var_name].append(prog)
        for var_name in list(var_info.keys()):
            if len(var_info[var_name]) == 1:
                var_info.pop(var_name)

        # write_info = {var_name: program}, where program is the only program
        # in which the var named var_name is written.
        write_info = dict()
        for var_name in var_info.keys():
            for prog in var_info[var_name]:
                block = prog.block(0)
                for op in block.ops:
                    if op.type == "dequeue": continue
                    # We have processed lr related vars
                    if op.attr(self._op_role_key) == int(
                            self._op_role.Optimize.LRSched):
                        continue
                    if var_name in op.desc.output_arg_names():
                        assert var_name not in write_info, (
                            "two sections write the same var({}): second "
                            "op {}.".format(var_name, op))
                        write_info[var_name] = prog
                        break

        for var_name in var_info.keys():
            # Case 1: read only variables, no special process
            if not var_name in write_info: continue

            # Case 2: one write multiple reads
            write_prog = write_info[var_name]
            write_block = write_prog.block(0)
            write_device = self._get_device_info(write_block)
            all_progs = var_info[var_name]
            for prog in all_progs:
                if prog == write_prog: continue

                queue_name = var_name + "_blocking_queue"
                queue_name = unique_name.generate(queue_name)
                queue_var = startup_prog.block(0).create_var(
                    name=queue_name,
                    persistable=True,
                    type=core.VarDesc.VarType.RAW)
                startup_prog.block(0).append_op(
                    type='queue_generator',
                    attrs={
                        'names': [queue_name],
                        'capacity': self._num_microbatches
                    })
                write_block._insert_op(
                    index=0,
                    type='enqueue',
                    inputs={'X': write_block.var(var_name), },
                    attrs={
                        'queue_name': queue_name,
                        self._op_device_key: write_device,
                        # A trick to make the role LRSched to avoid copy every
                        # microbatch
                        self._op_role_key: self._op_role.LRSched
                    })
                read_block = prog.block(0)
                read_device = self._get_device_info(read_block)
                read_block._insert_op(
                    index=0,
                    type='dequeue',
                    outputs={'Out': [read_block.var(var_name)]},
                    attrs={
                        self._op_device_key: read_device,
                        # A trick to make the role LRSched to avoid copy every
                        # microbatch
                        self._op_role_key: self._op_role.LRSched,
                        'queue_name': queue_name,
                    })
H
hutuxian 已提交
4396 4397 4398 4399 4400 4401

    def minimize(self,
                 loss,
                 startup_program=None,
                 parameter_list=None,
                 no_grad_set=None):
4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443
        main_block = loss.block
        if startup_program is None:
            startup_program = default_startup_program()
        optimize_ops, params_grads = self._optimizer.minimize(
            loss, startup_program, parameter_list, no_grad_set)
        self._update_param_device_map(params_grads, main_block)

        # Step1: add default op_device attribute for regulization and clip ops
        self._add_opdevice_attr_for_regularization_clip(main_block)

        # Step2: add default op_device attribute for ops whose op_device
        # attribute have not been set yet.
        self._add_default_opdevice_attr(main_block)
        device_specs = self._check_validation(main_block)

        # Step3: add enqueue and dequeue ops between section boundaries
        origin_prog = main_block.program.clone(for_test=False)
        origin_main_block = origin_prog.global_block()
        self._insert_enq_deq_ops_for_boundaries(main_block, origin_main_block,
                                                startup_program)

        # Step4: add a pair of enqueue and dequeueN for parameter gradients
        self._insert_enq_deq_ops_for_update(main_block, startup_program)

        main_program = main_block.program

        place_list = []
        place_id_list = []
        for dev_spec in device_specs:
            if dev_spec == "cpu":
                place_list.append(core.CPUPlace())
                place_id_list.append(-1)
            elif "gpu" in dev_spec and ":" in dev_spec:
                dev_index = dev_spec.split(":")[1]
                place_list.append(core.CUDAPlace(int(dev_index)))
                place_id_list.append(int(dev_index))
            else:
                raise ValueError("Unknown device type: %s", dev_spec)

        # Step5: split program into sections and add pairs of
        # enqueue and dequeue ops for data var.
        if len(place_list) == 0:
H
hutuxian 已提交
4444
            program_list = []
4445 4446 4447 4448 4449
            ptmp = {
                "program": main_program,
                "input_set": set(),
                "output_set": set()
            }
H
hutuxian 已提交
4450 4451
            program_list.append(ptmp)
        else:
4452
            program_list = self._split_program(main_program)
H
hutuxian 已提交
4453
            for p in program_list:
4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466
                self._create_vars(p["program"].block(0), main_program)
        self._insert_enq_deq_for_data_var(main_block, program_list,
                                          startup_program, device_specs)

        # Step6: Special Case: process persistable vars that exist in
        # multiple sections
        self._process_persistable_vars_in_multi_sections(
            main_program, startup_program, program_list)

        # Step7: Add sub blocks for section programs
        self._add_sub_blocks(main_block, program_list)

        main_program._pipeline_opt = {
H
hutuxian 已提交
4467 4468 4469
            "trainer": "PipelineTrainer",
            "device_worker": "Section",
            "section_program_list": program_list,
4470 4471 4472
            "place_list": place_list,
            "place_id_list": place_id_list,
            "sync_steps": -1,
L
lilong12 已提交
4473
            "num_microbatches": self._num_microbatches,
H
hutuxian 已提交
4474 4475
            "start_cpu_core_id": self._start_cpu_core_id,
        }
4476
        return optimize_ops, params_grads, program_list
M
mapingshuo 已提交
4477 4478


M
mapingshuo 已提交
4479 4480
class RecomputeOptimizer(Optimizer):
    """
4481
	:api_attr: Static Graph
S
swtkiwi 已提交
4482

M
mapingshuo 已提交
4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542
    Recompute Optimizer Wrapper

    Normally, a training step contains three sub-steps: first, run forward
    Operators to calculate the loss; second, run backward Operators to 
    calculate gradient of the parameters; third, apply optimization method
    to update the value of the parameters.

    In the forward computation process, all variables that are needed by 
    backward computation process will be kept in memory, which occupy a great
    amount of memory when the network becomes very deep.

    Recompute split the network to k segments. In each segment, It will 
    recompute the forward Operators, before running backward operators. It is
    very helpful for saving memory.
 
    The Variables that separate a network to segments are called as checkpoints,
    and users should set it manually. The usage is very simple:

    Args:
        optimizer (Optimizer): The optimizer that is applied to parameters.

    Examples:
        .. code-block:: python

            import paddle.fluid as fluid
            import numpy as np
            def gen_data():
                return {"x": np.random.random(size=(32, 32)).astype('float32'),
                "y": np.random.randint(2, size=(32, 1)).astype('int64')}
            def mlp(input_x, input_y, hid_dim=128, label_dim=2):
                print(input_x)
                fc_1 = fluid.layers.fc(input=input_x, size=hid_dim)
                prediction = fluid.layers.fc(input=[fc_1], size=label_dim, act='softmax')
                cost = fluid.layers.cross_entropy(input=prediction, label=input_y)
                sum_cost = fluid.layers.reduce_mean(cost)
                return sum_cost, fc_1, prediction
            input_x = fluid.layers.data(name="x", shape=[32], dtype='float32')
            input_y = fluid.layers.data(name="y", shape=[1], dtype='int64')
            cost, fc_1, pred = mlp(input_x, input_y)

            sgd = fluid.optimizer.Adam(learning_rate=0.01)
            sgd = fluid.optimizer.RecomputeOptimizer(sgd)
            sgd._set_checkpoints([fc_1, pred])
            sgd.minimize(cost)

            print("Finished optimize")
            place = fluid.CPUPlace()
            exe = fluid.Executor(place)
            exe.run(fluid.default_startup_program())
            step = 10

            for i in range(step):
                cost_val = exe.run(feed=gen_data(),
                       program=fluid.default_main_program(),
                       fetch_list=[cost.name])
                print("step=%d cost=%f" % (i, cost_val[0]))

    """

    def __init__(self, optimizer):
Z
zhongpu 已提交
4543 4544
        if framework.in_dygraph_mode():
            raise Exception("In dygraph, don't support RecomputeOptimizer.")
M
mapingshuo 已提交
4545 4546
        self._optimizer = optimizer
        self._checkpoints = None
M
mapingshuo 已提交
4547 4548
        self._learning_rate = self._optimizer._learning_rate
        self._learning_rate_map = self._optimizer._learning_rate_map
M
mapingshuo 已提交
4549 4550

    def _set_checkpoints(self, checkpoints):
4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561
        """
        Args:
            checkpoints (list): List of Variable or string    
        """
        assert isinstance(
            checkpoints, list
        ), "_checkpoints should be a list of Variable or a list of String"
        for ckpt in checkpoints:
            assert (
                isinstance(ckpt, six.string_types) or isinstance(ckpt, Variable)
            ), "_checkpoints should be a list of Variable or a list of String"
M
mapingshuo 已提交
4562 4563 4564 4565
        self._checkpoints = checkpoints

    def load(self, stat_dict):
        """
4566
	:api_attr: Static Graph
S
swtkiwi 已提交
4567

M
mapingshuo 已提交
4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634
        load function is not supported by Recompute Optimizer for now.
        :return: None

        Args:
            stat_dict: the dict load by load_persistable method

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import paddle.compat as cpt
                
                def mlp(input_x, input_y, hid_dim=128, label_dim=2):
                    fc_1 = fluid.layers.fc(input=input_x, size=hid_dim)
                    prediction = fluid.layers.fc(input=[fc_1], size=label_dim, act='softmax')
                    cost = fluid.layers.cross_entropy(input=prediction, label=input_y)
                    sum_cost = fluid.layers.reduce_mean(cost)
                    return sum_cost, fc_1, prediction
                
                input_x = fluid.layers.data(name="x", shape=[32], dtype='float32')
                input_y = fluid.layers.data(name="y", shape=[1], dtype='int64')
                cost, fc_1, pred = mlp(input_x, input_y)
                print("Finished FF")
                
                sgd = fluid.optimizer.Adam(learning_rate=0.01)
                sgd = fluid.optimizer.RecomputeOptimizer(sgd)
                sgd._set_checkpoints([fc_1, pred])
                try:
                    stat_dict = {}
                    sgd.load(stat_dict)
                except NotImplementedError as e:
                    print(cpt.get_exception_message(e))
        """
        raise NotImplementedError(
            "load function is not supported by Recompute Optimizer for now")

    def apply_gradients(self, params_grads):
        """
        call apply_gradients function of self._optimizer.

        Args:
            params_grads (list): list of (param, grad) pair to do optimization.

        Returns:
            list: A list of operators appended to the current program.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import paddle.fluid.framework as framework

                def mlp(input_x, input_y, hid_dim=128, label_dim=2):
                    fc_1 = fluid.layers.fc(input=input_x, size=hid_dim)
                    prediction = fluid.layers.fc(input=[fc_1], size=label_dim, act='softmax')
                    cost = fluid.layers.cross_entropy(input=prediction, label=input_y)
                    sum_cost = fluid.layers.reduce_mean(cost)
                    return sum_cost, fc_1, prediction


                input_x = fluid.layers.data(name="x", shape=[32], dtype='float32')
                input_y = fluid.layers.data(name="y", shape=[1], dtype='int64')
                cost, fc_1, pred = mlp(input_x, input_y)
                print("Finished FF")

                sgd = fluid.optimizer.Adam(learning_rate=0.01)
                sgd = fluid.optimizer.RecomputeOptimizer(sgd)
4635
                sgd._set_checkpoints([fc_1, pred])
M
mapingshuo 已提交
4636 4637 4638 4639
                params_grads = sgd.backward(
                    cost,
                    startup_program=None,
                    parameter_list=None,
4640
                    no_grad_set=None)
M
mapingshuo 已提交
4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655

                program = cost.block.program
                with framework.program_guard(program, None):
                    optimize_ops = sgd.apply_gradients(params_grads)

                print("Finished apply gradients")
        """

        return self._optimizer.apply_gradients(params_grads=params_grads)

    def backward(self,
                 loss,
                 startup_program=None,
                 parameter_list=None,
                 no_grad_set=None,
4656
                 callbacks=None):
M
mapingshuo 已提交
4657 4658 4659 4660 4661 4662 4663
        """
        call append_backward with checkpoints.

        Args:
            loss (Variable): loss variable to run optimizations.
            startup_program (Program): startup_program for initializing parameters
                in `parameter_list`.
4664 4665
            parameter_list (list): list of Variables or Variable.names to update.
            no_grad_set (set|None): set of Variables or Variables.names should be ignored.
M
mapingshuo 已提交
4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689
            callbacks (list|None): list of callables to run when appending backward
                operator for one parameter.
            checkpoints (list): list of Variables as checkpoints

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
    
                def mlp(input_x, input_y, hid_dim=128, label_dim=2):
                    fc_1 = fluid.layers.fc(input=input_x, size=hid_dim)
                    prediction = fluid.layers.fc(input=[fc_1], size=label_dim, act='softmax')
                    cost = fluid.layers.cross_entropy(input=prediction, label=input_y)
                    sum_cost = fluid.layers.reduce_mean(cost)
                    return sum_cost, fc_1, prediction
    
    
                input_x = fluid.layers.data(name="x", shape=[32], dtype='float32')
                input_y = fluid.layers.data(name="y", shape=[1], dtype='int64')
                cost, fc_1, pred = mlp(input_x, input_y)
                print("Finished FF")
    
                sgd = fluid.optimizer.Adam(learning_rate=0.01)
                sgd = fluid.optimizer.RecomputeOptimizer(sgd)
4690
                sgd._set_checkpoints([fc_1, pred])
M
mapingshuo 已提交
4691 4692 4693 4694
                params_grads = sgd.backward(
                    cost,
                    startup_program=None,
                    parameter_list=None,
4695
                    no_grad_set=None)
M
mapingshuo 已提交
4696 4697
                print("Finished backward")
        """
4698 4699
        assert (self._checkpoints is not None
                ), "You should call _set_checkpoints first"
M
mapingshuo 已提交
4700 4701 4702 4703 4704 4705 4706 4707

        if framework.in_dygraph_mode():
            raise NotImplementedError(
                "DyGraph current does not support recompute")

        self._dtype = loss.dtype
        program = loss.block.program
        with program_guard(program, startup_program):
4708 4709 4710 4711 4712 4713 4714
            checkpoint_vars = []
            for ckpt in self._checkpoints:
                if isinstance(ckpt, Variable):
                    checkpoint_vars.append(ckpt)
                else:
                    checkpoint_vars.append(loss.block.var(ckpt))

M
mapingshuo 已提交
4715
            params_grads = append_backward(
4716
                loss, parameter_list, no_grad_set, checkpoints=checkpoint_vars)
4717 4718
            # Note: since we can't use all_reduce_op now,
            #  dgc_op should be the last op of one grad.
M
mapingshuo 已提交
4719 4720
            if hasattr(self._optimizer, "_append_dgc_ops"):
                self._optimizer._append_dgc_ops(params_grads)
M
mapingshuo 已提交
4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739
        return params_grads

    def apply_optimize(self, loss, startup_program, params_grads):
        """
        call the apply_optimize function of self._optimizer
        Args:
            loss (Variable): loss variable to run optimizations.
            startup_program (Program): startup_program for initializing parameters
                in `parameter_list`.
            params_grads (list): list of (param, grad) pair to do optimization.
        Examples:
            .. code-block:: python
                import paddle.fluid as fluid
                
                def mlp(input_x, input_y, hid_dim=128, label_dim=2):
                    fc_1 = fluid.layers.fc(input=input_x, size=hid_dim)
                    prediction = fluid.layers.fc(input=[fc_1], size=label_dim, act='softmax')
                    cost = fluid.layers.cross_entropy(input=prediction, label=input_y)
                    sum_cost = fluid.layers.reduce_mean(cost)
M
mapingshuo 已提交
4740
                    return sum_cost, fc_1, prediction                
M
mapingshuo 已提交
4741 4742 4743 4744 4745 4746 4747 4748
                
                input_x = fluid.layers.data(name="x", shape=[32], dtype='float32')
                input_y = fluid.layers.data(name="y", shape=[1], dtype='int64')
                cost, fc_1, pred = mlp(input_x, input_y)
                print("Finished FF")
                
                sgd = fluid.optimizer.Adam(learning_rate=0.01)
                sgd = fluid.optimizer.RecomputeOptimizer(sgd)
4749
                sgd._set_checkpoints([fc_1, pred])
M
mapingshuo 已提交
4750 4751 4752 4753
                params_grads = sgd.backward(
                    cost,
                    startup_program=None,
                    parameter_list=None,
4754
                    no_grad_set=None)
M
mapingshuo 已提交
4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768
                
                optimize_ops = sgd.apply_optimize(
                    cost, startup_program=None, params_grads=params_grads)
                
                print("Finished apply_optimize")
        """

        return self._optimizer.apply_optimize(
            loss, startup_program=startup_program, params_grads=params_grads)

    def minimize(self,
                 loss,
                 startup_program=None,
                 parameter_list=None,
4769
                 no_grad_set=None):
4770
        assert isinstance(loss, Variable), "The loss should be an Variable."
M
mapingshuo 已提交
4771 4772 4773 4774 4775 4776 4777 4778 4779
        assert (self._checkpoints is not None
                ), "You should call _set_checkpoints first"
        if framework.in_dygraph_mode():
            raise NotImplementedError(
                "DyGraph current does not support recompute")
        params_grads = self.backward(
            loss,
            startup_program=startup_program,
            parameter_list=parameter_list,
4780
            no_grad_set=no_grad_set)
M
mapingshuo 已提交
4781 4782 4783 4784 4785 4786 4787

        optimize_ops = self.apply_optimize(
            loss, startup_program=startup_program, params_grads=params_grads)

        return optimize_ops, params_grads


M
mapingshuo 已提交
4788 4789
class LookaheadOptimizer(object):
    """
4790
	:api_attr: Static Graph
S
swtkiwi 已提交
4791

M
mapingshuo 已提交
4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844
    This implements the Lookahead optimizer of the
    paper : https://arxiv.org/abs/1907.08610.

    Lookahead keeps two sets of params: the fast_params and
    the slow_params. inner_optimizer update fast_params every 
    training step. Lookahead updates the slow_params and fast_params 
    every k training steps as follows:

    .. math::
        
        slow\_param_t &= slow\_param_{t-1} + \\alpha * (fast\_param_{t-1} - slow\_param_{t-1})
	
	fast\_param_t &=  slow\_param_t

    Args:
        inner_optimizer (Optimizer): The optimizer that update fast params step by step. 
        alpha (float): The learning rate of Lookahead.
        k (int): The slow params is updated every k steps.

    Examples:
        .. code-block:: python

            import paddle
            import paddle.fluid as fluid
            import numpy as np

	    x = fluid.layers.data(name='x', shape=[2], dtype='float32')
	    label = fluid.layers.data(name="label", shape=[1], dtype="int64")
	    y = fluid.layers.fc(input=[x], size=2, act="softmax")
	    loss = fluid.layers.cross_entropy(input=y, label=label)
	    loss = fluid.layers.mean(x=loss)
	    sgd = fluid.optimizer.SGD(learning_rate=0.01)
	    optimizer = fluid.optimizer.LookaheadOptimizer(sgd,
                                            alpha=0.5,
                                            k=5)
	    optimizer.minimize(loss)
	    main_program = fluid.default_main_program()
	    place = fluid.CPUPlace()
	    exe = fluid.Executor(place)
	    exe.run(fluid.default_startup_program())

	    feeder = fluid.DataFeeder(feed_list=[x, label], place=place)

	    step = 0
            while(step < 10):
                step += 1
		exe.run(fluid.default_main_program(),
            	feed=feeder.feed(batch_data))

    """

    def __init__(self, inner_optimizer, alpha=0.5, k=5):

Z
zhongpu 已提交
4845 4846
        if framework.in_dygraph_mode():
            raise Exception("In dygraph, don't support LookaheadOptimizer.")
M
mapingshuo 已提交
4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897
        assert (inner_optimizer is not None), "inner optimizer can not be None"
        assert (
            0.0 <= alpha <= 1.0
        ), "alpha should be larger or equal to 0.0, and less or equal than 1.0"
        assert (isinstance(k, int) and k > 0), "k should be a positive integer"

        self.inner_optimizer = inner_optimizer
        self.alpha = alpha
        self.k = k
        self.type = "lookahead"

    def minimize(self, loss, startup_program=None):

        # Apply inner optimizer to the main_program
        mini_out = self.inner_optimizer.minimize(
            loss, startup_program=startup_program)

        # Get startup_program and main_program
        if startup_program is None:
            startup_program = default_startup_program()
        main_block = loss.block

        # add some vars to the main_program
        params = [param.name for param in main_block.all_parameters()]
        param_to_slow = {}
        for param in params:
            fast_var = main_block.var(param)
            assert (fast_var is not None)
            slow_var = main_block.create_var(
                name=param + "@SLOW",
                shape=fast_var.shape,
                dtype=fast_var.dtype,
                persistable=True)
            param_to_slow[param] = slow_var

        # add some vars to the startup_program
        startup_block = startup_program.global_block()
        for param in params:
            fast_var = startup_block.var(param)
            assert (fast_var is not None)
            slow_var = startup_block.create_var(
                name=param + "@SLOW",
                shape=fast_var.shape,
                dtype=fast_var.dtype,
                persistable=True)

            startup_block.append_op(
                type="assign",
                inputs={"X": fast_var},
                outputs={"Out": slow_var})

4898 4899 4900 4901 4902 4903 4904 4905
        with framework.program_guard(main_block.program, startup_program):
            # Add Var k to main prog and startup prog
            k = layers.create_global_var(
                name="lookahead_k",
                shape=[1],
                value=int(self.k),
                dtype='int32',
                persistable=True)
M
mapingshuo 已提交
4906

4907 4908 4909 4910 4911 4912 4913
            # Add Var alpha to main prog and startup prog
            alpha = layers.create_global_var(
                name="lookahead_alpha",
                shape=[1],
                value=float(self.alpha),
                dtype='float32',
                persistable=True)
M
mapingshuo 已提交
4914

4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945
            # Add Var step
            step = layers.create_global_var(
                name="lookahead_step",
                shape=[1],
                value=int(0),
                dtype='int32',
                persistable=True)
            layers.increment(x=step, value=1.0, in_place=True)

            # lookahead
            zero_var = layers.fill_constant(
                shape=[1], dtype='float32', value=0.0)

            one_var = layers.fill_constant(
                shape=[1], dtype='float32', value=1.0)

            mod = layers.elementwise_mod(step, k)
            with layers.control_flow.Switch() as switch:
                with switch.case(mod == zero_var):
                    for param_name in params:
                        fast_var = main_block.var(param_name)
                        slow_var = param_to_slow[param_name]
                        tmp_var = layers.elementwise_add(
                            layers.elementwise_mul(fast_var, alpha),
                            layers.elementwise_mul(
                                slow_var,
                                layers.elementwise_sub(one_var, alpha)))
                        layers.assign(input=tmp_var, output=slow_var)
                        layers.assign(input=tmp_var, output=fast_var)
                with switch.default():
                    pass
M
mapingshuo 已提交
4946
        return mini_out
4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019


class GradientMergeOptimizer(object):
    """
    Gradient Merge, also called as Gradient Accumulation,
    is a training strategy for larger batches. With this strategy,
    the parameter will not be updated until specific steps.

    For each step, the forward network and the backward network
    will run to calculate the gradient of the parameters.

    For every k step, the optimization network will run,
    applying a specific optimization method (such as SGD, Adam)
    to the parameters.

    Args:
        inner_optimizer (Optimizer): The specific optimization (such as SGD, Adam)
            which update the parameters
        k_steps (int): the update period of the parameters
        avg (bool): whether to average the gradients of each mini-batch,
            the default value is `True`

    Examples:
        .. code-block:: python

        import paddle.fluid as fluid
        import numpy as np

        def gen_data(batch_size):
            return {"x": np.random.random(size=(batch_size, 32)).astype('float32'),
                    "y": np.random.random(size=(batch_size, 1)).astype('int64')}

        def mlp(input_x, input_y, hid_dim=128, label_dim=2):
            fc_1 = fluid.layers.fc(input=input_x, size=hid_dim)
            prediction = fluid.layers.fc(input=[fc_1], size=label_dim, act='softmax')
            cost = fluid.layers.cross_entropy(input=prediction, label=input_y)
            sum_cost = fluid.layers.reduce_mean(cost)
            return sum_cost, fc_1, prediction

        input_x = fluid.layers.data(name="x", shape=[32], dtype='float32')
        input_y = fluid.layers.data(name="y", shape=[1], dtype='int64')
        cost, fc_1, pred = mlp(input_x, input_y)
        sgd = fluid.optimizer.Adam(learning_rate=0.01)
        sgd = fluid.optimizer.GradientMergeOptimizer(sgd, k_steps=4, avg=True)
        sgd.minimize(cost)

        place = fluid.CPUPlace()
        exe = fluid.Executor(place)
        exe.run(fluid.default_startup_program())

        for i in range(10):
            cost_val = exe.run(feed=gen_data(32),
                       program=fluid.default_main_program(),
                       fetch_list=[cost.name])
            print("step=%d, cost=%f" % (i, cost_val[0]))
    """

    def __init__(self, inner_optimizer, k_steps=1, avg=True):
        if framework.in_dygraph_mode():
            raise Exception(
                "In dygraph, we don't support GradientMergeOptimizer."
                "You can do Gradient merge by yourself with k-times forward + backward, "
                "and one-time optimizer.minimize()")

        assert (inner_optimizer is not None), "inner optimizer can not be None"
        assert (isinstance(k_steps, int) and
                k_steps > 0), "k_steps should be a positive integer"

        self.inner_optimizer = inner_optimizer
        self.k_steps = k_steps
        self.type = "gradient_merge"
        self.avg = avg

5020 5021 5022 5023 5024 5025
    def _set_k_steps(self, k_steps):
        self.k_steps = k_steps

    def _set_avg(self, avg):
        self.avg = avg

5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172
    def minimize(self,
                 loss,
                 startup_program=None,
                 parameter_list=None,
                 no_grad_set=None):

        assert isinstance(loss, Variable), "The loss should be an Variable."
        assert (
            parameter_list is None
        ), "The parameter_list should be None when using GradientMergeOptimizer"
        assert (
            no_grad_set is None
        ), "The no_grad_set should be None when using GradientMergeOptimizer"

        params_grads = self.inner_optimizer.backward(
            loss, startup_program=startup_program)

        #TODO(mapingshuo) support sparse embedding
        for k, v in params_grads:
            assert (
                v.type != core.VarDesc.VarType.SELECTED_ROWS
            ), "SELECTED_ROWS is not supported in GradientMergeOptimizer for now"

        param_to_grad = {k.name: v for (k, v) in params_grads}

        # Get startup_program and main_program
        if startup_program is None:
            startup_program = default_startup_program()
        main_block = loss.block

        # add some vars to the main_program and startup_program
        startup_block = startup_program.global_block()
        param_names = param_to_grad.keys()
        param_to_gradient_merge = {}

        for param_name in param_names:
            param_var = main_block.var(param_name)
            assert (param_var is not None)
            gradient_merge_var = main_block.create_var(
                name=param_name + "@GRAD@GradientMerge",
                shape=param_var.shape,
                dtype=param_var.dtype,
                persistable=True)
            param_to_gradient_merge[param_name] = gradient_merge_var
            startup_gradient_merge_var = startup_block.create_var(
                name=param_name + "@GRAD@GradientMerge",
                shape=param_var.shape,
                dtype=param_var.dtype,
                persistable=True)
            startup_block.append_op(
                type="fill_constant",
                outputs={"Out": startup_gradient_merge_var},
                attrs={
                    "shape": param_var.shape,
                    "dtype": param_var.dtype,
                    "value": float(0),
                })

        with framework.program_guard(main_block.program, startup_program):
            # Add Var k to main prog and startup prog
            gradient_merge_k = layers.create_global_var(
                name="gradient_merge_k",
                shape=[1],
                value=int(self.k_steps),
                dtype='int32',
                persistable=True)

            # Add Var step
            gradient_merge_step = layers.create_global_var(
                name="gradient_merge_step",
                shape=[1],
                value=int(0),
                dtype='int32',
                persistable=True)
            layers.increment(x=gradient_merge_step, value=1.0, in_place=True)

            # gradient merge
            zero_var = layers.fill_constant(
                shape=[1], dtype='float32', value=0.0)
            one_var = layers.fill_constant(
                shape=[1], dtype='float32', value=1.0)

            mod = layers.elementwise_mod(gradient_merge_step, gradient_merge_k)
            with layers.control_flow.Switch() as switch:
                with switch.case(mod != zero_var):
                    # 1. update the gradient_merge_vars
                    #  gradient_merge_vars += gradient_vars
                    cur_block = main_block.program.current_block()
                    for param_name in param_names:
                        grad = param_to_grad[param_name]
                        grad_merge = param_to_gradient_merge[param_name]
                        cur_block.append_op(
                            type="elementwise_add",
                            inputs={'X': grad,
                                    'Y': grad_merge},
                            outputs={'Out': grad_merge},
                            attrs={'axis': -1,
                                   'use_mkldnn': False})

                with switch.default():
                    # 1. update the graient_vars
                    #     gradient_vars += gradient_merge_vars
                    cur_block_idx = main_block.program.current_block_idx
                    cur_block = main_block.program.current_block()
                    for param_name in param_names:
                        grad = param_to_grad[param_name]
                        grad_merge = param_to_gradient_merge[param_name]
                        if self.avg:
                            tmp_var = layers.elementwise_add(grad, grad_merge)
                            cur_block.append_op(
                                type='scale',
                                inputs={'X': tmp_var},
                                outputs={'Out': grad},
                                attrs={
                                    'scale': 1.0 / self.k_steps,
                                    'bias': 0.0,
                                    'bias_after_scale': False
                                })
                        else:
                            cur_block.append_op(
                                type="elementwise_add",
                                inputs={'X': grad,
                                        'Y': grad_merge},
                                outputs={'Out': grad},
                                attrs={'axis': -1,
                                       'use_mkldnn': False})

                    # 2. apply_optimize
                    target_grad_block = main_block.program._create_block(
                        parent_idx=cur_block.parent_idx)
                    target_grad_block._set_forward_block_idx(cur_block_idx)
                    main_block.program.current_block_idx = cur_block_idx

                    optimize_ops = self.inner_optimizer.apply_optimize(
                        loss,
                        startup_program=startup_program,
                        params_grads=params_grads)

                    # 3. clear gradient_merge_vars
                    for param_name in param_names:
                        grad_merge = param_to_gradient_merge[param_name]
                        layers.fill_constant(
                            shape=grad_merge.shape,
                            dtype=grad_merge.dtype,
                            value=0.0,
                            out=grad_merge)
        return optimize_ops, params_grads