optimizer.py 3.9 KB
Newer Older
Q
qiaolongfei 已提交
1
import py_paddle.swig_paddle as swig_api
Y
Yu Yang 已提交
2

Q
qiaolongfei 已提交
3
import paddle.trainer_config_helpers.config_parser_utils as config_parser_utils
Y
Yu Yang 已提交
4 5 6 7 8 9
import paddle.trainer_config_helpers.optimizers as v1_optimizers
"""
Optimizers(update equation) for SGD method.

TODO(yuyang18): Complete comments.
"""
Q
qiaolongfei 已提交
10

L
Luo Tao 已提交
11 12 13 14
__all__ = [
    'Momentum', 'Adam', 'Adamax', 'AdaGrad', 'DecayedAdaGrad', 'AdaDelta',
    'RMSProp', 'ModelAverage', 'L2Regularization'
]
Q
qiaolongfei 已提交
15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43


class Optimizer(object):
    def __init__(self, **kwargs):
        if 'batch_size' in kwargs:
            del kwargs['batch_size']  # not important for python library.

        def __impl__():
            v1_optimizers.settings(batch_size=1, **kwargs)

        self.__opt_conf_proto__ = config_parser_utils.parse_optimizer_config(
            __impl__)
        self.__opt_conf__ = swig_api.OptimizationConfig.createFromProto(
            self.__opt_conf_proto__)

    def enable_types(self):
        """
        get enable_types for each optimizer.
        enable_types = [value, gradient, momentum, etc]
        For each optimizer(SGD, Adam), GradientMachine should enable different
        buffers.
        """
        tmp = swig_api.ParameterOptimizer.create(self.__opt_conf__)
        assert isinstance(tmp, swig_api.ParameterOptimizer)
        return tmp.getParameterTypes()

    def create_local_updater(self):
        return swig_api.ParameterUpdater.createLocalUpdater(self.__opt_conf__)

Q
qiaolongfei 已提交
44
    def create_remote_updater(self, pass_num, use_sparse_updater):
Q
qiaolongfei 已提交
45
        return swig_api.ParameterUpdater.createRemoteUpdater(self.__opt_conf__,
Q
qiaolongfei 已提交
46
                                                             pass_num, use_sparse_updater)
Q
qiaolongfei 已提交
47 48


L
Luo Tao 已提交
49 50 51
class Momentum(Optimizer):
    def __init__(self, momentum=None, sparse=False, **kwargs):
        learning_method = v1_optimizers.MomentumOptimizer(
Y
Yu Yang 已提交
52
            momentum=momentum, sparse=sparse)
L
Luo Tao 已提交
53 54 55 56
        super(Momentum, self).__init__(
            learning_method=learning_method, **kwargs)


Q
qiaolongfei 已提交
57 58 59 60 61 62 63 64 65 66 67 68 69 70
class Adam(Optimizer):
    def __init__(self, beta1=0.9, beta2=0.999, epsilon=1e-8, **kwargs):
        learning_method = v1_optimizers.AdamOptimizer(
            beta1=beta1, beta2=beta2, epsilon=epsilon)
        super(Adam, self).__init__(learning_method=learning_method, **kwargs)


class Adamax(Optimizer):
    def __init__(self, beta1=0.9, beta2=0.999, **kwargs):
        learning_method = v1_optimizers.AdamaxOptimizer(
            beta1=beta1, beta2=beta2)
        super(Adamax, self).__init__(learning_method=learning_method, **kwargs)


L
Luo Tao 已提交
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
class AdaGrad(Optimizer):
    def __init__(self, **kwargs):
        learning_method = v1_optimizers.AdaGradOptimizer()
        super(AdaGrad, self).__init__(learning_method=learning_method, **kwargs)


class DecayedAdaGrad(Optimizer):
    def __init__(self, rho=0.95, epsilon=1e-06, **kwargs):
        learning_method = v1_optimizers.DecayedAdaGradOptimizer(
            rho=rho, epsilon=epsilon)
        super(DecayedAdaGrad, self).__init__(
            learning_method=learning_method, **kwargs)


class AdaDelta(Optimizer):
    def __init__(self, rho=0.95, epsilon=1e-06, **kwargs):
        learning_method = v1_optimizers.AdaDeltaOptimizer(
            rho=rho, epsilon=epsilon)
        super(AdaDelta, self).__init__(
            learning_method=learning_method, **kwargs)


class RMSProp(Optimizer):
    def __init__(self, rho=0.95, epsilon=1e-6, **kwargs):
        learning_method = v1_optimizers.RMSPropOptimizer(
            rho=rho, epsilon=epsilon)
        super(RMSProp, self).__init__(learning_method=learning_method, **kwargs)


ModelAverage = v1_optimizers.ModelAverage
L2Regularization = v1_optimizers.L2Regularization

Q
qiaolongfei 已提交
103 104
if __name__ == '__main__':
    swig_api.initPaddle('--use_gpu=false')
L
Luo Tao 已提交
105 106 107 108 109 110 111 112
    for opt in [
            Momentum(), Adam(), Adamax(), AdaGrad(), DecayedAdaGrad(),
            AdaDelta(), RMSProp(), Adam(
                model_average=ModelAverage(average_window=0.5),
                regularization=L2Regularization(rate=0.5),
                gradient_clipping_threshold=25)
    ]:
        print opt, opt.enable_types()