adadelta_optimizer.cc 1.9 KB
Newer Older
1 2
#include "adadelta_optimizer.h"
#include <algorithm>
3
#include <cmath>
4 5 6 7

namespace paddle {
namespace optimizer {

D
dzhwinter 已提交
8 9 10
void AdadeltaOptimizer::Update(const Tensor* gradient) {
  num_sample_passed_ += 1;
  double learning_rate = lr_policy_->LearningRate(num_sample_passed_);
11 12
  Tensor& param = *parameter_;
  const Tensor& grad = *gradient;
D
dzhwinter 已提交
13 14 15
  Tensor& accum_g = *accum_gradient_;
  Tensor& accum_d = *accum_delta_;
  Tensor& update_d = *update_delta_;
16
  for (size_t i = 0; i < param.size(); ++i) {
D
dzhwinter 已提交
17
    accum_g[i] = rho_ * accum_g[i] + (1.0 - rho_) * grad[i] * grad[i];
18

D
dzhwinter 已提交
19 20
    update_d[i] = std::sqrt(accum_d[i] + epsilon_) /
                  std::sqrt(accum_g[i] + epsilon_) * grad[i];
21

D
dzhwinter 已提交
22
    accum_d[i] = rho_ * accum_d[i] + (1.0 - rho_) * update_d[i] * update_d[i];
23

D
dzhwinter 已提交
24
    param[i] -= learning_rate * update_d[i] + learning_rate * decay_ * param[i];
25 26
  }
}
D
dzhwinter 已提交
27

28
std::string AdadeltaOptimizer::SerializeState() {
D
dzhwinter 已提交
29
  AdadeltaOptimizerState state;
D
dzhwinter 已提交
30
  state.set_num_sample_passed(num_sample_passed_);
31
  std::string lr_str = this->lr_policy_->SerializeState();
D
dongzhihong 已提交
32
  state.mutable_lr_state()->ParseFromString(lr_str);
D
dzhwinter 已提交
33 34 35 36 37

  TensorToProto(*parameter_, state.mutable_parameter());
  TensorToProto(*accum_gradient_, state.mutable_accum_gradient());
  TensorToProto(*accum_delta_, state.mutable_accum_delta());
  TensorToProto(*update_delta_, state.mutable_update_delta());
38
  return state.SerializeAsString();
D
dzhwinter 已提交
39 40
}

D
dzhwinter 已提交
41 42
void AdadeltaOptimizer::DeserializeState(const std::string& str) {
  AdadeltaOptimizerState state;
D
dzhwinter 已提交
43
  state.ParseFromString(str);
D
dongzhihong 已提交
44 45
  auto lr_state = state.lr_state();
  this->lr_policy_->DeserializeState(lr_state.SerializeAsString());
D
dzhwinter 已提交
46 47 48 49 50 51
  num_sample_passed_ = state.num_sample_passed();

  ProtoToTensor(state.parameter(), parameter_);
  ProtoToTensor(state.accum_gradient(), accum_gradient_);
  ProtoToTensor(state.accum_delta(), accum_delta_);
  ProtoToTensor(state.update_delta(), update_delta_);
D
dzhwinter 已提交
52
}
D
dzhwinter 已提交
53 54

}  // namespace optimizer
D
dzhwinter 已提交
55
}  // namespace paddle