adagrad_op.cc 2.8 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6 7 8 9 10 11 12 13 14

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

Q
QI JUN 已提交
15
#include <cmath>
H
hong 已提交
16
#include <vector>
Q
QI JUN 已提交
17

18
#include "paddle/fluid/framework/infershape_utils.h"
H
hong 已提交
19
#include "paddle/fluid/framework/op_registry.h"
Y
Yi Wang 已提交
20
#include "paddle/fluid/operators/math/selected_rows_functor.h"
H
hong 已提交
21 22
#include "paddle/phi/core/infermeta_utils.h"
#include "paddle/phi/infermeta/multiary.h"
23
#include "paddle/phi/kernels/funcs/math_function.h"
H
hong 已提交
24

25 26 27
namespace paddle {
namespace operators {

D
dzhwinter 已提交
28
using Tensor = framework::Tensor;
29 30 31 32
class AdagradOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

D
dzhwinter 已提交
33 34
  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext& ctx) const override {
35 36
    return framework::OpKernelType(
        OperatorWithKernel::IndicateVarDataType(ctx, "Param"), ctx.GetPlace());
D
dzhwinter 已提交
37
  }
38 39 40 41
};

class AdagradOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
Y
Yu Yang 已提交
42
  void Make() override {
K
Kexin Zhao 已提交
43 44 45 46 47 48 49 50 51 52 53 54
    AddInput("Param", "(Tensor) Input parameter");
    AddInput("Grad", "(Tensor) Input gradient");
    AddInput("Moment", "(Tensor) Second moment");
    AddInput("LearningRate", "(Tensor) Learning rate");

    AddOutput("ParamOut", "(Tensor) Output parameter");
    AddOutput("MomentOut", "(Tensor) Output second moment");

    AddAttr<float>("epsilon",
                   "(float, default 1.0e-6) "
                   "Constant for numerical stability")
        .SetDefault(1.0e-6f);
55 56 57 58
    AddComment(R"DOC(

Adaptive Gradient Algorithm (Adagrad).

59 60
The update is done as follows:

61 62
$$moment\_out = moment + grad * grad \\
param\_out = param - \frac{learning\_rate * grad}{\sqrt{moment\_out} + \epsilon}
63
$$
64 65

The original paper(http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf)
66 67 68
does not have the epsilon attribute. It is added here in our implementation
as also proposed here: http://cs231n.github.io/neural-networks-3/#ada
for numerical stability to avoid the division by zero error.
69 70 71 72

)DOC");
  }
};
Q
QI JUN 已提交
73

74 75 76 77
}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;
H
hong 已提交
78 79 80 81
DECLARE_INFER_SHAPE_FUNCTOR(adagrad, AdagradInferShapeFunctor,
                            PD_INFER_META(phi::AdagradInferMeta));
REGISTER_OP_WITHOUT_GRADIENT(adagrad, ops::AdagradOp, ops::AdagradOpMaker,
                             AdagradInferShapeFunctor);