softmax_with_cross_entropy_op.cc 8.1 KB
Newer Older
1 2 3 4 5 6
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

C
caoying03 已提交
7
http://www.apache.org/licenses/LICENSE-2.0
8

C
caoying03 已提交
9 10 11 12 13
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
14 15

#include "paddle/operators/softmax_with_cross_entropy_op.h"
Y
Yu Yang 已提交
16

17 18 19 20 21 22
namespace paddle {
namespace operators {

class SoftmaxWithCrossEntropyOpMaker
    : public framework::OpProtoAndCheckerMaker {
 public:
23
  SoftmaxWithCrossEntropyOpMaker(OpProto* proto, OpAttrChecker* op_checker)
24
      : OpProtoAndCheckerMaker(proto, op_checker) {
C
caoying03 已提交
25
    AddInput("Logits",
26
             "(Tensor, default: Tensor<float>), The unscaled log probabilities "
C
caoying03 已提交
27
             "which is a 2-D tensor with shape [N x K]. N is the batch_size, "
28 29
             "and K is the class number.");
    AddInput("Label",
C
caoying03 已提交
30 31 32 33
             "(Tensor) The ground truth which is a 2-D tensor. If soft_label "
             "is set to false, Label is a Tensor<int64> with shape [N x 1]. If "
             "soft_label is set to true, Label is a Tensor<float/double> with "
             "shape [N x K].");
C
caoying03 已提交
34 35
    AddOutput(
        "Softmax",
36
        "(Tensor, default: Tensor<float>), A 2-D tensor with shape [N x K]. "
C
caoying03 已提交
37 38
        "The outputs value of softmax activation by given the input batch, "
        "which will be used in backward calculation.")
C
caoying03 已提交
39
        .AsIntermediate();
C
caoying03 已提交
40
    AddOutput("Loss",
41
              "(Tensor, default: Tensor<float>), A 2-D tensor. The cross "
C
caoying03 已提交
42
              "entropy loss with shape [N x 1].");
C
caoying03 已提交
43
    AddAttr<bool>(
44
        "soft_label",
C
caoying03 已提交
45 46 47
        "(bool, default: false), A flag to indicate whether to interpretate "
        "the given labels as soft labels.")
        .SetDefault(false);
48
    AddComment(R"DOC(
49 50 51
Softmax With Cross Entropy Operator.

Cross entropy loss with softmax is used as the output layer extensively. This
52
operator computes the softmax normalized values for each row of the input
53
tensor, after which cross-entropy loss is computed. This provides a more
54 55
numerically stable gradient.

56 57 58
Because this operator performs a softmax on logits internally, it expects
unscaled logits. This operator should not be used with the output of
softmax operator since that would produce incorrect results.
59

C
caoying03 已提交
60
When the attribute soft_label is set false, this operators expects mutually
61 62
exclusive hard labels, each sample in a batch is in exactly one class with a
probability of 1.0. Each sample in the batch will have a single label.
63

64
The equation is as follows:
65

66
1) Hard label (one-hot label, so every sample has exactly one class)
67

68
$$Loss_j =  -\text{Logit}_{Label_j} +
69
\log\left(\sum_{i=0}^{K}\exp(\text{Logit}_i)\right),
70
j = 1,..., K$$
C
caoying03 已提交
71

72
2) Soft label (each sample can have a distribution over all classes)
C
caoying03 已提交
73

74
$$Loss_j =  -\sum_{i=0}^{K}\text{Label}_i \left(\text{Logit}_i -
75
\log\left(\sum_{i=0}^{K}\exp(\text{Logit}_i)\right)\right),
76
j = 1,...,K$$
C
caoying03 已提交
77 78

)DOC");
79 80 81 82 83 84 85
  }
};

class SoftmaxWithCrossEntropyOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

86
  void InferShape(framework::InferShapeContext* ctx) const override {
Q
qiaolongfei 已提交
87 88 89 90 91 92 93 94 95 96
    PADDLE_ENFORCE(ctx->HasInput("Logits"),
                   "Input(Logits) should be not null.");
    PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) should be not null.");

    PADDLE_ENFORCE(ctx->HasOutput("Softmax"),
                   "Output(Softmax) should be not null.");
    PADDLE_ENFORCE(ctx->HasOutput("Loss"), "Output(Loss) should be not null.");

    auto logits_dims = ctx->GetInputDim("Logits");
    auto labels_dims = ctx->GetInputDim("Label");
C
caoying03 已提交
97
    PADDLE_ENFORCE_EQ(
Q
qiaolongfei 已提交
98
        logits_dims.size(), 2UL,
99
        "The input of softmax_with_cross_entropy should be a 2-D tensor.");
Q
qiaolongfei 已提交
100
    PADDLE_ENFORCE_EQ(labels_dims.size(), 2UL,
C
caoying03 已提交
101
                      "The labels should be a 2-D tensor.");
102

103
    if (ctx->Attrs().Get<bool>("soft_label")) {
Q
qiaolongfei 已提交
104
      PADDLE_ENFORCE_EQ(logits_dims[1], labels_dims[1],
105
                        "If Attr(soft_label) == true, the 2nd dimension of "
106 107
                        "Input(X) and Input(Label) should be equal.");
    } else {
Q
qiaolongfei 已提交
108
      PADDLE_ENFORCE_EQ(labels_dims[1], 1UL,
109
                        "If Attr(soft_label) == false, the 2nd dimension of "
110 111 112
                        "Input(Label) should be 1.");
    }

Q
qiaolongfei 已提交
113 114
    ctx->SetOutputDim("Softmax", logits_dims);
    ctx->SetOutputDim("Loss", {logits_dims[0], 1});
115

Q
qiaolongfei 已提交
116 117
    ctx->ShareLoD("Logits", /*->*/ "Softmax");
    ctx->ShareLoD("Logits", /*->*/ "Loss");
C
caoying03 已提交
118
  }
Y
Yu Yang 已提交
119

120
 protected:
Q
Qiao Longfei 已提交
121
  framework::OpKernelType GetActualKernelType(
Y
Yu Yang 已提交
122
      const framework::ExecutionContext& ctx) const override {
Y
Yu Yang 已提交
123 124 125
    return framework::OpKernelType(
        framework::ToDataType(ctx.Input<Tensor>("Logits")->type()),
        ctx.device_context());
Y
Yu Yang 已提交
126
  }
C
caoying03 已提交
127 128 129 130 131 132
};

class SoftmaxWithCrossEntropyOpGrad : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

133
  void InferShape(framework::InferShapeContext* ctx) const override {
Q
qiaolongfei 已提交
134 135 136 137 138 139 140 141 142 143 144
    PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Loss")),
                   "Input(Loss@Grad) should not be null.");
    PADDLE_ENFORCE(ctx->HasInput("Softmax"),
                   "Input(Softmax) should be not null.");
    PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) should be not null.");
    PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("Logits")),
                   "Output(Logits@Grad) should be not null.");

    auto softmax_dims = ctx->GetInputDim("Softmax");
    auto labels_dims = ctx->GetInputDim("Label");
    PADDLE_ENFORCE_EQ(labels_dims.size(), 2UL,
C
caoying03 已提交
145
                      "The labels should be a 2-D tensor.");
146

147
    if (ctx->Attrs().Get<bool>("soft_label")) {
Q
qiaolongfei 已提交
148
      PADDLE_ENFORCE_EQ(softmax_dims[1], labels_dims[1],
149
                        "When Attr(soft_label) == true, the 2nd dimension of "
150 151
                        "Input(X) and Input(Label) should be equal.");
    } else {
Q
qiaolongfei 已提交
152
      PADDLE_ENFORCE_EQ(labels_dims[1], 1UL,
153
                        "When Attr(soft_label) == false, the 2nd dimension of "
154 155
                        "Input(Label) should be 1.");
    }
C
caoying03 已提交
156

Q
qiaolongfei 已提交
157 158
    ctx->SetOutputDim(framework::GradVarName("Logits"),
                      ctx->GetInputDim("Softmax"));
159
  }
Y
Yu Yang 已提交
160

161
 protected:
Q
Qiao Longfei 已提交
162
  framework::OpKernelType GetActualKernelType(
Y
Yu Yang 已提交
163
      const framework::ExecutionContext& ctx) const override {
Y
Yu Yang 已提交
164 165 166 167
    return framework::OpKernelType(
        framework::ToDataType(
            ctx.Input<Tensor>(framework::GradVarName("Loss"))->type()),
        ctx.device_context());
Y
Yu Yang 已提交
168
  }
169 170
};

171 172 173 174 175
class SoftmaxGradMaker : public framework::SingleGradOpDescMaker {
 public:
  using framework::SingleGradOpDescMaker::SingleGradOpDescMaker;

 protected:
Y
Yu Yang 已提交
176 177
  std::unique_ptr<framework::OpDesc> Apply() const override {
    auto* grad_op = new framework::OpDesc();
Y
Yu Yang 已提交
178 179 180 181 182 183 184 185
    grad_op->SetType("softmax_with_cross_entropy_grad");
    grad_op->SetInput("Label", Input("Label"));
    grad_op->SetInput("Softmax", Output("Softmax"));
    grad_op->SetInput("Loss", Output("Loss"));
    grad_op->SetInput(framework::GradVarName("Softmax"), OutputGrad("Softmax"));
    grad_op->SetInput(framework::GradVarName("Loss"), OutputGrad("Loss"));
    grad_op->SetOutput(framework::GradVarName("Logits"), InputGrad("Logits"));
    grad_op->SetAttrMap(Attrs());
Y
Yu Yang 已提交
186
    return std::unique_ptr<framework::OpDesc>(grad_op);
187 188 189
  }
};

190 191 192 193 194
}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;

195
REGISTER_OPERATOR(softmax_with_cross_entropy, ops::SoftmaxWithCrossEntropyOp,
Y
Yu Yang 已提交
196
                  ops::SoftmaxWithCrossEntropyOpMaker, ops::SoftmaxGradMaker);
197 198
REGISTER_OPERATOR(softmax_with_cross_entropy_grad,
                  ops::SoftmaxWithCrossEntropyOpGrad);
199
REGISTER_OP_CPU_KERNEL(softmax_with_cross_entropy,
C
caoying03 已提交
200 201
                       ops::SoftmaxWithCrossEntropyKernel<float>,
                       ops::SoftmaxWithCrossEntropyKernel<double>);
202
REGISTER_OP_CPU_KERNEL(softmax_with_cross_entropy_grad,
C
caoying03 已提交
203 204
                       ops::SoftmaxWithCrossEntropyGradKernel<float>,
                       ops::SoftmaxWithCrossEntropyGradKernel<double>);