precision_recall_op.cc 8.2 KB
Newer Older
Y
yangyaming 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

Y
yangyaming 已提交
15 16
#include "paddle/operators/precision_recall_op.h"

Y
yangyaming 已提交
17 18 19 20 21 22 23 24
namespace paddle {
namespace operators {

class PrecisionRecallOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

  void InferShape(framework::InferShapeContext *ctx) const override {
Y
yangyaming 已提交
25 26 27 28
    PADDLE_ENFORCE(ctx->HasInput("MaxProbs"),
                   "Input(MaxProbs) should not be null.");
    PADDLE_ENFORCE(ctx->HasInput("Indices"),
                   "Input(Indices) should not be null.");
Y
yangyaming 已提交
29 30 31 32 33 34 35 36 37
    PADDLE_ENFORCE(ctx->HasInput("Labels"),
                   "Input(Labels) should not be null.");
    PADDLE_ENFORCE(ctx->HasOutput("BatchMetrics"),
                   "Output(BatchMetrics) should not be null.");
    PADDLE_ENFORCE(ctx->HasOutput("AccumMetrics"),
                   "Output(AccumMetrics) should not be null.");
    PADDLE_ENFORCE(ctx->HasOutput("AccumStatesInfo"),
                   "Output(AccumStatesInfo) should not be null.");

Y
yangyaming 已提交
38 39 40
    int64_t cls_num =
        static_cast<int64_t>(ctx->Attrs().Get<int>("class_number"));
    auto max_probs_dims = ctx->GetInputDim("MaxProbs");
Y
yangyaming 已提交
41 42
    auto labels_dims = ctx->GetInputDim("Labels");

Y
yangyaming 已提交
43 44 45 46 47 48 49 50 51 52 53 54
    PADDLE_ENFORCE_EQ(max_probs_dims[1], 1,
                      "Each instance contains one max probability, so the "
                      "shape of Input(MaxProbs) should be [batch_size, 1].");
    PADDLE_ENFORCE_EQ(ctx->GetInputDim("Indices"), max_probs_dims,
                      "The shape of Input(Indices) should be [batch_size, 1].");
    PADDLE_ENFORCE_EQ(max_probs_dims[0], labels_dims[0],
                      "The 1st dimension of Input(MaxProbs) and "
                      "Input(Labels) both are batch_size and the shape should "
                      "be the same.");
    PADDLE_ENFORCE_EQ(labels_dims[1], 1,
                      "The 2nd dimension of Input(Labels) contains instance "
                      "label and the shape should be equal to 1.");
Y
yangyaming 已提交
55 56
    if (ctx->HasInput("Weights")) {
      auto weights_dims = ctx->GetInputDim("Weights");
Y
yangyaming 已提交
57
      PADDLE_ENFORCE_EQ(weights_dims,
Y
yangyaming 已提交
58
                        framework::make_ddim({max_probs_dims[0], 1}),
Y
yangyaming 已提交
59 60 61 62 63
                        "The shape of Input(Weights) should be "
                        "[batch_size, 1].");
    }
    if (ctx->HasInput("StatesInfo")) {
      auto states_dims = ctx->GetInputDim("StatesInfo");
Y
yangyaming 已提交
64
      PADDLE_ENFORCE_EQ(states_dims, framework::make_ddim({cls_num, 4}),
Y
yangyaming 已提交
65 66 67 68 69 70 71 72 73 74 75 76 77 78
                        "The shape of Input(StatesInfo) should be "
                        "[class_number, 4].");
    }

    // Layouts of BatchMetrics and AccumMetrics both are:
    // [
    //  macro average precision, macro average recall, macro average F1 score,
    //  micro average precision, micro average recall, micro average F1 score
    // ]
    ctx->SetOutputDim("BatchMetrics", {6});
    ctx->SetOutputDim("AccumMetrics", {6});
    // Shape of AccumStatesInfo is [class_number, 4]
    // The layout of each row is:
    // [ TP, FP, TN, FN ]
Y
yangyaming 已提交
79
    ctx->SetOutputDim("AccumStatesInfo", {cls_num, 4});
Y
yangyaming 已提交
80
  }
Y
yangyaming 已提交
81 82 83 84

 protected:
  framework::DataType IndicateDataType(
      const framework::ExecutionContext &ctx) const override {
Y
yangyaming 已提交
85
    return framework::ToDataType(ctx.Input<Tensor>("MaxProbs")->type());
Y
yangyaming 已提交
86
  }
Y
yangyaming 已提交
87 88 89 90 91 92 93
};

class PrecisionRecallOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
  PrecisionRecallOpMaker(framework::OpProto *proto,
                         framework::OpAttrChecker *op_checker)
      : OpProtoAndCheckerMaker(proto, op_checker) {
Y
yangyaming 已提交
94
    AddInput("MaxProbs",
K
kexinzhao 已提交
95
             "(Tensor, default Tensor<float>) A 2-D tensor with shape N x 1, "
Y
yangyaming 已提交
96 97 98 99
             "where N is the batch size. Each row contains the max probability "
             "of an instance which computed by the previous top_k (k=1) "
             "operator.");
    AddInput("Indices",
K
kexinzhao 已提交
100
             "(Tensor, default Tensor<int>) A 2-D tensor with shape N x 1, "
Y
yangyaming 已提交
101 102
             "where N is the batch size. Each row contains the corresponding "
             "index which computed by the previous top_k (k=1) operator.");
Y
yangyaming 已提交
103
    AddInput("Labels",
K
kexinzhao 已提交
104
             "(Tensor, default Tensor<int>) A 2-D tensor with shape N x 1, "
Y
yangyaming 已提交
105 106 107
             "where N is the batch size. Each element is a label and the "
             "value should be in [0, class_number - 1].");
    AddInput("Weights",
K
kexinzhao 已提交
108
             "(Tensor, default Tensor<float>) A 2-D tensor with shape N x 1, "
Y
yangyaming 已提交
109 110 111 112
             "where N is the batch size. This input is optional. If provided, "
             "weight of instance would be considered when computing metrics.")
        .AsDispensable();
    AddInput("StatesInfo",
K
kexinzhao 已提交
113
             "(Tensor, default Tensor<int>) A 2-D tensor with shape D x 4, "
Y
yangyaming 已提交
114 115
             "where D is the number of classes. This input is optional. If "
             "provided, current state will be accumulated to this state and "
K
kexinzhao 已提交
116
             "the accumulation state will be the output state.")
Y
yangyaming 已提交
117
        .AsDispensable();
Y
yangyaming 已提交
118
    AddOutput("BatchMetrics",
K
kexinzhao 已提交
119 120
              "(Tensor, default Tensor<float>) A 1-D tensor with shape {6}. "
              "This output tensor contains metrics for current batch data. "
Y
yangyaming 已提交
121 122
              "The layout is [macro average precision, macro average recall, "
              "macro f1 score, micro average precision, micro average recall, "
K
kexinzhao 已提交
123
              "micro f1 score].");
Y
yangyaming 已提交
124
    AddOutput("AccumMetrics",
K
kexinzhao 已提交
125 126
              "(Tensor, default Tensor<float>) A 1-D tensor with shape {6}. "
              "This output tensor contains metrics for accumulated data. "
Y
yangyaming 已提交
127 128
              "The layout is [macro average precision, macro average recall, "
              "macro f1 score, micro average precision, micro average recall, "
K
kexinzhao 已提交
129
              "micro f1 score].");
Y
yangyaming 已提交
130
    AddOutput("AccumStatesInfo",
K
kexinzhao 已提交
131
              "(Tensor, default Tensor<float>) A 2-D tensor with shape D x 4, "
Y
yangyaming 已提交
132 133 134 135
              "where D is equal to class number. This output tensor contains "
              "accumulated state variables used to compute metrics. The layout "
              "for each class is [true positives, false positives, "
              "true negatives, false negatives].");
K
kexinzhao 已提交
136
    AddAttr<int>("class_number", "(int) Number of classes to be evaluated.");
Y
yangyaming 已提交
137
    AddComment(R"DOC(
K
kexinzhao 已提交
138 139 140
Precision Recall Operator.

When given Input(Indices) and Input(Labels), this operator can be used
Y
yangyaming 已提交
141
to compute various metrics including:
K
kexinzhao 已提交
142 143 144 145 146 147
1. macro average precision
2. macro average recall
3. macro f1 score
4. micro average precision
5. micro average recall
6. micro f1 score
Y
yangyaming 已提交
148

149
To compute the above metrics, we need to do statistics for true positives,
K
kexinzhao 已提交
150
false positives and false negatives. Here the count of true negatives is not
151
necessary, but counting it may provide potential usage and the cost is
K
kexinzhao 已提交
152
trivial, so the operator also provides the count of true negatives.
Y
yangyaming 已提交
153

Y
yangyaming 已提交
154
We define state as a 2-D tensor with shape [class_number, 4]. Each row of a
Y
yangyaming 已提交
155 156
state contains statistic variables for corresponding class. Layout of each row
is: TP(true positives), FP(false positives), TN(true negatives),
K
kexinzhao 已提交
157 158
FN(false negatives). If Input(Weights) is provided, TP, FP, TN, FN will be
calculated by given weight instead of the instance count.
Y
yangyaming 已提交
159 160

This operator also supports metrics computing for cross-batch situation. To
K
kexinzhao 已提交
161 162
achieve this, Input(StatesInfo) should be provided. State of current batch
data will be accumulated to Input(StatesInfo) and Output(AccumStatesInfo)
Y
yangyaming 已提交
163 164
is the accumulation state.

K
kexinzhao 已提交
165 166
Output(BatchMetrics) is metrics of current batch data while
Output(AccumStatesInfo) is metrics of accumulation data.
Y
yangyaming 已提交
167

Y
yangyaming 已提交
168 169 170 171 172 173 174 175 176 177 178 179 180
)DOC");
  }
};

}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;
REGISTER_OP_WITHOUT_GRADIENT(precision_recall, ops::PrecisionRecallOp,
                             ops::PrecisionRecallOpMaker);
REGISTER_OP_CPU_KERNEL(
    precision_recall,
    ops::PrecisionRecallKernel<paddle::platform::CPUPlace, float>,
Y
yangyaming 已提交
181
    ops::PrecisionRecallKernel<paddle::platform::CPUPlace, double>);