Refine evaluator op types (#5208)

* refine evaluator op types

* update

* follow comments

* update

* fix v2 mnist case

* fix v2 mnist case

* update

* update

paddle/operators/accuracy_op.cc

@@ -22,23 +22,35 @@ class AccuracyOp : public framework::OperatorWithKernel {
   using framework::OperatorWithKernel::OperatorWithKernel;
 
   void InferShape(framework::InferShapeContext *ctx) const override {
-    PADDLE_ENFORCE(ctx->HasInput("Inference"),
-                   "Input(Inference) of AccuracyOp should not be null.");
+    PADDLE_ENFORCE(ctx->HasInput("Out"),
+                   "Input (Out) of accuracy op should not be null.");
+    PADDLE_ENFORCE(ctx->HasInput("Indices"),
+                   "Input (Indices) of accuracy op should not be null.");
     PADDLE_ENFORCE(ctx->HasInput("Label"),
-                   "Input(Label) of AccuracyOp should not be null.");
+                   "Input (Label) of accuracy op should not be null.");
     PADDLE_ENFORCE(ctx->HasOutput("Accuracy"),
-                   "Output(Accuracy) of AccuracyOp should not be null.");
+                   "Output (Accuracy) of AccuracyOp should not be null.");
 
-    auto inference_dim = ctx->GetInputDim("Inference");
+    auto inference_dim = ctx->GetInputDim("Out");
     auto label_dim = ctx->GetInputDim("Label");
+    // Assume indices has same shape with infernece, because
+    // it's the output of topk.
 
     PADDLE_ENFORCE_EQ(label_dim.size(), 2, "label's rank must be 2.");
     PADDLE_ENFORCE_EQ(label_dim[1], 1, "label's second dimension must be 1");
     PADDLE_ENFORCE_EQ(inference_dim[0], label_dim[0],
-                      "inference size must be the same as label size");
+                      "the inference tensor's num_rows must be"
+                      " the same as label.");
 
     ctx->SetOutputDim("Accuracy", {1});
-    ctx->ShareLoD("Inference", /*->*/ "Accuracy");
+    ctx->ShareLoD("Out", /*->*/ "Accuracy");
+  }
+
+ protected:
+  // IndicateDataType
+  framework::DataType IndicateDataType(
+      const framework::ExecutionContext &ctx) const override {
+    return framework::ToDataType(ctx.Input<Tensor>("Out")->type());
   }
 };
 
@@ -48,7 +60,8 @@ class AccuracyOpMaker : public framework::OpProtoAndCheckerMaker {
                   framework::OpAttrChecker *op_checker)
       : OpProtoAndCheckerMaker(proto, op_checker) {
     // TODO(typhoonzero): support both inference value and indices.
-    AddInput("Inference", "topk(indices) the network output");
+    AddInput("Out", "topk (inferences) the network output");
+    AddInput("Indices", "topk (indices) the network output");
     AddInput("Label", "Label of the training data");
     // TODO(typhoonzero): AddInput("Weight", ...
     AddOutput("Accuracy", "The accuracy of current batch");
@@ -59,7 +72,7 @@ The accuracy is:
 ..  math::
 accuracy = \\frac{NumOfCorrectPredicts}{NumOfAllSamples})
 
-Both the input `Inference` and `Label` can carry the LoD (Level of Details)
+Both the input `Out` and `Label` can carry the LoD (Level of Details)
 information, or not. But the output only shares the LoD with input `Inference`.
 )DOC");
   }
@@ -71,6 +84,8 @@ information, or not. But the output only shares the LoD with input `Inference`.
 namespace ops = paddle::operators;
 REGISTER_OPERATOR(accuracy, ops::AccuracyOp, ops::AccuracyOpMaker,
                   paddle::framework::EmptyGradOpMaker);
-REGISTER_OP_CPU_KERNEL(
-    accuracy, ops::AccuracyKernel<paddle::platform::CPUPlace, int>,
-    ops::AccuracyKernel<paddle::platform::CPUPlace, int64_t>);
+// FIXME(typhoonzero): types of T is for infernece data.
+// label data is always int.
+REGISTER_OP_CPU_KERNEL(accuracy,
+                       ops::AccuracyKernel<paddle::platform::CPUPlace, float>,
+                       ops::AccuracyKernel<paddle::platform::CPUPlace, double>);

paddle/operators/accuracy_op.cu

@@ -21,9 +21,10 @@ namespace paddle {
 namespace operators {
 using platform::PADDLE_CUDA_NUM_THREADS;
 
-template <typename T, int BlockSize>
-__global__ void AccuracyCudaKernel(const int N, const int D, const T* Xdata,
-                                   const T* labeldata, float* accuracy) {
+template <int BlockSize>
+__global__ void AccuracyCudaKernel(const int N, const int D,
+                                   const int64_t* Xdata,
+                                   const int64_t* labeldata, float* accuracy) {
   int count = 0;
   __shared__ int total[BlockSize];
 
@@ -52,13 +53,14 @@ class AccuracyOpCUDAKernel : public framework::OpKernel<T> {
   void Compute(const framework::ExecutionContext& ctx) const override {
     PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()),
                    "It must use GPUPlace.");
-    auto* inference = ctx.Input<Tensor>("Inference");
+    auto* inference = ctx.Input<Tensor>("Out");
+    auto* indices = ctx.Input<Tensor>("Indices");
     auto* label = ctx.Input<Tensor>("Label");
     auto* accuracy = ctx.Output<Tensor>("Accuracy");
     // FIXME(typhoonzero): only support indices currently
     // if add support for output values, how to detect the data type?
-    const T* inference_data = inference->data<T>();
-    const T* label_data = label->data<T>();
+    const int64_t* indices_data = indices->data<int64_t>();
+    const int64_t* label_data = label->data<int64_t>();
     float* accuracy_data = accuracy->mutable_data<float>(ctx.GetPlace());
 
     size_t num_samples = inference->dims()[0];
@@ -69,11 +71,11 @@ class AccuracyOpCUDAKernel : public framework::OpKernel<T> {
       return;
     }
 
-    AccuracyCudaKernel<T, PADDLE_CUDA_NUM_THREADS><<<
+    AccuracyCudaKernel<PADDLE_CUDA_NUM_THREADS><<<
         1, PADDLE_CUDA_NUM_THREADS, 0,
         reinterpret_cast<const platform::CUDADeviceContext&>(
             ctx.device_context())
-            .stream()>>>(num_samples, infer_width, inference_data, label_data,
+            .stream()>>>(num_samples, infer_width, indices_data, label_data,
                          accuracy_data);
   }
 };
@@ -81,5 +83,7 @@ class AccuracyOpCUDAKernel : public framework::OpKernel<T> {
 }  // namespace operators
 }  // namespace paddle
 
-REGISTER_OP_GPU_KERNEL(accuracy, paddle::operators::AccuracyOpCUDAKernel<int>,
-                       paddle::operators::AccuracyOpCUDAKernel<int64_t>);
+// FIXME(typhoonzero): types of T is for infernece data.
+// label data is always int
+REGISTER_OP_GPU_KERNEL(accuracy, paddle::operators::AccuracyOpCUDAKernel<float>,
+                       paddle::operators::AccuracyOpCUDAKernel<double>);

paddle/operators/accuracy_op.h

@@ -38,14 +38,15 @@ template <typename Place, typename T>
 class AccuracyKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& ctx) const override {
-    auto* inference = ctx.Input<Tensor>("Inference");
+    auto* inference = ctx.Input<Tensor>("Out");
+    auto* indices = ctx.Input<Tensor>("Indices");
     auto* label = ctx.Input<Tensor>("Label");
     auto* accuracy = ctx.Output<Tensor>("Accuracy");
 
     float* accuracy_data = accuracy->mutable_data<float>(ctx.GetPlace());
 
-    const T* inference_data = inference->data<T>();
-    const T* label_data = label->data<T>();
+    const int64_t* indices_data = indices->data<int64_t>();
+    const int64_t* label_data = label->data<int64_t>();
 
     size_t num_samples = inference->dims()[0];
     size_t class_dim = inference->dims()[1];
@@ -60,7 +61,7 @@ class AccuracyKernel : public framework::OpKernel<T> {
     for (size_t i = 0; i < num_samples; ++i) {
       PADDLE_ENFORCE_GE(label_data[i], 0, "label must >= 0");
       for (size_t j = 0; j < class_dim; ++j) {
-        if (inference_data[i * class_dim + j] == label_data[i]) {
+        if (indices_data[i * class_dim + j] == label_data[i]) {
           ++num_correct;
           break;
         }

paddle/operators/auc_op.cc

@@ -23,18 +23,26 @@ class AucOp : public framework::OperatorWithKernel {
 
  protected:
   void InferShape(framework::InferShapeContext *ctx) const override {
-    PADDLE_ENFORCE(ctx->HasInput("Inference"),
-                   "Input of Inference must be initialized.");
+    PADDLE_ENFORCE(ctx->HasInput("Out"), "Input of Out must be initialized.");
+    PADDLE_ENFORCE(ctx->HasInput("Indices"),
+                   "Input of Indices must be initialized.");
     PADDLE_ENFORCE(ctx->HasInput("Label"),
                    "Input of Label must be initialized.");
-    auto inference_dim = ctx->GetInputDim("Inference");
-    auto label_dim = ctx->GetInputDim("Label");
+    auto inference_height = ctx->GetInputDim("Out")[0];
+    auto label_height = ctx->GetInputDim("Label")[0];
 
-    PADDLE_ENFORCE_EQ(inference_dim, label_dim,
-                      "inference and label should have same shape");
+    PADDLE_ENFORCE_EQ(inference_height, label_height,
+                      "Out and Label should have same height.");
 
     ctx->SetOutputDim("AUC", {1});
-    ctx->ShareLoD("Inference", /*->*/ "AUC");
+    ctx->ShareLoD("Out", /*->*/ "AUC");
+  }
+
+ protected:
+  // IndicateDataType
+  framework::DataType IndicateDataType(
+      const framework::ExecutionContext &ctx) const override {
+    return framework::ToDataType(ctx.Input<Tensor>("Out")->type());
   }
 };
 
@@ -42,12 +50,18 @@ class AucOpMaker : public framework::OpProtoAndCheckerMaker {
  public:
   AucOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
       : OpProtoAndCheckerMaker(proto, op_checker) {
-    AddInput("Inference",
-             "A floating point tensor of arbitrary shape and whose values"
-             "are in the range [0, 1].");
+    AddInput("Out",
+             "A floating point 2D tensor, values are in the range [0, 1]."
+             "Each row is descend sorted. This input should be the"
+             "output of topk."
+             "Typically, this tensor indicates the probability of each label");
+    AddInput("Indices",
+             "An int 2D tensor, indicating the indices of original"
+             "tensor before sort. Typically, this tensor indicates which label"
+             "the probability stands for.");
     AddInput("Label",
-             "A tensor whose shape matches "
-             "Inference. Will be cast to bool.");
+             "A 2D int tensor indicating the label of the training data."
+             "The height is batch size and width is always 1.");
     // TODO(typhoonzero): support weight input
     AddOutput("AUC",
               "A scalar representing the "

paddle/operators/auc_op.h

@@ -29,7 +29,7 @@ template <typename Place, typename T>
 class AucKernel : public framework::OpKernel<T> {
  public:
   void Compute(const framework::ExecutionContext& ctx) const override {
-    auto* inference = ctx.Input<Tensor>("Inference");
+    auto* inference = ctx.Input<Tensor>("Out");
     auto* label = ctx.Input<Tensor>("Label");
     auto* auc = ctx.Output<Tensor>("AUC");
 
@@ -46,18 +46,11 @@ class AucKernel : public framework::OpKernel<T> {
     thresholds_list[0] = 0.0f - kEpsilon;
     thresholds_list[num_thresholds - 1] = 1.0f + kEpsilon;
 
-    size_t num_samples = inference->numel();
+    size_t batch_size = inference->dims()[0];
+    size_t inference_width = inference->dims()[1];
 
     const T* inference_data = inference->data<T>();
-    Tensor label_casted;
-    label_casted.Resize(label->dims());
-    bool* label_casted_data = label_casted.mutable_data<bool>(ctx.GetPlace());
-
-    const int* label_data = label->data<int>();
-    // cast label_data to bool
-    for (size_t i = 0; i < num_samples; i++) {
-      label_casted_data[i] = static_cast<bool>(label_data[i]);
-    }
+    const int64_t* label_data = label->data<int64_t>();
 
     // Create local tensor for storing the curve: TP, FN, TN, FP
     // TODO(typhoonzero): use eigen op to caculate these values.
@@ -68,23 +61,27 @@ class AucKernel : public framework::OpKernel<T> {
     true_negative.Resize({num_thresholds});
     false_positive.Resize({num_thresholds});
 
-    int* tp_data = true_positive.mutable_data<int>(ctx.GetPlace());
-    int* fn_data = false_negative.mutable_data<int>(ctx.GetPlace());
-    int* tn_data = true_negative.mutable_data<int>(ctx.GetPlace());
-    int* fp_data = false_positive.mutable_data<int>(ctx.GetPlace());
+    int64_t* tp_data = true_positive.mutable_data<int64_t>(ctx.GetPlace());
+    int64_t* fn_data = false_negative.mutable_data<int64_t>(ctx.GetPlace());
+    int64_t* tn_data = true_negative.mutable_data<int64_t>(ctx.GetPlace());
+    int64_t* fp_data = false_positive.mutable_data<int64_t>(ctx.GetPlace());
 
     for (int idx_thresh = 0; idx_thresh < num_thresholds; idx_thresh++) {
       // caculate TP, FN, TN, FP for current thresh
-      int tp = 0, fn = 0, tn = 0, fp = 0;
-      for (size_t i = 0; i < num_samples; i++) {
-        if (label_casted_data[i]) {
-          if (inference_data[i] >= (thresholds_list[idx_thresh])) {
+      int64_t tp = 0, fn = 0, tn = 0, fp = 0;
+      for (size_t i = 0; i < batch_size; i++) {
+        // NOTE: label_data used as bool, labels >0 will be treated as true.
+        if (label_data[i]) {
+          // use first(max) data in each row
+          if (inference_data[i * inference_width] >=
+              (thresholds_list[idx_thresh])) {
             tp++;
           } else {
             fn++;
           }
         } else {
-          if (inference_data[i] >= (thresholds_list[idx_thresh])) {
+          if (inference_data[i * inference_width] >=
+              (thresholds_list[idx_thresh])) {
             fp++;
           } else {
             tn++;

python/paddle/v2/framework/layers.py

@@ -243,8 +243,11 @@ def accuracy(input, label, k=1, **kwargs):
     acc_out = helper.create_tmp_variable(dtype=acc_out_dtype)
     helper.append_op(
         type="accuracy",
-        inputs={"Inference": [topk_indices],
-                "Label": [label]},
+        inputs={
+            "Out": [topk_out],
+            "Indices": [topk_indices],
+            "Label": [label]
+        },
         outputs={"Accuracy": [acc_out]})
     return acc_out
 

python/paddle/v2/framework/tests/test_accuracy_op.py

@@ -7,13 +7,14 @@ class TestAccuracyOp(OpTest):
     def setUp(self):
         self.op_type = "accuracy"
         n = 8192
-        infer = np.random.randint(0, 2, (n, 1)).astype("int")
-        label = np.random.randint(0, 2, (n, 1)).astype("int")
-        self.inputs = {'Inference': infer, "Label": label}
+        infer = np.random.random((n, 1)).astype("float32")
+        indices = np.random.randint(0, 2, (n, 1))
+        label = np.random.randint(0, 2, (n, 1))
+        self.inputs = {'Out': infer, 'Indices': indices, "Label": label}
         num_correct = 0
         for rowid in xrange(n):
-            for ele in infer[rowid]:
-                if ele == label[rowid][0]:
+            for ele in indices[rowid]:
+                if ele == label[rowid]:
                     num_correct += 1
                     break
         self.outputs = {

python/paddle/v2/framework/tests/test_auc_op.py

@@ -6,10 +6,11 @@ from op_test import OpTest
 class TestAucOp(OpTest):
     def setUp(self):
         self.op_type = "auc"
-        pred = np.random.random((128)).astype("float32")
-        labels = np.random.randint(0, 2, (128, ))
+        pred = np.random.random((128, 2)).astype("float32")
+        indices = np.random.randint(0, 2, (128, 2))
+        labels = np.random.randint(0, 2, (128, 1))
         num_thresholds = 200
-        self.inputs = {'Inference': pred, 'Label': labels}
+        self.inputs = {'Out': pred, 'Indices': indices, 'Label': labels}
         self.attrs = {'curve': 'ROC', 'num_thresholds': num_thresholds}
         # NOTE: sklearn use a different way to generate thresholds
         #       which will cause the result differs slightly:
@@ -31,12 +32,12 @@ class TestAucOp(OpTest):
             tp, fn, tn, fp = 0, 0, 0, 0
             for i, lbl in enumerate(labels):
                 if lbl:
-                    if pred[i] >= thresh:
+                    if pred[i, 0] >= thresh:
                         tp += 1
                     else:
                         fn += 1
                 else:
-                    if pred[i] >= thresh:
+                    if pred[i, 0] >= thresh:
                         fp += 1
                     else:
                         tn += 1
@@ -62,6 +63,5 @@ class TestAucOp(OpTest):
         self.check_output()
 
 
-# TODO(typhoonzero): add this back till we fix it
-#if __name__ == "__main__":
-#    unittest.main()
+if __name__ == "__main__":
+    unittest.main()