未验证 提交 878dd88f 编写于 作者: 武毅 提交者: GitHub

Refine evaluator op types (#5208)

* refine evaluator op types

* update

* follow comments

* update

* fix v2 mnist case

* fix v2 mnist case

* update

* update
上级 f122a5da
...@@ -22,23 +22,35 @@ class AccuracyOp : public framework::OperatorWithKernel { ...@@ -22,23 +22,35 @@ class AccuracyOp : public framework::OperatorWithKernel {
using framework::OperatorWithKernel::OperatorWithKernel; using framework::OperatorWithKernel::OperatorWithKernel;
void InferShape(framework::InferShapeContext *ctx) const override { void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Inference"), PADDLE_ENFORCE(ctx->HasInput("Out"),
"Input(Inference) of AccuracyOp should not be null."); "Input (Out) of accuracy op should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Indices"),
"Input (Indices) of accuracy op should not be null.");
PADDLE_ENFORCE(ctx->HasInput("Label"), PADDLE_ENFORCE(ctx->HasInput("Label"),
"Input(Label) of AccuracyOp should not be null."); "Input (Label) of accuracy op should not be null.");
PADDLE_ENFORCE(ctx->HasOutput("Accuracy"), PADDLE_ENFORCE(ctx->HasOutput("Accuracy"),
"Output(Accuracy) of AccuracyOp should not be null."); "Output (Accuracy) of AccuracyOp should not be null.");
auto inference_dim = ctx->GetInputDim("Inference"); auto inference_dim = ctx->GetInputDim("Out");
auto label_dim = ctx->GetInputDim("Label"); auto label_dim = ctx->GetInputDim("Label");
// Assume indices has same shape with infernece, because
// it's the output of topk.
PADDLE_ENFORCE_EQ(label_dim.size(), 2, "label's rank must be 2."); PADDLE_ENFORCE_EQ(label_dim.size(), 2, "label's rank must be 2.");
PADDLE_ENFORCE_EQ(label_dim[1], 1, "label's second dimension must be 1"); PADDLE_ENFORCE_EQ(label_dim[1], 1, "label's second dimension must be 1");
PADDLE_ENFORCE_EQ(inference_dim[0], label_dim[0], PADDLE_ENFORCE_EQ(inference_dim[0], label_dim[0],
"inference size must be the same as label size"); "the inference tensor's num_rows must be"
" the same as label.");
ctx->SetOutputDim("Accuracy", {1}); ctx->SetOutputDim("Accuracy", {1});
ctx->ShareLoD("Inference", /*->*/ "Accuracy"); ctx->ShareLoD("Out", /*->*/ "Accuracy");
}
protected:
// IndicateDataType
framework::DataType IndicateDataType(
const framework::ExecutionContext &ctx) const override {
return framework::ToDataType(ctx.Input<Tensor>("Out")->type());
} }
}; };
...@@ -48,7 +60,8 @@ class AccuracyOpMaker : public framework::OpProtoAndCheckerMaker { ...@@ -48,7 +60,8 @@ class AccuracyOpMaker : public framework::OpProtoAndCheckerMaker {
framework::OpAttrChecker *op_checker) framework::OpAttrChecker *op_checker)
: OpProtoAndCheckerMaker(proto, op_checker) { : OpProtoAndCheckerMaker(proto, op_checker) {
// TODO(typhoonzero): support both inference value and indices. // TODO(typhoonzero): support both inference value and indices.
AddInput("Inference", "topk(indices) the network output"); AddInput("Out", "topk (inferences) the network output");
AddInput("Indices", "topk (indices) the network output");
AddInput("Label", "Label of the training data"); AddInput("Label", "Label of the training data");
// TODO(typhoonzero): AddInput("Weight", ... // TODO(typhoonzero): AddInput("Weight", ...
AddOutput("Accuracy", "The accuracy of current batch"); AddOutput("Accuracy", "The accuracy of current batch");
...@@ -59,7 +72,7 @@ The accuracy is: ...@@ -59,7 +72,7 @@ The accuracy is:
.. math:: .. math::
accuracy = \\frac{NumOfCorrectPredicts}{NumOfAllSamples}) accuracy = \\frac{NumOfCorrectPredicts}{NumOfAllSamples})
Both the input `Inference` and `Label` can carry the LoD (Level of Details) Both the input `Out` and `Label` can carry the LoD (Level of Details)
information, or not. But the output only shares the LoD with input `Inference`. information, or not. But the output only shares the LoD with input `Inference`.
)DOC"); )DOC");
} }
...@@ -71,6 +84,8 @@ information, or not. But the output only shares the LoD with input `Inference`. ...@@ -71,6 +84,8 @@ information, or not. But the output only shares the LoD with input `Inference`.
namespace ops = paddle::operators; namespace ops = paddle::operators;
REGISTER_OPERATOR(accuracy, ops::AccuracyOp, ops::AccuracyOpMaker, REGISTER_OPERATOR(accuracy, ops::AccuracyOp, ops::AccuracyOpMaker,
paddle::framework::EmptyGradOpMaker); paddle::framework::EmptyGradOpMaker);
REGISTER_OP_CPU_KERNEL( // FIXME(typhoonzero): types of T is for infernece data.
accuracy, ops::AccuracyKernel<paddle::platform::CPUPlace, int>, // label data is always int.
ops::AccuracyKernel<paddle::platform::CPUPlace, int64_t>); REGISTER_OP_CPU_KERNEL(accuracy,
ops::AccuracyKernel<paddle::platform::CPUPlace, float>,
ops::AccuracyKernel<paddle::platform::CPUPlace, double>);
...@@ -21,9 +21,10 @@ namespace paddle { ...@@ -21,9 +21,10 @@ namespace paddle {
namespace operators { namespace operators {
using platform::PADDLE_CUDA_NUM_THREADS; using platform::PADDLE_CUDA_NUM_THREADS;
template <typename T, int BlockSize> template <int BlockSize>
__global__ void AccuracyCudaKernel(const int N, const int D, const T* Xdata, __global__ void AccuracyCudaKernel(const int N, const int D,
const T* labeldata, float* accuracy) { const int64_t* Xdata,
const int64_t* labeldata, float* accuracy) {
int count = 0; int count = 0;
__shared__ int total[BlockSize]; __shared__ int total[BlockSize];
...@@ -52,13 +53,14 @@ class AccuracyOpCUDAKernel : public framework::OpKernel<T> { ...@@ -52,13 +53,14 @@ class AccuracyOpCUDAKernel : public framework::OpKernel<T> {
void Compute(const framework::ExecutionContext& ctx) const override { void Compute(const framework::ExecutionContext& ctx) const override {
PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()),
"It must use GPUPlace."); "It must use GPUPlace.");
auto* inference = ctx.Input<Tensor>("Inference"); auto* inference = ctx.Input<Tensor>("Out");
auto* indices = ctx.Input<Tensor>("Indices");
auto* label = ctx.Input<Tensor>("Label"); auto* label = ctx.Input<Tensor>("Label");
auto* accuracy = ctx.Output<Tensor>("Accuracy"); auto* accuracy = ctx.Output<Tensor>("Accuracy");
// FIXME(typhoonzero): only support indices currently // FIXME(typhoonzero): only support indices currently
// if add support for output values, how to detect the data type? // if add support for output values, how to detect the data type?
const T* inference_data = inference->data<T>(); const int64_t* indices_data = indices->data<int64_t>();
const T* label_data = label->data<T>(); const int64_t* label_data = label->data<int64_t>();
float* accuracy_data = accuracy->mutable_data<float>(ctx.GetPlace()); float* accuracy_data = accuracy->mutable_data<float>(ctx.GetPlace());
size_t num_samples = inference->dims()[0]; size_t num_samples = inference->dims()[0];
...@@ -69,11 +71,11 @@ class AccuracyOpCUDAKernel : public framework::OpKernel<T> { ...@@ -69,11 +71,11 @@ class AccuracyOpCUDAKernel : public framework::OpKernel<T> {
return; return;
} }
AccuracyCudaKernel<T, PADDLE_CUDA_NUM_THREADS><<< AccuracyCudaKernel<PADDLE_CUDA_NUM_THREADS><<<
1, PADDLE_CUDA_NUM_THREADS, 0, 1, PADDLE_CUDA_NUM_THREADS, 0,
reinterpret_cast<const platform::CUDADeviceContext&>( reinterpret_cast<const platform::CUDADeviceContext&>(
ctx.device_context()) ctx.device_context())
.stream()>>>(num_samples, infer_width, inference_data, label_data, .stream()>>>(num_samples, infer_width, indices_data, label_data,
accuracy_data); accuracy_data);
} }
}; };
...@@ -81,5 +83,7 @@ class AccuracyOpCUDAKernel : public framework::OpKernel<T> { ...@@ -81,5 +83,7 @@ class AccuracyOpCUDAKernel : public framework::OpKernel<T> {
} // namespace operators } // namespace operators
} // namespace paddle } // namespace paddle
REGISTER_OP_GPU_KERNEL(accuracy, paddle::operators::AccuracyOpCUDAKernel<int>, // FIXME(typhoonzero): types of T is for infernece data.
paddle::operators::AccuracyOpCUDAKernel<int64_t>); // label data is always int
REGISTER_OP_GPU_KERNEL(accuracy, paddle::operators::AccuracyOpCUDAKernel<float>,
paddle::operators::AccuracyOpCUDAKernel<double>);
...@@ -38,14 +38,15 @@ template <typename Place, typename T> ...@@ -38,14 +38,15 @@ template <typename Place, typename T>
class AccuracyKernel : public framework::OpKernel<T> { class AccuracyKernel : public framework::OpKernel<T> {
public: public:
void Compute(const framework::ExecutionContext& ctx) const override { void Compute(const framework::ExecutionContext& ctx) const override {
auto* inference = ctx.Input<Tensor>("Inference"); auto* inference = ctx.Input<Tensor>("Out");
auto* indices = ctx.Input<Tensor>("Indices");
auto* label = ctx.Input<Tensor>("Label"); auto* label = ctx.Input<Tensor>("Label");
auto* accuracy = ctx.Output<Tensor>("Accuracy"); auto* accuracy = ctx.Output<Tensor>("Accuracy");
float* accuracy_data = accuracy->mutable_data<float>(ctx.GetPlace()); float* accuracy_data = accuracy->mutable_data<float>(ctx.GetPlace());
const T* inference_data = inference->data<T>(); const int64_t* indices_data = indices->data<int64_t>();
const T* label_data = label->data<T>(); const int64_t* label_data = label->data<int64_t>();
size_t num_samples = inference->dims()[0]; size_t num_samples = inference->dims()[0];
size_t class_dim = inference->dims()[1]; size_t class_dim = inference->dims()[1];
...@@ -60,7 +61,7 @@ class AccuracyKernel : public framework::OpKernel<T> { ...@@ -60,7 +61,7 @@ class AccuracyKernel : public framework::OpKernel<T> {
for (size_t i = 0; i < num_samples; ++i) { for (size_t i = 0; i < num_samples; ++i) {
PADDLE_ENFORCE_GE(label_data[i], 0, "label must >= 0"); PADDLE_ENFORCE_GE(label_data[i], 0, "label must >= 0");
for (size_t j = 0; j < class_dim; ++j) { for (size_t j = 0; j < class_dim; ++j) {
if (inference_data[i * class_dim + j] == label_data[i]) { if (indices_data[i * class_dim + j] == label_data[i]) {
++num_correct; ++num_correct;
break; break;
} }
......
...@@ -23,18 +23,26 @@ class AucOp : public framework::OperatorWithKernel { ...@@ -23,18 +23,26 @@ class AucOp : public framework::OperatorWithKernel {
protected: protected:
void InferShape(framework::InferShapeContext *ctx) const override { void InferShape(framework::InferShapeContext *ctx) const override {
PADDLE_ENFORCE(ctx->HasInput("Inference"), PADDLE_ENFORCE(ctx->HasInput("Out"), "Input of Out must be initialized.");
"Input of Inference must be initialized."); PADDLE_ENFORCE(ctx->HasInput("Indices"),
"Input of Indices must be initialized.");
PADDLE_ENFORCE(ctx->HasInput("Label"), PADDLE_ENFORCE(ctx->HasInput("Label"),
"Input of Label must be initialized."); "Input of Label must be initialized.");
auto inference_dim = ctx->GetInputDim("Inference"); auto inference_height = ctx->GetInputDim("Out")[0];
auto label_dim = ctx->GetInputDim("Label"); auto label_height = ctx->GetInputDim("Label")[0];
PADDLE_ENFORCE_EQ(inference_dim, label_dim, PADDLE_ENFORCE_EQ(inference_height, label_height,
"inference and label should have same shape"); "Out and Label should have same height.");
ctx->SetOutputDim("AUC", {1}); ctx->SetOutputDim("AUC", {1});
ctx->ShareLoD("Inference", /*->*/ "AUC"); ctx->ShareLoD("Out", /*->*/ "AUC");
}
protected:
// IndicateDataType
framework::DataType IndicateDataType(
const framework::ExecutionContext &ctx) const override {
return framework::ToDataType(ctx.Input<Tensor>("Out")->type());
} }
}; };
...@@ -42,12 +50,18 @@ class AucOpMaker : public framework::OpProtoAndCheckerMaker { ...@@ -42,12 +50,18 @@ class AucOpMaker : public framework::OpProtoAndCheckerMaker {
public: public:
AucOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) AucOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
: OpProtoAndCheckerMaker(proto, op_checker) { : OpProtoAndCheckerMaker(proto, op_checker) {
AddInput("Inference", AddInput("Out",
"A floating point tensor of arbitrary shape and whose values" "A floating point 2D tensor, values are in the range [0, 1]."
"are in the range [0, 1]."); "Each row is descend sorted. This input should be the"
"output of topk."
"Typically, this tensor indicates the probability of each label");
AddInput("Indices",
"An int 2D tensor, indicating the indices of original"
"tensor before sort. Typically, this tensor indicates which label"
"the probability stands for.");
AddInput("Label", AddInput("Label",
"A tensor whose shape matches " "A 2D int tensor indicating the label of the training data."
"Inference. Will be cast to bool."); "The height is batch size and width is always 1.");
// TODO(typhoonzero): support weight input // TODO(typhoonzero): support weight input
AddOutput("AUC", AddOutput("AUC",
"A scalar representing the " "A scalar representing the "
......
...@@ -29,7 +29,7 @@ template <typename Place, typename T> ...@@ -29,7 +29,7 @@ template <typename Place, typename T>
class AucKernel : public framework::OpKernel<T> { class AucKernel : public framework::OpKernel<T> {
public: public:
void Compute(const framework::ExecutionContext& ctx) const override { void Compute(const framework::ExecutionContext& ctx) const override {
auto* inference = ctx.Input<Tensor>("Inference"); auto* inference = ctx.Input<Tensor>("Out");
auto* label = ctx.Input<Tensor>("Label"); auto* label = ctx.Input<Tensor>("Label");
auto* auc = ctx.Output<Tensor>("AUC"); auto* auc = ctx.Output<Tensor>("AUC");
...@@ -46,18 +46,11 @@ class AucKernel : public framework::OpKernel<T> { ...@@ -46,18 +46,11 @@ class AucKernel : public framework::OpKernel<T> {
thresholds_list[0] = 0.0f - kEpsilon; thresholds_list[0] = 0.0f - kEpsilon;
thresholds_list[num_thresholds - 1] = 1.0f + kEpsilon; thresholds_list[num_thresholds - 1] = 1.0f + kEpsilon;
size_t num_samples = inference->numel(); size_t batch_size = inference->dims()[0];
size_t inference_width = inference->dims()[1];
const T* inference_data = inference->data<T>(); const T* inference_data = inference->data<T>();
Tensor label_casted; const int64_t* label_data = label->data<int64_t>();
label_casted.Resize(label->dims());
bool* label_casted_data = label_casted.mutable_data<bool>(ctx.GetPlace());
const int* label_data = label->data<int>();
// cast label_data to bool
for (size_t i = 0; i < num_samples; i++) {
label_casted_data[i] = static_cast<bool>(label_data[i]);
}
// Create local tensor for storing the curve: TP, FN, TN, FP // Create local tensor for storing the curve: TP, FN, TN, FP
// TODO(typhoonzero): use eigen op to caculate these values. // TODO(typhoonzero): use eigen op to caculate these values.
...@@ -68,23 +61,27 @@ class AucKernel : public framework::OpKernel<T> { ...@@ -68,23 +61,27 @@ class AucKernel : public framework::OpKernel<T> {
true_negative.Resize({num_thresholds}); true_negative.Resize({num_thresholds});
false_positive.Resize({num_thresholds}); false_positive.Resize({num_thresholds});
int* tp_data = true_positive.mutable_data<int>(ctx.GetPlace()); int64_t* tp_data = true_positive.mutable_data<int64_t>(ctx.GetPlace());
int* fn_data = false_negative.mutable_data<int>(ctx.GetPlace()); int64_t* fn_data = false_negative.mutable_data<int64_t>(ctx.GetPlace());
int* tn_data = true_negative.mutable_data<int>(ctx.GetPlace()); int64_t* tn_data = true_negative.mutable_data<int64_t>(ctx.GetPlace());
int* fp_data = false_positive.mutable_data<int>(ctx.GetPlace()); int64_t* fp_data = false_positive.mutable_data<int64_t>(ctx.GetPlace());
for (int idx_thresh = 0; idx_thresh < num_thresholds; idx_thresh++) { for (int idx_thresh = 0; idx_thresh < num_thresholds; idx_thresh++) {
// caculate TP, FN, TN, FP for current thresh // caculate TP, FN, TN, FP for current thresh
int tp = 0, fn = 0, tn = 0, fp = 0; int64_t tp = 0, fn = 0, tn = 0, fp = 0;
for (size_t i = 0; i < num_samples; i++) { for (size_t i = 0; i < batch_size; i++) {
if (label_casted_data[i]) { // NOTE: label_data used as bool, labels >0 will be treated as true.
if (inference_data[i] >= (thresholds_list[idx_thresh])) { if (label_data[i]) {
// use first(max) data in each row
if (inference_data[i * inference_width] >=
(thresholds_list[idx_thresh])) {
tp++; tp++;
} else { } else {
fn++; fn++;
} }
} else { } else {
if (inference_data[i] >= (thresholds_list[idx_thresh])) { if (inference_data[i * inference_width] >=
(thresholds_list[idx_thresh])) {
fp++; fp++;
} else { } else {
tn++; tn++;
......
...@@ -243,8 +243,11 @@ def accuracy(input, label, k=1, **kwargs): ...@@ -243,8 +243,11 @@ def accuracy(input, label, k=1, **kwargs):
acc_out = helper.create_tmp_variable(dtype=acc_out_dtype) acc_out = helper.create_tmp_variable(dtype=acc_out_dtype)
helper.append_op( helper.append_op(
type="accuracy", type="accuracy",
inputs={"Inference": [topk_indices], inputs={
"Label": [label]}, "Out": [topk_out],
"Indices": [topk_indices],
"Label": [label]
},
outputs={"Accuracy": [acc_out]}) outputs={"Accuracy": [acc_out]})
return acc_out return acc_out
......
...@@ -7,13 +7,14 @@ class TestAccuracyOp(OpTest): ...@@ -7,13 +7,14 @@ class TestAccuracyOp(OpTest):
def setUp(self): def setUp(self):
self.op_type = "accuracy" self.op_type = "accuracy"
n = 8192 n = 8192
infer = np.random.randint(0, 2, (n, 1)).astype("int") infer = np.random.random((n, 1)).astype("float32")
label = np.random.randint(0, 2, (n, 1)).astype("int") indices = np.random.randint(0, 2, (n, 1))
self.inputs = {'Inference': infer, "Label": label} label = np.random.randint(0, 2, (n, 1))
self.inputs = {'Out': infer, 'Indices': indices, "Label": label}
num_correct = 0 num_correct = 0
for rowid in xrange(n): for rowid in xrange(n):
for ele in infer[rowid]: for ele in indices[rowid]:
if ele == label[rowid][0]: if ele == label[rowid]:
num_correct += 1 num_correct += 1
break break
self.outputs = { self.outputs = {
......
...@@ -6,10 +6,11 @@ from op_test import OpTest ...@@ -6,10 +6,11 @@ from op_test import OpTest
class TestAucOp(OpTest): class TestAucOp(OpTest):
def setUp(self): def setUp(self):
self.op_type = "auc" self.op_type = "auc"
pred = np.random.random((128)).astype("float32") pred = np.random.random((128, 2)).astype("float32")
labels = np.random.randint(0, 2, (128, )) indices = np.random.randint(0, 2, (128, 2))
labels = np.random.randint(0, 2, (128, 1))
num_thresholds = 200 num_thresholds = 200
self.inputs = {'Inference': pred, 'Label': labels} self.inputs = {'Out': pred, 'Indices': indices, 'Label': labels}
self.attrs = {'curve': 'ROC', 'num_thresholds': num_thresholds} self.attrs = {'curve': 'ROC', 'num_thresholds': num_thresholds}
# NOTE: sklearn use a different way to generate thresholds # NOTE: sklearn use a different way to generate thresholds
# which will cause the result differs slightly: # which will cause the result differs slightly:
...@@ -31,12 +32,12 @@ class TestAucOp(OpTest): ...@@ -31,12 +32,12 @@ class TestAucOp(OpTest):
tp, fn, tn, fp = 0, 0, 0, 0 tp, fn, tn, fp = 0, 0, 0, 0
for i, lbl in enumerate(labels): for i, lbl in enumerate(labels):
if lbl: if lbl:
if pred[i] >= thresh: if pred[i, 0] >= thresh:
tp += 1 tp += 1
else: else:
fn += 1 fn += 1
else: else:
if pred[i] >= thresh: if pred[i, 0] >= thresh:
fp += 1 fp += 1
else: else:
tn += 1 tn += 1
...@@ -62,6 +63,5 @@ class TestAucOp(OpTest): ...@@ -62,6 +63,5 @@ class TestAucOp(OpTest):
self.check_output() self.check_output()
# TODO(typhoonzero): add this back till we fix it if __name__ == "__main__":
#if __name__ == "__main__": unittest.main()
# unittest.main()
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册