chunk_eval_op.cc 6.9 KB
Newer Older
G
guosheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/operators/chunk_eval_op.h"

namespace paddle {
namespace operators {

class ChunkEvalOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

  void InferShape(framework::InferShapeContext *ctx) const override {
    PADDLE_ENFORCE(ctx->HasInput("Inference"),
                   "Input(Inference) of ChunkEvalOp should not be null.");
    PADDLE_ENFORCE(ctx->HasInput("Label"),
                   "Input(Label) of ChunkEvalOp should not be null.");
    PADDLE_ENFORCE(ctx->HasOutput("Precision"),
                   "Output(Precision) of ChunkEvalOp should not be null.");
    PADDLE_ENFORCE(ctx->HasOutput("Recall"),
                   "Output(Recall) of ChunkEvalOp should not be null.");
    PADDLE_ENFORCE(ctx->HasOutput("F1-Score"),
                   "Output(F1-Score) of ChunkEvalOp should not be null.");
G
guosheng 已提交
35 36 37 38 39 40 41
    PADDLE_ENFORCE(ctx->HasOutput("NumInferChunks"),
                   "Output(NumInferChunks) of ChunkEvalOp should not be null.");
    PADDLE_ENFORCE(ctx->HasOutput("NumLabelChunks"),
                   "Output(NumLabelChunks) of ChunkEvalOp should not be null.");
    PADDLE_ENFORCE(
        ctx->HasOutput("NumCorrectChunks"),
        "Output(NumCorrectChunks) of ChunkEvalOp should not be null.");
G
guosheng 已提交
42 43 44 45 46 47 48 49 50 51

    auto inference_dim = ctx->GetInputDim("Inference");
    auto label_dim = ctx->GetInputDim("Label");

    PADDLE_ENFORCE(inference_dim == label_dim,
                   "Inference's shape must be the same as Label's shape.");

    ctx->SetOutputDim("Precision", {1});
    ctx->SetOutputDim("Recall", {1});
    ctx->SetOutputDim("F1-Score", {1});
G
guosheng 已提交
52 53 54
    ctx->SetOutputDim("NumInferChunks", {1});
    ctx->SetOutputDim("NumLabelChunks", {1});
    ctx->SetOutputDim("NumCorrectChunks", {1});
G
guosheng 已提交
55 56
  }

57
 protected:
58
  framework::OpKernelType GetExpectedKernelType(
G
guosheng 已提交
59
      const framework::ExecutionContext &ctx) const override {
60
    return framework::OpKernelType(framework::proto::DataType::FP32,
61
                                   platform::CPUPlace());
G
guosheng 已提交
62 63 64 65 66
  }
};

class ChunkEvalOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
67
  ChunkEvalOpMaker(OpProto *proto, OpAttrChecker *op_checker)
G
guosheng 已提交
68 69
      : OpProtoAndCheckerMaker(proto, op_checker) {
    AddInput("Inference",
Q
Qiao Longfei 已提交
70 71
             "(Tensor, default: Tensor<int64_t>). "
             "Predictions from the network.");
72
    AddInput("Label",
Q
Qiao Longfei 已提交
73
             "(Tensor, default: Tensor<int64_t>). The true tag sequences.");
74 75 76
    AddOutput("Precision",
              "(float). The evaluated precision (called positive predictive "
              "value) of chunks on the given mini-batch.");
G
guosheng 已提交
77
    AddOutput("Recall",
78 79
              "(float). The evaluated recall (true positive rate or "
              "sensitivity) of chunks on the given mini-batch.");
G
guosheng 已提交
80
    AddOutput("F1-Score",
81
              "(float). The evaluated F1-Score on the given mini-batch.");
82 83 84
    AddOutput("NumInferChunks",
              "(int64_t). The number of chunks in Inference on the given "
              "mini-batch.");
G
guosheng 已提交
85
    AddOutput(
86 87 88 89 90 91
        "NumLabelChunks",
        "(int64_t). The number of chunks in Label on the given mini-batch.");
    AddOutput(
        "NumCorrectChunks",
        "(int64_t). The number of chunks both in Inference and Label on the "
        "given mini-batch.");
92 93 94 95 96 97 98
    AddAttr<int>("num_chunk_types",
                 "(int). The number of chunk type. See below for details.");
    AddAttr<std::string>(
        "chunk_scheme",
        "(string, default IOB). The labeling scheme indicating "
        "how to encode the chunks. Must be IOB, IOE, IOBES or plain. See below "
        "for details.")
G
guosheng 已提交
99
        .SetDefault("IOB");
100 101 102 103
    AddAttr<std::vector<int>>("excluded_chunk_types",
                              "(list<int>) A list including chunk type ids "
                              "indicating chunk types that are not counted. "
                              "See below for details.")
G
guosheng 已提交
104 105
        .SetDefault(std::vector<int>{});
    AddComment(R"DOC(
Y
yangyaming 已提交
106
For some basics of chunking, please refer to
Q
Qiao Longfei 已提交
107
‘Chunking with Support Vector Machines <https://aclanthology.info/pdf/N/N01/N01-1025.pdf>’.
108 109


Y
yangyaming 已提交
110 111
CheckEvalOp computes the precision, recall, and F1-score of chunk detection,
and supports IOB, IOE, IOBES and IO (also known as plain) tagging schemes.
112 113 114 115 116 117 118 119
Here is a NER example of labeling for these tagging schemes:

 	     Li     Ming    works  at  Agricultural   Bank   of    China  in  Beijing.
  IO:    I-PER  I-PER   O      O   I-ORG          I-ORG  I-ORG I-ORG  O   I-LOC
  IOB:   B-PER  I-PER   O      O   B-ORG          I-ORG  I-ORG I-ORG  O   B-LOC
  IOE:   I-PER  E-PER   O      O   I-ORG          I-ORG  I-ORG E-ORG  O   E-LOC
  IOBES: B-PER  E-PER   O      O   I-ORG          I-ORG  I-ORG E-ORG  O   S-LOC

Q
Qiao Longfei 已提交
120
There are three chunk types(named entity types) including PER(person), ORG(organization)
121 122
and LOC(LOCATION), and we can see that the labels have the form <tag type>-<chunk type>.

Y
yangyaming 已提交
123 124 125
Since the calculations actually use label ids rather than labels, extra attention
should be paid when mapping labels to ids to make CheckEvalOp work. The key point
is that the listed equations are satisfied by ids.
126 127 128 129

    tag_type = label % num_tag_type
    chunk_type = label / num_tag_type

Y
yangyaming 已提交
130
where `num_tag_type` is the num of tag types in the tagging scheme, `num_chunk_type`
131
is the num of chunk types, and `tag_type` get its value from the following table.
G
guosheng 已提交
132 133

    Scheme Begin Inside End   Single
134 135 136 137
     plain   0     -      -     -
     IOB     0     1      -     -
     IOE     -     0      1     -
     IOBES   0     1      2     3
G
guosheng 已提交
138

Y
yangyaming 已提交
139
Still use NER as example, assuming the tagging scheme is IOB while chunk types are ORG,
140
PER and LOC. To satisfy the above equations, the label map can be like this:
G
guosheng 已提交
141 142 143 144 145 146 147 148 149

    B-ORG  0
    I-ORG  1
    B-PER  2
    I-PER  3
    B-LOC  4
    I-LOC  5
    O      6

Y
yangyaming 已提交
150 151 152
It’s not hard to verify the equations noting that the num of chunk types
is 3 and the num of tag types in IOB scheme is 2. For example, the label
id of I-LOC is 5, the tag type id of I-LOC is 1, and the chunk type id of
153
I-LOC is 2, which consistent with the results from the equations.
G
guosheng 已提交
154 155 156 157 158 159 160 161 162 163 164 165
)DOC");
  }
};

}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;
REGISTER_OP_WITHOUT_GRADIENT(chunk_eval, ops::ChunkEvalOp,
                             ops::ChunkEvalOpMaker);
REGISTER_OP_CPU_KERNEL(chunk_eval,
                       ops::ChunkEvalKernel<paddle::platform::CPUPlace, float>);