chunk_eval_op.cc 8.0 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
G
guosheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

Y
Yi Wang 已提交
15
#include "paddle/fluid/operators/chunk_eval_op.h"
16

17 18
#include <string>
#include <vector>
G
guosheng 已提交
19 20 21 22 23 24 25 26 27

namespace paddle {
namespace operators {

class ChunkEvalOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

  void InferShape(framework::InferShapeContext *ctx) const override {
28 29
    OP_INOUT_CHECK(
        ctx->HasInput("Inference"), "Input", "Inference", "chunk_eval");
30 31
    OP_INOUT_CHECK(ctx->HasInput("Label"), "Input", "Label", "chunk_eval");

32 33
    OP_INOUT_CHECK(
        ctx->HasOutput("Precision"), "Output", "Precision", "chunk_eval");
34
    OP_INOUT_CHECK(ctx->HasOutput("Recall"), "Output", "Recall", "chunk_eval");
35 36 37 38 39
    OP_INOUT_CHECK(
        ctx->HasOutput("F1-Score"), "Output", "F1-Score", "chunk_eval");
    OP_INOUT_CHECK(ctx->HasOutput("NumInferChunks"),
                   "Output",
                   "NumInferChunks",
40
                   "chunk_eval");
41 42 43
    OP_INOUT_CHECK(ctx->HasOutput("NumLabelChunks"),
                   "Output",
                   "NumLabelChunks",
44
                   "chunk_eval");
45 46 47
    OP_INOUT_CHECK(ctx->HasOutput("NumCorrectChunks"),
                   "Output",
                   "NumCorrectChunks",
48
                   "chunk_eval");
G
guosheng 已提交
49 50 51 52

    auto inference_dim = ctx->GetInputDim("Inference");
    auto label_dim = ctx->GetInputDim("Label");

53
    PADDLE_ENFORCE_EQ(
54 55
        inference_dim,
        label_dim,
56 57 58
        platform::errors::InvalidArgument(
            "Input(Inference)'s shape must be the same as Input(Label)'s "
            "shape, but received [%s] (Inference) vs [%s] (Label).",
59 60
            inference_dim,
            label_dim));
G
guosheng 已提交
61

62 63
    bool use_padding = ctx->HasInput("SeqLength");
    if (use_padding) {
64 65 66
      PADDLE_ENFORCE_EQ(
          (inference_dim.size() == 3 && inference_dim[2] == 1) ||
              inference_dim.size() == 2,
67 68 69 70 71 72
          true,
          platform::errors::InvalidArgument(
              "when Input(SeqLength) is provided, Input(Inference) "
              "should be of dim 3 (batch_size, bucket, 1) or dim 2 "
              "(batch_size, bucket), but received [%s].",
              inference_dim));
73
      auto seq_length_dim = ctx->GetInputDim("SeqLength");
74 75
      PADDLE_ENFORCE_LE(seq_length_dim.size(),
                        2,
76 77 78 79
                        platform::errors::InvalidArgument(
                            "Input(SeqLength)'s rank should not be greater "
                            "than 2, but received %d.",
                            seq_length_dim.size()));
80 81
    }

G
guosheng 已提交
82 83 84
    ctx->SetOutputDim("Precision", {1});
    ctx->SetOutputDim("Recall", {1});
    ctx->SetOutputDim("F1-Score", {1});
G
guosheng 已提交
85 86 87
    ctx->SetOutputDim("NumInferChunks", {1});
    ctx->SetOutputDim("NumLabelChunks", {1});
    ctx->SetOutputDim("NumCorrectChunks", {1});
G
guosheng 已提交
88 89
  }

90
 protected:
91
  framework::OpKernelType GetExpectedKernelType(
G
guosheng 已提交
92
      const framework::ExecutionContext &ctx) const override {
93
    return framework::OpKernelType(framework::proto::VarType::FP32,
94
                                   platform::CPUPlace());
G
guosheng 已提交
95 96 97 98 99
  }
};

class ChunkEvalOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
Y
Yu Yang 已提交
100
  void Make() override {
G
guosheng 已提交
101
    AddInput("Inference",
Q
Qiao Longfei 已提交
102 103
             "(Tensor, default: Tensor<int64_t>). "
             "Predictions from the network.");
104
    AddInput("Label",
Q
Qiao Longfei 已提交
105
             "(Tensor, default: Tensor<int64_t>). The true tag sequences.");
106 107 108 109
    AddInput("SeqLength",
             "(Tensor, default: Tensor<int64_t>). The length of each sequence, "
             "used when Inference and Label are Tensor type .")
        .AsDispensable();
110 111 112
    AddOutput("Precision",
              "(float). The evaluated precision (called positive predictive "
              "value) of chunks on the given mini-batch.");
G
guosheng 已提交
113
    AddOutput("Recall",
114 115
              "(float). The evaluated recall (true positive rate or "
              "sensitivity) of chunks on the given mini-batch.");
G
guosheng 已提交
116
    AddOutput("F1-Score",
117
              "(float). The evaluated F1-Score on the given mini-batch.");
118 119 120
    AddOutput("NumInferChunks",
              "(int64_t). The number of chunks in Inference on the given "
              "mini-batch.");
G
guosheng 已提交
121
    AddOutput(
122 123 124 125 126 127
        "NumLabelChunks",
        "(int64_t). The number of chunks in Label on the given mini-batch.");
    AddOutput(
        "NumCorrectChunks",
        "(int64_t). The number of chunks both in Inference and Label on the "
        "given mini-batch.");
128
    AddAttr<int>("num_chunk_types",
Y
yi.wu 已提交
129 130 131 132 133 134
                 "The number of chunk type. See the description for details.");
    AddAttr<std::string>("chunk_scheme",
                         "The labeling scheme indicating "
                         "how to encode the chunks. Must be IOB, IOE, IOBES or "
                         "plain. See the description"
                         "for details.")
G
guosheng 已提交
135
        .SetDefault("IOB");
136
    AddAttr<std::vector<int>>("excluded_chunk_types",
Y
yi.wu 已提交
137
                              "A list including chunk type ids "
138
                              "indicating chunk types that are not counted. "
Y
yi.wu 已提交
139
                              "See the description for details.")
G
guosheng 已提交
140 141
        .SetDefault(std::vector<int>{});
    AddComment(R"DOC(
Y
yangyaming 已提交
142
For some basics of chunking, please refer to
Y
yi.wu 已提交
143
'Chunking with Support Vector Machines <https://aclanthology.info/pdf/N/N01/N01-1025.pdf>'.
144

Y
yi.wu 已提交
145
ChunkEvalOp computes the precision, recall, and F1-score of chunk detection,
Y
yangyaming 已提交
146
and supports IOB, IOE, IOBES and IO (also known as plain) tagging schemes.
147
Here is a NER example of labeling for these tagging schemes:
148

Y
yi.wu 已提交
149 150 151 152 153
          Li     Ming    works  at  Agricultural   Bank   of    China  in  Beijing.
   IO     I-PER  I-PER   O      O   I-ORG          I-ORG  I-ORG I-ORG  O   I-LOC
   IOB    B-PER  I-PER   O      O   B-ORG          I-ORG  I-ORG I-ORG  O   B-LOC
   IOE    I-PER  E-PER   O      O   I-ORG          I-ORG  I-ORG E-ORG  O   E-LOC
   IOBES  B-PER  E-PER   O      O   I-ORG          I-ORG  I-ORG E-ORG  O   S-LOC
154

Q
Qiao Longfei 已提交
155
There are three chunk types(named entity types) including PER(person), ORG(organization)
156 157
and LOC(LOCATION), and we can see that the labels have the form <tag type>-<chunk type>.

Y
yangyaming 已提交
158 159 160
Since the calculations actually use label ids rather than labels, extra attention
should be paid when mapping labels to ids to make CheckEvalOp work. The key point
is that the listed equations are satisfied by ids.
161

Y
yi.wu 已提交
162 163
   tag_type = label % num_tag_type
   chunk_type = label / num_tag_type
164

Y
yangyaming 已提交
165
where `num_tag_type` is the num of tag types in the tagging scheme, `num_chunk_type`
166
is the num of chunk types, and `tag_type` get its value from the following table.
167

Y
yi.wu 已提交
168 169 170 171 172
   Scheme Begin Inside End   Single
    plain   0     -      -     -
    IOB     0     1      -     -
    IOE     -     0      1     -
    IOBES   0     1      2     3
G
guosheng 已提交
173

Y
yangyaming 已提交
174
Still use NER as example, assuming the tagging scheme is IOB while chunk types are ORG,
175
PER and LOC. To satisfy the above equations, the label map can be like this:
G
guosheng 已提交
176

Y
yi.wu 已提交
177 178 179 180 181 182 183
   B-ORG  0
   I-ORG  1
   B-PER  2
   I-PER  3
   B-LOC  4
   I-LOC  5
   O      6
G
guosheng 已提交
184

Y
yi.wu 已提交
185
It's not hard to verify the equations noting that the num of chunk types
Y
yangyaming 已提交
186 187
is 3 and the num of tag types in IOB scheme is 2. For example, the label
id of I-LOC is 5, the tag type id of I-LOC is 1, and the chunk type id of
188
I-LOC is 2, which consistent with the results from the equations.
G
guosheng 已提交
189 190 191 192 193 194 195 196
)DOC");
  }
};

}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;
197 198
REGISTER_OP_WITHOUT_GRADIENT(chunk_eval,
                             ops::ChunkEvalOp,
G
guosheng 已提交
199 200 201
                             ops::ChunkEvalOpMaker);
REGISTER_OP_CPU_KERNEL(chunk_eval,
                       ops::ChunkEvalKernel<paddle::platform::CPUPlace, float>);