diff --git a/paddle/operators/chunk_eval_op.cc b/paddle/operators/chunk_eval_op.cc index 2b40c1873ccbc273779a5657c43570a1b38bdc81..a3d0d996464910cf51dfe6e206e93f26fc63cd2b 100644 --- a/paddle/operators/chunk_eval_op.cc +++ b/paddle/operators/chunk_eval_op.cc @@ -21,7 +21,6 @@ class ChunkEvalOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; - protected: void InferShape(framework::InferShapeContext *ctx) const override { PADDLE_ENFORCE(ctx->HasInput("Inference"), "Input(Inference) of ChunkEvalOp should not be null."); @@ -45,6 +44,7 @@ class ChunkEvalOp : public framework::OperatorWithKernel { ctx->SetOutputDim("F1-Score", {1}); } + protected: framework::DataType IndicateDataType( const framework::ExecutionContext &ctx) const override { return framework::DataType::FP32; @@ -57,61 +57,66 @@ class ChunkEvalOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("Inference", - "(Tensor, default: Tensor) Predictions from the network."); - AddInput("Label", "(Tensor, default: Tensor) Labels of the data."); - AddOutput( - "Precision", - "(float) The precision ratio of the predictions on current data."); + "(Tensor, default: Tensor). Predictions from the network."); + AddInput("Label", + "(Tensor, default: Tensor). The true tag sequences."); + AddOutput("Precision", + "(float). The evaluated precision (called positive predictive " + "value) of chunks on the given mini-batch."); AddOutput("Recall", - "(float) The recall ratio of the predictions on current data."); + "(float). The evaluated recall (true positive rate or " + "sensitivity) of chunks on the given mini-batch."); AddOutput("F1-Score", - "(float) The F1-Score of the predictions on current data."); - AddAttr("num_chunk_types", "(int) The number of chunk type."); - AddAttr("chunk_scheme", - "(string, default IOB) The label scheme.") + "(float). The evaluated F1-Score on the given mini-batch."); + AddAttr("num_chunk_types", + "(int). The number of chunk type. See below for details."); + AddAttr( + "chunk_scheme", + "(string, default IOB). The labeling scheme indicating " + "how to encode the chunks. Must be IOB, IOE, IOBES or plain. See below " + "for details.") .SetDefault("IOB"); - AddAttr>( - "excluded_chunk_types", - "(list) A list indicating chunk types not to be counted.") + AddAttr>("excluded_chunk_types", + "(list) A list including chunk type ids " + "indicating chunk types that are not counted. " + "See below for details.") .SetDefault(std::vector{}); AddComment(R"DOC( -Chunk evaluator is used to evaluate segment labelling accuracy for a -sequence. It calculates precision, recall and F1 scores for the chunk detection. -To use chunk evaluator, several concepts need to be clarified firstly. -[Chunk type] is the type of the whole chunk and a chunk consists of one or several words. (For example in NER, ORG for organization name, PER for person name etc.) -[Tag type] indicates the position of a word in a chunk. (B for begin, I for inside, E for end, S for single) -We can name a label by combining tag type and chunk type. (ie. B-ORG for begining of an organization name) -The construction of label dictionary should obey the following rules: -- Use one of the listed labelling schemes. These schemes differ in ways indicating chunk boundry. - - Scheme Description - plain Use the same label for the whole chunk. - IOB Two labels for chunk type X, B-X for chunk begining and I-X for chunk inside. - IOE Two labels for chunk type X, E-X for chunk ending and I-X for chunk inside. - IOBES Four labels for chunk type X, B-X for chunk begining, I-X for chunk inside, E-X for chunk end and S-X for single word chunk. - -To make it clear, let's illustrate by an NER example. -Assuming that there are three named entity types including ORG, PER and LOC which are called 'chunk type' here, -if 'IOB' scheme were used, the label set will be extended to a set including B-ORG, I-ORG, B-PER, I-PER, B-LOC, I-LOC and O, -in which B-ORG for begining of ORG and I-ORG for inside of ORG. -Prefixes which are called 'tag type' here are added to chunk types and there are two tag types including B and I. -Of course, the training data should be labeled accordingly. -- Mapping is done correctly by the listed equations and assigning protocol. -The following table are equations to extract tag type and chunk type from a label. - - tagType = label % numTagType - chunkType = label / numTagType - otherChunkType = numChunkTypes - -The following table shows the mapping rule between tagType and tag type in each scheme. +For some basics of chunking, please refer to +‘Chunking with Support Vector Mechines ’. + + +CheckEvalOp computes the precision, recall, and F1-score of chunk detection, +and supports IOB, IOE, IOBES and IO (also known as plain) tagging schemes. +Here is a NER example of labeling for these tagging schemes: + + Li Ming works at Agricultural Bank of China in Beijing. + IO: I-PER I-PER O O I-ORG I-ORG I-ORG I-ORG O I-LOC + IOB: B-PER I-PER O O B-ORG I-ORG I-ORG I-ORG O B-LOC + IOE: I-PER E-PER O O I-ORG I-ORG I-ORG E-ORG O E-LOC + IOBES: B-PER E-PER O O I-ORG I-ORG I-ORG E-ORG O S-LOC + +There are three chunk types(named entity types) including PER(person), ORG(orgnazation) +and LOC(LOCATION), and we can see that the labels have the form -. + +Since the calculations actually use label ids rather than labels, extra attention +should be paid when mapping labels to ids to make CheckEvalOp work. The key point +is that the listed equations are satisfied by ids. + + tag_type = label % num_tag_type + chunk_type = label / num_tag_type + +where `num_tag_type` is the num of tag types in the tagging scheme, `num_chunk_type` +is the num of chunk types, and `tag_type` get its value from the following table. Scheme Begin Inside End Single - plain 0 - - - - IOB 0 1 - - - IOE - 0 1 - - IOBES 0 1 2 3 + plain 0 - - - + IOB 0 1 - - + IOE - 0 1 - + IOBES 0 1 2 3 -Continue the NER example, and the label dict should look like this to satify above equations: +Still use NER as example, assuming the tagging scheme is IOB while chunk types are ORG, +PER and LOC. To satisfy the above equations, the label map can be like this: B-ORG 0 I-ORG 1 @@ -121,11 +126,10 @@ Continue the NER example, and the label dict should look like this to satify abo I-LOC 5 O 6 -In this example, chunkType has three values: 0 for ORG, 1 for PER, 2 for LOC, because the scheme is -"IOB" so tagType has two values: 0 for B and 1 for I. -Here we will use I-LOC to explain the above mapping rules in detail. -For I-LOC, the label id is 5, so we can get tagType=1 and chunkType=2, which means I-LOC is a part of NER chunk LOC -and the tag is I. +It’s not hard to verify the equations noting that the num of chunk types +is 3 and the num of tag types in IOB scheme is 2. For example, the label +id of I-LOC is 5, the tag type id of I-LOC is 1, and the chunk type id of +I-LOC is 2, which consistent with the results from the equations. )DOC"); } }; diff --git a/paddle/operators/chunk_eval_op.h b/paddle/operators/chunk_eval_op.h index b29c97225d6eab68b1655c976f274486c36f9eeb..81aa07817b673b2ff85a35a51cc43742b7ad7fed 100644 --- a/paddle/operators/chunk_eval_op.h +++ b/paddle/operators/chunk_eval_op.h @@ -171,10 +171,10 @@ class ChunkEvalKernel : public framework::OpKernel { num_tag_types, other_chunk_type, tag_begin, tag_inside, tag_end, tag_single, excluded_chunk_types); } - *precision_data = - !num_output_segments ? 0 : (T)num_correct / num_output_segments; - *racall_data = - !num_label_segments ? 0 : (T)num_correct / num_label_segments; + *precision_data = !num_output_segments ? 0 : static_cast(num_correct) / + num_output_segments; + *racall_data = !num_label_segments ? 0 : static_cast(num_correct) / + num_label_segments; *f1_data = !num_correct ? 0 : 2 * (*precision_data) * (*racall_data) / ((*precision_data) + (*racall_data)); } diff --git a/python/paddle/v2/framework/tests/test_chunk_eval_op.py b/python/paddle/v2/framework/tests/test_chunk_eval_op.py index f22b8316ae896673394da398c57da3aa3603dd3f..48673296a67716c4de804da533f0fd2567f10e2e 100644 --- a/python/paddle/v2/framework/tests/test_chunk_eval_op.py +++ b/python/paddle/v2/framework/tests/test_chunk_eval_op.py @@ -3,15 +3,15 @@ import numpy as np from op_test import OpTest -class Segments(object): +class Segment(object): def __init__(self, chunk_type, start_idx, end_idx): self.chunk_type = chunk_type self.start_idx = start_idx self.end_idx = end_idx def __str__(self): - return '(Segments: %s, %s, %s)' % (self.chunk_type, self.start_idx, - self.end_idx) + return '(Segment: %s, %s, %s)' % (self.chunk_type, self.start_idx, + self.end_idx) __repr__ = __str__ @@ -71,7 +71,7 @@ class TestChunkEvalOp(OpTest): # generate chunks for chunk_pos in zip(chunk_begins, chunk_ends): chunk_type = np.random.randint(self.num_chunk_types) - chunks.append(Segments(chunk_type, *chunk_pos)) + chunks.append(Segment(chunk_type, *chunk_pos)) return chunks def gen_chunks(self, infer, label, starts): @@ -120,7 +120,7 @@ class TestChunkEvalOp(OpTest): self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks = 4, 5, 9 def set_data(self): - infer = np.zeros((self.batch_size, )).astype("int32") + infer = np.zeros((self.batch_size, )).astype('int32') infer.fill(self.num_chunk_types * self.num_tag_types) label = np.copy(infer) starts = np.random.choice( @@ -142,9 +142,12 @@ class TestChunkEvalOp(OpTest): f1 = float(2 * precision * recall) / ( precision + recall) if self.num_correct_chunks else 0 self.outputs = { - 'Precision': [precision], - 'Recall': [recall], - 'F1-Score': [f1] + 'Precision': np.asarray( + [precision], dtype='float32'), + 'Recall': np.asarray( + [recall], dtype='float32'), + 'F1-Score': np.asarray( + [f1], dtype='float32') } def setUp(self):