evaluators.py 24.3 KB
Newer Older
1
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
Z
zhangjinchao01 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from paddle.trainer.config_parser import *
from default_decorators import *

Q
qijun 已提交
18 19 20 21 22 23
__all__ = [
    "evaluator_base", "classification_error_evaluator", "auc_evaluator",
    "pnpair_evaluator", "precision_recall_evaluator", "ctc_error_evaluator",
    "chunk_evaluator", "sum_evaluator", "column_sum_evaluator",
    "value_printer_evaluator", "gradient_printer_evaluator",
    "maxid_printer_evaluator", "maxframe_printer_evaluator",
Y
yangyaming 已提交
24 25
    "seqtext_printer_evaluator", "classification_error_printer_evaluator",
    "detection_map_evaluator"
Q
qijun 已提交
26
]
Z
zhangjinchao01 已提交
27 28 29 30 31 32 33 34


class EvaluatorAttribute(object):
    FOR_CLASSIFICATION = 1
    FOR_REGRESSION = 1 << 1
    FOR_RANK = 1 << 2
    FOR_PRINT = 1 << 3
    FOR_UTILS = 1 << 4
Y
yangyaming 已提交
35
    FOR_DETECTION = 1 << 5
Z
zhangjinchao01 已提交
36 37

    KEYS = [
Q
qijun 已提交
38
        "for_classification", "for_regression", "for_rank", "for_print",
Y
yangyaming 已提交
39
        "for_utils", "for_detection"
Z
zhangjinchao01 已提交
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
    ]

    @staticmethod
    def to_key(idx):
        tmp = 1
        for i in xrange(0, len(EvaluatorAttribute.KEYS)):
            if idx == tmp:
                return EvaluatorAttribute.KEYS[i]
            else:
                tmp = (tmp << 1)


def evaluator(*attrs):
    def impl(method):
        for attr in attrs:
            setattr(method, EvaluatorAttribute.to_key(attr), True)
        method.is_evaluator = True
        return method
Q
qijun 已提交
58

Z
zhangjinchao01 已提交
59 60
    return impl

Q
qijun 已提交
61

Y
yangyaming 已提交
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
def evaluator_base(input,
                   type,
                   label=None,
                   weight=None,
                   name=None,
                   chunk_scheme=None,
                   num_chunk_types=None,
                   classification_threshold=None,
                   positive_label=None,
                   dict_file=None,
                   result_file=None,
                   num_results=None,
                   delimited=None,
                   top_k=None,
                   excluded_chunk_types=None,
                   overlap_threshold=None,
                   background_id=None,
                   evaluate_difficult=None,
                   ap_type=None):
Z
zhangjinchao01 已提交
81
    """
L
luotao02 已提交
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103
    Evaluator will evaluate the network status while training/testing.

    User can use evaluator by classify/regression job. For example.

    ..  code-block:: python

        classify(prediction, output, evaluator=classification_error_evaluator)

    And user could define evaluator separately as follow.

    ..  code-block:: python

        classification_error_evaluator("ErrorRate", prediction, label)

    The evaluator often contains a name parameter. It will also be printed when
    evaluating network. The printed information may look like the following.

    ..  code-block:: text

         Batch=200 samples=20000 AvgCost=0.679655 CurrentCost=0.662179 Eval:
         classification_error_evaluator=0.4486
         CurrentEval: ErrorRate=0.3964
104

Z
zhangjinchao01 已提交
105 106 107 108 109 110 111 112
    :param input: Input layers, a object of LayerOutput or a list of
                  LayerOutput.
    :type input: list|LayerOutput
    :param label: An input layer containing the ground truth label.
    :type label: LayerOutput|None
    :param weight: An input layer which is a weight for each sample.
                   Each evaluator may calculate differently to use this weight.
    :type weight: LayerOutput.
L
Liang Zhao 已提交
113 114
    :param top_k: number k in top-k error rate
    :type top_k: int
Y
yangyaming 已提交
115 116 117 118 119 120 121 122
    :param overlap_threshold: In detection tasks to filter detection results
    :type overlap_threshold: float
    :param background_id: Identifier of background class
    :type background_id: int
    :param evaluate_difficult: Whether to evaluate difficult objects
    :type evaluate_difficult: bool
    :param ap_type: How to calculate average persicion
    :type ap_type: str
Z
zhangjinchao01 已提交
123 124
    """
    # inputs type assertions.
125 126 127 128
    assert classification_threshold is None or isinstance(
        classification_threshold, float)
    assert positive_label is None or isinstance(positive_label, int)
    assert num_results is None or isinstance(num_results, int)
L
Liang Zhao 已提交
129
    assert top_k is None or isinstance(top_k, int)
Z
zhangjinchao01 已提交
130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148

    if not isinstance(input, list):
        input = [input]

    if label:
        input.append(label)
    if weight:
        input.append(weight)

    Evaluator(
        name=name,
        type=type,
        inputs=[i.name for i in input],
        chunk_scheme=chunk_scheme,
        num_chunk_types=num_chunk_types,
        classification_threshold=classification_threshold,
        positive_label=positive_label,
        dict_file=dict_file,
        result_file=result_file,
149
        delimited=delimited,
L
Liang Zhao 已提交
150 151
        num_results=num_results,
        top_k=top_k,
Y
yangyaming 已提交
152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
        excluded_chunk_types=excluded_chunk_types,
        overlap_threshold=overlap_threshold,
        background_id=background_id,
        evaluate_difficult=evaluate_difficult,
        ap_type=ap_type)


@evaluator(EvaluatorAttribute.FOR_DETECTION)
@wrap_name_default()
def detection_map_evaluator(input,
                            label,
                            overlap_threshold=0.5,
                            background_id=0,
                            evaluate_difficult=False,
                            ap_type="11point",
                            name=None):
    """
    Detection mAP Evaluator. It will print mean Average Precision for detection.

    The detection mAP Evaluator according to the detection_output's output count
    the true positive and the false positive bbox and integral them to get the
    mAP.

    The simple usage is:

    .. code-block:: python

       eval =  detection_map_evaluator(input=det_output,label=lbl)

    :param input: Input layer.
    :type input: LayerOutput
    :param label: Label layer.
    :type label: LayerOutput
    :param overlap_threshold: The bbox overlap threshold of a true positive.
    :type overlap_threshold: float
    :param background_id: The background class index.
    :type background_id: int
    :param evaluate_difficult: Wether evaluate a difficult ground truth.
    :type evaluate_difficult: bool
    """
    if not isinstance(input, list):
        input = [input]

    if label:
        input.append(label)

    evaluator_base(
        name=name,
        type="detection_map",
        input=input,
        label=label,
        overlap_threshold=overlap_threshold,
        background_id=background_id,
        evaluate_difficult=evaluate_difficult,
        ap_type=ap_type)
Z
zhangjinchao01 已提交
207

Q
qijun 已提交
208

Z
zhangjinchao01 已提交
209 210
@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
Q
qijun 已提交
211 212 213 214
def classification_error_evaluator(input,
                                   label,
                                   name=None,
                                   weight=None,
L
Liang Zhao 已提交
215
                                   top_k=None,
Q
qijun 已提交
216
                                   threshold=None):
Z
zhangjinchao01 已提交
217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
    """
    Classification Error Evaluator. It will print error rate for classification.

    The classification error is:

    ..  math::

        classification\\_error = \\frac{NumOfWrongPredicts}{NumOfAllSamples}

    The simple usage is:

    .. code-block:: python

       eval =  classification_error_evaluator(input=prob,label=lbl)

    :param name: Evaluator name.
    :type name: basestring
    :param input: Input Layer name. The output prediction of network.
    :type input: LayerOutput
    :param label: Label layer name.
    :type label: basestring
    :param weight: Weight Layer name. It should be a matrix with size
                  [sample_num, 1]. And will just multiply to NumOfWrongPredicts
                  and NumOfAllSamples. So, the elements of weight are all one,
                  then means not set weight. The larger weight it is, the more
                  important this sample is.
    :type weight: LayerOutput
L
Liang Zhao 已提交
244 245
    :param top_k: number k in top-k error rate
    :type top_k: int
Z
zhangjinchao01 已提交
246 247 248 249 250
    :param threshold: The classification threshold.
    :type threshold: float
    :return: None.
    """

Q
qijun 已提交
251 252 253 254 255 256
    evaluator_base(
        name=name,
        type="classification_error",
        input=input,
        label=label,
        weight=weight,
L
Liang Zhao 已提交
257
        top_k=top_k,
Q
qijun 已提交
258 259
        classification_threshold=threshold, )

Z
zhangjinchao01 已提交
260 261 262 263 264 265 266

@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
def auc_evaluator(
        input,
        label,
        name=None,
Q
qijun 已提交
267
        weight=None, ):
Z
zhangjinchao01 已提交
268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286
    """
    Auc Evaluator which adapts to binary classification.

    The simple usage:

    .. code-block:: python

       eval = auc_evaluator(input, label)

    :param name: Evaluator name.
    :type name: None|basestring
    :param input: Input Layer name. The output prediction of network.
    :type input: LayerOutput
    :param label: Label layer name.
    :type label: None|basestring
    :param weight: Weight Layer name. It should be a matrix with size
                  [sample_num, 1].
    :type weight: LayerOutput
    """
Q
qijun 已提交
287 288 289 290 291 292 293
    evaluator_base(
        name=name,
        type="last-column-auc",
        input=input,
        label=label,
        weight=weight)

Z
zhangjinchao01 已提交
294 295 296 297 298 299 300 301

@evaluator(EvaluatorAttribute.FOR_RANK)
@wrap_name_default()
def pnpair_evaluator(
        input,
        label,
        info,
        name=None,
Q
qijun 已提交
302
        weight=None, ):
Z
zhangjinchao01 已提交
303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
    """
    Positive-negative pair rate Evaluator which adapts to rank task like
    learning to rank. This evaluator must contain at least three layers.

    The simple usage:

    .. code-block:: python

       eval = pnpair_evaluator(input, info, label)

    :param name: Evaluator name.
    :type name: None|basestring
    :param input: Input Layer name. The output prediction of network.
    :type input: LayerOutput
    :param label: Label layer name.
    :type label: LayerOutput
    :param info: Label layer name. (TODO, explaination)
    :type info: LayerOutput
    :param weight: Weight Layer name. It should be a matrix with size
                  [sample_num, 1]. (TODO, explaination)
    :type weight: LayerOutput
    """
Q
qijun 已提交
325 326 327 328 329 330 331 332
    evaluator_base(
        name=name,
        type="pnpair",
        input=input,
        label=label,
        info=info,
        weight=weight)

Z
zhangjinchao01 已提交
333 334 335 336 337 338

@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
def precision_recall_evaluator(
        input,
        label,
339
        positive_label=None,
Z
zhangjinchao01 已提交
340
        weight=None,
Q
qijun 已提交
341
        name=None, ):
Z
zhangjinchao01 已提交
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
    """
    An Evaluator to calculate precision and recall, F1-score.
    It is adapt to the task with multiple labels.

    - If positive_label=-1, it will print the average precision, recall,
      F1-score of all labels.

    - If use specify positive_label, it will print the precision, recall,
      F1-score of this label.

    The simple usage:

    .. code-block:: python

       eval = precision_recall_evaluator(input, label)

    :param name: Evaluator name.
    :type name: None|basestring
    :param input: Input Layer name. The output prediction of network.
    :type input: LayerOutput
    :param label: Label layer name.
    :type label: LayerOutput
    :param positive_label: The input label layer.
    :type positive_label: LayerOutput.
    :param weight: Weight Layer name. It should be a matrix with size
                  [sample_num, 1]. (TODO, explaination)
    :type weight: LayerOutput
    """
Q
qijun 已提交
370 371 372 373 374 375 376 377
    evaluator_base(
        name=name,
        type="precision_recall",
        input=input,
        label=label,
        positive_label=positive_label,
        weight=weight)

Z
zhangjinchao01 已提交
378 379 380 381 382

@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
def ctc_error_evaluator(
        input,
383
        label,
Q
qijun 已提交
384
        name=None, ):
Z
zhangjinchao01 已提交
385 386 387 388 389 390 391
    """
    This evaluator is to calculate sequence-to-sequence edit distance.

    The simple usage is :

    .. code-block:: python

392
       eval = ctc_error_evaluator(input=input, label=lbl)
Z
zhangjinchao01 已提交
393 394 395

    :param name: Evaluator name.
    :type name: None|basestring
396
    :param input: Input Layer. Should be the same as the input for ctc_layer.
Z
zhangjinchao01 已提交
397
    :type input: LayerOutput
398 399
    :param label: input label, which is a data_layer. Should be the same as the
                  label for ctc_layer
400
    :type label: LayerOutput
Z
zhangjinchao01 已提交
401
    """
Q
qijun 已提交
402 403 404
    evaluator_base(
        name=name, type="ctc_edit_distance", input=input, label=label)

Z
zhangjinchao01 已提交
405 406 407 408 409

@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
def chunk_evaluator(
        input,
410 411 412
        label,
        chunk_scheme,
        num_chunk_types,
413 414
        name=None,
        excluded_chunk_types=None, ):
Z
zhangjinchao01 已提交
415 416
    """
    Chunk evaluator is used to evaluate segment labelling accuracy for a
417
    sequence. It calculates precision, recall and F1 scores for the chunk detection.
Z
zhangjinchao01 已提交
418

419
    To use chunk evaluator, several concepts need to be clarified firstly.
Y
yangyaming 已提交
420 421 422 423

    * **Chunk type** is the type of the whole chunk and a chunk consists of one or several words.  (For example in NER, ORG for organization name, PER for person name etc.)

    * **Tag type** indicates the position of a word in a chunk. (B for begin, I for inside, E for end, S for single)
424 425
    We can name a label by combining tag type and chunk type. (ie. B-ORG for begining of an organization name)

Y
yangyaming 已提交
426
    The construction of label dictionary should obey the following rules:
Z
zhangjinchao01 已提交
427

Y
yangyaming 已提交
428 429 430 431 432 433 434 435 436
    - Use one of the listed labelling schemes. These schemes differ in ways indicating chunk boundry.

    .. code-block:: text

        Scheme    Description                                                                                  
        plain    Use the same label for the whole chunk.
        IOB      Two labels for chunk type X, B-X for chunk begining and I-X for chunk inside. 
        IOE      Two labels for chunk type X, E-X for chunk ending and I-X for chunk inside.
        IOBES    Four labels for chunk type X, B-X for chunk begining, I-X for chunk inside, E-X for chunk end and S-X for single word chunk. 
437 438 439 440 441
   
    To make it clear, let's illustrate by an NER example.
    Assuming that there are three named entity types including ORG, PER and LOC which are called 'chunk type' here,
    if 'IOB' scheme were used, the label set will be extended to a set including B-ORG, I-ORG, B-PER, I-PER, B-LOC, I-LOC and O,
    in which B-ORG for begining of ORG and I-ORG for inside of ORG.
442 443
    Prefixes which are called 'tag type' here are added to chunk types and there are two tag types including B and I.
    Of course, the training data should be labeled accordingly.
Z
zhangjinchao01 已提交
444

Y
yangyaming 已提交
445
    - Mapping is done correctly by the listed equations and assigning protocol.
446 447

    The following table are equations to extract tag type and chunk type from a label.
Z
zhangjinchao01 已提交
448

Y
yangyaming 已提交
449 450 451 452 453
    .. code-block:: text

        tagType = label % numTagType
        chunkType = label / numTagType
        otherChunkType = numChunkTypes
454 455
    
    The following table shows the mapping rule between tagType and tag type in each scheme.
Z
zhangjinchao01 已提交
456

Y
yangyaming 已提交
457 458 459 460 461 462 463
    .. code-block:: text

        Scheme Begin Inside End   Single
        plain  0     -      -     -
        IOB    0     1      -     -
        IOE    -     0      1     -
        IOBES  0     1      2     3
464 465

    Continue the NER example, and the label dict should look like this to satify above equations:
466

Y
yangyaming 已提交
467 468 469 470 471 472 473 474 475
    .. code-block:: text

        B-ORG  0
        I-ORG  1
        B-PER  2
        I-PER  3
        B-LOC  4
        I-LOC  5
        O      6
Z
zhangjinchao01 已提交
476

477 478 479
    In this example, chunkType has three values: 0 for ORG, 1 for PER, 2 for LOC, because the scheme is
    "IOB" so tagType has two values: 0 for B and 1 for I. 
    Here we will use I-LOC to explain the above mapping rules in detail.
Y
yangyaming 已提交
480
    For I-LOC, the label id is 5, so we can get tagType=1 and chunkType=2, which means I-LOC is a part of NER chunk LOC
481
    and the tag is I.
Z
zhangjinchao01 已提交
482 483 484 485 486

    The simple usage is:

    .. code-block:: python

487
       eval = chunk_evaluator(input, label, chunk_scheme, num_chunk_types)
Z
zhangjinchao01 已提交
488

489
    
Z
zhangjinchao01 已提交
490 491
    :param input: The input layers.
    :type input: LayerOutput
492 493
    :param label: An input layer containing the ground truth label.
    :type label: LayerOutput
Z
zhangjinchao01 已提交
494
    :param chunk_scheme: The labelling schemes support 4 types. It is one of
495
                         "IOB", "IOE", "IOBES", "plain". It is required.
Z
zhangjinchao01 已提交
496 497
    :type chunk_scheme: basestring
    :param num_chunk_types: number of chunk types other than "other"
498 499
    :param name: The Evaluator name, it is optional.
    :type name: basename|None
500
    :param excluded_chunk_types: chunks of these types are not considered
P
Peng Li 已提交
501
    :type excluded_chunk_types: list of integer|None
Z
zhangjinchao01 已提交
502
    """
Q
qijun 已提交
503 504 505 506
    evaluator_base(
        name=name,
        type="chunk",
        input=input,
507
        label=label,
Q
qijun 已提交
508
        chunk_scheme=chunk_scheme,
509 510
        num_chunk_types=num_chunk_types,
        excluded_chunk_types=excluded_chunk_types, )
Q
qijun 已提交
511

Z
zhangjinchao01 已提交
512 513 514 515 516 517

@evaluator(EvaluatorAttribute.FOR_UTILS)
@wrap_name_default()
def sum_evaluator(
        input,
        name=None,
Q
qijun 已提交
518
        weight=None, ):
Z
zhangjinchao01 已提交
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535
    """
    An Evaluator to sum the result of input.

    The simple usage:

    .. code-block:: python

       eval = sum_evaluator(input)

    :param name: Evaluator name.
    :type name: None|basestring
    :param input: Input Layer name.
    :type input: LayerOutput
    :param weight: Weight Layer name. It should be a matrix with size
                  [sample_num, 1]. (TODO, explaination)
    :type weight: LayerOutput
    """
Q
qijun 已提交
536 537
    evaluator_base(name=name, type="sum", input=input, weight=weight)

Z
zhangjinchao01 已提交
538 539 540 541 542 543

@evaluator(EvaluatorAttribute.FOR_UTILS)
@wrap_name_default()
def column_sum_evaluator(
        input,
        name=None,
Q
qijun 已提交
544
        weight=None, ):
Z
zhangjinchao01 已提交
545 546 547 548 549 550 551 552 553 554 555 556 557 558
    """
    This Evaluator is used to sum the last column of input.

    The simple usage is:

    .. code-block:: python

       eval = column_sum_evaluator(input, label)

    :param name: Evaluator name.
    :type name: None|basestring
    :param input: Input Layer name.
    :type input: LayerOutput
    """
Q
qijun 已提交
559 560 561
    evaluator_base(
        name=name, type="last-column-sum", input=input, weight=weight)

Z
zhangjinchao01 已提交
562 563 564 565 566 567

"""
The following are printer Evaluators which are usually used to
print the result, like value or gradient of input layers, the
results generated in machine translation, the classification error etc.
"""
Q
qijun 已提交
568 569


Z
zhangjinchao01 已提交
570 571 572 573
@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def value_printer_evaluator(
        input,
Q
qijun 已提交
574
        name=None, ):
Z
zhangjinchao01 已提交
575 576 577 578 579 580 581 582 583 584 585 586 587 588 589
    """
    This Evaluator is used to print the values of input layers. It contains
    one or more input layers.

    The simple usage is:

    .. code-block:: python

       eval = value_printer_evaluator(input)

    :param input: One or more input layers.
    :type input: LayerOutput|list
    :param name: Evaluator name.
    :type name: None|basestring
    """
Q
qijun 已提交
590 591
    evaluator_base(name=name, type="value_printer", input=input)

Z
zhangjinchao01 已提交
592 593 594 595 596

@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def gradient_printer_evaluator(
        input,
Q
qijun 已提交
597
        name=None, ):
Z
zhangjinchao01 已提交
598 599 600 601 602 603 604 605 606 607 608 609 610 611 612
    """
    This Evaluator is used to print the gradient of input layers. It contains
    one or more input layers.

    The simple usage is:

    .. code-block:: python

       eval = gradient_printer_evaluator(input)

    :param input: One or more input layers.
    :type input: LayerOutput|list
    :param name: Evaluator name.
    :type name: None|basestring
    """
Q
qijun 已提交
613 614
    evaluator_base(name=name, type="gradient_printer", input=input)

L
Liang Zhao 已提交
615

Z
zhangjinchao01 已提交
616 617 618 619
@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def maxid_printer_evaluator(
        input,
620
        num_results=None,
Q
qijun 已提交
621
        name=None, ):
Z
zhangjinchao01 已提交
622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
    """
    This Evaluator is used to print maximum top k values and their indexes
    of each row of input layers. It contains one or more input layers.
    k is specified by num_results.

    The simple usage is:

    .. code-block:: python

       eval = maxid_printer_evaluator(input)

    :param input: Input Layer name.
    :type input: LayerOutput|list
    :param num_results: This number is used to specify the top k numbers.
                        It is 1 by default.
    :type num_results: int.
    :param name: Evaluator name.
    :type name: None|basestring
    """
Q
qijun 已提交
641 642 643
    evaluator_base(
        name=name, type="max_id_printer", input=input, num_results=num_results)

Z
zhangjinchao01 已提交
644 645 646 647 648

@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def maxframe_printer_evaluator(
        input,
649
        num_results=None,
Q
qijun 已提交
650
        name=None, ):
Z
zhangjinchao01 已提交
651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670
    """
    This Evaluator is used to print the top k frames of each input layers.
    The input layers should contain sequences info or sequences type.
    k is specified by num_results.
    It contains one or more input layers.

    Note:
        The width of each frame is 1.

    The simple usage is:

    .. code-block:: python

       eval = maxframe_printer_evaluator(input)

    :param input: Input Layer name.
    :type input: LayerOutput|list
    :param name: Evaluator name.
    :type name: None|basestring
    """
Q
qijun 已提交
671 672 673 674 675 676
    evaluator_base(
        name=name,
        type="max_frame_printer",
        input=input,
        num_results=num_results)

Z
zhangjinchao01 已提交
677 678 679 680 681

@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def seqtext_printer_evaluator(
        input,
682
        result_file,
683
        id_input=None,
684 685
        dict_file=None,
        delimited=None,
Q
qijun 已提交
686
        name=None, ):
Z
zhangjinchao01 已提交
687 688 689 690
    """
    Sequence text printer will print text according to index matrix and a
    dictionary. There can be multiple input to this layer:

691
    1. If there is no id_input, the input must be a matrix containing
Z
zhangjinchao01 已提交
692 693
    the sequence of indices;

694
    2. If there is id_input, it should be ids, and interpreted as sample ids.
Z
zhangjinchao01 已提交
695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724

    The output format will be:

    1. sequence without sub-sequence, and there is probability.

    .. code-block:: python

         id \t prob space_seperated_tokens_from_dictionary_according_to_seq

    2. sequence without sub-sequence, and there is not probability.

    .. code-block:: python

         id \t space_seperated_tokens_from_dictionary_according_to_seq

    3. sequence with sub-sequence, and there is not probability.

    .. code-block:: python

         id \t space_seperated_tokens_from_dictionary_according_to_sub_seq
         \t \t space_seperated_tokens_from_dictionary_according_to_sub_seq
         ...

    Typically SequenceTextPrinter layer takes output of maxid or RecurrentGroup
    with maxid (when generating) as an input.

    The simple usage is:

    .. code-block:: python

725 726
       eval = seqtext_printer_evaluator(input=maxid_layer,
                                        id_input=sample_id,
Z
zhangjinchao01 已提交
727 728 729 730 731
                                        dict_file=dict_file,
                                        result_file=result_file)

    :param input: Input Layer name.
    :type input: LayerOutput|list
732
    :param result_file: Path of the file to store the generated results.
Z
zhangjinchao01 已提交
733
    :type result_file: basestring
734 735 736 737 738 739 740 741 742 743
    :param id_input: Index of the input sequence, and the specified index will
                     be prited in the gereated results. This an optional
                     parameter.
    :type id_input: LayerOutput
    :param dict_file: Path of dictionary. This is an optional parameter.
                      Every line is a word in the dictionary with
                      (line number - 1) as the word index.
                      If this parameter is set to None, or to an empty string,
                      only word index are printed in the generated results.
    :type dict_file: basestring
Z
zhangjinchao01 已提交
744 745 746 747 748
    :param delimited: Whether to use space to separate output tokens.
                Default is True. No space is added if set to False.
    :type delimited: bool
    :param name: Evaluator name.
    :type name: None|basestring
749 750
    :return: The seq_text_printer that prints the generated sequence to a file.
    :rtype: evaluator
Z
zhangjinchao01 已提交
751
    """
752
    assert isinstance(result_file, basestring)
753 754 755 756 757 758
    if id_input is None:
        inputs = [input]
    else:
        inputs = [id_input, input]
        input.parents.append(id_input)

Q
qijun 已提交
759 760 761 762 763 764 765 766
    evaluator_base(
        name=name,
        type="seq_text_printer",
        input=inputs,
        dict_file=dict_file,
        result_file=result_file,
        delimited=delimited)

Z
zhangjinchao01 已提交
767 768 769 770 771 772 773

@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def classification_error_printer_evaluator(
        input,
        label,
        threshold=0.5,
Q
qijun 已提交
774
        name=None, ):
Z
zhangjinchao01 已提交
775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790
    """
    This Evaluator is used to print the classification error of each sample.

    The simple usage is:

    .. code-block:: python

       eval = classification_error_printer_evaluator(input)

    :param input: Input layer.
    :type input: LayerOutput
    :param label: Input label layer.
    :type label: LayerOutput
    :param name: Evaluator name.
    :type name: None|basestring
    """
Q
qijun 已提交
791 792 793 794 795 796
    evaluator_base(
        name=name,
        type="classification_error_printer",
        input=input,
        label=label,
        classification_threshold=threshold)