evaluators.py 24.7 KB
Newer Older
1
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from paddle.trainer.config_parser import *
from default_decorators import *

18
__all__ = [
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
    "evaluator_base",
    "classification_error_evaluator",
    "auc_evaluator",
    "pnpair_evaluator",
    "precision_recall_evaluator",
    "ctc_error_evaluator",
    "chunk_evaluator",
    "sum_evaluator",
    "column_sum_evaluator",
    "value_printer_evaluator",
    "gradient_printer_evaluator",
    "maxid_printer_evaluator",
    "maxframe_printer_evaluator",
    "seqtext_printer_evaluator",
    "classification_error_printer_evaluator",
    "detection_map_evaluator",
35
]
36 37 38 39 40 41 42 43


class EvaluatorAttribute(object):
    FOR_CLASSIFICATION = 1
    FOR_REGRESSION = 1 << 1
    FOR_RANK = 1 << 2
    FOR_PRINT = 1 << 3
    FOR_UTILS = 1 << 4
Y
yangyaming 已提交
44
    FOR_DETECTION = 1 << 5
45 46

    KEYS = [
47
        "for_classification", "for_regression", "for_rank", "for_print",
Y
yangyaming 已提交
48
        "for_utils", "for_detection"
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
    ]

    @staticmethod
    def to_key(idx):
        tmp = 1
        for i in xrange(0, len(EvaluatorAttribute.KEYS)):
            if idx == tmp:
                return EvaluatorAttribute.KEYS[i]
            else:
                tmp = (tmp << 1)


def evaluator(*attrs):
    def impl(method):
        for attr in attrs:
            setattr(method, EvaluatorAttribute.to_key(attr), True)
        method.is_evaluator = True
        return method
67

68 69
    return impl

70

Y
yangyaming 已提交
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
def evaluator_base(input,
                   type,
                   label=None,
                   weight=None,
                   name=None,
                   chunk_scheme=None,
                   num_chunk_types=None,
                   classification_threshold=None,
                   positive_label=None,
                   dict_file=None,
                   result_file=None,
                   num_results=None,
                   delimited=None,
                   top_k=None,
                   excluded_chunk_types=None,
                   overlap_threshold=None,
                   background_id=None,
                   evaluate_difficult=None,
                   ap_type=None):
90
    """
91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112
    Evaluator will evaluate the network status while training/testing.

    User can use evaluator by classify/regression job. For example.

    ..  code-block:: python

        classify(prediction, output, evaluator=classification_error_evaluator)

    And user could define evaluator separately as follow.

    ..  code-block:: python

        classification_error_evaluator("ErrorRate", prediction, label)

    The evaluator often contains a name parameter. It will also be printed when
    evaluating network. The printed information may look like the following.

    ..  code-block:: text

         Batch=200 samples=20000 AvgCost=0.679655 CurrentCost=0.662179 Eval:
         classification_error_evaluator=0.4486
         CurrentEval: ErrorRate=0.3964
113

114 115 116 117 118 119 120 121
    :param input: Input layers, a object of LayerOutput or a list of
                  LayerOutput.
    :type input: list|LayerOutput
    :param label: An input layer containing the ground truth label.
    :type label: LayerOutput|None
    :param weight: An input layer which is a weight for each sample.
                   Each evaluator may calculate differently to use this weight.
    :type weight: LayerOutput.
122 123
    :param top_k: number k in top-k error rate
    :type top_k: int
Y
yangyaming 已提交
124 125 126 127 128 129 130 131
    :param overlap_threshold: In detection tasks to filter detection results
    :type overlap_threshold: float
    :param background_id: Identifier of background class
    :type background_id: int
    :param evaluate_difficult: Whether to evaluate difficult objects
    :type evaluate_difficult: bool
    :param ap_type: How to calculate average persicion
    :type ap_type: str
132 133
    """
    # inputs type assertions.
134 135 136 137
    assert classification_threshold is None or isinstance(
        classification_threshold, float)
    assert positive_label is None or isinstance(positive_label, int)
    assert num_results is None or isinstance(num_results, int)
138
    assert top_k is None or isinstance(top_k, int)
139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157

    if not isinstance(input, list):
        input = [input]

    if label:
        input.append(label)
    if weight:
        input.append(weight)

    Evaluator(
        name=name,
        type=type,
        inputs=[i.name for i in input],
        chunk_scheme=chunk_scheme,
        num_chunk_types=num_chunk_types,
        classification_threshold=classification_threshold,
        positive_label=positive_label,
        dict_file=dict_file,
        result_file=result_file,
158
        delimited=delimited,
159 160
        num_results=num_results,
        top_k=top_k,
Y
yangyaming 已提交
161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177
        excluded_chunk_types=excluded_chunk_types,
        overlap_threshold=overlap_threshold,
        background_id=background_id,
        evaluate_difficult=evaluate_difficult,
        ap_type=ap_type)


@evaluator(EvaluatorAttribute.FOR_DETECTION)
@wrap_name_default()
def detection_map_evaluator(input,
                            label,
                            overlap_threshold=0.5,
                            background_id=0,
                            evaluate_difficult=False,
                            ap_type="11point",
                            name=None):
    """
Y
yangyaming 已提交
178
    Detection mAP Evaluator. It will print mean Average Precision (mAP) for detection.
Y
yangyaming 已提交
179

Y
yangyaming 已提交
180
    The detection mAP Evaluator based on the output of detection_output layer counts
Y
yangyaming 已提交
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
    the true positive and the false positive bbox and integral them to get the
    mAP.

    The simple usage is:

    .. code-block:: python

       eval =  detection_map_evaluator(input=det_output,label=lbl)

    :param input: Input layer.
    :type input: LayerOutput
    :param label: Label layer.
    :type label: LayerOutput
    :param overlap_threshold: The bbox overlap threshold of a true positive.
    :type overlap_threshold: float
    :param background_id: The background class index.
    :type background_id: int
Y
yangyaming 已提交
198
    :param evaluate_difficult: Whether evaluate a difficult ground truth.
Y
yangyaming 已提交
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215
    :type evaluate_difficult: bool
    """
    if not isinstance(input, list):
        input = [input]

    if label:
        input.append(label)

    evaluator_base(
        name=name,
        type="detection_map",
        input=input,
        label=label,
        overlap_threshold=overlap_threshold,
        background_id=background_id,
        evaluate_difficult=evaluate_difficult,
        ap_type=ap_type)
216

217

218 219
@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
220 221 222 223
def classification_error_evaluator(input,
                                   label,
                                   name=None,
                                   weight=None,
224
                                   top_k=None,
225
                                   threshold=None):
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
    """
    Classification Error Evaluator. It will print error rate for classification.

    The classification error is:

    ..  math::

        classification\\_error = \\frac{NumOfWrongPredicts}{NumOfAllSamples}

    The simple usage is:

    .. code-block:: python

       eval =  classification_error_evaluator(input=prob,label=lbl)

    :param name: Evaluator name.
    :type name: basestring
    :param input: Input Layer name. The output prediction of network.
    :type input: LayerOutput
    :param label: Label layer name.
    :type label: basestring
    :param weight: Weight Layer name. It should be a matrix with size
                  [sample_num, 1]. And will just multiply to NumOfWrongPredicts
                  and NumOfAllSamples. So, the elements of weight are all one,
                  then means not set weight. The larger weight it is, the more
                  important this sample is.
    :type weight: LayerOutput
253 254
    :param top_k: number k in top-k error rate
    :type top_k: int
255 256 257 258 259
    :param threshold: The classification threshold.
    :type threshold: float
    :return: None.
    """

260 261 262 263 264 265
    evaluator_base(
        name=name,
        type="classification_error",
        input=input,
        label=label,
        weight=weight,
266
        top_k=top_k,
267 268
        classification_threshold=threshold, )

269 270 271 272 273 274 275

@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
def auc_evaluator(
        input,
        label,
        name=None,
276
        weight=None, ):
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
    """
    Auc Evaluator which adapts to binary classification.

    The simple usage:

    .. code-block:: python

       eval = auc_evaluator(input, label)

    :param name: Evaluator name.
    :type name: None|basestring
    :param input: Input Layer name. The output prediction of network.
    :type input: LayerOutput
    :param label: Label layer name.
    :type label: None|basestring
    :param weight: Weight Layer name. It should be a matrix with size
                  [sample_num, 1].
    :type weight: LayerOutput
    """
296 297 298 299 300 301 302
    evaluator_base(
        name=name,
        type="last-column-auc",
        input=input,
        label=label,
        weight=weight)

303 304 305 306 307 308

@evaluator(EvaluatorAttribute.FOR_RANK)
@wrap_name_default()
def pnpair_evaluator(
        input,
        label,
309
        query_id,
W
wanghaoshuang 已提交
310
        weight=None,
W
wanghaoshuang 已提交
311
        name=None, ):
312 313 314 315 316 317 318 319
    """
    Positive-negative pair rate Evaluator which adapts to rank task like
    learning to rank. This evaluator must contain at least three layers.

    The simple usage:

    .. code-block:: python

320
       eval = pnpair_evaluator(input, label, query_id)
321 322 323 324 325

    :param input: Input Layer name. The output prediction of network.
    :type input: LayerOutput
    :param label: Label layer name.
    :type label: LayerOutput
326 327 328 329
    :param query_id: Query_id layer name. Query_id indicates that which query
     each sample belongs to. Its shape should be
     the same as output of Label layer.
    :type query_id: LayerOutput
330
    :param weight: Weight Layer name. It should be a matrix with size
331 332 333
                  [sample_num, 1] which indicates the weight of each sample.
                  The default weight of sample is 1 if the weight layer is None.
                  And the pair weight is the mean of the two samples' weight.
334
    :type weight: LayerOutput
W
wanghaoshuang 已提交
335 336
    :param name: Evaluator name.
    :type name: None|basestring
337
    """
W
wanghaoshuang 已提交
338 339 340 341
    if not isinstance(input, list):
        input = [input]
    if label:
        input.append(label)
342 343
    if query_id:
        input.append(query_id)
344 345
    evaluator_base(
        input=input,
W
wanghaoshuang 已提交
346 347 348
        type="pnpair",
        weight=weight,
        name=name, )
349

350 351 352 353 354 355

@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
def precision_recall_evaluator(
        input,
        label,
356
        positive_label=None,
357
        weight=None,
358
        name=None, ):
359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
    """
    An Evaluator to calculate precision and recall, F1-score.
    It is adapt to the task with multiple labels.

    - If positive_label=-1, it will print the average precision, recall,
      F1-score of all labels.

    - If use specify positive_label, it will print the precision, recall,
      F1-score of this label.

    The simple usage:

    .. code-block:: python

       eval = precision_recall_evaluator(input, label)

    :param name: Evaluator name.
    :type name: None|basestring
    :param input: Input Layer name. The output prediction of network.
    :type input: LayerOutput
    :param label: Label layer name.
    :type label: LayerOutput
    :param positive_label: The input label layer.
    :type positive_label: LayerOutput.
    :param weight: Weight Layer name. It should be a matrix with size
                  [sample_num, 1]. (TODO, explaination)
    :type weight: LayerOutput
    """
387 388 389 390 391 392 393 394
    evaluator_base(
        name=name,
        type="precision_recall",
        input=input,
        label=label,
        positive_label=positive_label,
        weight=weight)

395 396 397 398 399

@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
def ctc_error_evaluator(
        input,
400
        label,
401
        name=None, ):
402 403 404 405 406 407 408
    """
    This evaluator is to calculate sequence-to-sequence edit distance.

    The simple usage is :

    .. code-block:: python

409
       eval = ctc_error_evaluator(input=input, label=lbl)
410 411 412

    :param name: Evaluator name.
    :type name: None|basestring
413
    :param input: Input Layer. Should be the same as the input for ctc_layer.
414
    :type input: LayerOutput
415 416
    :param label: input label, which is a data_layer. Should be the same as the
                  label for ctc_layer
417
    :type label: LayerOutput
418
    """
419 420 421
    evaluator_base(
        name=name, type="ctc_edit_distance", input=input, label=label)

422 423 424 425 426

@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
def chunk_evaluator(
        input,
427 428 429
        label,
        chunk_scheme,
        num_chunk_types,
430 431
        name=None,
        excluded_chunk_types=None, ):
432 433
    """
    Chunk evaluator is used to evaluate segment labelling accuracy for a
434
    sequence. It calculates precision, recall and F1 scores for the chunk detection.
435

436
    To use chunk evaluator, several concepts need to be clarified firstly.
437

438
    * **Chunk type** is the type of the whole chunk and a chunk consists of one or several words.  (For example in NER, ORG for organization name, PER for person name etc.)
439

440
    * **Tag type** indicates the position of a word in a chunk. (B for begin, I for inside, E for end, S for single)
441
    We can name a label by combining tag type and chunk type. (ie. B-ORG for begining of an organization name)
442

443
    The construction of label dictionary should obey the following rules:
444

445
    - Use one of the listed labelling schemes. These schemes differ in ways indicating chunk boundry.
446

447 448
    .. code-block:: text

W
wanghaoshuang 已提交
449
        Scheme    Description
450
        plain    Use the same label for the whole chunk.
W
wanghaoshuang 已提交
451
        IOB      Two labels for chunk type X, B-X for chunk begining and I-X for chunk inside.
452
        IOE      Two labels for chunk type X, E-X for chunk ending and I-X for chunk inside.
W
wanghaoshuang 已提交
453 454
        IOBES    Four labels for chunk type X, B-X for chunk begining, I-X for chunk inside, E-X for chunk end and S-X for single word chunk.

455 456 457 458
    To make it clear, let's illustrate by an NER example.
    Assuming that there are three named entity types including ORG, PER and LOC which are called 'chunk type' here,
    if 'IOB' scheme were used, the label set will be extended to a set including B-ORG, I-ORG, B-PER, I-PER, B-LOC, I-LOC and O,
    in which B-ORG for begining of ORG and I-ORG for inside of ORG.
459 460
    Prefixes which are called 'tag type' here are added to chunk types and there are two tag types including B and I.
    Of course, the training data should be labeled accordingly.
461

462
    - Mapping is done correctly by the listed equations and assigning protocol.
463 464

    The following table are equations to extract tag type and chunk type from a label.
465

466 467 468 469 470
    .. code-block:: text

        tagType = label % numTagType
        chunkType = label / numTagType
        otherChunkType = numChunkTypes
W
wanghaoshuang 已提交
471

472
    The following table shows the mapping rule between tagType and tag type in each scheme.
473

474 475 476 477 478 479 480
    .. code-block:: text

        Scheme Begin Inside End   Single
        plain  0     -      -     -
        IOB    0     1      -     -
        IOE    -     0      1     -
        IOBES  0     1      2     3
481 482

    Continue the NER example, and the label dict should look like this to satify above equations:
483

484
    .. code-block:: text
485

486 487 488 489 490 491 492
        B-ORG  0
        I-ORG  1
        B-PER  2
        I-PER  3
        B-LOC  4
        I-LOC  5
        O      6
493

494
    In this example, chunkType has three values: 0 for ORG, 1 for PER, 2 for LOC, because the scheme is
W
wanghaoshuang 已提交
495
    "IOB" so tagType has two values: 0 for B and 1 for I.
496
    Here we will use I-LOC to explain the above mapping rules in detail.
497
    For I-LOC, the label id is 5, so we can get tagType=1 and chunkType=2, which means I-LOC is a part of NER chunk LOC
498
    and the tag is I.
499 500 501 502 503

    The simple usage is:

    .. code-block:: python

504
       eval = chunk_evaluator(input, label, chunk_scheme, num_chunk_types)
505

W
wanghaoshuang 已提交
506

507 508
    :param input: The input layers.
    :type input: LayerOutput
509 510
    :param label: An input layer containing the ground truth label.
    :type label: LayerOutput
511
    :param chunk_scheme: The labelling schemes support 4 types. It is one of
512
                         "IOB", "IOE", "IOBES", "plain". It is required.
513 514
    :type chunk_scheme: basestring
    :param num_chunk_types: number of chunk types other than "other"
515 516
    :param name: The Evaluator name, it is optional.
    :type name: basename|None
517
    :param excluded_chunk_types: chunks of these types are not considered
P
Peng Li 已提交
518
    :type excluded_chunk_types: list of integer|None
519
    """
520 521 522 523
    evaluator_base(
        name=name,
        type="chunk",
        input=input,
524
        label=label,
525
        chunk_scheme=chunk_scheme,
526 527
        num_chunk_types=num_chunk_types,
        excluded_chunk_types=excluded_chunk_types, )
528

529 530 531 532 533 534

@evaluator(EvaluatorAttribute.FOR_UTILS)
@wrap_name_default()
def sum_evaluator(
        input,
        name=None,
535
        weight=None, ):
536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552
    """
    An Evaluator to sum the result of input.

    The simple usage:

    .. code-block:: python

       eval = sum_evaluator(input)

    :param name: Evaluator name.
    :type name: None|basestring
    :param input: Input Layer name.
    :type input: LayerOutput
    :param weight: Weight Layer name. It should be a matrix with size
                  [sample_num, 1]. (TODO, explaination)
    :type weight: LayerOutput
    """
553 554
    evaluator_base(name=name, type="sum", input=input, weight=weight)

555 556 557 558 559 560

@evaluator(EvaluatorAttribute.FOR_UTILS)
@wrap_name_default()
def column_sum_evaluator(
        input,
        name=None,
561
        weight=None, ):
562 563 564 565 566 567 568 569 570 571 572 573 574 575
    """
    This Evaluator is used to sum the last column of input.

    The simple usage is:

    .. code-block:: python

       eval = column_sum_evaluator(input, label)

    :param name: Evaluator name.
    :type name: None|basestring
    :param input: Input Layer name.
    :type input: LayerOutput
    """
576 577 578
    evaluator_base(
        name=name, type="last-column-sum", input=input, weight=weight)

579 580 581 582 583 584

"""
The following are printer Evaluators which are usually used to
print the result, like value or gradient of input layers, the
results generated in machine translation, the classification error etc.
"""
585 586


587 588 589 590
@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def value_printer_evaluator(
        input,
591
        name=None, ):
592 593 594 595 596 597 598 599 600 601 602 603 604 605 606
    """
    This Evaluator is used to print the values of input layers. It contains
    one or more input layers.

    The simple usage is:

    .. code-block:: python

       eval = value_printer_evaluator(input)

    :param input: One or more input layers.
    :type input: LayerOutput|list
    :param name: Evaluator name.
    :type name: None|basestring
    """
607 608
    evaluator_base(name=name, type="value_printer", input=input)

609 610 611 612 613

@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def gradient_printer_evaluator(
        input,
614
        name=None, ):
615 616 617 618 619 620 621 622 623 624 625 626 627 628 629
    """
    This Evaluator is used to print the gradient of input layers. It contains
    one or more input layers.

    The simple usage is:

    .. code-block:: python

       eval = gradient_printer_evaluator(input)

    :param input: One or more input layers.
    :type input: LayerOutput|list
    :param name: Evaluator name.
    :type name: None|basestring
    """
630 631
    evaluator_base(name=name, type="gradient_printer", input=input)

L
Liang Zhao 已提交
632

633 634 635 636
@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def maxid_printer_evaluator(
        input,
637
        num_results=None,
638
        name=None, ):
639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657
    """
    This Evaluator is used to print maximum top k values and their indexes
    of each row of input layers. It contains one or more input layers.
    k is specified by num_results.

    The simple usage is:

    .. code-block:: python

       eval = maxid_printer_evaluator(input)

    :param input: Input Layer name.
    :type input: LayerOutput|list
    :param num_results: This number is used to specify the top k numbers.
                        It is 1 by default.
    :type num_results: int.
    :param name: Evaluator name.
    :type name: None|basestring
    """
658 659 660
    evaluator_base(
        name=name, type="max_id_printer", input=input, num_results=num_results)

661 662 663 664 665

@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def maxframe_printer_evaluator(
        input,
666
        num_results=None,
667
        name=None, ):
668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687
    """
    This Evaluator is used to print the top k frames of each input layers.
    The input layers should contain sequences info or sequences type.
    k is specified by num_results.
    It contains one or more input layers.

    Note:
        The width of each frame is 1.

    The simple usage is:

    .. code-block:: python

       eval = maxframe_printer_evaluator(input)

    :param input: Input Layer name.
    :type input: LayerOutput|list
    :param name: Evaluator name.
    :type name: None|basestring
    """
688 689 690 691 692 693
    evaluator_base(
        name=name,
        type="max_frame_printer",
        input=input,
        num_results=num_results)

694 695 696 697 698

@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def seqtext_printer_evaluator(
        input,
699
        result_file,
700
        id_input=None,
701 702
        dict_file=None,
        delimited=None,
703
        name=None, ):
704 705 706 707
    """
    Sequence text printer will print text according to index matrix and a
    dictionary. There can be multiple input to this layer:

708
    1. If there is no id_input, the input must be a matrix containing
709 710
    the sequence of indices;

711
    2. If there is id_input, it should be ids, and interpreted as sample ids.
712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741

    The output format will be:

    1. sequence without sub-sequence, and there is probability.

    .. code-block:: python

         id \t prob space_seperated_tokens_from_dictionary_according_to_seq

    2. sequence without sub-sequence, and there is not probability.

    .. code-block:: python

         id \t space_seperated_tokens_from_dictionary_according_to_seq

    3. sequence with sub-sequence, and there is not probability.

    .. code-block:: python

         id \t space_seperated_tokens_from_dictionary_according_to_sub_seq
         \t \t space_seperated_tokens_from_dictionary_according_to_sub_seq
         ...

    Typically SequenceTextPrinter layer takes output of maxid or RecurrentGroup
    with maxid (when generating) as an input.

    The simple usage is:

    .. code-block:: python

742 743
       eval = seqtext_printer_evaluator(input=maxid_layer,
                                        id_input=sample_id,
744 745 746 747 748
                                        dict_file=dict_file,
                                        result_file=result_file)

    :param input: Input Layer name.
    :type input: LayerOutput|list
749
    :param result_file: Path of the file to store the generated results.
750
    :type result_file: basestring
751 752 753 754 755 756 757 758 759 760
    :param id_input: Index of the input sequence, and the specified index will
                     be prited in the gereated results. This an optional
                     parameter.
    :type id_input: LayerOutput
    :param dict_file: Path of dictionary. This is an optional parameter.
                      Every line is a word in the dictionary with
                      (line number - 1) as the word index.
                      If this parameter is set to None, or to an empty string,
                      only word index are printed in the generated results.
    :type dict_file: basestring
761 762 763 764 765
    :param delimited: Whether to use space to separate output tokens.
                Default is True. No space is added if set to False.
    :type delimited: bool
    :param name: Evaluator name.
    :type name: None|basestring
766 767
    :return: The seq_text_printer that prints the generated sequence to a file.
    :rtype: evaluator
768
    """
769
    assert isinstance(result_file, basestring)
770 771 772 773 774 775
    if id_input is None:
        inputs = [input]
    else:
        inputs = [id_input, input]
        input.parents.append(id_input)

776 777 778 779 780 781 782 783
    evaluator_base(
        name=name,
        type="seq_text_printer",
        input=inputs,
        dict_file=dict_file,
        result_file=result_file,
        delimited=delimited)

784 785 786 787 788 789 790

@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def classification_error_printer_evaluator(
        input,
        label,
        threshold=0.5,
791
        name=None, ):
792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807
    """
    This Evaluator is used to print the classification error of each sample.

    The simple usage is:

    .. code-block:: python

       eval = classification_error_printer_evaluator(input)

    :param input: Input layer.
    :type input: LayerOutput
    :param label: Input label layer.
    :type label: LayerOutput
    :param name: Evaluator name.
    :type name: None|basestring
    """
808 809 810 811 812 813
    evaluator_base(
        name=name,
        type="classification_error_printer",
        input=input,
        label=label,
        classification_threshold=threshold)
新手
引导
客服 返回
顶部