evaluators.py 19.4 KB
Newer Older
1
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
Z
zhangjinchao01 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from paddle.trainer.config_parser import *
from default_decorators import *

Q
qijun 已提交
18 19 20 21 22 23 24 25
__all__ = [
    "evaluator_base", "classification_error_evaluator", "auc_evaluator",
    "pnpair_evaluator", "precision_recall_evaluator", "ctc_error_evaluator",
    "chunk_evaluator", "sum_evaluator", "column_sum_evaluator",
    "value_printer_evaluator", "gradient_printer_evaluator",
    "maxid_printer_evaluator", "maxframe_printer_evaluator",
    "seqtext_printer_evaluator", "classification_error_printer_evaluator"
]
Z
zhangjinchao01 已提交
26 27 28 29 30 31 32 33 34 35


class EvaluatorAttribute(object):
    FOR_CLASSIFICATION = 1
    FOR_REGRESSION = 1 << 1
    FOR_RANK = 1 << 2
    FOR_PRINT = 1 << 3
    FOR_UTILS = 1 << 4

    KEYS = [
Q
qijun 已提交
36
        "for_classification", "for_regression", "for_rank", "for_print",
Z
zhangjinchao01 已提交
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
        "for_utils"
    ]

    @staticmethod
    def to_key(idx):
        tmp = 1
        for i in xrange(0, len(EvaluatorAttribute.KEYS)):
            if idx == tmp:
                return EvaluatorAttribute.KEYS[i]
            else:
                tmp = (tmp << 1)


def evaluator(*attrs):
    def impl(method):
        for attr in attrs:
            setattr(method, EvaluatorAttribute.to_key(attr), True)
        method.is_evaluator = True
        return method
Q
qijun 已提交
56

Z
zhangjinchao01 已提交
57 58
    return impl

Q
qijun 已提交
59

60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
def evaluator_base(
        input,
        type,
        label=None,
        weight=None,
        name=None,
        chunk_scheme=None,
        num_chunk_types=None,
        classification_threshold=None,
        positive_label=None,
        dict_file=None,
        result_file=None,
        num_results=None,
        delimited=None,
        excluded_chunk_types=None, ):
Z
zhangjinchao01 已提交
75
    """
L
luotao02 已提交
76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97
    Evaluator will evaluate the network status while training/testing.

    User can use evaluator by classify/regression job. For example.

    ..  code-block:: python

        classify(prediction, output, evaluator=classification_error_evaluator)

    And user could define evaluator separately as follow.

    ..  code-block:: python

        classification_error_evaluator("ErrorRate", prediction, label)

    The evaluator often contains a name parameter. It will also be printed when
    evaluating network. The printed information may look like the following.

    ..  code-block:: text

         Batch=200 samples=20000 AvgCost=0.679655 CurrentCost=0.662179 Eval:
         classification_error_evaluator=0.4486
         CurrentEval: ErrorRate=0.3964
98

Z
zhangjinchao01 已提交
99 100 101 102 103 104 105 106 107 108
    :param input: Input layers, a object of LayerOutput or a list of
                  LayerOutput.
    :type input: list|LayerOutput
    :param label: An input layer containing the ground truth label.
    :type label: LayerOutput|None
    :param weight: An input layer which is a weight for each sample.
                   Each evaluator may calculate differently to use this weight.
    :type weight: LayerOutput.
    """
    # inputs type assertions.
109 110 111 112
    assert classification_threshold is None or isinstance(
        classification_threshold, float)
    assert positive_label is None or isinstance(positive_label, int)
    assert num_results is None or isinstance(num_results, int)
Z
zhangjinchao01 已提交
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131

    if not isinstance(input, list):
        input = [input]

    if label:
        input.append(label)
    if weight:
        input.append(weight)

    Evaluator(
        name=name,
        type=type,
        inputs=[i.name for i in input],
        chunk_scheme=chunk_scheme,
        num_chunk_types=num_chunk_types,
        classification_threshold=classification_threshold,
        positive_label=positive_label,
        dict_file=dict_file,
        result_file=result_file,
132 133
        delimited=delimited,
        excluded_chunk_types=excluded_chunk_types, )
Z
zhangjinchao01 已提交
134

Q
qijun 已提交
135

Z
zhangjinchao01 已提交
136 137
@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
Q
qijun 已提交
138 139 140 141 142
def classification_error_evaluator(input,
                                   label,
                                   name=None,
                                   weight=None,
                                   threshold=None):
Z
zhangjinchao01 已提交
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
    """
    Classification Error Evaluator. It will print error rate for classification.

    The classification error is:

    ..  math::

        classification\\_error = \\frac{NumOfWrongPredicts}{NumOfAllSamples}

    The simple usage is:

    .. code-block:: python

       eval =  classification_error_evaluator(input=prob,label=lbl)

    :param name: Evaluator name.
    :type name: basestring
    :param input: Input Layer name. The output prediction of network.
    :type input: LayerOutput
    :param label: Label layer name.
    :type label: basestring
    :param weight: Weight Layer name. It should be a matrix with size
                  [sample_num, 1]. And will just multiply to NumOfWrongPredicts
                  and NumOfAllSamples. So, the elements of weight are all one,
                  then means not set weight. The larger weight it is, the more
                  important this sample is.
    :type weight: LayerOutput
    :param threshold: The classification threshold.
    :type threshold: float
    :return: None.
    """

Q
qijun 已提交
175 176 177 178 179 180 181 182
    evaluator_base(
        name=name,
        type="classification_error",
        input=input,
        label=label,
        weight=weight,
        classification_threshold=threshold, )

Z
zhangjinchao01 已提交
183 184 185 186 187 188 189

@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
def auc_evaluator(
        input,
        label,
        name=None,
Q
qijun 已提交
190
        weight=None, ):
Z
zhangjinchao01 已提交
191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
    """
    Auc Evaluator which adapts to binary classification.

    The simple usage:

    .. code-block:: python

       eval = auc_evaluator(input, label)

    :param name: Evaluator name.
    :type name: None|basestring
    :param input: Input Layer name. The output prediction of network.
    :type input: LayerOutput
    :param label: Label layer name.
    :type label: None|basestring
    :param weight: Weight Layer name. It should be a matrix with size
                  [sample_num, 1].
    :type weight: LayerOutput
    """
Q
qijun 已提交
210 211 212 213 214 215 216
    evaluator_base(
        name=name,
        type="last-column-auc",
        input=input,
        label=label,
        weight=weight)

Z
zhangjinchao01 已提交
217 218 219 220 221 222 223 224

@evaluator(EvaluatorAttribute.FOR_RANK)
@wrap_name_default()
def pnpair_evaluator(
        input,
        label,
        info,
        name=None,
Q
qijun 已提交
225
        weight=None, ):
Z
zhangjinchao01 已提交
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
    """
    Positive-negative pair rate Evaluator which adapts to rank task like
    learning to rank. This evaluator must contain at least three layers.

    The simple usage:

    .. code-block:: python

       eval = pnpair_evaluator(input, info, label)

    :param name: Evaluator name.
    :type name: None|basestring
    :param input: Input Layer name. The output prediction of network.
    :type input: LayerOutput
    :param label: Label layer name.
    :type label: LayerOutput
    :param info: Label layer name. (TODO, explaination)
    :type info: LayerOutput
    :param weight: Weight Layer name. It should be a matrix with size
                  [sample_num, 1]. (TODO, explaination)
    :type weight: LayerOutput
    """
Q
qijun 已提交
248 249 250 251 252 253 254 255
    evaluator_base(
        name=name,
        type="pnpair",
        input=input,
        label=label,
        info=info,
        weight=weight)

Z
zhangjinchao01 已提交
256 257 258 259 260 261

@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
def precision_recall_evaluator(
        input,
        label,
262
        positive_label=None,
Z
zhangjinchao01 已提交
263
        weight=None,
Q
qijun 已提交
264
        name=None, ):
Z
zhangjinchao01 已提交
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292
    """
    An Evaluator to calculate precision and recall, F1-score.
    It is adapt to the task with multiple labels.

    - If positive_label=-1, it will print the average precision, recall,
      F1-score of all labels.

    - If use specify positive_label, it will print the precision, recall,
      F1-score of this label.

    The simple usage:

    .. code-block:: python

       eval = precision_recall_evaluator(input, label)

    :param name: Evaluator name.
    :type name: None|basestring
    :param input: Input Layer name. The output prediction of network.
    :type input: LayerOutput
    :param label: Label layer name.
    :type label: LayerOutput
    :param positive_label: The input label layer.
    :type positive_label: LayerOutput.
    :param weight: Weight Layer name. It should be a matrix with size
                  [sample_num, 1]. (TODO, explaination)
    :type weight: LayerOutput
    """
Q
qijun 已提交
293 294 295 296 297 298 299 300
    evaluator_base(
        name=name,
        type="precision_recall",
        input=input,
        label=label,
        positive_label=positive_label,
        weight=weight)

Z
zhangjinchao01 已提交
301 302 303 304 305

@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
def ctc_error_evaluator(
        input,
306
        label,
Q
qijun 已提交
307
        name=None, ):
Z
zhangjinchao01 已提交
308 309 310 311 312 313 314
    """
    This evaluator is to calculate sequence-to-sequence edit distance.

    The simple usage is :

    .. code-block:: python

315
       eval = ctc_error_evaluator(input=input, label=lbl)
Z
zhangjinchao01 已提交
316 317 318

    :param name: Evaluator name.
    :type name: None|basestring
319
    :param input: Input Layer. Should be the same as the input for ctc_layer.
Z
zhangjinchao01 已提交
320
    :type input: LayerOutput
321 322
    :param label: input label, which is a data_layer. Should be the same as the
                  label for ctc_layer
323
    :type label: LayerOutput
Z
zhangjinchao01 已提交
324
    """
Q
qijun 已提交
325 326 327
    evaluator_base(
        name=name, type="ctc_edit_distance", input=input, label=label)

Z
zhangjinchao01 已提交
328 329 330 331 332

@evaluator(EvaluatorAttribute.FOR_CLASSIFICATION)
@wrap_name_default()
def chunk_evaluator(
        input,
333 334 335
        label,
        chunk_scheme,
        num_chunk_types,
336 337
        name=None,
        excluded_chunk_types=None, ):
Z
zhangjinchao01 已提交
338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
    """
    Chunk evaluator is used to evaluate segment labelling accuracy for a
    sequence. It calculates the chunk detection F1 score.

    A chunk is correctly detected if its beginning, end and type are correct.
    Other chunk type is ignored.

    For each label in the label sequence, we have:

    .. code-block:: python

       tagType = label % numTagType
       chunkType = label / numTagType
       otherChunkType = numChunkTypes

    The total number of different labels is numTagType*numChunkTypes+1.
    We support 4 labelling scheme.
    The tag type for each of the scheme is shown as follows:

    .. code-block:: python

       Scheme Begin Inside End   Single
       plain  0     -      -     -
       IOB    0     1      -     -
       IOE    -     0      1     -
       IOBES  0     1      2     3

    'plain' means the whole chunk must contain exactly the same chunk label.

    The simple usage is:

    .. code-block:: python

371
       eval = chunk_evaluator(input, label, chunk_scheme, num_chunk_types)
Z
zhangjinchao01 已提交
372 373 374

    :param input: The input layers.
    :type input: LayerOutput
375 376
    :param label: An input layer containing the ground truth label.
    :type label: LayerOutput
Z
zhangjinchao01 已提交
377
    :param chunk_scheme: The labelling schemes support 4 types. It is one of
378
                         "IOB", "IOE", "IOBES", "plain". It is required.
Z
zhangjinchao01 已提交
379 380
    :type chunk_scheme: basestring
    :param num_chunk_types: number of chunk types other than "other"
381 382
    :param name: The Evaluator name, it is optional.
    :type name: basename|None
383
    :param excluded_chunk_types: chunks of these types are not considered
P
Peng Li 已提交
384
    :type excluded_chunk_types: list of integer|None
Z
zhangjinchao01 已提交
385
    """
Q
qijun 已提交
386 387 388 389
    evaluator_base(
        name=name,
        type="chunk",
        input=input,
390
        label=label,
Q
qijun 已提交
391
        chunk_scheme=chunk_scheme,
392 393
        num_chunk_types=num_chunk_types,
        excluded_chunk_types=excluded_chunk_types, )
Q
qijun 已提交
394

Z
zhangjinchao01 已提交
395 396 397 398 399 400

@evaluator(EvaluatorAttribute.FOR_UTILS)
@wrap_name_default()
def sum_evaluator(
        input,
        name=None,
Q
qijun 已提交
401
        weight=None, ):
Z
zhangjinchao01 已提交
402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418
    """
    An Evaluator to sum the result of input.

    The simple usage:

    .. code-block:: python

       eval = sum_evaluator(input)

    :param name: Evaluator name.
    :type name: None|basestring
    :param input: Input Layer name.
    :type input: LayerOutput
    :param weight: Weight Layer name. It should be a matrix with size
                  [sample_num, 1]. (TODO, explaination)
    :type weight: LayerOutput
    """
Q
qijun 已提交
419 420
    evaluator_base(name=name, type="sum", input=input, weight=weight)

Z
zhangjinchao01 已提交
421 422 423 424 425 426

@evaluator(EvaluatorAttribute.FOR_UTILS)
@wrap_name_default()
def column_sum_evaluator(
        input,
        name=None,
Q
qijun 已提交
427
        weight=None, ):
Z
zhangjinchao01 已提交
428 429 430 431 432 433 434 435 436 437 438 439 440 441
    """
    This Evaluator is used to sum the last column of input.

    The simple usage is:

    .. code-block:: python

       eval = column_sum_evaluator(input, label)

    :param name: Evaluator name.
    :type name: None|basestring
    :param input: Input Layer name.
    :type input: LayerOutput
    """
Q
qijun 已提交
442 443 444
    evaluator_base(
        name=name, type="last-column-sum", input=input, weight=weight)

Z
zhangjinchao01 已提交
445 446 447 448 449 450

"""
The following are printer Evaluators which are usually used to
print the result, like value or gradient of input layers, the
results generated in machine translation, the classification error etc.
"""
Q
qijun 已提交
451 452


Z
zhangjinchao01 已提交
453 454 455 456
@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def value_printer_evaluator(
        input,
Q
qijun 已提交
457
        name=None, ):
Z
zhangjinchao01 已提交
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
    """
    This Evaluator is used to print the values of input layers. It contains
    one or more input layers.

    The simple usage is:

    .. code-block:: python

       eval = value_printer_evaluator(input)

    :param input: One or more input layers.
    :type input: LayerOutput|list
    :param name: Evaluator name.
    :type name: None|basestring
    """
Q
qijun 已提交
473 474
    evaluator_base(name=name, type="value_printer", input=input)

Z
zhangjinchao01 已提交
475 476 477 478 479

@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def gradient_printer_evaluator(
        input,
Q
qijun 已提交
480
        name=None, ):
Z
zhangjinchao01 已提交
481 482 483 484 485 486 487 488 489 490 491 492 493 494 495
    """
    This Evaluator is used to print the gradient of input layers. It contains
    one or more input layers.

    The simple usage is:

    .. code-block:: python

       eval = gradient_printer_evaluator(input)

    :param input: One or more input layers.
    :type input: LayerOutput|list
    :param name: Evaluator name.
    :type name: None|basestring
    """
Q
qijun 已提交
496 497
    evaluator_base(name=name, type="gradient_printer", input=input)

Z
zhangjinchao01 已提交
498 499 500 501
@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def maxid_printer_evaluator(
        input,
502
        num_results=None,
Q
qijun 已提交
503
        name=None, ):
Z
zhangjinchao01 已提交
504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
    """
    This Evaluator is used to print maximum top k values and their indexes
    of each row of input layers. It contains one or more input layers.
    k is specified by num_results.

    The simple usage is:

    .. code-block:: python

       eval = maxid_printer_evaluator(input)

    :param input: Input Layer name.
    :type input: LayerOutput|list
    :param num_results: This number is used to specify the top k numbers.
                        It is 1 by default.
    :type num_results: int.
    :param name: Evaluator name.
    :type name: None|basestring
    """
Q
qijun 已提交
523 524 525
    evaluator_base(
        name=name, type="max_id_printer", input=input, num_results=num_results)

Z
zhangjinchao01 已提交
526 527 528 529 530

@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def maxframe_printer_evaluator(
        input,
531
        num_results=None,
Q
qijun 已提交
532
        name=None, ):
Z
zhangjinchao01 已提交
533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552
    """
    This Evaluator is used to print the top k frames of each input layers.
    The input layers should contain sequences info or sequences type.
    k is specified by num_results.
    It contains one or more input layers.

    Note:
        The width of each frame is 1.

    The simple usage is:

    .. code-block:: python

       eval = maxframe_printer_evaluator(input)

    :param input: Input Layer name.
    :type input: LayerOutput|list
    :param name: Evaluator name.
    :type name: None|basestring
    """
Q
qijun 已提交
553 554 555 556 557 558
    evaluator_base(
        name=name,
        type="max_frame_printer",
        input=input,
        num_results=num_results)

Z
zhangjinchao01 已提交
559 560 561 562 563

@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def seqtext_printer_evaluator(
        input,
564
        result_file,
565
        id_input=None,
566 567
        dict_file=None,
        delimited=None,
Q
qijun 已提交
568
        name=None, ):
Z
zhangjinchao01 已提交
569 570 571 572
    """
    Sequence text printer will print text according to index matrix and a
    dictionary. There can be multiple input to this layer:

573
    1. If there is no id_input, the input must be a matrix containing
Z
zhangjinchao01 已提交
574 575
    the sequence of indices;

576
    2. If there is id_input, it should be ids, and interpreted as sample ids.
Z
zhangjinchao01 已提交
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606

    The output format will be:

    1. sequence without sub-sequence, and there is probability.

    .. code-block:: python

         id \t prob space_seperated_tokens_from_dictionary_according_to_seq

    2. sequence without sub-sequence, and there is not probability.

    .. code-block:: python

         id \t space_seperated_tokens_from_dictionary_according_to_seq

    3. sequence with sub-sequence, and there is not probability.

    .. code-block:: python

         id \t space_seperated_tokens_from_dictionary_according_to_sub_seq
         \t \t space_seperated_tokens_from_dictionary_according_to_sub_seq
         ...

    Typically SequenceTextPrinter layer takes output of maxid or RecurrentGroup
    with maxid (when generating) as an input.

    The simple usage is:

    .. code-block:: python

607 608
       eval = seqtext_printer_evaluator(input=maxid_layer,
                                        id_input=sample_id,
Z
zhangjinchao01 已提交
609 610 611 612 613
                                        dict_file=dict_file,
                                        result_file=result_file)

    :param input: Input Layer name.
    :type input: LayerOutput|list
614
    :param result_file: Path of the file to store the generated results.
Z
zhangjinchao01 已提交
615
    :type result_file: basestring
616 617 618 619 620 621 622 623 624 625
    :param id_input: Index of the input sequence, and the specified index will
                     be prited in the gereated results. This an optional
                     parameter.
    :type id_input: LayerOutput
    :param dict_file: Path of dictionary. This is an optional parameter.
                      Every line is a word in the dictionary with
                      (line number - 1) as the word index.
                      If this parameter is set to None, or to an empty string,
                      only word index are printed in the generated results.
    :type dict_file: basestring
Z
zhangjinchao01 已提交
626 627 628 629 630
    :param delimited: Whether to use space to separate output tokens.
                Default is True. No space is added if set to False.
    :type delimited: bool
    :param name: Evaluator name.
    :type name: None|basestring
631 632
    :return: The seq_text_printer that prints the generated sequence to a file.
    :rtype: evaluator
Z
zhangjinchao01 已提交
633
    """
634
    assert isinstance(result_file, basestring)
635 636 637 638 639 640
    if id_input is None:
        inputs = [input]
    else:
        inputs = [id_input, input]
        input.parents.append(id_input)

Q
qijun 已提交
641 642 643 644 645 646 647 648
    evaluator_base(
        name=name,
        type="seq_text_printer",
        input=inputs,
        dict_file=dict_file,
        result_file=result_file,
        delimited=delimited)

Z
zhangjinchao01 已提交
649 650 651 652 653 654 655

@evaluator(EvaluatorAttribute.FOR_PRINT)
@wrap_name_default()
def classification_error_printer_evaluator(
        input,
        label,
        threshold=0.5,
Q
qijun 已提交
656
        name=None, ):
Z
zhangjinchao01 已提交
657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672
    """
    This Evaluator is used to print the classification error of each sample.

    The simple usage is:

    .. code-block:: python

       eval = classification_error_printer_evaluator(input)

    :param input: Input layer.
    :type input: LayerOutput
    :param label: Input label layer.
    :type label: LayerOutput
    :param name: Evaluator name.
    :type name: None|basestring
    """
Q
qijun 已提交
673 674 675 676 677 678
    evaluator_base(
        name=name,
        type="classification_error_printer",
        input=input,
        label=label,
        classification_threshold=threshold)