metrics.py 26.3 KB
Newer Older
D
dzhwinter 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Fluid Metrics
"""
17 18 19

from __future__ import print_function

D
dzhwinter 已提交
20 21 22
import numpy as np
import copy
import warnings
23
import six
D
dzhwinter 已提交
24

D
Dang Qingqing 已提交
25 26 27 28 29 30
from .layer_helper import LayerHelper
from .initializer import Constant
from . import unique_name
from .framework import Program, Variable, program_guard
from . import layers

D
dzhwinter 已提交
31 32 33
__all__ = [
    'MetricBase',
    'CompositeMetric',
34 35
    'Precision',
    'Recall',
D
dzhwinter 已提交
36 37 38 39 40 41 42 43 44 45 46 47 48
    'Accuracy',
    'ChunkEvaluator',
    'EditDistance',
    'DetectionMAP',
    'Auc',
]


def _is_numpy_(var):
    return isinstance(var, (np.ndarray, np.generic))


def _is_number_(var):
P
peizhilin 已提交
49 50
    return isinstance(var, int) or isinstance(var, np.int64) or isinstance(
        var, float) or (isinstance(var, np.ndarray) and var.shape == (1, ))
D
dzhwinter 已提交
51 52 53 54 55 56 57 58


def _is_number_or_matrix_(var):
    return _is_number_(var) or isinstance(var, np.ndarray)


class MetricBase(object):
    """
59 60 61 62 63 64 65 66 67
    Base Class for all Metrics.
    MetricBase define a group of interfaces for the
    model evaluation methods. Metrics accumulate metric states between
    consecutive minibatches, at every minibatch, use update
    interface to add current minibatch value to global states.
    Use eval to compute accumative metric value from last reset()
    or from scratch on.
    If you need to custom a new metric, please inherit from MetricBase and
    custom implementation.
D
dzhwinter 已提交
68 69

    Args:
70 71 72
        name(str): The name of metric instance. such as, "accuracy".
                  It needed if you want to distinct different metrics in a model.

D
dzhwinter 已提交
73 74
    """

75
    def __init__(self, name):
D
dzhwinter 已提交
76 77 78 79 80 81 82
        self._name = str(name) if name != None else self.__class__.__name__

    def __str__(self):
        return self._name

    def reset(self):
        """
83 84 85 86
        reset clear the states of metrics. By default, the states
        are the members who do not has _ prefix, reset set them to inital states.
        If you violate the implicit name rule, please also custom the reset
        interface.
D
dzhwinter 已提交
87 88 89
        """
        states = {
            attr: value
M
minqiyang 已提交
90
            for attr, value in six.iteritems(self.__dict__)
D
dzhwinter 已提交
91 92
            if not attr.startswith("_")
        }
M
minqiyang 已提交
93
        for attr, value in six.iteritems(states):
D
dzhwinter 已提交
94 95 96 97 98 99 100 101 102 103
            if isinstance(value, int):
                setattr(self, attr, 0)
            elif isinstance(value, float):
                setattr(self, attr, .0)
            elif isinstance(value, (np.ndarray, np.generic)):
                setattr(self, attr, np.zeros_like(value))
            else:
                setattr(self, attr, None)

    def get_config(self):
104 105 106 107 108 109 110 111 112 113
        """
        Get the metric and current states.
        The states are the members who do not has "_" prefix.

        Args:
            None

        Returns:
            dict: a dict of metric and states
        """
D
dzhwinter 已提交
114 115
        states = {
            attr: value
M
minqiyang 已提交
116
            for attr, value in six.iteritems(self.__dict__)
D
dzhwinter 已提交
117 118
            if not attr.startswith("_")
        }
119
        config = {}
D
dzhwinter 已提交
120 121 122
        config.update({"name": self._name, "states": copy.deepcopy(states)})
        return config

123 124 125 126 127 128 129 130 131 132 133 134 135
    def update(self, preds, labels):
        """
        Updates the metric states at every minibatch.
        One user can compute the minibatch metric via pure Python, or
        via a c++ operator.

        Args:
            preds(numpy.array): the predictions of current minibatch
            labels(numpy.array): the labels of current minibatch, if the label is one-hot
                               or soft-label, should custom the corresponding update rule.
        """
        raise NotImplementedError(
            "Should not use it directly, please extend it.")
D
dzhwinter 已提交
136 137

    def eval(self):
138 139 140 141 142 143 144 145
        """
        Evalute the current metrics based the accumulated states.

        Returns:
            float|list(float)|numpy.array: the metrics via Python.
        """
        raise NotImplementedError(
            "Should not use it directly, please extend it.")
D
dzhwinter 已提交
146 147 148 149


class CompositeMetric(MetricBase):
    """
150
    Composite multiple metrics in one instance.
D
dzhwinter 已提交
151
    for example, merge F1, accuracy, recall into one Metric.
152

153 154
    Examples:
        .. code-block:: python
155

156 157 158 159 160 161 162 163 164 165 166 167 168 169
          labels = fluid.layers.data(name="data", shape=[1], dtype="int32")
          data = fluid.layers.data(name="data", shape=[32, 32], dtype="int32")
          pred = fluid.layers.fc(input=data, size=1000, act="tanh")
          comp = fluid.metrics.CompositeMetric()
          acc = fluid.metrics.Precision()
          recall = fluid.metrics.Recall()
          comp.add_metric(acc)
          comp.add_metric(recall)
          for pass in range(PASSES):
            comp.reset()
            for data in train_reader():
                loss, preds, labels = exe.run(fetch_list=[cost, preds, labels])
            comp.update(preds=preds, labels=labels)
            numpy_acc, numpy_recall = comp.eval()
D
dzhwinter 已提交
170 171
    """

172 173
    def __init__(self, name=None):
        super(CompositeMetric, self).__init__(name)
D
dzhwinter 已提交
174 175
        self._metrics = []

Q
qiaolongfei 已提交
176
    def add_metric(self, metric):
177 178 179 180 181 182
        """
        add one metric instance to CompositeMetric.

        Args:
            metric: a instance of MetricBase.
        """
D
dzhwinter 已提交
183 184 185 186
        if not isinstance(metric, MetricBase):
            raise ValueError("SubMetric should be inherit from MetricBase.")
        self._metrics.append(metric)

187 188 189 190 191 192 193 194 195 196
    def update(self, preds, labels):
        """
        Update every metrics in sequence.

        Args:
            preds(numpy.array): the predictions of current minibatch
            labels(numpy.array): the labels of current minibatch, if the label is one-hot
                               or soft-label, should custom the corresponding update rule.
        """
        for m in self._metrics:
D
dzhwinter 已提交
197
            m.update(preds, labels)
198

D
dzhwinter 已提交
199
    def eval(self):
200 201 202 203 204 205
        """
        Evaluate every metrics in sequence.

        Returns:
            list(float|numpy.array): a list of metrics value in Python.
        """
D
dzhwinter 已提交
206 207 208 209 210 211
        ans = []
        for m in self._metrics:
            ans.append(m.eval())
        return ans


212 213 214 215 216 217 218 219 220 221 222 223 224
class Precision(MetricBase):
    """
    Precision (also called positive predictive value) is the fraction of
    relevant instances among the retrieved instances.
    https://en.wikipedia.org/wiki/Evaluation_of_binary_classifiers

    Note Precision is different with Accuracy in binary classifiers.
    accuracy = true positive / total instances
    precision = true positive / all positive instance

    Examples:
        .. code-block:: python

T
Tink_Y 已提交
225 226 227 228 229
            metric = fluid.metrics.Precision()
            for pass in range(PASSES):
                metric.reset()
                for data in train_reader():
                    loss, preds, labels = exe.run(fetch_list=[cost, preds, labels])
230
                    metric.update(preds=preds, labels=labels)
T
Tink_Y 已提交
231
                numpy_precision = metric.eval()
232 233 234 235 236 237 238 239 240 241 242 243
    """

    def __init__(self, name=None):
        super(Precision, self).__init__(name)
        self.tp = 0  # true positive
        self.fp = 0  # false positive

    def update(self, preds, labels):
        if not _is_numpy_(preds):
            raise ValueError("The 'preds' must be a numpy ndarray.")
        if not _is_numpy_(labels):
            raise ValueError("The 'labels' must be a numpy ndarray.")
244 245
        sample_num = labels.shape[0]
        preds = np.rint(preds).astype("int32")
246
        for i in range(sample_num):
247
            pred = preds[i]
248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
            label = labels[i]
            if label == 1:
                if pred == label:
                    self.tp += 1
                else:
                    self.fp += 1

    def eval(self):
        ap = self.tp + self.fp
        return float(self.tp) / ap if ap != 0 else .0


class Recall(MetricBase):
    """
    Recall (also known as sensitivity) is the fraction of
    relevant instances that have been retrieved over the
    total amount of relevant instances

    https://en.wikipedia.org/wiki/Precision_and_recall

    Examples:
        .. code-block:: python

T
Tink_Y 已提交
271 272 273 274 275 276 277
            metric = fluid.metrics.Recall()
            for pass in range(PASSES):
                metric.reset()
                for data in train_reader():
                    loss, preds, labels = exe.run(fetch_list=[cost, preds, labels])
                metric.update(preds=preds, labels=labels)
                numpy_recall = metric.eval()
278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
    """

    def __init__(self, name=None):
        super(Recall, self).__init__(name)
        self.tp = 0  # true positive
        self.fn = 0  # false negtive

    def update(self, preds, labels):
        if not _is_numpy_(preds):
            raise ValueError("The 'preds' must be a numpy ndarray.")
        if not _is_numpy_(labels):
            raise ValueError("The 'labels' must be a numpy ndarray.")
        sample_num = labels[0]
        for i in range(sample_num):
            pred = preds[i].astype("int32")
            label = labels[i]
            if label == 1:
                if pred == label:
                    self.tp += 1
            else:
                if pred != label:
                    self.fn += 1

    def eval(self):
        recall = self.tp + self.fn
        return float(self.tp) / recall if recall != 0 else .0


D
dzhwinter 已提交
306 307 308 309
class Accuracy(MetricBase):
    """
    Accumulate the accuracy from minibatches and compute the average accuracy
    for every pass.
310
    https://en.wikipedia.org/wiki/Accuracy_and_precision
D
dzhwinter 已提交
311 312 313 314

    Args:
       name: the metrics name

315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
    Examples:
        .. code-block:: python

            labels = fluid.layers.data(name="data", shape=[1], dtype="int32")
            data = fluid.layers.data(name="data", shape=[32, 32], dtype="int32")
            pred = fluid.layers.fc(input=data, size=1000, act="tanh")
            minibatch_accuracy = fluid.layers.accuracy(pred, label)
            accuracy_evaluator = fluid.metrics.Accuracy()
            for pass in range(PASSES):
                accuracy_evaluator.reset()
                for data in train_reader():
                    batch_size = data[0]
                    loss = exe.run(fetch_list=[cost, minibatch_accuracy])
                accuracy_evaluator.update(value=minibatch_accuracy, weight=batch_size)
                numpy_acc = accuracy_evaluator.eval()
D
dzhwinter 已提交
330 331 332 333 334 335 336 337
    """

    def __init__(self, name=None):
        super(Accuracy, self).__init__(name)
        self.value = .0
        self.weight = .0

    def update(self, value, weight):
338 339 340 341 342 343 344
        """
        Update minibatch states.

        Args:
            value(float|numpy.array): accuracy of one minibatch.
            weight(int|float): batch size.
        """
D
dzhwinter 已提交
345 346 347 348 349 350 351 352 353 354
        if not _is_number_or_matrix_(value):
            raise ValueError(
                "The 'value' must be a number(int, float) or a numpy ndarray.")
        if not _is_number_(weight):
            raise ValueError("The 'weight' must be a number(int, float).")
        self.value += value * weight
        self.weight += weight

    def eval(self):
        if self.weight == 0:
355 356
            raise ValueError("There is no data in Accuracy Metrics. \
                Please check layers.accuracy output has added to Accuracy.")
D
dzhwinter 已提交
357 358 359
        return self.value / self.weight


360
class ChunkEvaluator(MetricBase):
D
dzhwinter 已提交
361 362 363 364
    """
    Accumulate counter numbers output by chunk_eval from mini-batches and
    compute the precision recall and F1-score using the accumulated counter
    numbers.
H
haowang101779990 已提交
365 366
    For some basics of chunking, please refer to 
    `Chunking with Support Vector Machines <https://aclanthology.info/pdf/N/N01/N01-1025.pdf>`_ .
367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383
    ChunkEvalEvaluator computes the precision, recall, and F1-score of chunk detection,
    and supports IOB, IOE, IOBES and IO (also known as plain) tagging schemes.

    Examples:
        .. code-block:: python

            labels = fluid.layers.data(name="data", shape=[1], dtype="int32")
            data = fluid.layers.data(name="data", shape=[32, 32], dtype="int32")
            pred = fluid.layers.fc(input=data, size=1000, act="tanh")
            precision, recall, f1_score, num_infer_chunks, num_label_chunks, num_correct_chunks = layers.chunk_eval(
                input=pred,
                label=label)
            metric = fluid.metrics.ChunkEvaluator()
            for data in train_reader():
                loss, preds, labels = exe.run(fetch_list=[cost, preds, labels])
                metric.update(num_infer_chunks, num_label_chunks, num_correct_chunks)
                numpy_precision, numpy_recall, numpy_f1 = metric.eval()
D
dzhwinter 已提交
384 385 386
    """

    def __init__(self, name=None):
T
update  
typhoonzero 已提交
387
        super(ChunkEvaluator, self).__init__(name)
D
dzhwinter 已提交
388 389 390 391 392
        self.num_infer_chunks = 0
        self.num_label_chunks = 0
        self.num_correct_chunks = 0

    def update(self, num_infer_chunks, num_label_chunks, num_correct_chunks):
393 394
        """
        Update the states based on the layers.chunk_eval() ouputs.
H
haowang101779990 已提交
395

396 397 398 399 400 401
        Args:
            num_infer_chunks(int|numpy.array): The number of chunks in Inference on the given minibatch.
            num_label_chunks(int|numpy.array): The number of chunks in Label on the given mini-batch.
            num_correct_chunks(int|float|numpy.array): The number of chunks both in Inference and Label on the
                                                  given mini-batch.
        """
D
dzhwinter 已提交
402 403
        if not _is_number_or_matrix_(num_infer_chunks):
            raise ValueError(
404
                "The 'num_infer_chunks' must be a number(int) or a numpy ndarray."
D
dzhwinter 已提交
405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430
            )
        if not _is_number_or_matrix_(num_label_chunks):
            raise ValueError(
                "The 'num_label_chunks' must be a number(int, float) or a numpy ndarray."
            )
        if not _is_number_or_matrix_(num_correct_chunks):
            raise ValueError(
                "The 'num_correct_chunks' must be a number(int, float) or a numpy ndarray."
            )
        self.num_infer_chunks += num_infer_chunks
        self.num_label_chunks += num_label_chunks
        self.num_correct_chunks += num_correct_chunks

    def eval(self):
        precision = float(
            self.num_correct_chunks
        ) / self.num_infer_chunks if self.num_infer_chunks else 0
        recall = float(self.num_correct_chunks
                       ) / self.num_label_chunks if self.num_label_chunks else 0
        f1_score = float(2 * precision * recall) / (
            precision + recall) if self.num_correct_chunks else 0
        return precision, recall, f1_score


class EditDistance(MetricBase):
    """
431 432 433 434 435
    Edit distance is a way of quantifying how dissimilar two strings
    (e.g., words) are to one another by counting the minimum number
    of operations required to transform one string into the other.
    Refer to https://en.wikipedia.org/wiki/Edit_distance

D
dzhwinter 已提交
436 437 438 439 440 441
    Accumulate edit distance sum and sequence number from mini-batches and
    compute the average edit_distance and instance error of all batches.

    Args:
        name: the metrics name

442 443 444 445 446 447 448 449 450 451 452
    Examples:
        .. code-block:: python

            distances, seq_num = fluid.layers.edit_distance(input, label)
            distance_evaluator = fluid.metrics.EditDistance()
            for epoch in PASS_NUM:
                distance_evaluator.reset()
                for data in batches:
                    loss = exe.run(fetch_list=[cost] + list(edit_distance_metrics))
                distance_evaluator.update(distances, seq_num)
                distance, instance_error = distance_evaluator.eval()
D
dzhwinter 已提交
453

T
Tink_Y 已提交
454 455
    In the above example:

H
haowang101779990 已提交
456 457
        - 'distance' is the average of the edit distance in a pass.
        - 'instance_error' is the instance error rate in a pass.
D
dzhwinter 已提交
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477

    """

    def __init__(self, name):
        super(EditDistance, self).__init__(name)
        self.total_distance = .0
        self.seq_num = 0
        self.instance_error = 0

    def update(self, distances, seq_num):
        if not _is_numpy_(distances):
            raise ValueError("The 'distances' must be a numpy ndarray.")
        if not _is_number_(seq_num):
            raise ValueError("The 'seq_num' must be a number(int, float).")
        seq_right_count = np.sum(distances == 0)
        total_distance = np.sum(distances)
        self.seq_num += seq_num
        self.instance_error += seq_num - seq_right_count
        self.total_distance += total_distance

Q
qiaolongfei 已提交
478
    def eval(self):
D
dzhwinter 已提交
479 480 481 482 483
        if self.seq_num == 0:
            raise ValueError(
                "There is no data in EditDistance Metric. Please check layers.edit_distance output has been added to EditDistance."
            )
        avg_distance = self.total_distance / self.seq_num
S
sneaxiy 已提交
484
        avg_instance_error = self.instance_error / float(self.seq_num)
D
dzhwinter 已提交
485 486 487 488 489
        return avg_distance, avg_instance_error


class Auc(MetricBase):
    """
490 491 492
    Auc metric adapts to the binary classification.
    Refer to https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve
    Need to note that auc metric compute the value via Python natively.
D
dzhwinter 已提交
493 494 495
    If you concern the speed, please use the fluid.layers.auc instead.

    The `auc` function creates four local variables, `true_positives`,
496 497 498 499 500 501
    `true_negatives`, `false_positives` and `false_negatives` that are used to
    compute the AUC. To discretize the AUC curve, a linearly spaced set of
    thresholds is used to compute pairs of recall and precision values. The area
    under the ROC-curve is therefore computed using the height of the recall
    values by the false positive rate, while the area under the PR-curve is the
    computed using the height of the precision values by the recall.
D
dzhwinter 已提交
502 503 504 505 506 507 508

    Args:
        name: metric name
        curve: Specifies the name of the curve to be computed, 'ROC' [default] or
          'PR' for the Precision-Recall-curve.

    "NOTE: only implement the ROC curve type via Python now."
509 510 511 512 513 514 515 516 517 518

    Examples:
        .. code-block:: python

            pred = fluid.layers.fc(input=data, size=1000, act="tanh")
            metric = fluid.metrics.Auc()
            for data in train_reader():
                loss, preds, labels = exe.run(fetch_list=[cost, preds, labels])
                metric.update(preds, labels)
                numpy_auc = metric.eval()
D
dzhwinter 已提交
519 520
    """

T
tangwei12 已提交
521
    def __init__(self, name, curve='ROC', num_thresholds=4095):
Q
fix auc  
qiaolongfei 已提交
522
        super(Auc, self).__init__(name=name)
D
dzhwinter 已提交
523 524
        self._curve = curve
        self._num_thresholds = num_thresholds
T
tangwei12 已提交
525 526 527 528

        _num_pred_buckets = num_thresholds + 1
        self._stat_pos = [0] * _num_pred_buckets
        self._stat_neg = [0] * _num_pred_buckets
D
dzhwinter 已提交
529

Q
qiaolongfei 已提交
530
    def update(self, preds, labels):
D
dzhwinter 已提交
531 532
        if not _is_numpy_(labels):
            raise ValueError("The 'labels' must be a numpy ndarray.")
Q
qiaolongfei 已提交
533
        if not _is_numpy_(preds):
D
dzhwinter 已提交
534 535
            raise ValueError("The 'predictions' must be a numpy ndarray.")

T
tangwei12 已提交
536 537 538 539 540 541 542 543 544 545 546 547
        for i, lbl in enumerate(labels):
            value = preds[i, 1]
            bin_idx = int(value * self._num_thresholds)
            assert bin_idx <= self._num_thresholds
            if lbl:
                self._stat_pos[bin_idx] += 1.0
            else:
                self._stat_neg[bin_idx] += 1.0

    @staticmethod
    def trapezoid_area(x1, x2, y1, y2):
        return abs(x1 - x2) * (y1 + y2) / 2.0
D
dzhwinter 已提交
548 549

    def eval(self):
T
tangwei12 已提交
550 551 552 553 554 555 556 557 558 559 560 561 562 563 564
        tot_pos = 0.0
        tot_neg = 0.0
        auc = 0.0

        idx = self._num_thresholds
        while idx >= 0:
            tot_pos_prev = tot_pos
            tot_neg_prev = tot_neg
            tot_pos += self._stat_pos[idx]
            tot_neg += self._stat_neg[idx]
            auc += self.trapezoid_area(tot_neg, tot_neg_prev, tot_pos,
                                       tot_pos_prev)
            idx -= 1

        return auc / tot_pos / tot_neg if tot_pos > 0.0 and tot_neg > 0.0 else 0.0
565 566 567 568 569 570 571


class DetectionMAP(object):
    """
    Calculate the detection mean average precision (mAP).

    The general steps are as follows:
H
haowang101779990 已提交
572

573
    1. calculate the true positive and false positive according to the input
H
haowang101779990 已提交
574
       of detection and labels.
575 576 577
    2. calculate mAP value, support two versions: '11 point' and 'integral'.

    Please get more information from the following articles:
H
haowang101779990 已提交
578

579
      https://sanchom.wordpress.com/tag/average-precision/
H
haowang101779990 已提交
580

581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610
      https://arxiv.org/abs/1512.02325

    Args:
        input (Variable): The detection results, which is a LoDTensor with shape
            [M, 6]. The layout is [label, confidence, xmin, ymin, xmax, ymax].
        gt_label (Variable): The ground truth label index, which is a LoDTensor
            with shape [N, 1].
        gt_box (Variable): The ground truth bounding box (bbox), which is a
            LoDTensor with shape [N, 4]. The layout is [xmin, ymin, xmax, ymax].
        gt_difficult (Variable|None): Whether this ground truth is a difficult
            bounding bbox, which can be a LoDTensor [N, 1] or not set. If None,
            it means all the ground truth labels are not difficult bbox.
        class_num (int): The class number.
        background_label (int): The index of background label, the background
            label will be ignored. If set to -1, then all categories will be
            considered, 0 by defalut.
        overlap_threshold (float): The threshold for deciding true/false
            positive, 0.5 by defalut.
        evaluate_difficult (bool): Whether to consider difficult ground truth
            for evaluation, True by defalut. This argument does not work when
            gt_difficult is None.
        ap_version (string): The average precision calculation ways, it must be
            'integral' or '11point'. Please check
            https://sanchom.wordpress.com/tag/average-precision/ for details.
            - 11point: the 11-point interpolated average precision.
            - integral: the natural integral of the precision-recall curve.

    Examples:
        .. code-block:: python

D
Dang Qingqing 已提交
611
            exe = fluid.Executor(place)
612 613 614 615 616 617 618 619 620
            map_evaluator = fluid.Evaluator.DetectionMAP(input,
                gt_label, gt_box, gt_difficult)
            cur_map, accum_map = map_evaluator.get_map_var()
            fetch = [cost, cur_map, accum_map]
            for epoch in PASS_NUM:
                map_evaluator.reset(exe)
                for data in batches:
                    loss, cur_map_v, accum_map_v = exe.run(fetch_list=fetch)

H
haowang101779990 已提交
621 622 623 624
    In the above example:

            - 'cur_map_v' is the mAP of current mini-batch.
            - 'accum_map_v' is the accumulative mAP of one pass.
625

H
haowang101779990 已提交
626
 
627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719
    """

    def __init__(self,
                 input,
                 gt_label,
                 gt_box,
                 gt_difficult=None,
                 class_num=None,
                 background_label=0,
                 overlap_threshold=0.5,
                 evaluate_difficult=True,
                 ap_version='integral'):

        self.helper = LayerHelper('map_eval')
        gt_label = layers.cast(x=gt_label, dtype=gt_box.dtype)
        if gt_difficult:
            gt_difficult = layers.cast(x=gt_difficult, dtype=gt_box.dtype)
            label = layers.concat([gt_label, gt_difficult, gt_box], axis=1)
        else:
            label = layers.concat([gt_label, gt_box], axis=1)

        # calculate mean average precision (mAP) of current mini-batch
        map = layers.detection_map(
            input,
            label,
            class_num,
            background_label,
            overlap_threshold=overlap_threshold,
            evaluate_difficult=evaluate_difficult,
            ap_version=ap_version)

        states = []
        states.append(
            self._create_state(
                dtype='int32', shape=None, suffix='accum_pos_count'))
        states.append(
            self._create_state(
                dtype='float32', shape=None, suffix='accum_true_pos'))
        states.append(
            self._create_state(
                dtype='float32', shape=None, suffix='accum_false_pos'))
        var = self._create_state(dtype='int32', shape=[1], suffix='has_state')
        self.helper.set_variable_initializer(
            var, initializer=Constant(value=int(0)))
        self.has_state = var

        # calculate accumulative mAP
        accum_map = layers.detection_map(
            input,
            label,
            class_num,
            background_label,
            overlap_threshold=overlap_threshold,
            evaluate_difficult=evaluate_difficult,
            has_state=self.has_state,
            input_states=states,
            out_states=states,
            ap_version=ap_version)

        layers.fill_constant(
            shape=self.has_state.shape,
            value=1,
            dtype=self.has_state.dtype,
            out=self.has_state)

        self.cur_map = map
        self.accum_map = accum_map

    def _create_state(self, suffix, dtype, shape):
        """
        Create state variable.
        Args:
            suffix(str): the state suffix.
            dtype(str|core.VarDesc.VarType): the state data type
            shape(tuple|list): the shape of state
        Returns: State variable
        """
        state = self.helper.create_variable(
            name="_".join([unique_name.generate(self.helper.name), suffix]),
            persistable=True,
            dtype=dtype,
            shape=shape)
        return state

    def get_map_var(self):
        """
        Returns: mAP variable of current mini-batch and
            accumulative mAP variable cross mini-batches.
        """
        return self.cur_map, self.accum_map

    def reset(self, executor, reset_program=None):
        """
D
Dang Qingqing 已提交
720
        Reset metric states at the begin of each pass/user specified batch.
721 722

        Args:
D
Dang Qingqing 已提交
723
            executor(Executor): a executor for executing
724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745
                the reset_program.
            reset_program(Program|None): a single Program for reset process.
                If None, will create a Program.
        """

        def _clone_var_(block, var):
            assert isinstance(var, Variable)
            return block.create_var(
                name=var.name,
                shape=var.shape,
                dtype=var.dtype,
                type=var.type,
                lod_level=var.lod_level,
                persistable=var.persistable)

        if reset_program is None:
            reset_program = Program()
        with program_guard(main_program=reset_program):
            var = _clone_var_(reset_program.current_block(), self.has_state)
            layers.fill_constant(
                shape=var.shape, value=0, dtype=var.dtype, out=var)
        executor.run(reset_program)