test_detection_map_op.py

#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest
import numpy as np
import sys
import collections
import math
from op_test import OpTest


class TestDetectionMAPOp(OpTest):
    def set_data(self):
        self.class_num = 4
        self.init_test_case()
        self.mAP = [self.calc_map(self.tf_pos, self.tf_pos_lod)]
        self.label = np.array(self.label).astype('float32')
        self.detect = np.array(self.detect).astype('float32')
        self.mAP = np.array(self.mAP).astype('float32')

        if (len(self.class_pos_count) > 0):
            self.class_pos_count = np.array(self.class_pos_count).astype(
                'int32')
            self.true_pos = np.array(self.true_pos).astype('float32')
            self.false_pos = np.array(self.false_pos).astype('float32')
            self.has_state = np.array([1]).astype('int32')

            self.inputs = {
                'Label': (self.label, self.label_lod),
                'DetectRes': (self.detect, self.detect_lod),
                'HasState': self.has_state,
                'PosCount': self.class_pos_count,
                'TruePos': (self.true_pos, self.true_pos_lod),
                'FalsePos': (self.false_pos, self.false_pos_lod)
            }
        else:
            self.inputs = {
                'Label': (self.label, self.label_lod),
                'DetectRes': (self.detect, self.detect_lod),
            }

        self.attrs = {
            'overlap_threshold': self.overlap_threshold,
            'evaluate_difficult': self.evaluate_difficult,
            'ap_type': self.ap_type,
            'class_num': self.class_num
        }

        self.out_class_pos_count = np.array(self.out_class_pos_count).astype(
            'int')
        self.out_true_pos = np.array(self.out_true_pos).astype('float32')
        self.out_false_pos = np.array(self.out_false_pos).astype('float32')

        self.outputs = {
            'MAP': self.mAP,
            'AccumPosCount': self.out_class_pos_count,
            'AccumTruePos': (self.out_true_pos, self.out_true_pos_lod),
            'AccumFalsePos': (self.out_false_pos, self.out_false_pos_lod)
        }

    def init_test_case(self):
        self.overlap_threshold = 0.3
        self.evaluate_difficult = True
        self.ap_type = "integral"

        self.label_lod = [[0, 2, 4]]
        # label difficult xmin ymin xmax ymax
        self.label = [[1, 0, 0.1, 0.1, 0.3, 0.3], [1, 1, 0.6, 0.6, 0.8, 0.8],
                      [2, 0, 0.3, 0.3, 0.6, 0.5], [1, 0, 0.7, 0.1, 0.9, 0.3]]

        # label score xmin ymin xmax ymax difficult
        self.detect_lod = [[0, 3, 7]]
        self.detect = [
            [1, 0.3, 0.1, 0.0, 0.4, 0.3], [1, 0.7, 0.0, 0.1, 0.2, 0.3],
            [1, 0.9, 0.7, 0.6, 0.8, 0.8], [2, 0.8, 0.2, 0.1, 0.4, 0.4],
            [2, 0.1, 0.4, 0.3, 0.7, 0.5], [1, 0.2, 0.8, 0.1, 1.0, 0.3],
            [3, 0.2, 0.8, 0.1, 1.0, 0.3]
        ]

        # label score true_pos false_pos
        self.tf_pos_lod = [[0, 3, 7]]
        self.tf_pos = [[1, 0.9, 1, 0], [1, 0.7, 1, 0], [1, 0.3, 0, 1],
                       [1, 0.2, 1, 0], [2, 0.8, 0, 1], [2, 0.1, 1, 0],
                       [3, 0.2, 0, 1]]

        self.class_pos_count = []
        self.true_pos_lod = [[]]
        self.true_pos = [[]]
        self.false_pos_lod = [[]]
        self.false_pos = [[]]

    def calc_map(self, tf_pos, tf_pos_lod):
        mAP = 0.0
        count = 0

        def get_input_pos(class_pos_count, true_pos, true_pos_lod, false_pos,
                          false_pos_lod):
            class_pos_count_dict = collections.Counter()
            true_pos_dict = collections.defaultdict(list)
            false_pos_dict = collections.defaultdict(list)
            for i, count in enumerate(class_pos_count):
                class_pos_count_dict[i] = count

            for i in range(len(true_pos_lod[0]) - 1):
                start = true_pos_lod[0][i]
                end = true_pos_lod[0][i + 1]
                for j in range(start, end):
                    true_pos_dict[i].append(true_pos[j])

            for i in range(len(false_pos_lod[0]) - 1):
                start = false_pos_lod[0][i]
                end = false_pos_lod[0][i + 1]
                for j in range(start, end):
                    false_pos_dict[i].append(false_pos[j])

            return class_pos_count_dict, true_pos_dict, false_pos_dict

        def get_output_pos(label_count, true_pos, false_pos):
            label_number = self.class_num

            out_class_pos_count = []
            out_true_pos_lod = [0]
            out_true_pos = []
            out_false_pos_lod = [0]
            out_false_pos = []

            for i in range(label_number):
                out_class_pos_count.append([label_count[i]])
                true_pos_list = true_pos[i]
                out_true_pos += true_pos_list
                out_true_pos_lod.append(len(out_true_pos))
                false_pos_list = false_pos[i]
                out_false_pos += false_pos_list
                out_false_pos_lod.append(len(out_false_pos))

            return out_class_pos_count, out_true_pos, [
                out_true_pos_lod
            ], out_false_pos, [out_false_pos_lod]

        def get_accumulation(pos_list):
            sorted_list = sorted(pos_list, key=lambda pos: pos[0], reverse=True)
            sum = 0
            accu_list = []
            for (score, count) in sorted_list:
                sum += count
                accu_list.append(sum)
            return accu_list

        label_count, true_pos, false_pos = get_input_pos(
            self.class_pos_count, self.true_pos, self.true_pos_lod,
            self.false_pos, self.false_pos_lod)
        for (label, difficult, xmin, ymin, xmax, ymax) in self.label:
            if self.evaluate_difficult:
                label_count[label] += 1
            elif not difficult:
                label_count[label] += 1

        true_pos = collections.defaultdict(list)
        false_pos = collections.defaultdict(list)
        for (label, score, tp, fp) in tf_pos:
            true_pos[label].append([score, tp])
            false_pos[label].append([score, fp])

        for (label, label_pos_num) in label_count.items():
            if label_pos_num == 0 or label not in true_pos: continue
            label_true_pos = true_pos[label]
            label_false_pos = false_pos[label]

            accu_tp_sum = get_accumulation(label_true_pos)
            accu_fp_sum = get_accumulation(label_false_pos)

            precision = []
            recall = []

            for i in range(len(accu_tp_sum)):
                precision.append(
                    float(accu_tp_sum[i]) /
                    float(accu_tp_sum[i] + accu_fp_sum[i]))
                recall.append(float(accu_tp_sum[i]) / label_pos_num)

            if self.ap_type == "11point":
                max_precisions = [0.0] * 11
                start_idx = len(accu_tp_sum) - 1
                for j in range(10, -1, -1):
                    for i in range(start_idx, -1, -1):
                        if recall[i] < float(j) / 10.0:
                            start_idx = i
                            if j > 0:
                                max_precisions[j - 1] = max_precisions[j]
                                break
                        else:
                            if max_precisions[j] < precision[i]:
                                max_precisions[j] = precision[i]
                for j in range(10, -1, -1):
                    mAP += max_precisions[j] / 11
                count += 1
            elif self.ap_type == "integral":
                average_precisions = 0.0
                prev_recall = 0.0
                for i in range(len(accu_tp_sum)):
                    if math.fabs(recall[i] - prev_recall) > 1e-6:
                        average_precisions += precision[i] * \
                            math.fabs(recall[i] - prev_recall)
                        prev_recall = recall[i]

                mAP += average_precisions
                count += 1
        pcnt, tp, tp_lod, fp, fp_lod = get_output_pos(label_count, true_pos,
                                                      false_pos)
        self.out_class_pos_count = pcnt
        self.out_true_pos = tp
        self.out_true_pos_lod = tp_lod
        self.out_false_pos = fp
        self.out_false_pos_lod = fp_lod
        if count != 0:
            mAP /= count
        return mAP

    def setUp(self):
        self.op_type = "detection_map"
        self.set_data()

    def test_check_output(self):
        self.check_output()


class TestDetectionMAPOpSkipDiff(TestDetectionMAPOp):
    def init_test_case(self):
        super(TestDetectionMAPOpSkipDiff, self).init_test_case()

        self.evaluate_difficult = False

        self.tf_pos_lod = [[0, 2, 6]]
        # label score true_pos false_pos
        self.tf_pos = [[1, 0.7, 1, 0], [1, 0.3, 0, 1], [1, 0.2, 1, 0],
                       [2, 0.8, 0, 1], [2, 0.1, 1, 0], [3, 0.2, 0, 1]]


class TestDetectionMAPOp11Point(TestDetectionMAPOp):
    def init_test_case(self):
        super(TestDetectionMAPOp11Point, self).init_test_case()

        self.ap_type = "11point"


class TestDetectionMAPOpMultiBatch(TestDetectionMAPOp):
    def init_test_case(self):
        super(TestDetectionMAPOpMultiBatch, self).init_test_case()
        self.class_pos_count = [0, 2, 1]
        self.true_pos_lod = [[0, 0, 3, 5]]
        self.true_pos = [[0.7, 1.], [0.3, 0.], [0.2, 1.], [0.8, 0.], [0.1, 1.]]
        self.false_pos_lod = [[0, 0, 3, 5]]
        self.false_pos = [[0.7, 0.], [0.3, 1.], [0.2, 0.], [0.8, 1.], [0.1, 0.]]


if __name__ == '__main__':
    unittest.main()