eval_coco_map.py

import os
import time
import numpy as np
import argparse
import functools

import paddle
import paddle.fluid as fluid
import reader
from mobilenet_ssd import mobile_net
from utility import add_arguments, print_arguments

# A special mAP metric for COCO dataset, which averages AP in different IoUs.
# To use this eval_cocoMAP.py, [cocoapi](https://github.com/cocodataset/cocoapi) is needed.
import json
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval

parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser)
# yapf: disable
add_arg('dataset',          str,   'coco2014',  "coco2014, coco2017.")
add_arg('batch_size',       int,   32,        "Minibatch size.")
add_arg('use_gpu',          bool,  True,      "Whether use GPU.")
add_arg('data_dir',         str,   '',        "The data root path.")
add_arg('test_list',        str,   '',        "The testing data lists.")
add_arg('model_dir',        str,   '',     "The model path.")
add_arg('nms_threshold',    float, 0.5,    "NMS threshold.")
add_arg('ap_version',       str,   'cocoMAP',   "cocoMAP.")
add_arg('resize_h',         int,   300,    "The resized image height.")
add_arg('resize_w',         int,   300,    "The resized image height.")
add_arg('mean_value_B',     float, 127.5,  "Mean value for B channel which will be subtracted.")  #123.68
add_arg('mean_value_G',     float, 127.5,  "Mean value for G channel which will be subtracted.")  #116.78
add_arg('mean_value_R',     float, 127.5,  "Mean value for R channel which will be subtracted.")  #103.94
# yapf: enable


def eval(args, data_args, test_list, batch_size, model_dir=None):
    image_shape = [3, data_args.resize_h, data_args.resize_w]
    num_classes = 91

    image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
    gt_box = fluid.layers.data(
        name='gt_box', shape=[4], dtype='float32', lod_level=1)
    gt_label = fluid.layers.data(
        name='gt_label', shape=[1], dtype='int32', lod_level=1)
    gt_iscrowd = fluid.layers.data(
        name='gt_iscrowd', shape=[1], dtype='int32', lod_level=1)
    gt_image_info = fluid.layers.data(
        name='gt_image_id', shape=[3], dtype='int32', lod_level=1)

    locs, confs, box, box_var = mobile_net(num_classes, image, image_shape)
    nmsed_out = fluid.layers.detection_output(
        locs, confs, box, box_var, nms_threshold=args.nms_threshold)
    loss = fluid.layers.ssd_loss(locs, confs, gt_box, gt_label, box, box_var)
    loss = fluid.layers.reduce_sum(loss)

    place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
    exe = fluid.Executor(place)
    # yapf: disable
    if model_dir:
        def if_exist(var):
            return os.path.exists(os.path.join(model_dir, var.name))
        fluid.io.load_vars(exe, model_dir, predicate=if_exist)
    # yapf: enable
    test_reader = paddle.batch(
        reader.test(data_args, test_list), batch_size=batch_size)
    feeder = fluid.DataFeeder(
        place=place,
        feed_list=[image, gt_box, gt_label, gt_iscrowd, gt_image_info])

    def get_dt_res(nmsed_out_v, data):
        dts_res = []
        lod = nmsed_out_v[0].lod()[0]
        nmsed_out_v = np.array(nmsed_out_v[0])
        real_batch_size = min(batch_size, len(data))
        assert (len(lod) == real_batch_size + 1), \
        "Error Lod Tensor offset dimension. Lod({}) vs. batch_size({})".format(len(lod), batch_size)
        k = 0
        for i in range(real_batch_size):
            dt_num_this_img = lod[i + 1] - lod[i]
            image_id = int(data[i][4][0])
            image_width = int(data[i][4][1])
            image_height = int(data[i][4][2])
            for j in range(dt_num_this_img):
                dt = nmsed_out_v[k]
                k = k + 1
                category_id, score, xmin, ymin, xmax, ymax = dt.tolist()
                xmin = max(min(xmin, 1.0), 0.0) * image_width
                ymin = max(min(ymin, 1.0), 0.0) * image_height
                xmax = max(min(xmax, 1.0), 0.0) * image_width
                ymax = max(min(ymax, 1.0), 0.0) * image_height
                w = xmax - xmin
                h = ymax - ymin
                bbox = [xmin, ymin, w, h]
                dt_res = {
                    'image_id': image_id,
                    'category_id': category_id,
                    'bbox': bbox,
                    'score': score
                }
                dts_res.append(dt_res)
        return dts_res

    def test():
        dts_res = []

        for batch_id, data in enumerate(test_reader()):
            nmsed_out_v = exe.run(fluid.default_main_program(),
                                  feed=feeder.feed(data),
                                  fetch_list=[nmsed_out],
                                  return_numpy=False)
            if batch_id % 20 == 0:
                print("Batch {0}".format(batch_id))
            dts_res += get_dt_res(nmsed_out_v, data)

        with open("detection_result.json", 'w') as outfile:
            json.dump(dts_res, outfile)
        print("start evaluate using coco api")
        cocoGt = COCO(os.path.join(data_args.data_dir, test_list))
        cocoDt = cocoGt.loadRes("detection_result.json")
        cocoEval = COCOeval(cocoGt, cocoDt, "bbox")
        cocoEval.evaluate()
        cocoEval.accumulate()
        cocoEval.summarize()

    test()


if __name__ == '__main__':
    args = parser.parse_args()
    print_arguments(args)

    data_dir = './data/coco'
    if '2014' in args.dataset:
        test_list = 'annotations/instances_val2014.json'
    elif '2017' in args.dataset:
        test_list = 'annotations/instances_val2017.json'

    data_args = reader.Settings(
        dataset=args.dataset,
        data_dir=args.data_dir if len(args.data_dir) > 0 else data_dir,
        label_file='',
        resize_h=args.resize_h,
        resize_w=args.resize_w,
        mean_value=[args.mean_value_B, args.mean_value_G, args.mean_value_R],
        apply_distort=False,
        apply_expand=False,
        ap_version=args.ap_version,
        toy=0)
    eval(
        args,
        data_args=data_args,
        test_list=args.test_list if len(args.test_list) > 0 else test_list,
        batch_size=args.batch_size,
        model_dir=args.model_dir)