score.py

#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
# os.environ["CUDA_VISIBLE_DEVICES"] = "0"
# os.environ["FLAGS_fraction_of_gpu_memory_to_use"] = "0.3"
import sys
sys.path.insert(0, ".")
import argparse
import functools

import paddle.fluid as fluid
import reader
from utils import *
import json
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
import tempfile

parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser)
# yapf: disable
add_arg('batch_size',       int,   32,        "Minibatch size.")
add_arg('data_dir',         str,   '',        "The data root path.")
add_arg('test_list',        str,   '',        "The testing data lists.")
add_arg('model_dir',        str,   '',     "The model path.")
add_arg('nms_threshold',    float, 0.45,    "NMS threshold.")
add_arg('ap_version',       str,   'cocoMAP',   "cocoMAP.")
add_arg('mean_value_B',     float, 127.5,  "Mean value for B channel which will be subtracted.")  #123.68
add_arg('mean_value_G',     float, 127.5,  "Mean value for G channel which will be subtracted.")  #116.78
add_arg('mean_value_R',     float, 127.5,  "Mean value for R channel which will be subtracted.")  #103.94

def use_coco_api_compute_mAP(data_args, test_list, num_classes, test_reader, exe, infer_program,
                             feeded_var_names, feeder, target_var, batch_size):
    cocoGt = COCO(os.path.join(data_args.data_dir, test_list))
    json_category_id_to_contiguous_id = {
        v: i + 1
        for i, v in enumerate(cocoGt.getCatIds())
    }
    contiguous_category_id_to_json_id = {
        v: k
        for k, v in json_category_id_to_contiguous_id.items()
    }

    dts_res = []

    executor = fluid.Executor(fluid.CUDAPlace(0))
    test_program = fluid.Program()
    with fluid.program_guard(test_program):
        boxes = fluid.layers.data(
            name='boxes', shape=[-1, -1, 4], dtype='float32')
        scores = fluid.layers.data(
            name='scores', shape=[-1, num_classes, -1], dtype='float32')
        pred_result = fluid.layers.multiclass_nms(
            bboxes=boxes,
            scores=scores,
            score_threshold=0.01,
            nms_top_k=-1,
            nms_threshold=0.45,
            keep_top_k=-1,
            normalized=False)

    executor.run(fluid.default_startup_program())

    for batch_id, data in enumerate(test_reader()):
        boxes_np, scores_np = exe.run(program=infer_program,
                                      feed={feeded_var_names[0]: feeder.feed(data)['image']},
                                      fetch_list=target_var)

        nms_out = executor.run(
            program=test_program,
            feed={
                'boxes': boxes_np,
                'scores': scores_np
            },
            fetch_list=[pred_result], return_numpy=False)
        if batch_id % 20 == 0:
            print("Batch {0}".format(batch_id))
        dts_res += get_batch_dt_res(nms_out, data, contiguous_category_id_to_json_id, batch_size)

    _, tmp_file = tempfile.mkstemp()
    with open(tmp_file, 'w') as outfile:
        json.dump(dts_res, outfile)
    print("start evaluate using coco api")
    cocoDt = cocoGt.loadRes(tmp_file)
    cocoEval = COCOeval(cocoGt, cocoDt, "bbox")
    cocoEval.evaluate()
    cocoEval.accumulate()
    cocoEval.summarize()

    mAP = cocoEval.stats[0]
    return mAP

def compute_score(model_dir, data_dir, test_list='annotations/instances_val2017.json', batch_size=32, height=300, width=300, num_classes=81,
                          mean_value=[127.5, 127.5, 127.5]):
    """
        compute score, mAP, flops of a model

        Args:
            model_dir (string): directory of model
            data_dir (string): directory of coco dataset, like '/your/path/to/coco', '/work/datasets/coco'

        Returns:
            tuple: score, mAP, flops.

        """

    place = fluid.CUDAPlace(0)
    exe = fluid.Executor(place)
    exe.run(fluid.default_startup_program())

    [infer_program, feeded_var_names, target_var] = fluid.io.load_inference_model(dirname=model_dir, executor=exe)

    image_shape = [3, height, width]

    data_args = reader.Settings(
            dataset='coco2017',
            data_dir=data_dir,
            resize_h=height,
            resize_w=width,
            mean_value=mean_value,
            apply_distort=False,
            apply_expand=False,
            ap_version='cocoMAP')

    image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
    gt_box = fluid.layers.data(
        name='gt_box', shape=[4], dtype='float32', lod_level=1)
    gt_label = fluid.layers.data(
        name='gt_label', shape=[1], dtype='int32', lod_level=1)
    gt_iscrowd = fluid.layers.data(
        name='gt_iscrowd', shape=[1], dtype='int32', lod_level=1)
    gt_image_info = fluid.layers.data(
        name='gt_image_id', shape=[3], dtype='int32')

    test_reader = reader.test(data_args, test_list, batch_size)
    feeder = fluid.DataFeeder(
        place=place,
        feed_list=[image, gt_box, gt_label, gt_iscrowd, gt_image_info])

    mAP = use_coco_api_compute_mAP(data_args, test_list, num_classes, test_reader, exe, infer_program,
                             feeded_var_names, feeder, target_var, batch_size)
    total_flops_params, is_quantize = summary(infer_program)
    MAdds = np.sum(total_flops_params['flops']) / 2000000.0

    if is_quantize:
        MAdds /= 2.0

    print('mAP:', mAP)
    print('MAdds:', MAdds)

    if MAdds < 160.0:
        MAdds = 160.0

    if MAdds > 1300.0:
        score = 0.0
    else:
        score = mAP * 100 - (5.1249 * np.log(MAdds) - 14.499)

    print('score:', score)

    return score, mAP, MAdds


if __name__ == '__main__':
    args = parser.parse_args()
    print_arguments(args)
    score, mAP, flops = compute_score(args.model_dir, args.data_dir, batch_size=args.batch_size)