From 73acfad8c91d427b63ee8a2091ec18f913993523 Mon Sep 17 00:00:00 2001 From: Guanghua Yu <742925032@qq.com> Date: Mon, 15 Mar 2021 10:22:38 +0800 Subject: [PATCH] support compute per-category AP and PR curve (#2346) --- dygraph/docs/tutorials/GETTING_STARTED_cn.md | 1 + dygraph/ppdet/engine/trainer.py | 10 ++- dygraph/ppdet/metrics/coco_utils.py | 55 +++++++++++- dygraph/ppdet/metrics/map_utils.py | 92 ++++++++++++++++++-- dygraph/ppdet/metrics/metrics.py | 32 ++++--- dygraph/requirements.txt | 1 + dygraph/tools/eval.py | 6 ++ 7 files changed, 176 insertions(+), 21 deletions(-) diff --git a/dygraph/docs/tutorials/GETTING_STARTED_cn.md b/dygraph/docs/tutorials/GETTING_STARTED_cn.md index a0e3fd2f2..8c8dbe34a 100644 --- a/dygraph/docs/tutorials/GETTING_STARTED_cn.md +++ b/dygraph/docs/tutorials/GETTING_STARTED_cn.md @@ -30,6 +30,7 @@ PaddleDetection在[tools](https://github.com/PaddlePaddle/PaddleDetection/tree/m | --draw_threshold | infer | 可视化时分数阈值 | 0.5 | 可选，`--draw_threshold=0.7` | | --infer_dir | infer | 用于预测的图片文件夹路径 | None | 可选 | | --infer_img | infer | 用于预测的图片路径 | None | 可选，`--infer_img`和`--infer_dir`必须至少设置一个 | +| --classwise | eval | 是否评估单类AP和绘制单类PR曲线 | False | 可选 | ### 训练 diff --git a/dygraph/ppdet/engine/trainer.py b/dygraph/ppdet/engine/trainer.py index 5b562d5c6..bfa373f0f 100644 --- a/dygraph/ppdet/engine/trainer.py +++ b/dygraph/ppdet/engine/trainer.py @@ -115,19 +115,23 @@ class Trainer(object): if self.mode == 'test': self._metrics = [] return + classwise = self.cfg['classwise'] if 'classwise' in self.cfg else False if self.cfg.metric == 'COCO': # TODO: bias should be unified bias = self.cfg['bias'] if 'bias' in self.cfg else 0 self._metrics = [ COCOMetric( - anno_file=self.dataset.get_anno(), bias=bias) + anno_file=self.dataset.get_anno(), + classwise=classwise, + bias=bias) ] elif self.cfg.metric == 'VOC': self._metrics = [ VOCMetric( - anno_file=self.dataset.get_anno(), + label_list=self.dataset.get_label_list(), class_num=self.cfg.num_classes, - map_type=self.cfg.map_type) + map_type=self.cfg.map_type, + classwise=classwise) ] elif self.cfg.metric == 'WiderFace': multi_scale = self.cfg.multi_scale_eval if 'multi_scale_eval' in self.cfg else True diff --git a/dygraph/ppdet/metrics/coco_utils.py b/dygraph/ppdet/metrics/coco_utils.py index 6a14fd6fa..115268168 100644 --- a/dygraph/ppdet/metrics/coco_utils.py +++ b/dygraph/ppdet/metrics/coco_utils.py @@ -17,8 +17,12 @@ from __future__ import division from __future__ import print_function import os +import sys +import numpy as np +import itertools from ppdet.py_op.post_process import get_det_res, get_seg_res, get_solov2_segm_res +from ppdet.metrics.map_utils import draw_pr_curve from ppdet.utils.logger import setup_logger logger = setup_logger(__name__) @@ -59,7 +63,8 @@ def cocoapi_eval(jsonfile, style, coco_gt=None, anno_file=None, - max_dets=(100, 300, 1000)): + max_dets=(100, 300, 1000), + classwise=False): """ Args: jsonfile: Evaluation json file, eg: bbox.json, mask.json. @@ -68,6 +73,7 @@ def cocoapi_eval(jsonfile, eg: coco_gt = COCO(anno_file) anno_file: COCO annotations file. max_dets: COCO evaluation maxDets. + classwise: whether per-category AP and draw P-R Curve or not. """ assert coco_gt != None or anno_file != None from pycocotools.coco import COCO @@ -86,4 +92,51 @@ def cocoapi_eval(jsonfile, coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() + if classwise: + # Compute per-category AP and PR curve + try: + from terminaltables import AsciiTable + except Exception as e: + logger.error( + 'terminaltables not found, plaese install terminaltables. ' + 'for example: `pip install terminaltables`.') + raise e + precisions = coco_eval.eval['precision'] + cat_ids = coco_gt.getCatIds() + # precision: (iou, recall, cls, area range, max dets) + assert len(cat_ids) == precisions.shape[2] + results_per_category = [] + for idx, catId in enumerate(cat_ids): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + nm = coco_gt.loadCats(catId)[0] + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + if precision.size: + ap = np.mean(precision) + else: + ap = float('nan') + results_per_category.append( + (str(nm["name"]), '{:0.3f}'.format(float(ap)))) + pr_array = precisions[0, :, idx, 0, 2] + recall_array = np.arange(0.0, 1.01, 0.01) + draw_pr_curve( + pr_array, + recall_array, + out_dir=style + '_pr_curve', + file_name='{}_precision_recall_curve.jpg'.format(nm["name"])) + + num_columns = min(6, len(results_per_category) * 2) + results_flatten = list(itertools.chain(*results_per_category)) + headers = ['category', 'AP'] * (num_columns // 2) + results_2d = itertools.zip_longest( + * [results_flatten[i::num_columns] for i in range(num_columns)]) + table_data = [headers] + table_data += [result for result in results_2d] + table = AsciiTable(table_data) + logger.info('Per-category of {} AP: \n{}'.format(style, table.table)) + logger.info("per-category PR curve has output to {} folder.".format( + style + '_pr_curve')) + # flush coco evaluation result + sys.stdout.flush() return coco_eval.stats diff --git a/dygraph/ppdet/metrics/map_utils.py b/dygraph/ppdet/metrics/map_utils.py index 620200be9..86039e1f0 100644 --- a/dygraph/ppdet/metrics/map_utils.py +++ b/dygraph/ppdet/metrics/map_utils.py @@ -17,13 +17,42 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals +import os import sys import numpy as np +import itertools from ppdet.utils.logger import setup_logger logger = setup_logger(__name__) -__all__ = ['bbox_area', 'jaccard_overlap', 'prune_zero_padding', 'DetectionMAP'] +__all__ = [ + 'draw_pr_curve', 'bbox_area', 'jaccard_overlap', 'prune_zero_padding', + 'DetectionMAP' +] + + +def draw_pr_curve(precision, + recall, + iou=0.5, + out_dir='pr_curve', + file_name='precision_recall_curve.jpg'): + if not os.path.exists(out_dir): + os.makedirs(out_dir) + output_path = os.path.join(out_dir, file_name) + try: + import matplotlib.pyplot as plt + except Exception as e: + logger.error('Matplotlib not found, plaese install matplotlib.' + 'for example: `pip install matplotlib`.') + raise e + plt.cla() + plt.figure('P-R Curve') + plt.title('Precision/Recall Curve(IoU={})'.format(iou)) + plt.xlabel('Recall') + plt.ylabel('Precision') + plt.grid(True) + plt.plot(recall, precision) + plt.savefig(output_path) def bbox_area(bbox, is_bbox_normalized): @@ -84,6 +113,8 @@ class DetectionMAP(object): is normalized to range[0, 1]. Default False. evaluate_difficult (bool): whether to evaluate difficult bounding boxes. Default False. + classwise (bool): whether per-category AP and draw + P-R Curve or not. """ def __init__(self, @@ -91,7 +122,9 @@ class DetectionMAP(object): overlap_thresh=0.5, map_type='11point', is_bbox_normalized=False, - evaluate_difficult=False): + evaluate_difficult=False, + catid2name=None, + classwise=False): self.class_num = class_num self.overlap_thresh = overlap_thresh assert map_type in ['11point', 'integral'], \ @@ -100,6 +133,10 @@ class DetectionMAP(object): self.map_type = map_type self.is_bbox_normalized = is_bbox_normalized self.evaluate_difficult = evaluate_difficult + self.classwise = classwise + self.classes = [] + for cname in catid2name.values(): + self.classes.append(cname) self.reset() def update(self, bbox, score, label, gt_box, gt_label, difficult=None): @@ -155,6 +192,7 @@ class DetectionMAP(object): """ mAP = 0. valid_cnt = 0 + eval_results = [] for score_pos, count in zip(self.class_score_poss, self.class_gt_counts): if count == 0: continue @@ -170,6 +208,7 @@ class DetectionMAP(object): precision.append(float(ac_tp) / (ac_tp + ac_fp)) recall.append(float(ac_tp) / count) + one_class_ap = 0.0 if self.map_type == '11point': max_precisions = [0.] * 11 start_idx = len(precision) - 1 @@ -183,23 +222,29 @@ class DetectionMAP(object): else: if max_precisions[j] < precision[i]: max_precisions[j] = precision[i] - mAP += sum(max_precisions) / 11. + one_class_ap = sum(max_precisions) / 11. + mAP += one_class_ap valid_cnt += 1 elif self.map_type == 'integral': import math - ap = 0. prev_recall = 0. for i in range(len(precision)): recall_gap = math.fabs(recall[i] - prev_recall) if recall_gap > 1e-6: - ap += precision[i] * recall_gap + one_class_ap += precision[i] * recall_gap prev_recall = recall[i] - mAP += ap + mAP += one_class_ap valid_cnt += 1 else: logger.error("Unspported mAP type {}".format(self.map_type)) sys.exit(1) - + eval_results.append({ + 'class': self.classes[valid_cnt - 1], + 'ap': one_class_ap, + 'precision': precision, + 'recall': recall, + }) + self.eval_results = eval_results self.mAP = mAP / float(valid_cnt) if valid_cnt > 0 else mAP def get_map(self): @@ -208,6 +253,39 @@ class DetectionMAP(object): """ if self.mAP is None: logger.error("mAP is not calculated.") + if self.classwise: + # Compute per-category AP and PR curve + try: + from terminaltables import AsciiTable + except Exception as e: + logger.error( + 'terminaltables not found, plaese install terminaltables. ' + 'for example: `pip install terminaltables`.') + raise e + results_per_category = [] + for eval_result in self.eval_results: + results_per_category.append( + (str(eval_result['class']), + '{:0.3f}'.format(float(eval_result['ap'])))) + draw_pr_curve( + eval_result['precision'], + eval_result['recall'], + out_dir='voc_pr_curve', + file_name='{}_precision_recall_curve.jpg'.format( + eval_result['class'])) + + num_columns = min(6, len(results_per_category) * 2) + results_flatten = list(itertools.chain(*results_per_category)) + headers = ['category', 'AP'] * (num_columns // 2) + results_2d = itertools.zip_longest(* [ + results_flatten[i::num_columns] for i in range(num_columns) + ]) + table_data = [headers] + table_data += [result for result in results_2d] + table = AsciiTable(table_data) + logger.info('Per-category of VOC AP: \n{}'.format(table.table)) + logger.info( + "per-category PR curve has output to voc_pr_curve folder.") return self.mAP def _get_tp_fp_accum(self, score_pos_list): diff --git a/dygraph/ppdet/metrics/metrics.py b/dygraph/ppdet/metrics/metrics.py index bed5ae18a..f5827052f 100644 --- a/dygraph/ppdet/metrics/metrics.py +++ b/dygraph/ppdet/metrics/metrics.py @@ -63,6 +63,7 @@ class COCOMetric(Metric): "anno_file {} not a file".format(anno_file) self.anno_file = anno_file self.clsid2catid, self.catid2name = get_categories('COCO', anno_file) + self.classwise = kwargs.get('classwise', False) # TODO: bias should be unified self.bias = kwargs.get('bias', 0) self.reset() @@ -98,7 +99,10 @@ class COCOMetric(Metric): logger.info('The bbox result is saved to bbox.json.') bbox_stats = cocoapi_eval( - 'bbox.json', 'bbox', anno_file=self.anno_file) + 'bbox.json', + 'bbox', + anno_file=self.anno_file, + classwise=self.classwise) self.eval_results['bbox'] = bbox_stats sys.stdout.flush() @@ -108,7 +112,10 @@ class COCOMetric(Metric): logger.info('The mask result is saved to mask.json.') seg_stats = cocoapi_eval( - 'mask.json', 'segm', anno_file=self.anno_file) + 'mask.json', + 'segm', + anno_file=self.anno_file, + classwise=self.classwise) self.eval_results['mask'] = seg_stats sys.stdout.flush() @@ -118,7 +125,10 @@ class COCOMetric(Metric): logger.info('The segm result is saved to segm.json.') seg_stats = cocoapi_eval( - 'segm.json', 'segm', anno_file=self.anno_file) + 'segm.json', + 'segm', + anno_file=self.anno_file, + classwise=self.classwise) self.eval_results['mask'] = seg_stats sys.stdout.flush() @@ -131,16 +141,16 @@ class COCOMetric(Metric): class VOCMetric(Metric): def __init__(self, - anno_file, + label_list, class_num=20, overlap_thresh=0.5, map_type='11point', is_bbox_normalized=False, - evaluate_difficult=False): - assert os.path.isfile(anno_file), \ - "anno_file {} not a file".format(anno_file) - self.anno_file = anno_file - self.clsid2catid, self.catid2name = get_categories('VOC', anno_file) + evaluate_difficult=False, + classwise=False): + assert os.path.isfile(label_list), \ + "label_list {} not a file".format(label_list) + self.clsid2catid, self.catid2name = get_categories('VOC', label_list) self.overlap_thresh = overlap_thresh self.map_type = map_type @@ -150,7 +160,9 @@ class VOCMetric(Metric): overlap_thresh=overlap_thresh, map_type=map_type, is_bbox_normalized=is_bbox_normalized, - evaluate_difficult=evaluate_difficult) + evaluate_difficult=evaluate_difficult, + catid2name=self.catid2name, + classwise=classwise) self.reset() diff --git a/dygraph/requirements.txt b/dygraph/requirements.txt index c50d347b2..1b08e21b9 100644 --- a/dygraph/requirements.txt +++ b/dygraph/requirements.txt @@ -5,3 +5,4 @@ opencv-python PyYAML shapely scipy +terminaltables \ No newline at end of file diff --git a/dygraph/tools/eval.py b/dygraph/tools/eval.py index 690cc5501..44bc2973e 100755 --- a/dygraph/tools/eval.py +++ b/dygraph/tools/eval.py @@ -64,6 +64,11 @@ def parse_args(): action="store_true", help="whether add bias or not while getting w and h") + parser.add_argument( + "--classwise", + action="store_true", + help="whether per-category AP and draw P-R Curve or not.") + args = parser.parse_args() return args @@ -88,6 +93,7 @@ def main(): cfg = load_config(FLAGS.config) # TODO: bias should be unified cfg['bias'] = 1 if FLAGS.bias else 0 + cfg['classwise'] = True if FLAGS.classwise else False merge_config(FLAGS.opt) if FLAGS.slim_config: slim_cfg = load_config(FLAGS.slim_config) -- GitLab