check.py 23.2 KB
Newer Older
C
chenguowei01 已提交
1
# coding: utf8
W
wuzewu 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import os
import sys
import pprint
import argparse
import cv2
from tqdm import tqdm
import imghdr
C
chenguowei01 已提交
15
import logging
W
wuzewu 已提交
16 17 18

from utils.config import cfg

L
LutaoChu 已提交
19

W
wuzewu 已提交
20 21 22 23
def init_global_variable():
    """
    初始化全局变量
    """
C
chenguowei01 已提交
24
    global png_format_right_num  # 格式正确的标注图数量
C
chenguowei01 已提交
25 26
    global png_format_wrong_num  # 格式错误的标注图数量
    global total_grt_classes  # 总的标注类别
W
wuzewu 已提交
27
    global total_num_of_each_class  # 每个类别总的像素数
C
chenguowei01 已提交
28 29 30 31 32 33 34
    global shape_unequal_image  # 图片和标注shape不一致列表
    global png_format_wrong_image  # 标注格式错误列表
    global max_width  # 图片最长宽
    global max_height  # 图片最长高
    global min_aspectratio  # 图片最小宽高比
    global max_aspectratio  # 图片最大宽高比
    global img_dim  # 图片的通道数
L
LutaoChu 已提交
35 36
    global list_wrong  # 文件名格式错误列表
    global imread_failed  # 图片读取失败列表, 二元列表
C
chenguowei01 已提交
37
    global label_wrong  # 标注图片出错列表
C
chenguowei01 已提交
38
    global label_gray_wrong  # 标注图非灰度图列表
W
wuzewu 已提交
39 40 41 42 43

    png_format_right_num = 0
    png_format_wrong_num = 0
    total_grt_classes = []
    total_num_of_each_class = []
C
chenguowei01 已提交
44 45 46 47 48 49 50 51 52 53
    shape_unequal_image = []
    png_format_wrong_image = []
    max_width = 0
    max_height = 0
    min_aspectratio = sys.float_info.max
    max_aspectratio = 0
    img_dim = []
    list_wrong = []
    imread_failed = []
    label_wrong = []
C
chenguowei01 已提交
54
    label_gray_wrong = []
W
wuzewu 已提交
55

L
LutaoChu 已提交
56

W
wuzewu 已提交
57 58 59
def parse_args():
    parser = argparse.ArgumentParser(description='PaddleSeg check')
    parser.add_argument(
L
LutaoChu 已提交
60 61 62 63 64
        '--cfg',
        dest='cfg_file',
        help='Config file for training (and optionally testing)',
        default=None,
        type=str)
W
wuzewu 已提交
65 66
    return parser.parse_args()

L
LutaoChu 已提交
67

C
chenguowei01 已提交
68 69 70
def error_print(str):
    return "".join(["\nNOT PASS ", str])

L
LutaoChu 已提交
71

C
chenguowei01 已提交
72 73
def correct_print(str):
    return "".join(["\nPASS ", str])
W
wuzewu 已提交
74

L
LutaoChu 已提交
75

W
wuzewu 已提交
76
def cv2_imread(file_path, flag=cv2.IMREAD_COLOR):
C
chenguowei01 已提交
77 78 79
    """
    解决 cv2.imread 在window平台打开中文路径的问题.
    """
W
wuzewu 已提交
80 81
    return cv2.imdecode(np.fromfile(file_path, dtype=np.uint8), flag)

L
LutaoChu 已提交
82

C
chenguowei01 已提交
83
def get_image_max_height_width(img):
C
chenguowei01 已提交
84
    """获取图片最大宽和高"""
C
chenguowei01 已提交
85
    global max_width, max_height
W
wuzewu 已提交
86 87 88 89 90
    img_shape = img.shape
    height, width = img_shape[0], img_shape[1]
    max_height = max(height, max_height)
    max_width = max(width, max_width)

L
LutaoChu 已提交
91

C
chenguowei01 已提交
92
def get_image_min_max_aspectratio(img):
C
chenguowei01 已提交
93
    """计算图片最大宽高比"""
C
chenguowei01 已提交
94
    global min_aspectratio, max_aspectratio
W
wuzewu 已提交
95 96
    img_shape = img.shape
    height, width = img_shape[0], img_shape[1]
L
LutaoChu 已提交
97 98
    min_aspectratio = min(width / height, min_aspectratio)
    max_aspectratio = max(width / height, max_aspectratio)
W
wuzewu 已提交
99 100
    return min_aspectratio, max_aspectratio

L
LutaoChu 已提交
101

C
chenguowei01 已提交
102
def get_image_dim(img):
C
chenguowei01 已提交
103
    """获取图像的通道数"""
W
wuzewu 已提交
104 105 106 107
    img_shape = img.shape
    if img_shape[-1] not in img_dim:
        img_dim.append(img_shape[-1])

L
LutaoChu 已提交
108

C
chenguowei01 已提交
109 110 111 112 113 114 115 116
def is_label_gray(grt):
    """判断标签是否为灰度图"""
    grt_shape = grt.shape
    if len(grt_shape) == 2:
        return True
    else:
        return False

L
LutaoChu 已提交
117

C
chenguowei01 已提交
118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
def image_label_shape_check(img, grt):
    """
    验证图像和标注的大小是否匹配
    """

    flag = True
    img_height = img.shape[0]
    img_width = img.shape[1]
    grt_height = grt.shape[0]
    grt_width = grt.shape[1]

    if img_height != grt_height or img_width != grt_width:
        flag = False
    return flag

L
LutaoChu 已提交
133

C
chenguowei01 已提交
134 135 136
def ground_truth_check(grt, grt_path):
    """
    验证标注图像的格式
C
chenguowei01 已提交
137
    统计标注图类别和像素数
C
chenguowei01 已提交
138 139 140 141 142
    params:
        grt: 标注图
        grt_path: 标注图路径
    return:
        png_format: 返回是否是png格式图片
C
chenguowei01 已提交
143 144
        unique: 返回标注类别
        counts: 返回标注的像素数
C
chenguowei01 已提交
145 146 147 148 149 150 151 152 153
    """
    if imghdr.what(grt_path) == "png":
        png_format = True
    else:
        png_format = False

    unique, counts = np.unique(grt, return_counts=True)

    return png_format, unique, counts
W
wuzewu 已提交
154

L
LutaoChu 已提交
155

W
wuzewu 已提交
156 157
def sum_gt_check(png_format, grt_classes, num_of_each_class):
    """
C
chenguowei01 已提交
158
    统计所有标注图上的格式、类别和每个类别的像素数
W
wuzewu 已提交
159
    params:
C
chenguowei01 已提交
160
        png_format: 是否是png格式图片
C
chenguowei01 已提交
161
        grt_classes: 标注类别
W
wuzewu 已提交
162 163
        num_of_each_class: 各个类别的像素数目
    """
C
chenguowei01 已提交
164
    is_label_correct = True
W
wuzewu 已提交
165 166 167 168 169 170 171 172
    global png_format_right_num, png_format_wrong_num, total_grt_classes, total_num_of_each_class

    if png_format:
        png_format_right_num += 1
    else:
        png_format_wrong_num += 1

    if cfg.DATASET.IGNORE_INDEX in grt_classes:
L
LutaoChu 已提交
173 174
        grt_classes2 = np.delete(
            grt_classes, np.where(grt_classes == cfg.DATASET.IGNORE_INDEX))
C
chenguowei01 已提交
175 176
    else:
        grt_classes2 = grt_classes
W
wuzewu 已提交
177
    if min(grt_classes2) < 0 or max(grt_classes2) > cfg.DATASET.NUM_CLASSES - 1:
C
chenguowei01 已提交
178
        is_label_correct = False
W
wuzewu 已提交
179 180 181 182 183 184 185 186 187 188 189 190
    add_class = []
    add_num = []
    for i in range(len(grt_classes)):
        gi = grt_classes[i]
        if gi in total_grt_classes:
            j = total_grt_classes.index(gi)
            total_num_of_each_class[j] += num_of_each_class[i]
        else:
            add_class.append(gi)
            add_num.append(num_of_each_class[i])
    total_num_of_each_class += add_num
    total_grt_classes += add_class
C
chenguowei01 已提交
191
    return is_label_correct
W
wuzewu 已提交
192

L
LutaoChu 已提交
193

W
wuzewu 已提交
194 195
def gt_check():
    """
C
chenguowei01 已提交
196
    对标注图像进行校验,输出校验结果
W
wuzewu 已提交
197 198
    """
    if png_format_wrong_num == 0:
C
chenguowei01 已提交
199 200 201 202 203 204
        if png_format_right_num:
            logger.info(correct_print("label format check"))
        else:
            logger.info(error_print("label format check"))
            logger.info("No label image to check")
            return
W
wuzewu 已提交
205
    else:
C
chenguowei01 已提交
206
        logger.info(error_print("label format check"))
L
LutaoChu 已提交
207 208 209
    logger.info(
        "total {} label images are png format, {} label images are not png "
        "format".format(png_format_right_num, png_format_wrong_num))
C
chenguowei01 已提交
210 211 212
    if len(png_format_wrong_image) > 0:
        for i in png_format_wrong_image:
            logger.debug(i)
W
wuzewu 已提交
213

L
LutaoChu 已提交
214 215 216 217 218 219 220
    total_ratio = total_num_of_each_class / sum(total_num_of_each_class)
    total_ratio = np.around(total_ratio, decimals=4)
    total_nc = sorted(
        zip(total_grt_classes, total_num_of_each_class, total_ratio))
    logger.info(
        "\nDoing label pixel statistics:\n"
        "(label class, total pixel number, percentage) = {} ".format(total_nc))
W
wuzewu 已提交
221

C
chenguowei01 已提交
222 223
    if len(label_wrong) == 0 and not total_nc[0][0]:
        logger.info(correct_print("label class check!"))
W
wuzewu 已提交
224
    else:
C
chenguowei01 已提交
225 226 227 228
        logger.info(error_print("label class check!"))
        if total_nc[0][0]:
            logger.info("Warning: label classes should start from 0")
        if len(label_wrong) > 0:
L
LutaoChu 已提交
229 230 231
            logger.info(
                "fatal error: label class is out of range [0, {}]".format(
                    cfg.DATASET.NUM_CLASSES - 1))
C
chenguowei01 已提交
232 233
            for i in label_wrong:
                logger.debug(i)
W
wuzewu 已提交
234 235


L
LutaoChu 已提交
236 237
def eval_crop_size_check(max_height, max_width, min_aspectratio,
                         max_aspectratio):
W
wuzewu 已提交
238 239 240 241 242 243
    """
    判断eval_crop_siz与验证集及测试集的max_height, max_width的关系
    param
        max_height: 数据集的最大高
        max_width: 数据集的最大宽
    """
C
chenguowei01 已提交
244

W
wuzewu 已提交
245
    if cfg.AUG.AUG_METHOD == "stepscaling":
L
LutaoChu 已提交
246 247
        if max_width <= cfg.EVAL_CROP_SIZE[
                0] and max_height <= cfg.EVAL_CROP_SIZE[1]:
C
chenguowei01 已提交
248
            logger.info(correct_print("EVAL_CROP_SIZE check"))
L
LutaoChu 已提交
249 250 251 252
            logger.info(
                "satisfy current EVAL_CROP_SIZE: ({},{}) >= max width and max height of images: ({},{})"
                .format(cfg.EVAL_CROP_SIZE[0], cfg.EVAL_CROP_SIZE[1], max_width,
                        max_height))
C
chenguowei01 已提交
253 254 255
        else:
            logger.info(error_print("EVAL_CROP_SIZE check"))
            if max_width > cfg.EVAL_CROP_SIZE[0]:
L
LutaoChu 已提交
256 257 258
                logger.info(
                    "EVAL_CROP_SIZE[0]: {} should >= max width of images {}!".
                    format(cfg.EVAL_CROP_SIZE[0], max_width))
C
chenguowei01 已提交
259
            if max_height > cfg.EVAL_CROP_SIZE[1]:
L
LutaoChu 已提交
260 261 262
                logger.info(
                    "EVAL_CROP_SIZE[1]: {} should >= max height of images {}!".
                    format(cfg.EVAL_CROP_SIZE[1], max_height))
C
chenguowei01 已提交
263

W
wuzewu 已提交
264 265
    elif cfg.AUG.AUG_METHOD == "rangescaling":
        if min_aspectratio <= 1 and max_aspectratio >= 1:
L
LutaoChu 已提交
266 267
            if cfg.EVAL_CROP_SIZE[0] >= cfg.AUG.INF_RESIZE_VALUE \
                    and cfg.EVAL_CROP_SIZE[1] >= cfg.AUG.INF_RESIZE_VALUE:
C
chenguowei01 已提交
268
                logger.info(correct_print("EVAL_CROP_SIZE check"))
L
LutaoChu 已提交
269 270 271 272
                logger.info(
                    "satisfy current EVAL_CROP_SIZE: ({},{}) >= ({},{}) ".
                    format(cfg.EVAL_CROP_SIZE[0], cfg.EVAL_CROP_SIZE[1],
                           cfg.AUG.INF_RESIZE_VALUE, cfg.AUG.INF_RESIZE_VALUE))
W
wuzewu 已提交
273
            else:
C
chenguowei01 已提交
274
                logger.info(error_print("EVAL_CROP_SIZE check"))
L
LutaoChu 已提交
275 276 277 278
                logger.info(
                    "EVAL_CROP_SIZE must >= img size({},{}), current EVAL_CROP_SIZE is ({},{})"
                    .format(cfg.AUG.INF_RESIZE_VALUE, cfg.AUG.INF_RESIZE_VALUE,
                            cfg.EVAL_CROP_SIZE[0], cfg.EVAL_CROP_SIZE[1]))
W
wuzewu 已提交
279 280 281
        elif min_aspectratio > 1:
            max_height_rangscaling = cfg.AUG.INF_RESIZE_VALUE / min_aspectratio
            max_height_rangscaling = round(max_height_rangscaling)
L
LutaoChu 已提交
282 283 284
            if cfg.EVAL_CROP_SIZE[
                    0] >= cfg.AUG.INF_RESIZE_VALUE and cfg.EVAL_CROP_SIZE[
                        1] >= max_height_rangscaling:
C
chenguowei01 已提交
285
                logger.info(correct_print("EVAL_CROP_SIZE check"))
L
LutaoChu 已提交
286 287 288 289
                logger.info(
                    "satisfy current EVAL_CROP_SIZE: ({},{}) >= ({},{}) ".
                    format(cfg.EVAL_CROP_SIZE[0], cfg.EVAL_CROP_SIZE[1],
                           cfg.AUG.INF_RESIZE_VALUE, max_height_rangscaling))
W
wuzewu 已提交
290
            else:
C
chenguowei01 已提交
291
                logger.info(error_print("EVAL_CROP_SIZE check"))
L
LutaoChu 已提交
292 293 294 295
                logger.info(
                    "EVAL_CROP_SIZE must >= img size({},{}), current EVAL_CROP_SIZE is ({},{})"
                    .format(cfg.AUG.INF_RESIZE_VALUE, max_height_rangscaling,
                            cfg.EVAL_CROP_SIZE[0], cfg.EVAL_CROP_SIZE[1]))
W
wuzewu 已提交
296 297 298
        elif max_aspectratio < 1:
            max_width_rangscaling = cfg.AUG.INF_RESIZE_VALUE * max_aspectratio
            max_width_rangscaling = round(max_width_rangscaling)
L
LutaoChu 已提交
299 300 301
            if cfg.EVAL_CROP_SIZE[
                    0] >= max_width_rangscaling and cfg.EVAL_CROP_SIZE[
                        1] >= cfg.AUG.INF_RESIZE_VALUE:
C
chenguowei01 已提交
302
                logger.info(correct_print("EVAL_CROP_SIZE check"))
L
LutaoChu 已提交
303 304 305 306
                logger.info(
                    "satisfy current EVAL_CROP_SIZE: ({},{}) >= ({},{}) ".
                    format(cfg.EVAL_CROP_SIZE[0], cfg.EVAL_CROP_SIZE[1],
                           max_height_rangscaling, cfg.AUG.INF_RESIZE_VALUE))
W
wuzewu 已提交
307
            else:
C
chenguowei01 已提交
308
                logger.info(error_print("EVAL_CROP_SIZE check"))
L
LutaoChu 已提交
309 310 311 312
                logger.info(
                    "EVAL_CROP_SIZE must >= img size({},{}), current EVAL_CROP_SIZE is ({},{})"
                    .format(max_width_rangscaling, cfg.AUG.INF_RESIZE_VALUE,
                            cfg.EVAL_CROP_SIZE[0], cfg.EVAL_CROP_SIZE[1]))
W
wuzewu 已提交
313
    elif cfg.AUG.AUG_METHOD == "unpadding":
314 315
        if len(cfg.AUG.FIX_RESIZE_SIZE) != 2:
            logger.info(error_print("EVAL_CROP_SIZE check"))
L
LutaoChu 已提交
316 317 318 319 320
            logger.info(
                "you set AUG.AUG_METHOD = 'unpadding', but AUG.FIX_RESIZE_SIZE is wrong. "
                "AUG.FIX_RESIZE_SIZE should be a tuple of length 2")
        elif cfg.EVAL_CROP_SIZE[0] >= cfg.AUG.FIX_RESIZE_SIZE[0] \
                and cfg.EVAL_CROP_SIZE[1] >= cfg.AUG.FIX_RESIZE_SIZE[1]:
C
chenguowei01 已提交
321
            logger.info(correct_print("EVAL_CROP_SIZE check"))
L
LutaoChu 已提交
322 323 324 325
            logger.info(
                "satisfy current EVAL_CROP_SIZE: ({},{}) >= AUG.FIX_RESIZE_SIZE: ({},{}) "
                .format(cfg.EVAL_CROP_SIZE[0], cfg.EVAL_CROP_SIZE[1],
                        cfg.AUG.FIX_RESIZE_SIZE[0], cfg.AUG.FIX_RESIZE_SIZE[1]))
W
wuzewu 已提交
326
        else:
C
chenguowei01 已提交
327
            logger.info(error_print("EVAL_CROP_SIZE check"))
L
LutaoChu 已提交
328 329 330 331
            logger.info(
                "EVAL_CROP_SIZE: ({},{}) must >= AUG.FIX_RESIZE_SIZE: ({},{})".
                format(cfg.EVAL_CROP_SIZE[0], cfg.EVAL_CROP_SIZE[1],
                       cfg.AUG.FIX_RESIZE_SIZE[0], cfg.AUG.FIX_RESIZE_SIZE[1]))
W
wuzewu 已提交
332
    else:
L
LutaoChu 已提交
333 334 335 336
        logger.info(
            "\nERROR! cfg.AUG.AUG_METHOD setting wrong, it should be one of "
            "[unpadding, stepscaling, rangescaling]")

W
wuzewu 已提交
337 338 339 340 341

def inf_resize_value_check():
    if cfg.AUG.AUG_METHOD == "rangescaling":
        if cfg.AUG.INF_RESIZE_VALUE < cfg.AUG.MIN_RESIZE_VALUE or \
                cfg.AUG.INF_RESIZE_VALUE > cfg.AUG.MIN_RESIZE_VALUE:
L
LutaoChu 已提交
342 343 344 345 346 347 348
            logger.info(
                "\nWARNING! you set AUG.AUG_METHOD = 'rangescaling'"
                "AUG.INF_RESIZE_VALUE: {} not in [AUG.MIN_RESIZE_VALUE, AUG.MAX_RESIZE_VALUE]: "
                "[{}, {}].".format(cfg.AUG.INF_RESIZE_VALUE,
                                   cfg.AUG.MIN_RESIZE_VALUE,
                                   cfg.AUG.MAX_RESIZE_VALUE))

W
wuzewu 已提交
349 350 351 352 353 354 355 356 357

def image_type_check(img_dim):
    """
    验证图像的格式与DATASET.IMAGE_TYPE是否一致
    param
        img_dim: 图像包含的通道数
    return
    """
    if (1 in img_dim or 3 in img_dim) and cfg.DATASET.IMAGE_TYPE == 'rgba':
C
chenguowei01 已提交
358 359
        logger.info(error_print("DATASET.IMAGE_TYPE check"))
        logger.info("DATASET.IMAGE_TYPE is {} but the type of image has "
L
LutaoChu 已提交
360 361 362
                    "gray or rgb\n".format(cfg.DATASET.IMAGE_TYPE))
    elif (1 not in img_dim and 3 not in img_dim
          and 4 in img_dim) and cfg.DATASET.IMAGE_TYPE == 'rgb':
C
chenguowei01 已提交
363
        logger.info(correct_print("DATASET.IMAGE_TYPE check"))
L
LutaoChu 已提交
364 365 366
        logger.info(
            "\nWARNING: DATASET.IMAGE_TYPE is {} but the type of all image is rgba"
            .format(cfg.DATASET.IMAGE_TYPE))
W
wuzewu 已提交
367
    else:
C
chenguowei01 已提交
368
        logger.info(correct_print("DATASET.IMAGE_TYPE check"))
W
wuzewu 已提交
369

L
LutaoChu 已提交
370

C
chenguowei01 已提交
371 372 373 374 375 376 377
def shape_check():
    """输出shape校验结果"""
    if len(shape_unequal_image) == 0:
        logger.info(correct_print("shape check"))
        logger.info("All images are the same shape as the labels")
    else:
        logger.info(error_print("shape check"))
L
LutaoChu 已提交
378 379
        logger.info(
            "Some images are not the same shape as the labels as follow: ")
C
chenguowei01 已提交
380 381
        for i in shape_unequal_image:
            logger.debug(i)
W
wuzewu 已提交
382 383


C
chenguowei01 已提交
384 385 386
def file_list_check(list_name):
    """检查分割符是否复合要求"""
    if len(list_wrong) == 0:
L
LutaoChu 已提交
387 388 389
        logger.info(
            correct_print(
                list_name.split(os.sep)[-1] + " DATASET.SEPARATOR check"))
C
chenguowei01 已提交
390
    else:
L
LutaoChu 已提交
391 392 393 394 395
        logger.info(
            error_print(
                list_name.split(os.sep)[-1] + " DATASET.SEPARATOR check"))
        logger.info("The following list is not separated by {}".format(
            cfg.DATASET.SEPARATOR))
C
chenguowei01 已提交
396 397 398
        for i in list_wrong:
            logger.debug(i)

L
LutaoChu 已提交
399

C
chenguowei01 已提交
400 401 402 403 404 405 406 407 408
def imread_check():
    if len(imread_failed) == 0:
        logger.info(correct_print("dataset reading check"))
        logger.info("All images can be read successfully")
    else:
        logger.info(error_print("dataset reading check"))
        logger.info("Failed to read {} images".format(len(imread_failed)))
        for i in imread_failed:
            logger.debug(i)
W
wuzewu 已提交
409

L
LutaoChu 已提交
410

C
chenguowei01 已提交
411 412 413 414 415 416
def label_gray_check():
    if len(label_gray_wrong) == 0:
        logger.info(correct_print("label gray check"))
        logger.info("All label images are gray")
    else:
        logger.info(error_print("label gray check"))
L
LutaoChu 已提交
417 418 419
        logger.info(
            "{} label images are not gray\nLabel pixel statistics may be insignificant"
            .format(len(label_gray_wrong)))
C
chenguowei01 已提交
420 421 422
        for i in label_gray_wrong:
            logger.debug(i)

W
wuzewu 已提交
423

L
LutaoChu 已提交
424 425 426 427 428
def max_img_size_statistics():
    logger.info("\nDoing max image size statistics:")
    logger.info("max width and max height of images are ({},{})".format(
        max_width, max_height))

W
wuyefeilin 已提交
429 430 431 432 433 434 435 436 437
def num_classes_loss_matching_check():
    loss_type = cfg.SOLVER.LOSS
    num_classes = cfg.DATASET.NUM_CLASSES
    if num_classes > 2 and (("dice_loss" in loss_type) or ("bce_loss" in loss_type)):
        logger.info(error_print("loss check."
            " Dice loss and bce loss is only applicable to binary classfication"))
    else:
        logger.info(correct_print("loss check"))

W
wuzewu 已提交
438 439

def check_train_dataset():
C
chenguowei01 已提交
440 441 442
    list_file = cfg.DATASET.TRAIN_FILE_LIST
    logger.info("-----------------------------\n1. Check train dataset...")
    with open(list_file, 'r') as fid:
W
wuzewu 已提交
443 444
        lines = fid.readlines()
        for line in tqdm(lines):
C
chenguowei01 已提交
445 446
            line = line.strip()
            parts = line.split(cfg.DATASET.SEPARATOR)
W
wuzewu 已提交
447
            if len(parts) != 2:
C
chenguowei01 已提交
448
                list_wrong.append(line)
W
wuzewu 已提交
449 450 451 452
                continue
            img_name, grt_name = parts[0], parts[1]
            img_path = os.path.join(cfg.DATASET.DATA_DIR, img_name)
            grt_path = os.path.join(cfg.DATASET.DATA_DIR, grt_name)
C
chenguowei01 已提交
453 454
            try:
                img = cv2_imread(img_path, cv2.IMREAD_UNCHANGED)
C
chenguowei01 已提交
455
                grt = cv2_imread(grt_path, cv2.IMREAD_UNCHANGED)
C
chenguowei01 已提交
456 457 458
            except Exception as e:
                imread_failed.append((line, str(e)))
                continue
W
wuzewu 已提交
459

C
chenguowei01 已提交
460 461
            is_gray = is_label_gray(grt)
            if not is_gray:
C
chenguowei01 已提交
462
                label_gray_wrong.append(line)
C
chenguowei01 已提交
463
                grt = cv2.cvtColor(grt, cv2.COLOR_BGR2GRAY)
L
LutaoChu 已提交
464
            get_image_max_height_width(img)
C
chenguowei01 已提交
465
            get_image_dim(img)
W
wuzewu 已提交
466 467
            is_equal_img_grt_shape = image_label_shape_check(img, grt)
            if not is_equal_img_grt_shape:
C
chenguowei01 已提交
468
                shape_unequal_image.append(line)
W
wuzewu 已提交
469

L
LutaoChu 已提交
470 471
            png_format, grt_classes, num_of_each_class = ground_truth_check(
                grt, grt_path)
C
chenguowei01 已提交
472 473
            if not png_format:
                png_format_wrong_image.append(line)
L
LutaoChu 已提交
474 475
            is_label_correct = sum_gt_check(png_format, grt_classes,
                                            num_of_each_class)
C
chenguowei01 已提交
476 477
            if not is_label_correct:
                label_wrong.append(line)
W
wuzewu 已提交
478

C
chenguowei01 已提交
479 480
        file_list_check(list_file)
        imread_check()
C
chenguowei01 已提交
481
        label_gray_check()
W
wuzewu 已提交
482 483
        gt_check()
        image_type_check(img_dim)
L
LutaoChu 已提交
484
        max_img_size_statistics()
C
chenguowei01 已提交
485
        shape_check()
W
wuyefeilin 已提交
486
        num_classes_loss_matching_check()
C
chenguowei01 已提交
487 488


W
wuzewu 已提交
489
def check_val_dataset():
C
chenguowei01 已提交
490 491 492
    list_file = cfg.DATASET.VAL_FILE_LIST
    logger.info("\n-----------------------------\n2. Check val dataset...")
    with open(list_file) as fid:
W
wuzewu 已提交
493 494
        lines = fid.readlines()
        for line in tqdm(lines):
C
chenguowei01 已提交
495 496
            line = line.strip()
            parts = line.split(cfg.DATASET.SEPARATOR)
W
wuzewu 已提交
497
            if len(parts) != 2:
C
chenguowei01 已提交
498
                list_wrong.append(line)
W
wuzewu 已提交
499 500 501 502
                continue
            img_name, grt_name = parts[0], parts[1]
            img_path = os.path.join(cfg.DATASET.DATA_DIR, img_name)
            grt_path = os.path.join(cfg.DATASET.DATA_DIR, grt_name)
C
chenguowei01 已提交
503 504
            try:
                img = cv2_imread(img_path, cv2.IMREAD_UNCHANGED)
C
chenguowei01 已提交
505
                grt = cv2_imread(grt_path, cv2.IMREAD_UNCHANGED)
C
chenguowei01 已提交
506
            except Exception as e:
L
LutaoChu 已提交
507 508
                imread_failed.append((line, str(e)))
                continue
C
chenguowei01 已提交
509

C
chenguowei01 已提交
510 511
            is_gray = is_label_gray(grt)
            if not is_gray:
C
chenguowei01 已提交
512
                label_gray_wrong.append(line)
C
chenguowei01 已提交
513
                grt = cv2.cvtColor(grt, cv2.COLOR_BGR2GRAY)
C
chenguowei01 已提交
514 515 516
            get_image_max_height_width(img)
            get_image_min_max_aspectratio(img)
            get_image_dim(img)
W
wuzewu 已提交
517 518
            is_equal_img_grt_shape = image_label_shape_check(img, grt)
            if not is_equal_img_grt_shape:
C
chenguowei01 已提交
519
                shape_unequal_image.append(line)
L
LutaoChu 已提交
520 521
            png_format, grt_classes, num_of_each_class = ground_truth_check(
                grt, grt_path)
C
chenguowei01 已提交
522 523
            if not png_format:
                png_format_wrong_image.append(line)
L
LutaoChu 已提交
524 525
            is_label_correct = sum_gt_check(png_format, grt_classes,
                                            num_of_each_class)
C
chenguowei01 已提交
526 527 528 529 530
            if not is_label_correct:
                label_wrong.append(line)

        file_list_check(list_file)
        imread_check()
C
chenguowei01 已提交
531
        label_gray_check()
W
wuzewu 已提交
532 533
        gt_check()
        image_type_check(img_dim)
L
LutaoChu 已提交
534
        max_img_size_statistics()
C
chenguowei01 已提交
535
        shape_check()
L
LutaoChu 已提交
536 537 538
        eval_crop_size_check(max_height, max_width, min_aspectratio,
                             max_aspectratio)

W
wuzewu 已提交
539 540

def check_test_dataset():
C
chenguowei01 已提交
541 542 543 544
    list_file = cfg.DATASET.TEST_FILE_LIST
    has_label = False
    with open(list_file) as fid:
        logger.info("\n-----------------------------\n3. Check test dataset...")
W
wuzewu 已提交
545 546
        lines = fid.readlines()
        for line in tqdm(lines):
C
chenguowei01 已提交
547 548
            line = line.strip()
            parts = line.split(cfg.DATASET.SEPARATOR)
W
wuzewu 已提交
549 550
            if len(parts) == 1:
                img_name = parts
C
chenguowei01 已提交
551 552 553 554 555 556
                img_path = os.path.join(cfg.DATASET.DATA_DIR, img_name[0])
                try:
                    img = cv2_imread(img_path, cv2.IMREAD_UNCHANGED)
                except Exception as e:
                    imread_failed.append((line, str(e)))
                    continue
W
wuzewu 已提交
557
            elif len(parts) == 2:
C
chenguowei01 已提交
558
                has_label = True
W
wuzewu 已提交
559 560 561
                img_name, grt_name = parts[0], parts[1]
                img_path = os.path.join(cfg.DATASET.DATA_DIR, img_name)
                grt_path = os.path.join(cfg.DATASET.DATA_DIR, grt_name)
C
chenguowei01 已提交
562 563
                try:
                    img = cv2_imread(img_path, cv2.IMREAD_UNCHANGED)
C
chenguowei01 已提交
564
                    grt = cv2_imread(grt_path, cv2.IMREAD_UNCHANGED)
C
chenguowei01 已提交
565
                except Exception as e:
L
LutaoChu 已提交
566
                    imread_failed.append((line, str(e)))
C
chenguowei01 已提交
567
                    continue
C
chenguowei01 已提交
568

C
chenguowei01 已提交
569 570
                is_gray = is_label_gray(grt)
                if not is_gray:
C
chenguowei01 已提交
571
                    label_gray_wrong.append(line)
C
chenguowei01 已提交
572
                    grt = cv2.cvtColor(grt, cv2.COLOR_BGR2GRAY)
W
wuzewu 已提交
573 574
                is_equal_img_grt_shape = image_label_shape_check(img, grt)
                if not is_equal_img_grt_shape:
C
chenguowei01 已提交
575
                    shape_unequal_image.append(line)
L
LutaoChu 已提交
576 577
                png_format, grt_classes, num_of_each_class = ground_truth_check(
                    grt, grt_path)
C
chenguowei01 已提交
578 579
                if not png_format:
                    png_format_wrong_image.append(line)
L
LutaoChu 已提交
580 581
                is_label_correct = sum_gt_check(png_format, grt_classes,
                                                num_of_each_class)
C
chenguowei01 已提交
582 583
                if not is_label_correct:
                    label_wrong.append(line)
W
wuzewu 已提交
584
            else:
C
chenguowei01 已提交
585
                list_wrong.append(lines)
W
wuzewu 已提交
586
                continue
C
chenguowei01 已提交
587 588 589 590 591 592
            get_image_max_height_width(img)
            get_image_min_max_aspectratio(img)
            get_image_dim(img)

        file_list_check(list_file)
        imread_check()
C
chenguowei01 已提交
593 594
        if has_label:
            label_gray_check()
C
chenguowei01 已提交
595 596
        if has_label:
            gt_check()
W
wuzewu 已提交
597
        image_type_check(img_dim)
L
LutaoChu 已提交
598
        max_img_size_statistics()
C
chenguowei01 已提交
599 600
        if has_label:
            shape_check()
L
LutaoChu 已提交
601 602 603
        eval_crop_size_check(max_height, max_width, min_aspectratio,
                             max_aspectratio)

W
wuzewu 已提交
604 605 606 607

def main(args):
    if args.cfg_file is not None:
        cfg.update_from_file(args.cfg_file)
L
LutaoChu 已提交
608
    cfg.check_and_infer()
C
chenguowei01 已提交
609
    logger.info(pprint.pformat(cfg))
W
wuzewu 已提交
610 611 612 613 614 615 616 617 618 619 620 621

    init_global_variable()
    check_train_dataset()

    init_global_variable()
    check_val_dataset()

    init_global_variable()
    check_test_dataset()

    inf_resize_value_check()

L
LutaoChu 已提交
622 623 624
    print("\nDetailed error information can be viewed in detail.log file.")


W
wuzewu 已提交
625 626
if __name__ == "__main__":
    args = parse_args()
C
chenguowei01 已提交
627 628 629 630 631 632 633 634 635 636 637
    logger = logging.getLogger()
    logger.setLevel('DEBUG')
    BASIC_FORMAT = "%(message)s"
    formatter = logging.Formatter(BASIC_FORMAT)
    sh = logging.StreamHandler()
    sh.setFormatter(formatter)
    sh.setLevel('INFO')
    th = logging.FileHandler('detail.log', 'w')
    th.setFormatter(formatter)
    logger.addHandler(sh)
    logger.addHandler(th)
W
wuzewu 已提交
638
    main(args)